diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -1,5348 +1,14454 @@
 {
-  "best_global_step": null,
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 1.8740617180984154,
-  "eval_steps": 500,
-  "global_step": 280,
+  "best_global_step": 600,
+  "best_metric": 0.24665305,
+  "best_model_checkpoint": "/data2/kdd/crag/cjz/output/dpo_phase2/v1-20250528-012105/checkpoint-600",
+  "epoch": 1.6139863670790204,
+  "eval_steps": 300,
+  "global_step": 800,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0234375,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 14.6328125,
-      "completions/min_length": 2.0,
-      "epoch": 0.0110803324099723,
-      "grad_norm": 1.5475432762019699,
-      "kl": 0.0,
-      "learning_rate": 2.2222222222222223e-05,
-      "loss": 5.587935447692871e-09,
-      "memory(GiB)": 77.28,
-      "reward": 0.25,
-      "reward_std": 0.7772719860076904,
-      "rewards/Response_no_think_reward/mean": 0.25,
-      "rewards/Response_no_think_reward/std": 1.4795188903808594,
+      "epoch": 0.002019691996970462,
+      "grad_norm": 12.761810302734375,
+      "learning_rate": 4.000000000000001e-06,
+      "logits/chosen": -0.5775864720344543,
+      "logits/rejected": -0.7030954360961914,
+      "logps/chosen": -6.236894130706787,
+      "logps/rejected": -24.46524429321289,
+      "loss": 1.2380319833755493,
+      "memory(GiB)": 32.64,
+      "nll_loss": 0.5448847413063049,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
       "step": 1,
-      "train_speed(iter/s)": 0.001043
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "epoch": 0.0221606648199446,
-      "grad_norm": 1.552548658770951,
-      "kl": 0.0,
-      "learning_rate": 4.4444444444444447e-05,
-      "loss": 5.587935447692871e-09,
-      "memory(GiB)": 78.06,
+      "train_speed(iter/s)": 0.0187
+    },
+    {
+      "epoch": 0.004039383993940924,
+      "grad_norm": 8.313699722290039,
+      "learning_rate": 8.000000000000001e-06,
+      "logits/chosen": -0.8238492012023926,
+      "logits/rejected": -0.8350682258605957,
+      "logps/chosen": -5.732274055480957,
+      "logps/rejected": -11.927969932556152,
+      "loss": 1.4877614974975586,
+      "memory(GiB)": 32.64,
+      "nll_loss": 0.7946141958236694,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
       "step": 2,
-      "train_speed(iter/s)": 0.001737
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.03125,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 17.8828125,
-      "completions/min_length": 3.0,
-      "epoch": 0.0332409972299169,
-      "grad_norm": 1.291839688026328,
-      "kl": 0.0012106498143111821,
-      "learning_rate": 6.666666666666667e-05,
-      "loss": 0.0004578572406899184,
-      "memory(GiB)": 78.06,
-      "reward": 0.25,
-      "reward_std": 0.8250656127929688,
-      "rewards/Response_no_think_reward/mean": 0.25,
-      "rewards/Response_no_think_reward/std": 1.4634658098220825,
+      "train_speed(iter/s)": 0.020044
+    },
+    {
+      "epoch": 0.006059075990911386,
+      "grad_norm": 21.77679443359375,
+      "learning_rate": 1.2e-05,
+      "logits/chosen": -0.6877153515815735,
+      "logits/rejected": -0.7822633981704712,
+      "logps/chosen": -7.046360015869141,
+      "logps/rejected": -14.666805267333984,
+      "loss": 1.5761629343032837,
+      "memory(GiB)": 35.3,
+      "nll_loss": 0.8832842707633972,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.0033746182452887297,
+      "rewards/margins": 0.0006902526365593076,
+      "rewards/rejected": 0.0026843654923141003,
       "step": 3,
-      "train_speed(iter/s)": 0.001451
-    },
-    {
-      "clip_ratio/high_max": 0.004197790374746546,
-      "clip_ratio/high_mean": 0.004197790374746546,
-      "clip_ratio/low_mean": 0.012020835361909121,
-      "clip_ratio/low_min": 0.012020835361909121,
-      "clip_ratio/region_mean": 0.01621862585307099,
-      "epoch": 0.0443213296398892,
-      "grad_norm": 1.0301802967661304,
-      "kl": 0.005412253500253428,
-      "learning_rate": 8.888888888888889e-05,
-      "loss": -0.005840146914124489,
-      "memory(GiB)": 78.06,
+      "train_speed(iter/s)": 0.020481
+    },
+    {
+      "epoch": 0.008078767987881848,
+      "grad_norm": 12.999642372131348,
+      "learning_rate": 1.6000000000000003e-05,
+      "logits/chosen": -0.7259287238121033,
+      "logits/rejected": -0.8152387142181396,
+      "logps/chosen": -4.8744587898254395,
+      "logps/rejected": -14.308876991271973,
+      "loss": 1.4366666078567505,
+      "memory(GiB)": 35.3,
+      "nll_loss": 0.7484119534492493,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0021331249736249447,
+      "rewards/margins": 0.009978920221328735,
+      "rewards/rejected": -0.007845795713365078,
       "step": 4,
-      "train_speed(iter/s)": 0.001769
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.002237339736893773,
-      "clip_ratio/low_min": 0.002237339736893773,
-      "clip_ratio/region_mean": 0.002237339736893773,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 12.5546875,
-      "completions/min_length": 3.0,
-      "epoch": 0.055401662049861494,
-      "grad_norm": 2.027395284218205,
-      "kl": 0.014184385421685874,
-      "learning_rate": 0.00011111111111111112,
-      "loss": 0.0008896891959011555,
-      "memory(GiB)": 78.06,
-      "reward": 0.8515625,
-      "reward_std": 0.5118520259857178,
-      "rewards/Response_no_think_reward/mean": 0.8515625,
-      "rewards/Response_no_think_reward/std": 1.4147136211395264,
+      "train_speed(iter/s)": 0.020724
+    },
+    {
+      "epoch": 0.01009845998485231,
+      "grad_norm": 12.841984748840332,
+      "learning_rate": 2e-05,
+      "logits/chosen": -0.681520402431488,
+      "logits/rejected": -0.7851653695106506,
+      "logps/chosen": -5.029726982116699,
+      "logps/rejected": -16.70825958251953,
+      "loss": 1.2887729406356812,
+      "memory(GiB)": 35.3,
+      "nll_loss": 0.6066412329673767,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.02295457012951374,
+      "rewards/margins": 0.022426389157772064,
+      "rewards/rejected": 0.0005281821941025555,
       "step": 5,
-      "train_speed(iter/s)": 0.00164
-    },
-    {
-      "clip_ratio/high_max": 0.013844632252585143,
-      "clip_ratio/high_mean": 0.013844632252585143,
-      "clip_ratio/low_mean": 0.043920744908973575,
-      "clip_ratio/low_min": 0.043920744908973575,
-      "clip_ratio/region_mean": 0.05776537861675024,
-      "epoch": 0.0664819944598338,
-      "grad_norm": 6.212168397356187,
-      "kl": 1.7603500080003869,
-      "learning_rate": 0.00013333333333333334,
-      "loss": 0.01842591166496277,
-      "memory(GiB)": 79.98,
+      "train_speed(iter/s)": 0.020852
+    },
+    {
+      "epoch": 0.012118151981822771,
+      "grad_norm": 13.818325996398926,
+      "learning_rate": 2.4e-05,
+      "logits/chosen": -0.6980650424957275,
+      "logits/rejected": -0.8158392310142517,
+      "logps/chosen": -4.0964484214782715,
+      "logps/rejected": -15.3903169631958,
+      "loss": 1.3540700674057007,
+      "memory(GiB)": 35.3,
+      "nll_loss": 0.6896780133247375,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.06763897091150284,
+      "rewards/margins": 0.06137145683169365,
+      "rewards/rejected": 0.00626751035451889,
       "step": 6,
-      "train_speed(iter/s)": 0.00185
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.001959657238330692,
-      "clip_ratio/low_min": 0.001959657238330692,
-      "clip_ratio/region_mean": 0.001959657238330692,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 70.0,
-      "completions/mean_length": 10.359375,
-      "completions/min_length": 2.0,
-      "epoch": 0.07756232686980609,
-      "grad_norm": 3.537674599924329,
-      "kl": 0.33545287084416486,
-      "learning_rate": 0.00015555555555555556,
-      "loss": 0.0018048422643914819,
-      "memory(GiB)": 79.98,
-      "reward": 0.53125,
-      "reward_std": 0.6121620535850525,
-      "rewards/Response_no_think_reward/mean": 0.53125,
-      "rewards/Response_no_think_reward/std": 1.3685873746871948,
+      "train_speed(iter/s)": 0.020943
+    },
+    {
+      "epoch": 0.014137843978793235,
+      "grad_norm": 8.670878410339355,
+      "learning_rate": 2.8000000000000003e-05,
+      "logits/chosen": -0.6072264909744263,
+      "logits/rejected": -0.7461887001991272,
+      "logps/chosen": -6.871906757354736,
+      "logps/rejected": -19.193540573120117,
+      "loss": 1.1981984376907349,
+      "memory(GiB)": 35.3,
+      "nll_loss": 0.5265603065490723,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.04815364256501198,
+      "rewards/margins": 0.053483281284570694,
+      "rewards/rejected": -0.005329643841832876,
       "step": 7,
-      "train_speed(iter/s)": 0.001773
-    },
-    {
-      "clip_ratio/high_max": 0.014042179333046079,
-      "clip_ratio/high_mean": 0.014042179333046079,
-      "clip_ratio/low_mean": 0.03925089433323592,
-      "clip_ratio/low_min": 0.03925089433323592,
-      "clip_ratio/region_mean": 0.053293074015527964,
-      "epoch": 0.0886426592797784,
-      "grad_norm": 2.64201486377639,
-      "kl": 1.295830228133127,
-      "learning_rate": 0.00017777777777777779,
-      "loss": 0.011679837480187416,
-      "memory(GiB)": 79.98,
+      "train_speed(iter/s)": 0.020996
+    },
+    {
+      "epoch": 0.016157535975763696,
+      "grad_norm": 4.796175003051758,
+      "learning_rate": 3.2000000000000005e-05,
+      "logits/chosen": -0.7197842597961426,
+      "logits/rejected": -0.7792064547538757,
+      "logps/chosen": -7.072351455688477,
+      "logps/rejected": -14.618125915527344,
+      "loss": 1.1894290447235107,
+      "memory(GiB)": 35.3,
+      "nll_loss": 0.4693164825439453,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.13033029437065125,
+      "rewards/margins": -0.01030611153692007,
+      "rewards/rejected": 0.1406363844871521,
       "step": 8,
-      "train_speed(iter/s)": 0.001932
-    },
-    {
-      "clip_ratio/high_max": 0.0011001251987181604,
-      "clip_ratio/high_mean": 0.0011001251987181604,
-      "clip_ratio/low_mean": 0.008499634364852682,
-      "clip_ratio/low_min": 0.008499634364852682,
-      "clip_ratio/region_mean": 0.009599759563570842,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 62.0,
-      "completions/mean_length": 11.4375,
-      "completions/min_length": 2.0,
-      "epoch": 0.0997229916897507,
-      "grad_norm": 4.1929387792795065,
-      "kl": 2.5014250652166083,
-      "learning_rate": 0.0002,
-      "loss": 0.016260012984275818,
-      "memory(GiB)": 79.98,
-      "reward": 0.2734375,
-      "reward_std": 0.8412302732467651,
-      "rewards/Response_no_think_reward/mean": 0.2734375,
-      "rewards/Response_no_think_reward/std": 1.384474277496338,
+      "train_speed(iter/s)": 0.021046
+    },
+    {
+      "epoch": 0.018177227972734158,
+      "grad_norm": 6.6790852546691895,
+      "learning_rate": 3.6e-05,
+      "logits/chosen": -0.689906656742096,
+      "logits/rejected": -0.8183166980743408,
+      "logps/chosen": -1.6468329429626465,
+      "logps/rejected": -13.756692886352539,
+      "loss": 0.8348377346992493,
+      "memory(GiB)": 35.3,
+      "nll_loss": 0.2248959094285965,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.1438780128955841,
+      "rewards/margins": 0.22025901079177856,
+      "rewards/rejected": -0.07638098299503326,
       "step": 9,
-      "train_speed(iter/s)": 0.001887
-    },
-    {
-      "clip_ratio/high_max": 0.02779325417941436,
-      "clip_ratio/high_mean": 0.02779325417941436,
-      "clip_ratio/low_mean": 0.07855538238072768,
-      "clip_ratio/low_min": 0.07855538238072768,
-      "clip_ratio/region_mean": 0.10634863609448075,
-      "epoch": 0.11080332409972299,
-      "grad_norm": 990.7609804346584,
-      "kl": 1.6530181597918272,
-      "learning_rate": 0.00019998312416333227,
-      "loss": 1.336751103401184,
-      "memory(GiB)": 79.98,
+      "train_speed(iter/s)": 0.021091
+    },
+    {
+      "epoch": 0.02019691996970462,
+      "grad_norm": 11.680863380432129,
+      "learning_rate": 4e-05,
+      "logits/chosen": -0.7386271953582764,
+      "logits/rejected": -0.8098344206809998,
+      "logps/chosen": -3.001471996307373,
+      "logps/rejected": -16.38292694091797,
+      "loss": 1.0789214372634888,
+      "memory(GiB)": 35.3,
+      "nll_loss": 0.5013705492019653,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.18567466735839844,
+      "rewards/margins": 0.40445780754089355,
+      "rewards/rejected": -0.21878314018249512,
       "step": 10,
-      "train_speed(iter/s)": 0.002015
-    },
-    {
-      "clip_ratio/high_max": 0.001354054023977369,
-      "clip_ratio/high_mean": 0.001354054023977369,
-      "clip_ratio/low_mean": 0.004174399145995267,
-      "clip_ratio/low_min": 0.004174399145995267,
-      "clip_ratio/region_mean": 0.005528453169972636,
-      "completions/clipped_ratio": 0.0546875,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 26.453125,
-      "completions/min_length": 3.0,
-      "epoch": 0.12188365650969529,
-      "grad_norm": 1.115304569915306,
-      "kl": 0.3547552889212966,
-      "learning_rate": 0.00019993250234920636,
-      "loss": 0.005897670052945614,
-      "memory(GiB)": 79.98,
-      "reward": 0.5234375,
-      "reward_std": 0.876558780670166,
-      "rewards/Response_no_think_reward/mean": 0.5234375,
-      "rewards/Response_no_think_reward/std": 1.4251114130020142,
+      "train_speed(iter/s)": 0.021135
+    },
+    {
+      "epoch": 0.02221661196667508,
+      "grad_norm": 6.016479015350342,
+      "learning_rate": 4.4000000000000006e-05,
+      "logits/chosen": -0.6199511289596558,
+      "logits/rejected": -0.7641665935516357,
+      "logps/chosen": -1.786482810974121,
+      "logps/rejected": -23.814146041870117,
+      "loss": 0.6213081479072571,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.17753979563713074,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1519775092601776,
+      "rewards/margins": 0.6798387169837952,
+      "rewards/rejected": -0.5278611779212952,
       "step": 11,
-      "train_speed(iter/s)": 0.001964
-    },
-    {
-      "clip_ratio/high_max": 0.02347446102066897,
-      "clip_ratio/high_mean": 0.02347446102066897,
-      "clip_ratio/low_mean": 0.0428259114123648,
-      "clip_ratio/low_min": 0.0428259114123648,
-      "clip_ratio/region_mean": 0.06630037224385887,
-      "epoch": 0.1329639889196676,
-      "grad_norm": 2.183295109715178,
-      "kl": 0.6218942860141397,
-      "learning_rate": 0.00019984815164333163,
-      "loss": 0.007074224762618542,
-      "memory(GiB)": 79.98,
+      "train_speed(iter/s)": 0.021205
+    },
+    {
+      "epoch": 0.024236303963645543,
+      "grad_norm": 19.226726531982422,
+      "learning_rate": 4.8e-05,
+      "logits/chosen": -0.7735041975975037,
+      "logits/rejected": -0.8182462453842163,
+      "logps/chosen": -6.024149417877197,
+      "logps/rejected": -14.866905212402344,
+      "loss": 1.467666506767273,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.8242200016975403,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.03484102711081505,
+      "rewards/margins": 0.42743903398513794,
+      "rewards/rejected": -0.4622800350189209,
       "step": 12,
-      "train_speed(iter/s)": 0.00207
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.001206563669256866,
-      "clip_ratio/low_min": 0.001206563669256866,
-      "clip_ratio/region_mean": 0.001206563669256866,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 83.0,
-      "completions/mean_length": 14.0234375,
-      "completions/min_length": 4.0,
-      "epoch": 0.1440443213296399,
-      "grad_norm": 1.0424369292135751,
-      "kl": 0.40942037590866676,
-      "learning_rate": 0.00019973010051548275,
-      "loss": 0.004908258095383644,
-      "memory(GiB)": 79.98,
-      "reward": 0.5625,
-      "reward_std": 0.7038782835006714,
-      "rewards/Response_no_think_reward/mean": 0.5625,
-      "rewards/Response_no_think_reward/std": 1.2720495462417603,
+      "train_speed(iter/s)": 0.021226
+    },
+    {
+      "epoch": 0.026255995960616008,
+      "grad_norm": 3.4483253955841064,
+      "learning_rate": 5.2000000000000004e-05,
+      "logits/chosen": -0.754300594329834,
+      "logits/rejected": -0.7772647738456726,
+      "logps/chosen": -3.619762659072876,
+      "logps/rejected": -17.524477005004883,
+      "loss": 0.7115153074264526,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.2013324648141861,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.10463516414165497,
+      "rewards/margins": 0.7878110408782959,
+      "rewards/rejected": -0.6831759214401245,
       "step": 13,
-      "train_speed(iter/s)": 0.002042
-    },
-    {
-      "clip_ratio/high_max": 0.017414433998055756,
-      "clip_ratio/high_mean": 0.017414433998055756,
-      "clip_ratio/low_mean": 0.028266766399610788,
-      "clip_ratio/low_min": 0.028266766399610788,
-      "clip_ratio/region_mean": 0.04568120092153549,
-      "epoch": 0.15512465373961218,
-      "grad_norm": 0.650161789807741,
-      "kl": 0.4890678570009186,
-      "learning_rate": 0.00019957838880989078,
-      "loss": -0.0075666941702365875,
-      "memory(GiB)": 79.98,
+      "train_speed(iter/s)": 0.021242
+    },
+    {
+      "epoch": 0.02827568795758647,
+      "grad_norm": 15.451835632324219,
+      "learning_rate": 5.6000000000000006e-05,
+      "logits/chosen": -0.7475101947784424,
+      "logits/rejected": -0.8498263359069824,
+      "logps/chosen": -3.9441192150115967,
+      "logps/rejected": -21.486186981201172,
+      "loss": 0.9213531613349915,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.5798130035400391,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.09564964473247528,
+      "rewards/margins": 1.039682149887085,
+      "rewards/rejected": -0.9440325498580933,
       "step": 14,
-      "train_speed(iter/s)": 0.002135
-    },
-    {
-      "clip_ratio/high_max": 0.0018629207770572975,
-      "clip_ratio/high_mean": 0.0018629207770572975,
-      "clip_ratio/low_mean": 0.0069178942940197885,
-      "clip_ratio/low_min": 0.0069178942940197885,
-      "clip_ratio/region_mean": 0.00878081505652517,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 17.1953125,
-      "completions/min_length": 4.0,
-      "epoch": 0.16620498614958448,
-      "grad_norm": 2.2362713456466605,
-      "kl": 0.23374600778333843,
-      "learning_rate": 0.00019939306773179497,
-      "loss": 0.0009349192259833217,
-      "memory(GiB)": 79.98,
-      "reward": 0.6875,
-      "reward_std": 0.8529120683670044,
-      "rewards/Response_no_think_reward/mean": 0.6875,
-      "rewards/Response_no_think_reward/std": 1.3265905380249023,
+      "train_speed(iter/s)": 0.021265
+    },
+    {
+      "epoch": 0.03029537995455693,
+      "grad_norm": 3.8979263305664062,
+      "learning_rate": 6e-05,
+      "logits/chosen": -0.6522198915481567,
+      "logits/rejected": -0.7548784017562866,
+      "logps/chosen": -2.765364408493042,
+      "logps/rejected": -21.602596282958984,
+      "loss": 0.6480428576469421,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.23036827147006989,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.13951444625854492,
+      "rewards/margins": 0.8476398587226868,
+      "rewards/rejected": -0.7081253528594971,
       "step": 15,
-      "train_speed(iter/s)": 0.002091
-    },
-    {
-      "clip_ratio/high_max": 0.03511151310522109,
-      "clip_ratio/high_mean": 0.03511151310522109,
-      "clip_ratio/low_mean": 0.028677020454779267,
-      "clip_ratio/low_min": 0.028677020454779267,
-      "clip_ratio/region_mean": 0.06378853344358504,
-      "epoch": 0.1772853185595568,
-      "grad_norm": 3.3548685452904934,
-      "kl": 0.25148704896855634,
-      "learning_rate": 0.00019917419983016025,
-      "loss": 0.017475975677371025,
-      "memory(GiB)": 79.98,
+      "train_speed(iter/s)": 0.021272
+    },
+    {
+      "epoch": 0.03231507195152739,
+      "grad_norm": 7.998473644256592,
+      "learning_rate": 6.400000000000001e-05,
+      "logits/chosen": -0.7795235514640808,
+      "logits/rejected": -0.8571889400482178,
+      "logps/chosen": -2.1100573539733887,
+      "logps/rejected": -13.082208633422852,
+      "loss": 0.8766142129898071,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.41500476002693176,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.11589755117893219,
+      "rewards/margins": 0.6957570910453796,
+      "rewards/rejected": -0.5798596143722534,
       "step": 16,
-      "train_speed(iter/s)": 0.002171
-    },
-    {
-      "clip_ratio/high_max": 0.000469924823846668,
-      "clip_ratio/high_mean": 0.000469924823846668,
-      "clip_ratio/low_mean": 0.0014688223309349269,
-      "clip_ratio/low_min": 0.0014688223309349269,
-      "clip_ratio/region_mean": 0.0019387471547815949,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 18.2265625,
-      "completions/min_length": 6.0,
-      "epoch": 0.1883656509695291,
-      "grad_norm": 0.6996032916694407,
-      "kl": 0.138063008276049,
-      "learning_rate": 0.00019892185897656578,
-      "loss": 0.0021217623725533485,
-      "memory(GiB)": 79.98,
-      "reward": 0.5,
-      "reward_std": 0.5098158717155457,
-      "rewards/Response_no_think_reward/mean": 0.5,
-      "rewards/Response_no_think_reward/std": 1.4086347818374634,
+      "train_speed(iter/s)": 0.021293
+    },
+    {
+      "epoch": 0.034334763948497854,
+      "grad_norm": 2.908738613128662,
+      "learning_rate": 6.800000000000001e-05,
+      "logits/chosen": -0.7900448441505432,
+      "logits/rejected": -0.8519578576087952,
+      "logps/chosen": -3.0429553985595703,
+      "logps/rejected": -13.749216079711914,
+      "loss": 0.7060399651527405,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.27812597155570984,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.40308868885040283,
+      "rewards/margins": 0.7511462569236755,
+      "rewards/rejected": -0.3480575382709503,
       "step": 17,
-      "train_speed(iter/s)": 0.002127
-    },
-    {
-      "clip_ratio/high_max": 0.01048101403284818,
-      "clip_ratio/high_mean": 0.01048101403284818,
-      "clip_ratio/low_mean": 0.028478411200921983,
-      "clip_ratio/low_min": 0.028478411200921983,
-      "clip_ratio/region_mean": 0.03895942587405443,
-      "epoch": 0.1994459833795014,
-      "grad_norm": 0.7437416506383379,
-      "kl": 0.2438321103884391,
-      "learning_rate": 0.00019863613034027224,
-      "loss": -0.007616878021508455,
-      "memory(GiB)": 79.98,
+      "train_speed(iter/s)": 0.021313
+    },
+    {
+      "epoch": 0.036354455945468316,
+      "grad_norm": 4.66619348526001,
+      "learning_rate": 7.2e-05,
+      "logits/chosen": -0.7611753344535828,
+      "logits/rejected": -0.8464637398719788,
+      "logps/chosen": -3.652045488357544,
+      "logps/rejected": -19.29679298400879,
+      "loss": 0.9751962423324585,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.4190734326839447,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.08509862422943115,
+      "rewards/margins": 0.42228081822395325,
+      "rewards/rejected": -0.3371821641921997,
       "step": 18,
-      "train_speed(iter/s)": 0.002199
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0009733444603625685,
-      "clip_ratio/low_min": 0.0009733444603625685,
-      "clip_ratio/region_mean": 0.0009733444603625685,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 20.703125,
-      "completions/min_length": 6.0,
-      "epoch": 0.21052631578947367,
-      "grad_norm": 0.8303775863701255,
-      "kl": 0.09481111937202513,
-      "learning_rate": 0.0001983171103594755,
-      "loss": 0.0022672354243695736,
-      "memory(GiB)": 79.98,
-      "reward": -0.078125,
-      "reward_std": 0.6301807761192322,
-      "rewards/Response_no_think_reward/mean": -0.078125,
-      "rewards/Response_no_think_reward/std": 1.3666982650756836,
+      "train_speed(iter/s)": 0.021323
+    },
+    {
+      "epoch": 0.03837414794243878,
+      "grad_norm": 4.54453706741333,
+      "learning_rate": 7.6e-05,
+      "logits/chosen": -0.7284001708030701,
+      "logits/rejected": -0.7670996189117432,
+      "logps/chosen": -6.018934726715088,
+      "logps/rejected": -13.253512382507324,
+      "loss": 1.2025885581970215,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.45651721954345703,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0025453297421336174,
+      "rewards/margins": 0.006808534264564514,
+      "rewards/rejected": -0.004263207316398621,
       "step": 19,
-      "train_speed(iter/s)": 0.002152
-    },
-    {
-      "clip_ratio/high_max": 0.007191620621597394,
-      "clip_ratio/high_mean": 0.007191620621597394,
-      "clip_ratio/low_mean": 0.03550086636096239,
-      "clip_ratio/low_min": 0.03550086636096239,
-      "clip_ratio/region_mean": 0.0426924874773249,
-      "epoch": 0.22160664819944598,
-      "grad_norm": 1.7213808531720889,
-      "kl": 1.6638920252444223,
-      "learning_rate": 0.0001979649067087574,
-      "loss": 0.006861768197268248,
-      "memory(GiB)": 79.98,
+      "train_speed(iter/s)": 0.021331
+    },
+    {
+      "epoch": 0.04039383993940924,
+      "grad_norm": 2.965843677520752,
+      "learning_rate": 8e-05,
+      "logits/chosen": -0.674504280090332,
+      "logits/rejected": -0.7847579121589661,
+      "logps/chosen": -3.122269630432129,
+      "logps/rejected": -16.265222549438477,
+      "loss": 0.9356863498687744,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.2812672257423401,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.12974591553211212,
+      "rewards/margins": 0.18470463156700134,
+      "rewards/rejected": -0.05495870113372803,
       "step": 20,
-      "train_speed(iter/s)": 0.002217
-    },
-    {
-      "clip_ratio/high_max": 0.0004032258002553135,
-      "clip_ratio/high_mean": 0.0004032258002553135,
-      "clip_ratio/low_mean": 0.003614576125983149,
-      "clip_ratio/low_min": 0.003614576125983149,
-      "clip_ratio/region_mean": 0.004017801926238462,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 17.390625,
-      "completions/min_length": 6.0,
-      "epoch": 0.23268698060941828,
-      "grad_norm": 1.5808946015405074,
-      "kl": 0.1687323283404112,
-      "learning_rate": 0.00019757963826274357,
-      "loss": 0.003855248913168907,
-      "memory(GiB)": 79.98,
-      "reward": 0.5546875,
-      "reward_std": 0.8033354878425598,
-      "rewards/Response_no_think_reward/mean": 0.5546875,
-      "rewards/Response_no_think_reward/std": 1.3328590393066406,
+      "train_speed(iter/s)": 0.021338
+    },
+    {
+      "epoch": 0.0424135319363797,
+      "grad_norm": 3.544822931289673,
+      "learning_rate": 8.4e-05,
+      "logits/chosen": -0.8259390592575073,
+      "logits/rejected": -0.8455062508583069,
+      "logps/chosen": -6.160022258758545,
+      "logps/rejected": -8.875776290893555,
+      "loss": 1.0958666801452637,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.4123613238334656,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.17430700361728668,
+      "rewards/margins": 0.11852678656578064,
+      "rewards/rejected": 0.05578019842505455,
       "step": 21,
-      "train_speed(iter/s)": 0.002189
-    },
-    {
-      "clip_ratio/high_max": 0.012200821249280125,
-      "clip_ratio/high_mean": 0.012200821249280125,
-      "clip_ratio/low_mean": 0.04007338697556406,
-      "clip_ratio/low_min": 0.04007338697556406,
-      "clip_ratio/region_mean": 0.05227420857409015,
-      "epoch": 0.24376731301939059,
-      "grad_norm": 0.8850881940917656,
-      "kl": 0.48105100472457707,
-      "learning_rate": 0.0001971614350559814,
-      "loss": -0.004205920733511448,
-      "memory(GiB)": 79.98,
+      "train_speed(iter/s)": 0.021349
+    },
+    {
+      "epoch": 0.04443322393335016,
+      "grad_norm": 2.8963093757629395,
+      "learning_rate": 8.800000000000001e-05,
+      "logits/chosen": -0.7398238778114319,
+      "logits/rejected": -0.83761066198349,
+      "logps/chosen": -2.3067612648010254,
+      "logps/rejected": -13.408963203430176,
+      "loss": 0.9400858879089355,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.26946064829826355,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.1603003889322281,
+      "rewards/margins": 0.15373794734477997,
+      "rewards/rejected": 0.00656244158744812,
       "step": 22,
-      "train_speed(iter/s)": 0.002248
-    },
-    {
-      "clip_ratio/high_max": 0.0008194574620574713,
-      "clip_ratio/high_mean": 0.0008194574620574713,
-      "clip_ratio/low_mean": 0.000811688310932368,
-      "clip_ratio/low_min": 0.000811688310932368,
-      "clip_ratio/region_mean": 0.0016311457729898393,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 15.28125,
-      "completions/min_length": 6.0,
-      "epoch": 0.2548476454293629,
-      "grad_norm": 0.6954813297644332,
-      "kl": 0.11993603070732206,
-      "learning_rate": 0.0001967104382390511,
-      "loss": 0.001324990182183683,
-      "memory(GiB)": 79.98,
-      "reward": 0.78125,
-      "reward_std": 0.5077463984489441,
-      "rewards/Response_no_think_reward/mean": 0.78125,
-      "rewards/Response_no_think_reward/std": 1.3970582485198975,
+      "train_speed(iter/s)": 0.021358
+    },
+    {
+      "epoch": 0.046452915930320623,
+      "grad_norm": 3.5623438358306885,
+      "learning_rate": 9.200000000000001e-05,
+      "logits/chosen": -0.6750966906547546,
+      "logits/rejected": -0.7796332836151123,
+      "logps/chosen": -3.6001622676849365,
+      "logps/rejected": -14.151020050048828,
+      "loss": 1.0653876066207886,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.4920450747013092,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.2231697142124176,
+      "rewards/margins": 0.34812790155410767,
+      "rewards/rejected": -0.12495820969343185,
       "step": 23,
-      "train_speed(iter/s)": 0.002219
-    },
-    {
-      "clip_ratio/high_max": 0.014543175988364965,
-      "clip_ratio/high_mean": 0.014543175988364965,
-      "clip_ratio/low_mean": 0.027941336738877,
-      "clip_ratio/low_min": 0.027941336738877,
-      "clip_ratio/region_mean": 0.04248451231978834,
-      "epoch": 0.2659279778393352,
-      "grad_norm": 0.46891914008721675,
-      "kl": 0.11759324668673798,
-      "learning_rate": 0.00019622680003092503,
-      "loss": -0.009693928062915802,
-      "memory(GiB)": 79.98,
+      "train_speed(iter/s)": 0.021362
+    },
+    {
+      "epoch": 0.048472607927291085,
+      "grad_norm": 3.5425710678100586,
+      "learning_rate": 9.6e-05,
+      "logits/chosen": -0.7490979433059692,
+      "logits/rejected": -0.8129041194915771,
+      "logps/chosen": -2.327343463897705,
+      "logps/rejected": -10.883130073547363,
+      "loss": 0.921205461025238,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.2920321822166443,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.09307915717363358,
+      "rewards/margins": 0.1912679672241211,
+      "rewards/rejected": -0.09818882495164871,
       "step": 24,
-      "train_speed(iter/s)": 0.002273
-    },
-    {
-      "clip_ratio/high_max": 0.0006530559621751308,
-      "clip_ratio/high_mean": 0.0006530559621751308,
-      "clip_ratio/low_mean": 0.0019866162620019168,
-      "clip_ratio/low_min": 0.0019866162620019168,
-      "clip_ratio/region_mean": 0.002639672253280878,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 12.9609375,
-      "completions/min_length": 4.0,
-      "epoch": 0.2770083102493075,
-      "grad_norm": 1.333182473379268,
-      "kl": 0.22027045290451497,
-      "learning_rate": 0.00019571068366759143,
-      "loss": 0.0026610023342072964,
-      "memory(GiB)": 79.98,
-      "reward": 0.4765625,
-      "reward_std": 0.8257243633270264,
-      "rewards/Response_no_think_reward/mean": 0.4765625,
-      "rewards/Response_no_think_reward/std": 1.3219220638275146,
+      "train_speed(iter/s)": 0.021367
+    },
+    {
+      "epoch": 0.05049229992426155,
+      "grad_norm": 4.050752639770508,
+      "learning_rate": 0.0001,
+      "logits/chosen": -0.6667495965957642,
+      "logits/rejected": -0.7579994201660156,
+      "logps/chosen": -6.037406921386719,
+      "logps/rejected": -19.315378189086914,
+      "loss": 0.8225780725479126,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.3186819851398468,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3055117130279541,
+      "rewards/margins": 0.4999930262565613,
+      "rewards/rejected": -0.19448129832744598,
       "step": 25,
-      "train_speed(iter/s)": 0.002248
-    },
-    {
-      "clip_ratio/high_max": 0.02311275008833036,
-      "clip_ratio/high_mean": 0.02311275008833036,
-      "clip_ratio/low_mean": 0.03803046100074425,
-      "clip_ratio/low_min": 0.03803046100074425,
-      "clip_ratio/region_mean": 0.06114321056520566,
-      "epoch": 0.2880886426592798,
-      "grad_norm": 0.7053648057917502,
-      "kl": 0.2949459235333052,
-      "learning_rate": 0.0001951622633469592,
-      "loss": -0.015641074627637863,
-      "memory(GiB)": 79.98,
+      "train_speed(iter/s)": 0.021364
+    },
+    {
+      "epoch": 0.052511991921232015,
+      "grad_norm": 3.7214102745056152,
+      "learning_rate": 0.00010400000000000001,
+      "logits/chosen": -0.7550854086875916,
+      "logits/rejected": -0.8332770466804504,
+      "logps/chosen": -3.076296329498291,
+      "logps/rejected": -13.947905540466309,
+      "loss": 1.0035364627838135,
+      "memory(GiB)": 38.2,
+      "nll_loss": 0.3730461597442627,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.03216666728258133,
+      "rewards/margins": 0.19776780903339386,
+      "rewards/rejected": -0.16560114920139313,
       "step": 26,
-      "train_speed(iter/s)": 0.002297
-    },
-    {
-      "clip_ratio/high_max": 0.0004111842135898769,
-      "clip_ratio/high_mean": 0.0004111842135898769,
-      "clip_ratio/low_mean": 0.0027908546617254615,
-      "clip_ratio/low_min": 0.0027908546617254615,
-      "clip_ratio/region_mean": 0.0032020388753153384,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 85.0,
-      "completions/mean_length": 16.875,
-      "completions/min_length": 4.0,
-      "epoch": 0.29916897506925205,
-      "grad_norm": 1.544635006921068,
-      "kl": 0.20233443519100547,
-      "learning_rate": 0.00019458172417006347,
-      "loss": 0.0030936466064304113,
-      "memory(GiB)": 79.98,
-      "reward": 0.6875,
-      "reward_std": 0.9086803197860718,
-      "rewards/Response_no_think_reward/mean": 0.6875,
-      "rewards/Response_no_think_reward/std": 1.3265905380249023,
+      "train_speed(iter/s)": 0.02137
+    },
+    {
+      "epoch": 0.05453168391820248,
+      "grad_norm": 7.799258708953857,
+      "learning_rate": 0.00010800000000000001,
+      "logits/chosen": -0.6102021932601929,
+      "logits/rejected": -0.7568979263305664,
+      "logps/chosen": -5.391008377075195,
+      "logps/rejected": -21.710432052612305,
+      "loss": 1.2860000133514404,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.6317353844642639,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.007743582129478455,
+      "rewards/margins": 0.258750855922699,
+      "rewards/rejected": -0.2510072588920593,
       "step": 27,
-      "train_speed(iter/s)": 0.002272
-    },
-    {
-      "clip_ratio/high_max": 0.03639351949095726,
-      "clip_ratio/high_mean": 0.03639351949095726,
-      "clip_ratio/low_mean": 0.024818695266731083,
-      "clip_ratio/low_min": 0.024818695266731083,
-      "clip_ratio/region_mean": 0.06121221440844238,
-      "epoch": 0.31024930747922436,
-      "grad_norm": 2.0012174050572105,
-      "kl": 0.24312824057415128,
-      "learning_rate": 0.00019396926207859084,
-      "loss": -0.008060472086071968,
-      "memory(GiB)": 79.98,
+      "train_speed(iter/s)": 0.021361
+    },
+    {
+      "epoch": 0.05655137591517294,
+      "grad_norm": 3.4134862422943115,
+      "learning_rate": 0.00011200000000000001,
+      "logits/chosen": -0.681388795375824,
+      "logits/rejected": -0.7808853983879089,
+      "logps/chosen": -5.752387523651123,
+      "logps/rejected": -19.651212692260742,
+      "loss": 1.1130492687225342,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.4609695374965668,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.029274294152855873,
+      "rewards/margins": 0.19754451513290405,
+      "rewards/rejected": -0.16827021539211273,
       "step": 28,
-      "train_speed(iter/s)": 0.002319
-    },
-    {
-      "clip_ratio/high_max": 0.0007382866751868278,
-      "clip_ratio/high_mean": 0.0007382866751868278,
-      "clip_ratio/low_mean": 0.0024169938114937395,
-      "clip_ratio/low_min": 0.0024169938114937395,
-      "clip_ratio/region_mean": 0.0031552804866805673,
-      "completions/clipped_ratio": 0.0234375,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 23.4140625,
-      "completions/min_length": 3.0,
-      "epoch": 0.32132963988919666,
-      "grad_norm": 1.0333627248580772,
-      "kl": 0.19911292963661253,
-      "learning_rate": 0.0001933250837887457,
-      "loss": 0.004103388637304306,
-      "memory(GiB)": 79.98,
-      "reward": 0.6640625,
-      "reward_std": 0.7995060086250305,
-      "rewards/Response_no_think_reward/mean": 0.6640625,
-      "rewards/Response_no_think_reward/std": 1.4323447942733765,
+      "train_speed(iter/s)": 0.021358
+    },
+    {
+      "epoch": 0.0585710679121434,
+      "grad_norm": 2.6125388145446777,
+      "learning_rate": 0.000116,
+      "logits/chosen": -0.6846833229064941,
+      "logits/rejected": -0.7732559442520142,
+      "logps/chosen": -2.147914171218872,
+      "logps/rejected": -14.99377727508545,
+      "loss": 0.6967631578445435,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.18463140726089478,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.3106439709663391,
+      "rewards/margins": 0.524000346660614,
+      "rewards/rejected": -0.2133564054965973,
       "step": 29,
-      "train_speed(iter/s)": 0.002285
-    },
-    {
-      "clip_ratio/high_max": 0.021052728639915586,
-      "clip_ratio/high_mean": 0.021052728639915586,
-      "clip_ratio/low_mean": 0.03886253567179665,
-      "clip_ratio/low_min": 0.03886253567179665,
-      "clip_ratio/region_mean": 0.05991526402067393,
-      "epoch": 0.33240997229916897,
-      "grad_norm": 0.4744156832935017,
-      "kl": 0.2653088476508856,
-      "learning_rate": 0.00019264940672148018,
-      "loss": -0.014724130742251873,
-      "memory(GiB)": 79.98,
+      "train_speed(iter/s)": 0.021362
+    },
+    {
+      "epoch": 0.06059075990911386,
+      "grad_norm": 3.1886954307556152,
+      "learning_rate": 0.00012,
+      "logits/chosen": -0.7671568393707275,
+      "logits/rejected": -0.8747203946113586,
+      "logps/chosen": -1.0704294443130493,
+      "logps/rejected": -15.217928886413574,
+      "loss": 0.6793683171272278,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.11626588553190231,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.2649898827075958,
+      "rewards/margins": 0.3756120800971985,
+      "rewards/rejected": -0.11062214523553848,
       "step": 30,
-      "train_speed(iter/s)": 0.002328
-    },
-    {
-      "clip_ratio/high_max": 0.0008336337341461331,
-      "clip_ratio/high_mean": 0.0008336337341461331,
-      "clip_ratio/low_mean": 0.0028033541166223586,
-      "clip_ratio/low_min": 0.0028033541166223586,
-      "clip_ratio/region_mean": 0.0036369878507684916,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 17.8125,
-      "completions/min_length": 3.0,
-      "epoch": 0.34349030470914127,
-      "grad_norm": 1.3971369328491496,
-      "kl": 0.3057649952825159,
-      "learning_rate": 0.0001919424589291108,
-      "loss": 0.0056169466115534306,
-      "memory(GiB)": 80.03,
-      "reward": 0.4296875,
-      "reward_std": 0.8740850687026978,
-      "rewards/Response_no_think_reward/mean": 0.4296875,
-      "rewards/Response_no_think_reward/std": 1.3555577993392944,
+      "train_speed(iter/s)": 0.02137
+    },
+    {
+      "epoch": 0.06261045190608432,
+      "grad_norm": 3.672961711883545,
+      "learning_rate": 0.000124,
+      "logits/chosen": -0.6060948967933655,
+      "logits/rejected": -0.7454618811607361,
+      "logps/chosen": -3.801156997680664,
+      "logps/rejected": -23.42005157470703,
+      "loss": 1.0564124584197998,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.4279017746448517,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.18183766305446625,
+      "rewards/margins": 0.23221838474273682,
+      "rewards/rejected": -0.050380729138851166,
       "step": 31,
-      "train_speed(iter/s)": 0.002304
-    },
-    {
-      "clip_ratio/high_max": 0.013716876972466707,
-      "clip_ratio/high_mean": 0.013716876972466707,
-      "clip_ratio/low_mean": 0.05669466912513599,
-      "clip_ratio/low_min": 0.05669466912513599,
-      "clip_ratio/region_mean": 0.07041154580656439,
-      "epoch": 0.3545706371191136,
-      "grad_norm": 0.7437646652660285,
-      "kl": 0.3698675720952451,
-      "learning_rate": 0.00019120447901834706,
-      "loss": -0.013283709064126015,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021365
+    },
+    {
+      "epoch": 0.06463014390305478,
+      "grad_norm": 5.934133529663086,
+      "learning_rate": 0.00012800000000000002,
+      "logits/chosen": -0.7708523869514465,
+      "logits/rejected": -0.81435227394104,
+      "logps/chosen": -2.663968801498413,
+      "logps/rejected": -11.808816909790039,
+      "loss": 0.9843453168869019,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.37907925248146057,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.18125101923942566,
+      "rewards/margins": 0.27721551060676575,
+      "rewards/rejected": -0.0959644690155983,
       "step": 32,
-      "train_speed(iter/s)": 0.002345
-    },
-    {
-      "clip_ratio/high_max": 0.0013764221512246877,
-      "clip_ratio/high_mean": 0.0013764221512246877,
-      "clip_ratio/low_mean": 0.0013627433363581076,
-      "clip_ratio/low_min": 0.0013627433363581076,
-      "clip_ratio/region_mean": 0.0027391654875827953,
-      "completions/clipped_ratio": 0.0625,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 24.609375,
-      "completions/min_length": 4.0,
-      "epoch": 0.3656509695290859,
-      "grad_norm": 0.8795930368140337,
-      "kl": 0.22658177139237523,
-      "learning_rate": 0.00019043571606975777,
-      "loss": 0.0019196830689907074,
-      "memory(GiB)": 80.03,
-      "reward": 0.4140625,
-      "reward_std": 0.8300054669380188,
-      "rewards/Response_no_think_reward/mean": 0.4140625,
-      "rewards/Response_no_think_reward/std": 1.258216142654419,
+      "train_speed(iter/s)": 0.021371
+    },
+    {
+      "epoch": 0.06664983590002524,
+      "grad_norm": 3.0649428367614746,
+      "learning_rate": 0.000132,
+      "logits/chosen": -0.651940107345581,
+      "logits/rejected": -0.7574964761734009,
+      "logps/chosen": -1.5061438083648682,
+      "logps/rejected": -21.272098541259766,
+      "loss": 0.670626163482666,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.23393775522708893,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.312156617641449,
+      "rewards/margins": 0.6968778967857361,
+      "rewards/rejected": -0.38472118973731995,
       "step": 33,
-      "train_speed(iter/s)": 0.002323
-    },
-    {
-      "clip_ratio/high_max": 0.016442283987998962,
-      "clip_ratio/high_mean": 0.016442283987998962,
-      "clip_ratio/low_mean": 0.05633212422253564,
-      "clip_ratio/low_min": 0.05633212422253564,
-      "clip_ratio/region_mean": 0.07277440826874226,
-      "epoch": 0.3767313019390582,
-      "grad_norm": 0.5188690402432328,
-      "kl": 0.23683911142870784,
-      "learning_rate": 0.00018963642955370201,
-      "loss": -0.016352392733097076,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021373
+    },
+    {
+      "epoch": 0.06866952789699571,
+      "grad_norm": 3.6389997005462646,
+      "learning_rate": 0.00013600000000000003,
+      "logits/chosen": -0.6423748731613159,
+      "logits/rejected": -0.6685101985931396,
+      "logps/chosen": -6.548682689666748,
+      "logps/rejected": -13.626079559326172,
+      "loss": 1.1170402765274048,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.480682373046875,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.2068958729505539,
+      "rewards/margins": 0.31287699937820435,
+      "rewards/rejected": -0.10598116368055344,
       "step": 34,
-      "train_speed(iter/s)": 0.002362
-    },
-    {
-      "clip_ratio/high_max": 0.0025397460121894255,
-      "clip_ratio/high_mean": 0.0025397460121894255,
-      "clip_ratio/low_mean": 0.00378477135382127,
-      "clip_ratio/low_min": 0.00378477135382127,
-      "clip_ratio/region_mean": 0.006324517366010696,
-      "completions/clipped_ratio": 0.015625,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 23.375,
-      "completions/min_length": 3.0,
-      "epoch": 0.3878116343490305,
-      "grad_norm": 0.7751350107093148,
-      "kl": 0.21053988160565495,
-      "learning_rate": 0.00018880688924275378,
-      "loss": 0.001979985274374485,
-      "memory(GiB)": 80.03,
-      "reward": 0.25,
-      "reward_std": 0.8051205277442932,
-      "rewards/Response_no_think_reward/mean": 0.25,
-      "rewards/Response_no_think_reward/std": 1.3101662397384644,
+      "train_speed(iter/s)": 0.021372
+    },
+    {
+      "epoch": 0.07068921989396618,
+      "grad_norm": 2.625232219696045,
+      "learning_rate": 0.00014,
+      "logits/chosen": -0.5362560749053955,
+      "logits/rejected": -0.7028357982635498,
+      "logps/chosen": -3.359612464904785,
+      "logps/rejected": -33.268436431884766,
+      "loss": 0.8328565955162048,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.28516510128974915,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.05316687747836113,
+      "rewards/margins": 0.685168981552124,
+      "rewards/rejected": -0.6320021748542786,
       "step": 35,
-      "train_speed(iter/s)": 0.00234
-    },
-    {
-      "clip_ratio/high_max": 0.02383261898648925,
-      "clip_ratio/high_mean": 0.02383261898648925,
-      "clip_ratio/low_mean": 0.02296329999808222,
-      "clip_ratio/low_min": 0.02296329999808222,
-      "clip_ratio/region_mean": 0.046795917907729745,
-      "epoch": 0.3988919667590028,
-      "grad_norm": 0.6919043847160915,
-      "kl": 0.2092050458304584,
-      "learning_rate": 0.0001879473751206489,
-      "loss": -0.016633104532957077,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.02137
+    },
+    {
+      "epoch": 0.07270891189093663,
+      "grad_norm": 2.7919955253601074,
+      "learning_rate": 0.000144,
+      "logits/chosen": -0.6247209906578064,
+      "logits/rejected": -0.7358378767967224,
+      "logps/chosen": -4.632122039794922,
+      "logps/rejected": -18.708053588867188,
+      "loss": 0.719286322593689,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.25582897663116455,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.32600465416908264,
+      "rewards/margins": 0.7712743282318115,
+      "rewards/rejected": -0.44526970386505127,
       "step": 36,
-      "train_speed(iter/s)": 0.002376
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0003396739193703979,
-      "clip_ratio/low_min": 0.0003396739193703979,
-      "clip_ratio/region_mean": 0.0003396739193703979,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 16.8828125,
-      "completions/min_length": 3.0,
-      "epoch": 0.4099722991689751,
-      "grad_norm": 1.0548464052821087,
-      "kl": 0.20961805986007676,
-      "learning_rate": 0.00018705817728778624,
-      "loss": 0.003657686058431864,
-      "memory(GiB)": 80.03,
-      "reward": 0.953125,
-      "reward_std": 0.6187193393707275,
-      "rewards/Response_no_think_reward/mean": 0.953125,
-      "rewards/Response_no_think_reward/std": 1.1961840391159058,
+      "train_speed(iter/s)": 0.021371
+    },
+    {
+      "epoch": 0.0747286038879071,
+      "grad_norm": 2.5269482135772705,
+      "learning_rate": 0.000148,
+      "logits/chosen": -0.6388774514198303,
+      "logits/rejected": -0.7476755976676941,
+      "logps/chosen": -1.6513766050338745,
+      "logps/rejected": -20.447532653808594,
+      "loss": 0.652652382850647,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.181876540184021,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2968735098838806,
+      "rewards/margins": 0.7243574857711792,
+      "rewards/rejected": -0.4274839758872986,
       "step": 37,
-      "train_speed(iter/s)": 0.002358
-    },
-    {
-      "clip_ratio/high_max": 0.02387295541120693,
-      "clip_ratio/high_mean": 0.02387295541120693,
-      "clip_ratio/low_mean": 0.030282753868959844,
-      "clip_ratio/low_min": 0.030282753868959844,
-      "clip_ratio/region_mean": 0.05415570852346718,
-      "epoch": 0.42105263157894735,
-      "grad_norm": 0.49371052672691246,
-      "kl": 0.24616074899677187,
-      "learning_rate": 0.00018613959586331362,
-      "loss": -0.012075964361429214,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021373
+    },
+    {
+      "epoch": 0.07674829588487755,
+      "grad_norm": 2.419584274291992,
+      "learning_rate": 0.000152,
+      "logits/chosen": -0.5964392423629761,
+      "logits/rejected": -0.7218829393386841,
+      "logps/chosen": -1.5039414167404175,
+      "logps/rejected": -27.090576171875,
+      "loss": 0.6621359586715698,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.14563477039337158,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.28821805119514465,
+      "rewards/margins": 0.77447509765625,
+      "rewards/rejected": -0.48625698685646057,
       "step": 38,
-      "train_speed(iter/s)": 0.002393
-    },
-    {
-      "clip_ratio/high_max": 0.0013275333330966532,
-      "clip_ratio/high_mean": 0.0013275333330966532,
-      "clip_ratio/low_mean": 0.0013904034567531198,
-      "clip_ratio/low_min": 0.0013904034567531198,
-      "clip_ratio/region_mean": 0.002717936789849773,
-      "completions/clipped_ratio": 0.03125,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 21.6015625,
-      "completions/min_length": 4.0,
-      "epoch": 0.43213296398891965,
-      "grad_norm": 0.972842541028031,
-      "kl": 0.16011726018041372,
-      "learning_rate": 0.00018519194088383273,
-      "loss": 0.001440724590793252,
-      "memory(GiB)": 80.03,
-      "reward": 0.625,
-      "reward_std": 0.6670520305633545,
-      "rewards/Response_no_think_reward/mean": 0.625,
-      "rewards/Response_no_think_reward/std": 1.3457428216934204,
+      "train_speed(iter/s)": 0.021373
+    },
+    {
+      "epoch": 0.07876798788184802,
+      "grad_norm": 3.088440179824829,
+      "learning_rate": 0.00015600000000000002,
+      "logits/chosen": -0.7012370824813843,
+      "logits/rejected": -0.8179476261138916,
+      "logps/chosen": -1.9244059324264526,
+      "logps/rejected": -21.519695281982422,
+      "loss": 0.6807724237442017,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.244890034198761,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.15054234862327576,
+      "rewards/margins": 0.9000046849250793,
+      "rewards/rejected": -0.749462366104126,
       "step": 39,
-      "train_speed(iter/s)": 0.002371
-    },
-    {
-      "clip_ratio/high_max": 0.03226059180451557,
-      "clip_ratio/high_mean": 0.03226059180451557,
-      "clip_ratio/low_mean": 0.03314233338460326,
-      "clip_ratio/low_min": 0.03314233338460326,
-      "clip_ratio/region_mean": 0.0654029252473265,
-      "epoch": 0.44321329639889195,
-      "grad_norm": 0.5253408277855696,
-      "kl": 0.18682625680230558,
-      "learning_rate": 0.00018421553219875658,
-      "loss": -0.013099671341478825,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021371
+    },
+    {
+      "epoch": 0.08078767987881848,
+      "grad_norm": 2.713752508163452,
+      "learning_rate": 0.00016,
+      "logits/chosen": -0.6608636379241943,
+      "logits/rejected": -0.7343001365661621,
+      "logps/chosen": -4.701538562774658,
+      "logps/rejected": -27.00352668762207,
+      "loss": 0.6774569749832153,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.2570435106754303,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.13876499235630035,
+      "rewards/margins": 1.0153288841247559,
+      "rewards/rejected": -0.8765639066696167,
       "step": 40,
-      "train_speed(iter/s)": 0.002404
-    },
-    {
-      "clip_ratio/high_max": 0.00029620854184031487,
-      "clip_ratio/high_mean": 0.00029620854184031487,
-      "clip_ratio/low_mean": 0.0009311849280493334,
-      "clip_ratio/low_min": 0.0009311849280493334,
-      "clip_ratio/region_mean": 0.0012273934698896483,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 88.0,
-      "completions/mean_length": 17.5625,
-      "completions/min_length": 5.0,
-      "epoch": 0.45429362880886426,
-      "grad_norm": 1.2919745010410586,
-      "kl": 0.2366366102360189,
-      "learning_rate": 0.00018321069936235503,
-      "loss": 0.001985038397833705,
-      "memory(GiB)": 80.03,
-      "reward": 0.859375,
-      "reward_std": 0.6107450723648071,
-      "rewards/Response_no_think_reward/mean": 0.859375,
-      "rewards/Response_no_think_reward/std": 1.4071491956710815,
+      "train_speed(iter/s)": 0.021373
+    },
+    {
+      "epoch": 0.08280737187578895,
+      "grad_norm": 4.948164463043213,
+      "learning_rate": 0.000164,
+      "logits/chosen": -0.6391008496284485,
+      "logits/rejected": -0.6671096682548523,
+      "logps/chosen": -8.970189094543457,
+      "logps/rejected": -14.413579940795898,
+      "loss": 1.0303566455841064,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.41976141929626465,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0029628686606884003,
+      "rewards/margins": 0.573236882686615,
+      "rewards/rejected": -0.5761997699737549,
       "step": 41,
-      "train_speed(iter/s)": 0.002379
-    },
-    {
-      "clip_ratio/high_max": 0.01783788768807426,
-      "clip_ratio/high_mean": 0.01783788768807426,
-      "clip_ratio/low_mean": 0.044289187353570014,
-      "clip_ratio/low_min": 0.044289187353570014,
-      "clip_ratio/region_mean": 0.06212707597296685,
-      "epoch": 0.46537396121883656,
-      "grad_norm": 1.7719321167278534,
-      "kl": 0.6719344789162278,
-      "learning_rate": 0.0001821777815225245,
-      "loss": -0.008167732506990433,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021375
+    },
+    {
+      "epoch": 0.0848270638727594,
+      "grad_norm": 4.40416955947876,
+      "learning_rate": 0.000168,
+      "logits/chosen": -0.6820927858352661,
+      "logits/rejected": -0.7476735711097717,
+      "logps/chosen": -4.897199630737305,
+      "logps/rejected": -17.570022583007812,
+      "loss": 0.9847887754440308,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.42955508828163147,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.08176113665103912,
+      "rewards/margins": 0.5534911155700684,
+      "rewards/rejected": -0.47172996401786804,
       "step": 42,
-      "train_speed(iter/s)": 0.00241
-    },
-    {
-      "clip_ratio/high_max": 0.00036231885314919055,
-      "clip_ratio/high_mean": 0.00036231885314919055,
-      "clip_ratio/low_mean": 0.002238474931800738,
-      "clip_ratio/low_min": 0.002238474931800738,
-      "clip_ratio/region_mean": 0.0026007937849499285,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 97.0,
-      "completions/mean_length": 15.875,
-      "completions/min_length": 6.0,
-      "epoch": 0.47645429362880887,
-      "grad_norm": 1.1682612472028975,
-      "kl": 0.2416230053640902,
-      "learning_rate": 0.00018111712730632022,
-      "loss": 0.0018095734994858503,
-      "memory(GiB)": 80.03,
-      "reward": 0.0859375,
-      "reward_std": 0.39637458324432373,
-      "rewards/Response_no_think_reward/mean": 0.0859375,
-      "rewards/Response_no_think_reward/std": 0.9962713122367859,
+      "train_speed(iter/s)": 0.021376
+    },
+    {
+      "epoch": 0.08684675586972987,
+      "grad_norm": 5.150269508361816,
+      "learning_rate": 0.000172,
+      "logits/chosen": -0.7095463275909424,
+      "logits/rejected": -0.7945126891136169,
+      "logps/chosen": -4.1012396812438965,
+      "logps/rejected": -20.216747283935547,
+      "loss": 0.956037700176239,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.5109948515892029,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.19462859630584717,
+      "rewards/margins": 0.853682279586792,
+      "rewards/rejected": -0.6590536832809448,
       "step": 43,
-      "train_speed(iter/s)": 0.002397
-    },
-    {
-      "clip_ratio/high_max": 0.007687599485507235,
-      "clip_ratio/high_mean": 0.007687599485507235,
-      "clip_ratio/low_mean": 0.042899149731965736,
-      "clip_ratio/low_min": 0.042899149731965736,
-      "clip_ratio/region_mean": 0.05058674863539636,
-      "epoch": 0.48753462603878117,
-      "grad_norm": 17.524605028595435,
-      "kl": 15.062655651359819,
-      "learning_rate": 0.00018002909470228842,
-      "loss": 0.09481670707464218,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021378
+    },
+    {
+      "epoch": 0.08886644786670032,
+      "grad_norm": 4.477358341217041,
+      "learning_rate": 0.00017600000000000002,
+      "logits/chosen": -0.7657253742218018,
+      "logits/rejected": -0.8225359320640564,
+      "logps/chosen": -2.71124267578125,
+      "logps/rejected": -13.512589454650879,
+      "loss": 0.8599216341972351,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.27771878242492676,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.13035942614078522,
+      "rewards/margins": 0.38512665033340454,
+      "rewards/rejected": -0.25476720929145813,
       "step": 44,
-      "train_speed(iter/s)": 0.002427
-    },
-    {
-      "clip_ratio/high_max": 0.0012922932510264218,
-      "clip_ratio/high_mean": 0.0012922932510264218,
-      "clip_ratio/low_mean": 0.002554390055593103,
-      "clip_ratio/low_min": 0.002554390055593103,
-      "clip_ratio/region_mean": 0.003846683306619525,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 32.0,
-      "completions/mean_length": 8.546875,
-      "completions/min_length": 4.0,
-      "epoch": 0.4986149584487535,
-      "grad_norm": 1.474323820635427,
-      "kl": 0.7951482257340103,
-      "learning_rate": 0.00017891405093963938,
-      "loss": 0.008442065678536892,
-      "memory(GiB)": 80.03,
-      "reward": 0.171875,
-      "reward_std": 0.540536105632782,
-      "rewards/Response_no_think_reward/mean": 0.171875,
-      "rewards/Response_no_think_reward/std": 0.9648089408874512,
+      "train_speed(iter/s)": 0.021381
+    },
+    {
+      "epoch": 0.09088613986367079,
+      "grad_norm": 5.4391255378723145,
+      "learning_rate": 0.00018,
+      "logits/chosen": -0.6582808494567871,
+      "logits/rejected": -0.7074710726737976,
+      "logps/chosen": -5.520257472991943,
+      "logps/rejected": -18.50946044921875,
+      "loss": 0.8143137693405151,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.2541295886039734,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.06929375976324081,
+      "rewards/margins": 0.6602391004562378,
+      "rewards/rejected": -0.590945303440094,
       "step": 45,
-      "train_speed(iter/s)": 0.002416
-    },
-    {
-      "clip_ratio/high_max": 0.01782548933988437,
-      "clip_ratio/high_mean": 0.01782548933988437,
-      "clip_ratio/low_mean": 0.04517949424916878,
-      "clip_ratio/low_min": 0.04517949424916878,
-      "clip_ratio/region_mean": 0.06300498393829912,
-      "epoch": 0.5096952908587258,
-      "grad_norm": 23.18489815639863,
-      "kl": 20.399557466851547,
-      "learning_rate": 0.0001777723723643014,
-      "loss": 0.15100935101509094,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.02138
+    },
+    {
+      "epoch": 0.09290583186064125,
+      "grad_norm": 4.585366249084473,
+      "learning_rate": 0.00018400000000000003,
+      "logits/chosen": -0.722661018371582,
+      "logits/rejected": -0.7770288586616516,
+      "logps/chosen": -3.6351494789123535,
+      "logps/rejected": -10.902822494506836,
+      "loss": 1.1608294248580933,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.47840094566345215,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": 0.09331849962472916,
+      "rewards/margins": 0.1440279632806778,
+      "rewards/rejected": -0.05070946365594864,
       "step": 46,
-      "train_speed(iter/s)": 0.002445
-    },
-    {
-      "clip_ratio/high_max": 0.0001338329748250544,
-      "clip_ratio/high_mean": 0.0001338329748250544,
-      "clip_ratio/low_mean": 0.0020056332577951252,
-      "clip_ratio/low_min": 0.0020056332577951252,
-      "clip_ratio/region_mean": 0.0021394662326201797,
-      "completions/clipped_ratio": 0.015625,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 11.8203125,
-      "completions/min_length": 4.0,
-      "epoch": 0.5207756232686981,
-      "grad_norm": 1.9101810703272282,
-      "kl": 0.8325624349527061,
-      "learning_rate": 0.0001766044443118978,
-      "loss": 0.01317879930138588,
-      "memory(GiB)": 80.03,
-      "reward": 0.21875,
-      "reward_std": 0.5618736743927002,
-      "rewards/Response_no_think_reward/mean": 0.21875,
-      "rewards/Response_no_think_reward/std": 0.9301384091377258,
+      "train_speed(iter/s)": 0.021384
+    },
+    {
+      "epoch": 0.09492552385761172,
+      "grad_norm": 4.578927040100098,
+      "learning_rate": 0.000188,
+      "logits/chosen": -0.6195976734161377,
+      "logits/rejected": -0.7779173254966736,
+      "logps/chosen": -0.9245270490646362,
+      "logps/rejected": -30.03981590270996,
+      "loss": 0.5132064819335938,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.10489758849143982,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2798718214035034,
+      "rewards/margins": 0.944153904914856,
+      "rewards/rejected": -0.6642820835113525,
       "step": 47,
-      "train_speed(iter/s)": 0.002435
-    },
-    {
-      "clip_ratio/high_max": 0.01570215745596215,
-      "clip_ratio/high_mean": 0.01570215745596215,
-      "clip_ratio/low_mean": 0.01864259922876954,
-      "clip_ratio/low_min": 0.01864259922876954,
-      "clip_ratio/region_mean": 0.03434475651010871,
-      "epoch": 0.5318559556786704,
-      "grad_norm": 5.211743253538076,
-      "kl": 0.4838231955654919,
-      "learning_rate": 0.00017541066097768963,
-      "loss": 0.03044246882200241,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021382
+    },
+    {
+      "epoch": 0.09694521585458217,
+      "grad_norm": 6.127289772033691,
+      "learning_rate": 0.000192,
+      "logits/chosen": -0.6809214949607849,
+      "logits/rejected": -0.7704883813858032,
+      "logps/chosen": -1.4336328506469727,
+      "logps/rejected": -16.669113159179688,
+      "loss": 0.8413453102111816,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.27693358063697815,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.1910504847764969,
+      "rewards/margins": 0.4971596598625183,
+      "rewards/rejected": -0.30610913038253784,
       "step": 48,
-      "train_speed(iter/s)": 0.002462
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.002905012297560461,
-      "clip_ratio/low_min": 0.002905012297560461,
-      "clip_ratio/region_mean": 0.002905012297560461,
-      "completions/clipped_ratio": 0.0390625,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 17.7265625,
-      "completions/min_length": 4.0,
-      "epoch": 0.5429362880886427,
-      "grad_norm": 0.9459153229739418,
-      "kl": 0.33009129445417784,
-      "learning_rate": 0.00017419142528352817,
-      "loss": 0.004404420033097267,
-      "memory(GiB)": 80.03,
-      "reward": -0.0859375,
-      "reward_std": 0.547653079032898,
-      "rewards/Response_no_think_reward/mean": -0.0859375,
-      "rewards/Response_no_think_reward/std": 0.9138250946998596,
+      "train_speed(iter/s)": 0.021382
+    },
+    {
+      "epoch": 0.09896490785155264,
+      "grad_norm": 5.920216083526611,
+      "learning_rate": 0.000196,
+      "logits/chosen": -0.744787335395813,
+      "logits/rejected": -0.7133767604827881,
+      "logps/chosen": -8.577417373657227,
+      "logps/rejected": -15.383255958557129,
+      "loss": 1.23928964138031,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.5441281199455261,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1016860231757164,
+      "rewards/margins": 0.13402265310287476,
+      "rewards/rejected": -0.23570865392684937,
       "step": 49,
-      "train_speed(iter/s)": 0.002452
-    },
-    {
-      "clip_ratio/high_max": 0.00849696435034275,
-      "clip_ratio/high_mean": 0.00849696435034275,
-      "clip_ratio/low_mean": 0.059440263896249235,
-      "clip_ratio/low_min": 0.059440263896249235,
-      "clip_ratio/region_mean": 0.06793722766451538,
-      "epoch": 0.554016620498615,
-      "grad_norm": 1.5192885268898135,
-      "kl": 0.5284001431518845,
-      "learning_rate": 0.0001729471487418621,
-      "loss": -0.012666964903473854,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021382
+    },
+    {
+      "epoch": 0.1009845998485231,
+      "grad_norm": 5.736071586608887,
+      "learning_rate": 0.0002,
+      "logits/chosen": -0.7741532325744629,
+      "logits/rejected": -0.8851832747459412,
+      "logps/chosen": -4.090970039367676,
+      "logps/rejected": -21.758840560913086,
+      "loss": 1.0278420448303223,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.5550558567047119,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.15652738511562347,
+      "rewards/margins": 0.8236760497093201,
+      "rewards/rejected": -0.6671487092971802,
       "step": 50,
-      "train_speed(iter/s)": 0.002478
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0012755101779475808,
-      "clip_ratio/low_min": 0.0012755101779475808,
-      "clip_ratio/region_mean": 0.0012755101779475808,
-      "completions/clipped_ratio": 0.0078125,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 12.640625,
-      "completions/min_length": 3.0,
-      "epoch": 0.5650969529085873,
-      "grad_norm": 1.7932835802898264,
-      "kl": 0.582448753528297,
-      "learning_rate": 0.00017167825131684513,
-      "loss": 0.008432275615632534,
-      "memory(GiB)": 80.03,
-      "reward": 0.15625,
-      "reward_std": 0.7085258960723877,
-      "rewards/Response_no_think_reward/mean": 0.15625,
-      "rewards/Response_no_think_reward/std": 1.0228685140609741,
+      "train_speed(iter/s)": 0.021383
+    },
+    {
+      "epoch": 0.10300429184549356,
+      "grad_norm": 4.201787948608398,
+      "learning_rate": 0.0001999994415122672,
+      "logits/chosen": -0.8024938106536865,
+      "logits/rejected": -0.8299172520637512,
+      "logps/chosen": -6.9087419509887695,
+      "logps/rejected": -10.72697639465332,
+      "loss": 1.1162101030349731,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.4278947114944458,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.060360271483659744,
+      "rewards/margins": 0.15178537368774414,
+      "rewards/rejected": -0.0914250910282135,
       "step": 51,
-      "train_speed(iter/s)": 0.002465
-    },
-    {
-      "clip_ratio/high_max": 0.006597792147658765,
-      "clip_ratio/high_mean": 0.006597792147658765,
-      "clip_ratio/low_mean": 0.049801092012785375,
-      "clip_ratio/low_min": 0.049801092012785375,
-      "clip_ratio/region_mean": 0.05639888462610543,
-      "epoch": 0.5761772853185596,
-      "grad_norm": 0.8422307837338449,
-      "kl": 0.7035694038495421,
-      "learning_rate": 0.00017038516128259115,
-      "loss": -0.011608053930103779,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021239
+    },
+    {
+      "epoch": 0.10502398384246403,
+      "grad_norm": 3.155092477798462,
+      "learning_rate": 0.0001999977660553069,
+      "logits/chosen": -0.6624072790145874,
+      "logits/rejected": -0.8125584125518799,
+      "logps/chosen": -1.1845072507858276,
+      "logps/rejected": -28.393592834472656,
+      "loss": 0.5598468780517578,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.15286707878112793,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2649957835674286,
+      "rewards/margins": 1.0407716035842896,
+      "rewards/rejected": -0.7757757902145386,
       "step": 52,
-      "train_speed(iter/s)": 0.00249
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0005558648990700021,
-      "clip_ratio/low_min": 0.0005558648990700021,
-      "clip_ratio/region_mean": 0.0005558648990700021,
-      "completions/clipped_ratio": 0.015625,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 12.5,
-      "completions/min_length": 4.0,
-      "epoch": 0.5872576177285319,
-      "grad_norm": 1.0211580850819229,
-      "kl": 0.7616109761074767,
-      "learning_rate": 0.00016906831507862443,
-      "loss": 0.005634145811200142,
-      "memory(GiB)": 80.03,
-      "reward": 0.1640625,
-      "reward_std": 0.70704185962677,
-      "rewards/Response_no_think_reward/mean": 0.1640625,
-      "rewards/Response_no_think_reward/std": 0.9702450633049011,
+      "train_speed(iter/s)": 0.02124
+    },
+    {
+      "epoch": 0.10704367583943449,
+      "grad_norm": 3.9601948261260986,
+      "learning_rate": 0.0001999949736478336,
+      "logits/chosen": -0.8051952719688416,
+      "logits/rejected": -0.8654646873474121,
+      "logps/chosen": -3.793642997741699,
+      "logps/rejected": -17.214599609375,
+      "loss": 0.738736629486084,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.23666247725486755,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.17774169147014618,
+      "rewards/margins": 0.6443970799446106,
+      "rewards/rejected": -0.4666553735733032,
       "step": 53,
-      "train_speed(iter/s)": 0.002477
-    },
-    {
-      "clip_ratio/high_max": 0.02744574609096162,
-      "clip_ratio/high_mean": 0.02744574609096162,
-      "clip_ratio/low_mean": 0.04221567645436153,
-      "clip_ratio/low_min": 0.04221567645436153,
-      "clip_ratio/region_mean": 0.06966142146848142,
-      "epoch": 0.5983379501385041,
-      "grad_norm": 0.5808426269905869,
-      "kl": 0.801087921798171,
-      "learning_rate": 0.00016772815716257412,
-      "loss": -0.01366308145225048,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021244
+    },
+    {
+      "epoch": 0.10906336783640495,
+      "grad_norm": 2.747492790222168,
+      "learning_rate": 0.0001999910643210378,
+      "logits/chosen": -0.7448392510414124,
+      "logits/rejected": -0.9058634042739868,
+      "logps/chosen": -1.6127703189849854,
+      "logps/rejected": -30.806520462036133,
+      "loss": 0.5075910091400146,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.17231889069080353,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.4633626639842987,
+      "rewards/margins": 1.3208293914794922,
+      "rewards/rejected": -0.8574665784835815,
       "step": 54,
-      "train_speed(iter/s)": 0.002502
-    },
-    {
-      "clip_ratio/high_max": 0.0003881987649947405,
-      "clip_ratio/high_mean": 0.0003881987649947405,
-      "clip_ratio/low_mean": 0.001932911021867767,
-      "clip_ratio/low_min": 0.001932911021867767,
-      "clip_ratio/region_mean": 0.0023211097868625075,
-      "completions/clipped_ratio": 0.0390625,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 18.9921875,
-      "completions/min_length": 6.0,
-      "epoch": 0.6094182825484764,
-      "grad_norm": 1.1265096543789663,
-      "kl": 0.5226203491911292,
-      "learning_rate": 0.00016636513986016213,
-      "loss": 0.007094700820744038,
-      "memory(GiB)": 80.03,
-      "reward": 0.1875,
-      "reward_std": 0.7182328104972839,
-      "rewards/Response_no_think_reward/mean": 0.1875,
-      "rewards/Response_no_think_reward/std": 1.0480577945709229,
+      "train_speed(iter/s)": 0.021246
+    },
+    {
+      "epoch": 0.11108305983337541,
+      "grad_norm": 3.485903739929199,
+      "learning_rate": 0.00019998603811858571,
+      "logits/chosen": -0.6839566230773926,
+      "logits/rejected": -0.8555458784103394,
+      "logps/chosen": -2.3723244667053223,
+      "logps/rejected": -34.97600173950195,
+      "loss": 0.7429671883583069,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.3656916618347168,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.24485386908054352,
+      "rewards/margins": 1.4793577194213867,
+      "rewards/rejected": -1.2345038652420044,
       "step": 55,
-      "train_speed(iter/s)": 0.00249
-    },
-    {
-      "clip_ratio/high_max": 0.011908911721548066,
-      "clip_ratio/high_mean": 0.011908911721548066,
-      "clip_ratio/low_mean": 0.05817525731981732,
-      "clip_ratio/low_min": 0.05817525731981732,
-      "clip_ratio/region_mean": 0.07008416869211942,
-      "epoch": 0.6204986149584487,
-      "grad_norm": 0.5560920258062684,
-      "kl": 0.5747523186728358,
-      "learning_rate": 0.000164979723212536,
-      "loss": -0.014954826794564724,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021247
+    },
+    {
+      "epoch": 0.11310275183034588,
+      "grad_norm": 4.815752029418945,
+      "learning_rate": 0.0001999798950966188,
+      "logits/chosen": -0.7993583679199219,
+      "logits/rejected": -0.8806532025337219,
+      "logps/chosen": -5.939935207366943,
+      "logps/rejected": -18.04892349243164,
+      "loss": 1.1477632522583008,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.5634173154830933,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.026777414605021477,
+      "rewards/margins": 0.612076461315155,
+      "rewards/rejected": -0.5852990746498108,
       "step": 56,
-      "train_speed(iter/s)": 0.002513
-    },
-    {
-      "clip_ratio/high_max": 0.002692167239729315,
-      "clip_ratio/high_mean": 0.002692167239729315,
-      "clip_ratio/low_mean": 0.0006428283377317712,
-      "clip_ratio/low_min": 0.0006428283377317712,
-      "clip_ratio/region_mean": 0.003334995577461086,
-      "completions/clipped_ratio": 0.0390625,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 16.53125,
-      "completions/min_length": 6.0,
-      "epoch": 0.631578947368421,
-      "grad_norm": 1.5596809103877662,
-      "kl": 0.6855434570461512,
-      "learning_rate": 0.00016357237482099684,
-      "loss": 0.0062956069596111774,
-      "memory(GiB)": 80.03,
-      "reward": 0.53125,
-      "reward_std": 0.5907745361328125,
-      "rewards/Response_no_think_reward/mean": 0.53125,
-      "rewards/Response_no_think_reward/std": 1.0790598392486572,
+      "train_speed(iter/s)": 0.021251
+    },
+    {
+      "epoch": 0.11512244382731633,
+      "grad_norm": 2.9355173110961914,
+      "learning_rate": 0.00019997263532375303,
+      "logits/chosen": -0.7389899492263794,
+      "logits/rejected": -0.8667410612106323,
+      "logps/chosen": -2.2958154678344727,
+      "logps/rejected": -32.464534759521484,
+      "loss": 0.6630210876464844,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.3285917043685913,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1826460361480713,
+      "rewards/margins": 1.4345746040344238,
+      "rewards/rejected": -1.251928448677063,
       "step": 57,
-      "train_speed(iter/s)": 0.002501
-    },
-    {
-      "clip_ratio/high_max": 0.0316368987550959,
-      "clip_ratio/high_mean": 0.0316368987550959,
-      "clip_ratio/low_mean": 0.015557856269879267,
-      "clip_ratio/low_min": 0.015557856269879267,
-      "clip_ratio/region_mean": 0.04719475514139049,
-      "epoch": 0.6426592797783933,
-      "grad_norm": 0.950735686233936,
-      "kl": 0.5240823943167925,
-      "learning_rate": 0.00016214356968917648,
-      "loss": -0.015531505458056927,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021253
+    },
+    {
+      "epoch": 0.1171421358242868,
+      "grad_norm": 4.644790172576904,
+      "learning_rate": 0.0001999642588810784,
+      "logits/chosen": -0.8502639532089233,
+      "logits/rejected": -0.9041623473167419,
+      "logps/chosen": -4.175015449523926,
+      "logps/rejected": -20.228912353515625,
+      "loss": 1.007567048072815,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.4676174223423004,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.053391821682453156,
+      "rewards/margins": 0.8722233176231384,
+      "rewards/rejected": -0.925615131855011,
       "step": 58,
-      "train_speed(iter/s)": 0.002524
-    },
-    {
-      "clip_ratio/high_max": 0.00011467889999039471,
-      "clip_ratio/high_mean": 0.00011467889999039471,
-      "clip_ratio/low_mean": 0.0004376750875962898,
-      "clip_ratio/low_min": 0.0004376750875962898,
-      "clip_ratio/region_mean": 0.0005523539875866845,
-      "completions/clipped_ratio": 0.078125,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 23.9140625,
-      "completions/min_length": 6.0,
-      "epoch": 0.6537396121883656,
-      "grad_norm": 1.3162721482480142,
-      "kl": 0.5214177745606321,
-      "learning_rate": 0.00016069379006271566,
-      "loss": 0.004921327345073223,
-      "memory(GiB)": 80.03,
-      "reward": -0.0625,
-      "reward_std": 0.6303451657295227,
-      "rewards/Response_no_think_reward/mean": -0.0625,
-      "rewards/Response_no_think_reward/std": 1.0019665956497192,
+      "train_speed(iter/s)": 0.021258
+    },
+    {
+      "epoch": 0.11916182782125725,
+      "grad_norm": 4.834179878234863,
+      "learning_rate": 0.00019995476586215762,
+      "logits/chosen": -0.8684061169624329,
+      "logits/rejected": -0.9059044718742371,
+      "logps/chosen": -9.152887344360352,
+      "logps/rejected": -15.517383575439453,
+      "loss": 1.118729591369629,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.6267417073249817,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.0668405294418335,
+      "rewards/margins": 0.7597751021385193,
+      "rewards/rejected": -0.6929346323013306,
       "step": 59,
-      "train_speed(iter/s)": 0.002511
-    },
-    {
-      "clip_ratio/high_max": 0.00810479320352897,
-      "clip_ratio/high_mean": 0.00810479320352897,
-      "clip_ratio/low_mean": 0.05729365168372169,
-      "clip_ratio/low_min": 0.05729365168372169,
-      "clip_ratio/region_mean": 0.06539844395592809,
-      "epoch": 0.6648199445983379,
-      "grad_norm": 0.5733548474442324,
-      "kl": 0.6363338100200053,
-      "learning_rate": 0.00015922352526649803,
-      "loss": -0.021113965660333633,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021264
+    },
+    {
+      "epoch": 0.12118151981822772,
+      "grad_norm": 3.052858591079712,
+      "learning_rate": 0.00019994415637302547,
+      "logits/chosen": -0.7785470485687256,
+      "logits/rejected": -0.8464182019233704,
+      "logps/chosen": -2.411245107650757,
+      "logps/rejected": -17.925155639648438,
+      "loss": 0.7822959423065186,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.25340673327445984,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.19467662274837494,
+      "rewards/margins": 0.8461788892745972,
+      "rewards/rejected": -0.6515023112297058,
       "step": 60,
-      "train_speed(iter/s)": 0.002532
-    },
-    {
-      "clip_ratio/high_max": 0.0005996339968987741,
-      "clip_ratio/high_mean": 0.0005996339968987741,
-      "clip_ratio/low_mean": 0.004901745676761493,
-      "clip_ratio/low_min": 0.004901745676761493,
-      "clip_ratio/region_mean": 0.00550137969548814,
-      "completions/clipped_ratio": 0.0859375,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 22.375,
-      "completions/min_length": 2.0,
-      "epoch": 0.6759002770083102,
-      "grad_norm": 1.887884786225259,
-      "kl": 2.07791917472332,
-      "learning_rate": 0.00015773327153949465,
-      "loss": 0.008165515027940273,
-      "memory(GiB)": 80.03,
-      "reward": 0.40625,
-      "reward_std": 0.5973243117332458,
-      "rewards/Response_no_think_reward/mean": 0.40625,
-      "rewards/Response_no_think_reward/std": 1.186787486076355,
+      "train_speed(iter/s)": 0.021268
+    },
+    {
+      "epoch": 0.12320121181519818,
+      "grad_norm": 3.3108043670654297,
+      "learning_rate": 0.0001999324305321873,
+      "logits/chosen": -0.762412428855896,
+      "logits/rejected": -0.8276241421699524,
+      "logps/chosen": -2.0741934776306152,
+      "logps/rejected": -14.743797302246094,
+      "loss": 0.8085264563560486,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.25528597831726074,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.13805074989795685,
+      "rewards/margins": 0.4351329803466797,
+      "rewards/rejected": -0.29708221554756165,
       "step": 61,
-      "train_speed(iter/s)": 0.002506
-    },
-    {
-      "clip_ratio/high_max": 0.017942053091246635,
-      "clip_ratio/high_mean": 0.017942053091246635,
-      "clip_ratio/low_mean": 0.06714771036058664,
-      "clip_ratio/low_min": 0.06714771036058664,
-      "clip_ratio/region_mean": 0.08508976292796433,
-      "epoch": 0.6869806094182825,
-      "grad_norm": 11.312330346219078,
-      "kl": 0.5888316835043952,
-      "learning_rate": 0.00015622353186727544,
-      "loss": 0.04619387909770012,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021273
+    },
+    {
+      "epoch": 0.12522090381216863,
+      "grad_norm": 3.6591594219207764,
+      "learning_rate": 0.00019991958847061784,
+      "logits/chosen": -0.6577974557876587,
+      "logits/rejected": -0.7462488412857056,
+      "logps/chosen": -5.370046138763428,
+      "logps/rejected": -15.76892375946045,
+      "loss": 0.9229806661605835,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.3679361343383789,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.14663854241371155,
+      "rewards/margins": 0.4843655228614807,
+      "rewards/rejected": -0.33772704005241394,
       "step": 62,
-      "train_speed(iter/s)": 0.002527
-    },
-    {
-      "clip_ratio/high_max": 0.00020032051543239504,
-      "clip_ratio/high_mean": 0.00020032051543239504,
-      "clip_ratio/low_mean": 0.001202335930429399,
-      "clip_ratio/low_min": 0.001202335930429399,
-      "clip_ratio/region_mean": 0.001402656445861794,
-      "completions/clipped_ratio": 0.0546875,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 18.5859375,
-      "completions/min_length": 2.0,
-      "epoch": 0.6980609418282548,
-      "grad_norm": 1.9804020858052087,
-      "kl": 6.258792589243967,
-      "learning_rate": 0.00015469481581224272,
-      "loss": 0.014128579758107662,
-      "memory(GiB)": 80.03,
-      "reward": 0.78125,
-      "reward_std": 0.5936684608459473,
-      "rewards/Response_no_think_reward/mean": 0.78125,
-      "rewards/Response_no_think_reward/std": 1.235546350479126,
+      "train_speed(iter/s)": 0.021275
+    },
+    {
+      "epoch": 0.1272405958091391,
+      "grad_norm": 3.069516658782959,
+      "learning_rate": 0.00019990563033175983,
+      "logits/chosen": -0.6640661954879761,
+      "logits/rejected": -0.7672103643417358,
+      "logps/chosen": -2.8094382286071777,
+      "logps/rejected": -18.480703353881836,
+      "loss": 0.7683196663856506,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.2581484913825989,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2657431662082672,
+      "rewards/margins": 0.5364934802055359,
+      "rewards/rejected": -0.27075034379959106,
       "step": 63,
-      "train_speed(iter/s)": 0.002486
-    },
-    {
-      "clip_ratio/high_max": 0.020077986438991502,
-      "clip_ratio/high_mean": 0.020077986438991502,
-      "clip_ratio/low_mean": 0.12254823022522032,
-      "clip_ratio/low_min": 0.12254823022522032,
-      "clip_ratio/region_mean": 0.142626216635108,
-      "epoch": 0.7091412742382271,
-      "grad_norm": 1.5460664241155249,
-      "kl": 5.241092938755173,
-      "learning_rate": 0.0001531476393416456,
-      "loss": -0.0033248686231672764,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021277
+    },
+    {
+      "epoch": 0.12926028780610957,
+      "grad_norm": 4.207716464996338,
+      "learning_rate": 0.0001998905562715222,
+      "logits/chosen": -0.7866019010543823,
+      "logits/rejected": -0.8437267541885376,
+      "logps/chosen": -6.750940799713135,
+      "logps/rejected": -15.358600616455078,
+      "loss": 1.2175216674804688,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.6082065105438232,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.004789266735315323,
+      "rewards/margins": 0.4517655074596405,
+      "rewards/rejected": -0.44697627425193787,
       "step": 64,
-      "train_speed(iter/s)": 0.002506
-    },
-    {
-      "clip_ratio/high_max": 0.0006774475477868691,
-      "clip_ratio/high_mean": 0.0006774475477868691,
-      "clip_ratio/low_mean": 0.0009178321633953601,
-      "clip_ratio/low_min": 0.0009178321633953601,
-      "clip_ratio/region_mean": 0.0015952797257341444,
-      "completions/clipped_ratio": 0.0390625,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 15.1484375,
-      "completions/min_length": 6.0,
-      "epoch": 0.7202216066481995,
-      "grad_norm": 1.528383760465238,
-      "kl": 1.888367731589824,
-      "learning_rate": 0.00015158252465343242,
-      "loss": 0.010861902497708797,
-      "memory(GiB)": 80.03,
-      "reward": 0.59375,
-      "reward_std": 0.5738716125488281,
-      "rewards/Response_no_think_reward/mean": 0.59375,
-      "rewards/Response_no_think_reward/std": 1.1462881565093994,
+      "train_speed(iter/s)": 0.021281
+    },
+    {
+      "epoch": 0.13127997980308004,
+      "grad_norm": 2.928544282913208,
+      "learning_rate": 0.0001998743664582786,
+      "logits/chosen": -0.7847710251808167,
+      "logits/rejected": -0.8567973971366882,
+      "logps/chosen": -2.9689300060272217,
+      "logps/rejected": -19.40102767944336,
+      "loss": 0.9175847768783569,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.4591127634048462,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.08043977618217468,
+      "rewards/margins": 0.8204008340835571,
+      "rewards/rejected": -0.7399609088897705,
       "step": 65,
-      "train_speed(iter/s)": 0.002444
-    },
-    {
-      "clip_ratio/high_max": 0.05604529404081404,
-      "clip_ratio/high_mean": 0.05604529404081404,
-      "clip_ratio/low_mean": 0.012011443439405411,
-      "clip_ratio/low_min": 0.012011443439405411,
-      "clip_ratio/region_mean": 0.06805673893541098,
-      "epoch": 0.7313019390581718,
-      "grad_norm": 0.9299498266912626,
-      "kl": 1.0373663480422692,
-      "learning_rate": 0.00015000000000000001,
-      "loss": -0.001186850480735302,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021285
+    },
+    {
+      "epoch": 0.13329967180005048,
+      "grad_norm": 3.2160611152648926,
+      "learning_rate": 0.00019985706107286514,
+      "logits/chosen": -0.737562358379364,
+      "logits/rejected": -0.8171918988227844,
+      "logps/chosen": -1.0520405769348145,
+      "logps/rejected": -13.78677749633789,
+      "loss": 0.6010702252388,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.1751595437526703,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.34129399061203003,
+      "rewards/margins": 0.7596877813339233,
+      "rewards/rejected": -0.41839370131492615,
       "step": 66,
-      "train_speed(iter/s)": 0.002463
-    },
-    {
-      "clip_ratio/high_max": 0.006279462773818523,
-      "clip_ratio/high_mean": 0.006279462773818523,
-      "clip_ratio/low_mean": 0.006620214961003512,
-      "clip_ratio/low_min": 0.006620214961003512,
-      "clip_ratio/region_mean": 0.012899677676614374,
-      "completions/clipped_ratio": 0.0546875,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 17.5234375,
-      "completions/min_length": 2.0,
-      "epoch": 0.7423822714681441,
-      "grad_norm": 3.0195311219652377,
-      "kl": 1.4996049946639687,
-      "learning_rate": 0.0001484005995098999,
-      "loss": 0.0103817880153656,
-      "memory(GiB)": 80.03,
-      "reward": 0.2421875,
-      "reward_std": 0.5205168724060059,
-      "rewards/Response_no_think_reward/mean": 0.2421875,
-      "rewards/Response_no_think_reward/std": 1.2595843076705933,
+      "train_speed(iter/s)": 0.021288
+    },
+    {
+      "epoch": 0.13531936379702095,
+      "grad_norm": 3.3036932945251465,
+      "learning_rate": 0.00019983864030857882,
+      "logits/chosen": -0.8425466418266296,
+      "logits/rejected": -0.8621857762336731,
+      "logps/chosen": -3.0595273971557617,
+      "logps/rejected": -10.996197700500488,
+      "loss": 0.8735880851745605,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.35458827018737793,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.18186795711517334,
+      "rewards/margins": 0.6314299702644348,
+      "rewards/rejected": -0.4495620131492615,
       "step": 67,
-      "train_speed(iter/s)": 0.002399
-    },
-    {
-      "clip_ratio/high_max": 0.030079254298470914,
-      "clip_ratio/high_mean": 0.030079254298470914,
-      "clip_ratio/low_mean": 0.10815927106887102,
-      "clip_ratio/low_min": 0.10815927106887102,
-      "clip_ratio/region_mean": 0.1382385252509266,
-      "epoch": 0.7534626038781164,
-      "grad_norm": 7.7164901906176375,
-      "kl": 4.463950714329258,
-      "learning_rate": 0.0001467848630075608,
-      "loss": 0.0073772999458014965,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021292
+    },
+    {
+      "epoch": 0.13733905579399142,
+      "grad_norm": 4.266996383666992,
+      "learning_rate": 0.000199819104371175,
+      "logits/chosen": -0.780097246170044,
+      "logits/rejected": -0.8485308289527893,
+      "logps/chosen": -2.523211717605591,
+      "logps/rejected": -16.111953735351562,
+      "loss": 0.8541173934936523,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.3557310998439789,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.15383680164813995,
+      "rewards/margins": 0.7679665684700012,
+      "rewards/rejected": -0.6141297817230225,
       "step": 68,
-      "train_speed(iter/s)": 0.002418
-    },
-    {
-      "clip_ratio/high_max": 0.0009369817780680023,
-      "clip_ratio/high_mean": 0.0009369817780680023,
-      "clip_ratio/low_mean": 0.006284750299528241,
-      "clip_ratio/low_min": 0.006284750299528241,
-      "clip_ratio/region_mean": 0.007221732055768371,
-      "completions/clipped_ratio": 0.1484375,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 26.9296875,
-      "completions/min_length": 3.0,
-      "epoch": 0.7645429362880887,
-      "grad_norm": 2.034534758963297,
-      "kl": 2.321827916195616,
-      "learning_rate": 0.00014515333583108896,
-      "loss": 0.01671188324689865,
-      "memory(GiB)": 80.03,
-      "reward": 0.5390625,
-      "reward_std": 0.4874863028526306,
-      "rewards/Response_no_think_reward/mean": 0.5390625,
-      "rewards/Response_no_think_reward/std": 1.3095791339874268,
+      "train_speed(iter/s)": 0.021296
+    },
+    {
+      "epoch": 0.13935874779096188,
+      "grad_norm": 4.107142925262451,
+      "learning_rate": 0.0001997984534788654,
+      "logits/chosen": -0.8730877041816711,
+      "logits/rejected": -0.9365532398223877,
+      "logps/chosen": -3.6425538063049316,
+      "logps/rejected": -14.239816665649414,
+      "loss": 0.9654451608657837,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.5073843002319336,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.17483213543891907,
+      "rewards/margins": 0.7276958227157593,
+      "rewards/rejected": -0.5528636574745178,
       "step": 69,
-      "train_speed(iter/s)": 0.002373
-    },
-    {
-      "clip_ratio/high_max": 0.012032406637445092,
-      "clip_ratio/high_mean": 0.012032406637445092,
-      "clip_ratio/low_mean": 0.11593639780767262,
-      "clip_ratio/low_min": 0.11593639780767262,
-      "clip_ratio/region_mean": 0.1279688044451177,
-      "epoch": 0.775623268698061,
-      "grad_norm": 1.497770517232466,
-      "kl": 3.323778461664915,
-      "learning_rate": 0.00014350656864820733,
-      "loss": 0.0011269270908087492,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021301
+    },
+    {
+      "epoch": 0.14137843978793235,
+      "grad_norm": 6.40019416809082,
+      "learning_rate": 0.00019977668786231534,
+      "logits/chosen": -0.8017210960388184,
+      "logits/rejected": -0.8579236268997192,
+      "logps/chosen": -3.737541437149048,
+      "logps/rejected": -21.258663177490234,
+      "loss": 0.9284271001815796,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.45301198959350586,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.02088707685470581,
+      "rewards/margins": 1.0906426906585693,
+      "rewards/rejected": -1.0697555541992188,
       "step": 70,
-      "train_speed(iter/s)": 0.002392
-    },
-    {
-      "clip_ratio/high_max": 0.000255623715929687,
-      "clip_ratio/high_mean": 0.000255623715929687,
-      "clip_ratio/low_mean": 0.005516766890650615,
-      "clip_ratio/low_min": 0.005516766890650615,
-      "clip_ratio/region_mean": 0.005772390664787963,
-      "completions/clipped_ratio": 0.09375,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 17.25,
-      "completions/min_length": 2.0,
-      "epoch": 0.7867036011080333,
-      "grad_norm": 2.3636642819394384,
-      "kl": 2.6716066980734468,
-      "learning_rate": 0.00014184511727039612,
-      "loss": 0.020374953746795654,
-      "memory(GiB)": 80.03,
-      "reward": 0.34375,
-      "reward_std": 0.5405020713806152,
-      "rewards/Response_no_think_reward/mean": 0.34375,
-      "rewards/Response_no_think_reward/std": 1.2323558330535889,
+      "train_speed(iter/s)": 0.021303
+    },
+    {
+      "epoch": 0.1433981317849028,
+      "grad_norm": 2.583921432495117,
+      "learning_rate": 0.0001997538077646414,
+      "logits/chosen": -0.7329834699630737,
+      "logits/rejected": -0.8454375863075256,
+      "logps/chosen": -1.1047859191894531,
+      "logps/rejected": -17.00843620300293,
+      "loss": 0.5504162907600403,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.13712885975837708,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.37011057138442993,
+      "rewards/margins": 0.8445523381233215,
+      "rewards/rejected": -0.4744417369365692,
       "step": 71,
-      "train_speed(iter/s)": 0.00236
-    },
-    {
-      "clip_ratio/high_max": 0.004156995622906834,
-      "clip_ratio/high_mean": 0.004156995622906834,
-      "clip_ratio/low_mean": 0.11334922257810831,
-      "clip_ratio/low_min": 0.11334922257810831,
-      "clip_ratio/region_mean": 0.11750621721148491,
-      "epoch": 0.7977839335180056,
-      "grad_norm": 1.5028809890146027,
-      "kl": 3.652272095438093,
-      "learning_rate": 0.00014016954246529696,
-      "loss": 0.00900588370859623,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021305
+    },
+    {
+      "epoch": 0.14541782378187326,
+      "grad_norm": 2.9413509368896484,
+      "learning_rate": 0.00019972981344140874,
+      "logits/chosen": -0.7188448309898376,
+      "logits/rejected": -0.8286901116371155,
+      "logps/chosen": -2.631627321243286,
+      "logps/rejected": -22.459745407104492,
+      "loss": 0.7896307706832886,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.3527805805206299,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.12620781362056732,
+      "rewards/margins": 1.1612414121627808,
+      "rewards/rejected": -1.0350335836410522,
       "step": 72,
-      "train_speed(iter/s)": 0.002378
-    },
-    {
-      "clip_ratio/high_max": 0.0024562697508372366,
-      "clip_ratio/high_mean": 0.0024562697508372366,
-      "clip_ratio/low_mean": 0.00900937442202121,
-      "clip_ratio/low_min": 0.00900937442202121,
-      "clip_ratio/region_mean": 0.01146564440568909,
-      "completions/clipped_ratio": 0.1015625,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 20.9609375,
-      "completions/min_length": 6.0,
-      "epoch": 0.8088642659279779,
-      "grad_norm": 1.9262088024122541,
-      "kl": 2.068145776007441,
-      "learning_rate": 0.00013848040976744457,
-      "loss": 0.009554898366332054,
-      "memory(GiB)": 80.03,
-      "reward": 0.7734375,
-      "reward_std": 0.6535134315490723,
-      "rewards/Response_no_think_reward/mean": 0.7734375,
-      "rewards/Response_no_think_reward/std": 1.224518895149231,
+      "train_speed(iter/s)": 0.021307
+    },
+    {
+      "epoch": 0.14743751577884373,
+      "grad_norm": 3.0875403881073,
+      "learning_rate": 0.000199704705160628,
+      "logits/chosen": -0.7547197341918945,
+      "logits/rejected": -0.8351807594299316,
+      "logps/chosen": -5.00991153717041,
+      "logps/rejected": -30.629638671875,
+      "loss": 0.8512266874313354,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.39438357949256897,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1695525050163269,
+      "rewards/margins": 1.1737316846847534,
+      "rewards/rejected": -1.0041792392730713,
       "step": 73,
-      "train_speed(iter/s)": 0.002347
-    },
-    {
-      "clip_ratio/high_max": 0.05884167249314487,
-      "clip_ratio/high_mean": 0.05884167249314487,
-      "clip_ratio/low_mean": 0.014398490195162594,
-      "clip_ratio/low_min": 0.014398490195162594,
-      "clip_ratio/region_mean": 0.0732401623390615,
-      "epoch": 0.8199445983379502,
-      "grad_norm": 3.7794078536832525,
-      "kl": 1.1005137297324836,
-      "learning_rate": 0.00013677828928738934,
-      "loss": 0.027932219207286835,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021308
+    },
+    {
+      "epoch": 0.1494572077758142,
+      "grad_norm": 3.0828444957733154,
+      "learning_rate": 0.0001996784832027525,
+      "logits/chosen": -0.7492446899414062,
+      "logits/rejected": -0.8275682926177979,
+      "logps/chosen": -3.108285903930664,
+      "logps/rejected": -18.224658966064453,
+      "loss": 0.7956384420394897,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.2778869569301605,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1457613706588745,
+      "rewards/margins": 0.7900225520133972,
+      "rewards/rejected": -0.6442610621452332,
       "step": 74,
-      "train_speed(iter/s)": 0.002365
-    },
-    {
-      "clip_ratio/high_max": 0.0020199596765451133,
-      "clip_ratio/high_mean": 0.0020199596765451133,
-      "clip_ratio/low_mean": 0.009859619050985202,
-      "clip_ratio/low_min": 0.009859619050985202,
-      "clip_ratio/region_mean": 0.0118795787129784,
-      "completions/clipped_ratio": 0.09375,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 19.734375,
-      "completions/min_length": 2.0,
-      "epoch": 0.8310249307479224,
-      "grad_norm": 2.6931881467895598,
-      "kl": 1.9703011065721512,
-      "learning_rate": 0.00013506375551927547,
-      "loss": 0.010023066774010658,
-      "memory(GiB)": 80.03,
-      "reward": 0.4140625,
-      "reward_std": 0.6804871559143066,
-      "rewards/Response_no_think_reward/mean": 0.4140625,
-      "rewards/Response_no_think_reward/std": 1.2005729675292969,
+      "train_speed(iter/s)": 0.021311
+    },
+    {
+      "epoch": 0.15147689977278464,
+      "grad_norm": 2.5818111896514893,
+      "learning_rate": 0.00019965114786067516,
+      "logits/chosen": -0.7824280858039856,
+      "logits/rejected": -0.8512482643127441,
+      "logps/chosen": -0.3878008723258972,
+      "logps/rejected": -19.119585037231445,
+      "loss": 0.3888098895549774,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.06809913367033005,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3290511965751648,
+      "rewards/margins": 1.1828463077545166,
+      "rewards/rejected": -0.853795051574707,
       "step": 75,
-      "train_speed(iter/s)": 0.00233
-    },
-    {
-      "clip_ratio/high_max": 0.004052987133036368,
-      "clip_ratio/high_mean": 0.004052987133036368,
-      "clip_ratio/low_mean": 0.14585177681874484,
-      "clip_ratio/low_min": 0.14585177681874484,
-      "clip_ratio/region_mean": 0.14990476449020207,
-      "epoch": 0.8421052631578947,
-      "grad_norm": 1.4459331642351634,
-      "kl": 4.161040774546564,
-      "learning_rate": 0.00013333738714693956,
-      "loss": -0.005643587093800306,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021315
+    },
+    {
+      "epoch": 0.1534965917697551,
+      "grad_norm": 3.1177899837493896,
+      "learning_rate": 0.000199622699439725,
+      "logits/chosen": -0.7851688861846924,
+      "logits/rejected": -0.8108853101730347,
+      "logps/chosen": -1.9916081428527832,
+      "logps/rejected": -18.17394256591797,
+      "loss": 0.6110791563987732,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.25529026985168457,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.44155535101890564,
+      "rewards/margins": 1.2954609394073486,
+      "rewards/rejected": -0.8539055585861206,
       "step": 76,
-      "train_speed(iter/s)": 0.002347
-    },
-    {
-      "clip_ratio/high_max": 0.00046641789958812296,
-      "clip_ratio/high_mean": 0.00046641789958812296,
-      "clip_ratio/low_mean": 0.0038580247201025486,
-      "clip_ratio/low_min": 0.0038580247201025486,
-      "clip_ratio/region_mean": 0.0043244426196906716,
-      "completions/clipped_ratio": 0.0859375,
-      "completions/max_length": 99.0,
-      "completions/mean_length": 14.8828125,
-      "completions/min_length": 2.0,
-      "epoch": 0.853185595567867,
-      "grad_norm": 2.393092790871116,
-      "kl": 2.344057558570057,
-      "learning_rate": 0.00013159976684859527,
-      "loss": 0.014792806468904018,
-      "memory(GiB)": 80.03,
-      "reward": 0.8046875,
-      "reward_std": 0.515557050704956,
-      "rewards/Response_no_think_reward/mean": 0.8046875,
-      "rewards/Response_no_think_reward/std": 1.2862604856491089,
+      "train_speed(iter/s)": 0.021318
+    },
+    {
+      "epoch": 0.15551628376672558,
+      "grad_norm": 4.25754976272583,
+      "learning_rate": 0.00019959313825766386,
+      "logits/chosen": -0.6911004185676575,
+      "logits/rejected": -0.7695882320404053,
+      "logps/chosen": -2.4477198123931885,
+      "logps/rejected": -31.02164077758789,
+      "loss": 0.7074019908905029,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.29494914412498474,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.19254879653453827,
+      "rewards/margins": 1.7098207473754883,
+      "rewards/rejected": -1.5172719955444336,
       "step": 77,
-      "train_speed(iter/s)": 0.002292
-    },
-    {
-      "clip_ratio/high_max": 0.005874009046237916,
-      "clip_ratio/high_mean": 0.005874009046237916,
-      "clip_ratio/low_mean": 0.0609364231931977,
-      "clip_ratio/low_min": 0.0609364231931977,
-      "clip_ratio/region_mean": 0.0668104327050969,
-      "epoch": 0.8642659279778393,
-      "grad_norm": 0.9235795825109938,
-      "kl": 3.164612793829292,
-      "learning_rate": 0.00012985148110016947,
-      "loss": 0.001517204917035997,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.02132
+    },
+    {
+      "epoch": 0.15753597576369605,
+      "grad_norm": 7.972916603088379,
+      "learning_rate": 0.00019956246464468294,
+      "logits/chosen": -0.6392947435379028,
+      "logits/rejected": -0.7352097630500793,
+      "logps/chosen": -4.085236549377441,
+      "logps/rejected": -42.829315185546875,
+      "loss": 0.8604206442832947,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.34664490818977356,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.04581055790185928,
+      "rewards/margins": 1.8653144836425781,
+      "rewards/rejected": -1.9111250638961792,
       "step": 78,
-      "train_speed(iter/s)": 0.002309
-    },
-    {
-      "clip_ratio/high_max": 0.002277967141708359,
-      "clip_ratio/high_mean": 0.002277967141708359,
-      "clip_ratio/low_mean": 0.0037592062435578555,
-      "clip_ratio/low_min": 0.0037592062435578555,
-      "clip_ratio/region_mean": 0.006037173385266215,
-      "completions/clipped_ratio": 0.0625,
-      "completions/max_length": 87.0,
-      "completions/mean_length": 15.703125,
-      "completions/min_length": 2.0,
-      "epoch": 0.8753462603878116,
-      "grad_norm": 1.666555946903183,
-      "kl": 2.2231151023879647,
-      "learning_rate": 0.00012809311997735696,
-      "loss": 0.012878447771072388,
-      "memory(GiB)": 80.03,
-      "reward": 0.546875,
-      "reward_std": 0.7733994126319885,
-      "rewards/Response_no_think_reward/mean": 0.546875,
-      "rewards/Response_no_think_reward/std": 1.3032931089401245,
+      "train_speed(iter/s)": 0.021318
+    },
+    {
+      "epoch": 0.1595556677606665,
+      "grad_norm": 9.261566162109375,
+      "learning_rate": 0.00019953067894339896,
+      "logits/chosen": -0.7612121105194092,
+      "logits/rejected": -0.7584162950515747,
+      "logps/chosen": -22.329971313476562,
+      "logps/rejected": -21.844179153442383,
+      "loss": 1.6152374744415283,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.7144608497619629,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.5578466653823853,
+      "rewards/margins": 0.5714378952980042,
+      "rewards/rejected": -1.1292846202850342,
       "step": 79,
-      "train_speed(iter/s)": 0.002271
-    },
-    {
-      "clip_ratio/high_max": 0.03329824731918052,
-      "clip_ratio/high_mean": 0.03329824731918052,
-      "clip_ratio/low_mean": 0.017765316180884838,
-      "clip_ratio/low_min": 0.017765316180884838,
-      "clip_ratio/region_mean": 0.05106356361648068,
-      "epoch": 0.8864265927977839,
-      "grad_norm": 1.0568349375834465,
-      "kl": 1.923786539278808,
-      "learning_rate": 0.00012632527695645993,
-      "loss": -0.0005231135291978717,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.02132
+    },
+    {
+      "epoch": 0.16157535975763695,
+      "grad_norm": 2.517524242401123,
+      "learning_rate": 0.00019949778150885042,
+      "logits/chosen": -0.7058742046356201,
+      "logits/rejected": -0.8091444373130798,
+      "logps/chosen": -2.800210952758789,
+      "logps/rejected": -20.613197326660156,
+      "loss": 0.5419849157333374,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.18958702683448792,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1262124478816986,
+      "rewards/margins": 1.2596018314361572,
+      "rewards/rejected": -1.1333893537521362,
       "step": 80,
-      "train_speed(iter/s)": 0.002287
-    },
-    {
-      "clip_ratio/high_max": 0.00021551724057644606,
-      "clip_ratio/high_mean": 0.00021551724057644606,
-      "clip_ratio/low_mean": 0.006801646784879267,
-      "clip_ratio/low_min": 0.006801646784879267,
-      "clip_ratio/region_mean": 0.007017164025455713,
-      "completions/clipped_ratio": 0.046875,
-      "completions/max_length": 90.0,
-      "completions/mean_length": 14.03125,
-      "completions/min_length": 2.0,
-      "epoch": 0.8975069252077562,
-      "grad_norm": 9.477113742654934,
-      "kl": 3.0111945159733295,
-      "learning_rate": 0.00012454854871407994,
-      "loss": 0.016959797590970993,
-      "memory(GiB)": 80.03,
-      "reward": 0.2578125,
-      "reward_std": 0.5107755661010742,
-      "rewards/Response_no_think_reward/mean": 0.2578125,
-      "rewards/Response_no_think_reward/std": 1.2874077558517456,
+      "train_speed(iter/s)": 0.02132
+    },
+    {
+      "epoch": 0.16359505175460742,
+      "grad_norm": 2.3371551036834717,
+      "learning_rate": 0.00019946377270849356,
+      "logits/chosen": -0.6530874967575073,
+      "logits/rejected": -0.7903028130531311,
+      "logps/chosen": -1.8574573993682861,
+      "logps/rejected": -34.57234573364258,
+      "loss": 0.47842535376548767,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.20502685010433197,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.21239632368087769,
+      "rewards/margins": 1.7234400510787964,
+      "rewards/rejected": -1.5110437870025635,
       "step": 81,
-      "train_speed(iter/s)": 0.002263
-    },
-    {
-      "clip_ratio/high_max": 0.012387449765810743,
-      "clip_ratio/high_mean": 0.012387449765810743,
-      "clip_ratio/low_mean": 0.04806764563545585,
-      "clip_ratio/low_min": 0.04806764563545585,
-      "clip_ratio/region_mean": 0.06045509548857808,
-      "epoch": 0.9085872576177285,
-      "grad_norm": 1.1639747912015215,
-      "kl": 3.7243148013949394,
-      "learning_rate": 0.00012276353492572935,
-      "loss": 0.0026693246327340603,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021321
+    },
+    {
+      "epoch": 0.1656147437515779,
+      "grad_norm": 2.623552083969116,
+      "learning_rate": 0.00019942865292219838,
+      "logits/chosen": -0.5469449162483215,
+      "logits/rejected": -0.7048972249031067,
+      "logps/chosen": -4.390412330627441,
+      "logps/rejected": -31.797592163085938,
+      "loss": 0.6188596487045288,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.28328126668930054,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1721397191286087,
+      "rewards/margins": 1.405639410018921,
+      "rewards/rejected": -1.2334996461868286,
       "step": 82,
-      "train_speed(iter/s)": 0.002279
-    },
-    {
-      "clip_ratio/high_max": 0.0004032258002553135,
-      "clip_ratio/high_mean": 0.0004032258002553135,
-      "clip_ratio/low_mean": 0.0061736139468848705,
-      "clip_ratio/low_min": 0.0061736139468848705,
-      "clip_ratio/region_mean": 0.006576839747140184,
-      "completions/clipped_ratio": 0.03125,
-      "completions/max_length": 99.0,
-      "completions/mean_length": 12.625,
-      "completions/min_length": 2.0,
-      "epoch": 0.9196675900277008,
-      "grad_norm": 7.345333737975268,
-      "kl": 1.6264500059187412,
-      "learning_rate": 0.00012097083806343103,
-      "loss": 0.02225027047097683,
-      "memory(GiB)": 80.03,
-      "reward": 0.6640625,
-      "reward_std": 0.4172249436378479,
-      "rewards/Response_no_think_reward/mean": 0.6640625,
-      "rewards/Response_no_think_reward/std": 1.1588573455810547,
+      "train_speed(iter/s)": 0.02132
+    },
+    {
+      "epoch": 0.16763443574854833,
+      "grad_norm": 3.757638931274414,
+      "learning_rate": 0.00019939242254224423,
+      "logits/chosen": -0.7363824248313904,
+      "logits/rejected": -0.8039526343345642,
+      "logps/chosen": -4.921553134918213,
+      "logps/rejected": -18.160133361816406,
+      "loss": 1.017354965209961,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.4084692597389221,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.042609840631484985,
+      "rewards/margins": 0.6757681965827942,
+      "rewards/rejected": -0.7183780074119568,
       "step": 83,
-      "train_speed(iter/s)": 0.00227
-    },
-    {
-      "clip_ratio/high_max": 0.0035271961241960526,
-      "clip_ratio/high_mean": 0.0035271961241960526,
-      "clip_ratio/low_mean": 0.02831025089835748,
-      "clip_ratio/low_min": 0.02831025089835748,
-      "clip_ratio/region_mean": 0.03183744702255353,
-      "epoch": 0.9307479224376731,
-      "grad_norm": 84.3494363093827,
-      "kl": 10.616167868487537,
-      "learning_rate": 0.00011917106319237386,
-      "loss": 0.17172452807426453,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021321
+    },
+    {
+      "epoch": 0.1696541277455188,
+      "grad_norm": 2.6959855556488037,
+      "learning_rate": 0.00019935508197331555,
+      "logits/chosen": -0.601901650428772,
+      "logits/rejected": -0.7518936395645142,
+      "logps/chosen": -2.1843678951263428,
+      "logps/rejected": -32.7392578125,
+      "loss": 0.5553274154663086,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.21038973331451416,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.22294606268405914,
+      "rewards/margins": 1.4514588117599487,
+      "rewards/rejected": -1.2285128831863403,
       "step": 84,
-      "train_speed(iter/s)": 0.002286
-    },
-    {
-      "clip_ratio/high_max": 0.0011701860348694026,
-      "clip_ratio/high_mean": 0.0011701860348694026,
-      "clip_ratio/low_mean": 0.014238029951229692,
-      "clip_ratio/low_min": 0.014238029951229692,
-      "clip_ratio/region_mean": 0.015408215986099094,
-      "completions/clipped_ratio": 0.140625,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 19.7578125,
-      "completions/min_length": 4.0,
-      "epoch": 0.9418282548476454,
-      "grad_norm": 1.9907484000781221,
-      "kl": 2.995624510163907,
-      "learning_rate": 0.00011736481776669306,
-      "loss": 0.01750401221215725,
-      "memory(GiB)": 80.03,
-      "reward": 0.4765625,
-      "reward_std": 0.4854952394962311,
-      "rewards/Response_no_think_reward/mean": 0.4765625,
-      "rewards/Response_no_think_reward/std": 1.1080580949783325,
+      "train_speed(iter/s)": 0.02132
+    },
+    {
+      "epoch": 0.17167381974248927,
+      "grad_norm": 3.8080639839172363,
+      "learning_rate": 0.00019931663163249742,
+      "logits/chosen": -0.8070834875106812,
+      "logits/rejected": -0.8214148283004761,
+      "logps/chosen": -4.387559413909912,
+      "logps/rejected": -13.731513977050781,
+      "loss": 0.8594411611557007,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.2712303698062897,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.08239315450191498,
+      "rewards/margins": 0.5998476147651672,
+      "rewards/rejected": -0.5174545049667358,
       "step": 85,
-      "train_speed(iter/s)": 0.002274
-    },
-    {
-      "clip_ratio/high_max": 0.008182557401596569,
-      "clip_ratio/high_mean": 0.008182557401596569,
-      "clip_ratio/low_mean": 0.11124554229900241,
-      "clip_ratio/low_min": 0.11124554229900241,
-      "clip_ratio/region_mean": 0.11942810016626026,
-      "epoch": 0.9529085872576177,
-      "grad_norm": 1.9564641583381899,
-      "kl": 4.675610944104847,
-      "learning_rate": 0.00011555271142444433,
-      "loss": 0.009754904545843601,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021322
+    },
+    {
+      "epoch": 0.17369351173945974,
+      "grad_norm": 4.74073600769043,
+      "learning_rate": 0.00019927707194927066,
+      "logits/chosen": -0.770719051361084,
+      "logits/rejected": -0.802936315536499,
+      "logps/chosen": -3.8553550243377686,
+      "logps/rejected": -22.488487243652344,
+      "loss": 0.7815805077552795,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.432535856962204,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1827356368303299,
+      "rewards/margins": 1.2188729047775269,
+      "rewards/rejected": -1.0361372232437134,
       "step": 86,
-      "train_speed(iter/s)": 0.002289
-    },
-    {
-      "clip_ratio/high_max": 0.0004139889351790771,
-      "clip_ratio/high_mean": 0.0004139889351790771,
-      "clip_ratio/low_mean": 0.009557914454489946,
-      "clip_ratio/low_min": 0.009557914454489946,
-      "clip_ratio/region_mean": 0.0099719034624286,
-      "completions/clipped_ratio": 0.0859375,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 17.90625,
-      "completions/min_length": 2.0,
-      "epoch": 0.96398891966759,
-      "grad_norm": 1.805689670049186,
-      "kl": 3.4809365491382778,
-      "learning_rate": 0.00011373535578184082,
-      "loss": 0.0213579460978508,
-      "memory(GiB)": 80.03,
-      "reward": 0.5859375,
-      "reward_std": 0.5313136577606201,
-      "rewards/Response_no_think_reward/mean": 0.5859375,
-      "rewards/Response_no_think_reward/std": 1.1939964294433594,
+      "train_speed(iter/s)": 0.021324
+    },
+    {
+      "epoch": 0.1757132037364302,
+      "grad_norm": 6.638978481292725,
+      "learning_rate": 0.0001992364033655072,
+      "logits/chosen": -0.6007230281829834,
+      "logits/rejected": -0.7821131944656372,
+      "logps/chosen": -0.698825478553772,
+      "logps/rejected": -31.5311279296875,
+      "loss": 0.4713582396507263,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.09712451696395874,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.23644955456256866,
+      "rewards/margins": 1.2481129169464111,
+      "rewards/rejected": -1.0116633176803589,
       "step": 87,
-      "train_speed(iter/s)": 0.002277
-    },
-    {
-      "clip_ratio/high_max": 0.009385127894347534,
-      "clip_ratio/high_mean": 0.009385127894347534,
-      "clip_ratio/low_mean": 0.08647578035015613,
-      "clip_ratio/low_min": 0.08647578035015613,
-      "clip_ratio/region_mean": 0.095860909903422,
-      "epoch": 0.9750692520775623,
-      "grad_norm": 2.411018383110019,
-      "kl": 4.761912747140741,
-      "learning_rate": 0.00011191336422682237,
-      "loss": 0.01895134523510933,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021324
+    },
+    {
+      "epoch": 0.17773289573340065,
+      "grad_norm": 5.0127739906311035,
+      "learning_rate": 0.00019919462633546519,
+      "logits/chosen": -0.7654692530632019,
+      "logits/rejected": -0.8514058589935303,
+      "logps/chosen": -4.281129837036133,
+      "logps/rejected": -17.92319107055664,
+      "loss": 1.0033936500549316,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.5217462182044983,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.14046959578990936,
+      "rewards/margins": 0.7611122131347656,
+      "rewards/rejected": -0.6206426620483398,
       "step": 88,
-      "train_speed(iter/s)": 0.002291
-    },
-    {
-      "clip_ratio/high_max": 0.0031819915457163006,
-      "clip_ratio/high_mean": 0.0031819915457163006,
-      "clip_ratio/low_mean": 0.012216789647936821,
-      "clip_ratio/low_min": 0.012216789647936821,
-      "clip_ratio/region_mean": 0.015398780989926308,
-      "completions/clipped_ratio": 0.0625,
-      "completions/max_length": 64.0,
-      "completions/mean_length": 11.28125,
-      "completions/min_length": 2.0,
-      "epoch": 0.9861495844875346,
-      "grad_norm": 2.2626010793409574,
-      "kl": 6.020935451146215,
-      "learning_rate": 0.00011008735171202684,
-      "loss": 0.031282562762498856,
-      "memory(GiB)": 80.03,
-      "reward": 0.2890625,
-      "reward_std": 0.847247302532196,
-      "rewards/Response_no_think_reward/mean": 0.2890625,
-      "rewards/Response_no_think_reward/std": 1.1915208101272583,
+      "train_speed(iter/s)": 0.021326
+    },
+    {
+      "epoch": 0.17975258773037112,
+      "grad_norm": 5.285451889038086,
+      "learning_rate": 0.00019915174132578378,
+      "logits/chosen": -0.6645584106445312,
+      "logits/rejected": -0.789848804473877,
+      "logps/chosen": -5.77445650100708,
+      "logps/rejected": -37.78670883178711,
+      "loss": 0.6340041160583496,
+      "memory(GiB)": 41.88,
+      "nll_loss": 0.26252368092536926,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.07033467292785645,
+      "rewards/margins": 1.5086185932159424,
+      "rewards/rejected": -1.4382836818695068,
       "step": 89,
-      "train_speed(iter/s)": 0.002254
-    },
-    {
-      "clip_ratio/high_max": 0.024421937006991357,
-      "clip_ratio/high_mean": 0.024421937006991357,
-      "clip_ratio/low_mean": 0.0521851975354366,
-      "clip_ratio/low_min": 0.0521851975354366,
-      "clip_ratio/region_mean": 0.07660713430959731,
-      "epoch": 0.997229916897507,
-      "grad_norm": 1.8302536272850396,
-      "kl": 6.024846433196217,
-      "learning_rate": 0.00010825793454723325,
-      "loss": 0.017346249893307686,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021326
+    },
+    {
+      "epoch": 0.18177227972734158,
+      "grad_norm": 6.486772060394287,
+      "learning_rate": 0.000199107748815478,
+      "logits/chosen": -0.5895614624023438,
+      "logits/rejected": -0.7669257521629333,
+      "logps/chosen": -2.622659206390381,
+      "logps/rejected": -42.33251953125,
+      "loss": 0.7003304958343506,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.30153194069862366,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2086510956287384,
+      "rewards/margins": 1.978740930557251,
+      "rewards/rejected": -1.770089864730835,
       "step": 90,
-      "train_speed(iter/s)": 0.002268
-    },
-    {
-      "clip_ratio/high_max": 0.0011776478204410523,
-      "clip_ratio/high_mean": 0.0011776478204410523,
-      "clip_ratio/low_mean": 0.0035608798498287797,
-      "clip_ratio/low_min": 0.0035608798498287797,
-      "clip_ratio/region_mean": 0.004738527670269832,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 24.046875,
-      "completions/min_length": 3.0,
-      "epoch": 1.0110803324099722,
-      "grad_norm": 2.3698869054826233,
-      "kl": 5.333947827733937,
-      "learning_rate": 0.00010642573019134703,
-      "loss": 0.028505954891443253,
-      "memory(GiB)": 80.03,
-      "reward": 0.84375,
-      "reward_std": 0.6655995845794678,
-      "rewards/Response_no_think_reward/mean": 0.84375,
-      "rewards/Response_no_think_reward/std": 1.2065274715423584,
+      "train_speed(iter/s)": 0.021324
+    },
+    {
+      "epoch": 0.18379197172431205,
+      "grad_norm": 4.038703918457031,
+      "learning_rate": 0.00019906264929593347,
+      "logits/chosen": -0.6498143076896667,
+      "logits/rejected": -0.7417197823524475,
+      "logps/chosen": -6.643020153045654,
+      "logps/rejected": -26.659536361694336,
+      "loss": 0.5661642551422119,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23169369995594025,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.25588488578796387,
+      "rewards/margins": 1.4719629287719727,
+      "rewards/rejected": -1.2160780429840088,
       "step": 91,
-      "train_speed(iter/s)": 0.002232
-    },
-    {
-      "clip_ratio/high_max": 0.02221274602925405,
-      "clip_ratio/high_mean": 0.02221274602925405,
-      "clip_ratio/low_mean": 0.04682085904642008,
-      "clip_ratio/low_min": 0.04682085904642008,
-      "clip_ratio/region_mean": 0.0690336050465703,
-      "epoch": 1.0221606648199446,
-      "grad_norm": 1.377879165623118,
-      "kl": 6.767704317186144,
-      "learning_rate": 0.00010459135704399718,
-      "loss": 0.01659482903778553,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021323
+    },
+    {
+      "epoch": 0.1858116637212825,
+      "grad_norm": 2.8225672245025635,
+      "learning_rate": 0.00019901644327090064,
+      "logits/chosen": -0.7718486189842224,
+      "logits/rejected": -0.8817813992500305,
+      "logps/chosen": -2.4756038188934326,
+      "logps/rejected": -29.404544830322266,
+      "loss": 0.4890742599964142,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24528944492340088,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2983011305332184,
+      "rewards/margins": 1.8501620292663574,
+      "rewards/rejected": -1.5518608093261719,
       "step": 92,
-      "train_speed(iter/s)": 0.002246
-    },
-    {
-      "clip_ratio/high_max": 0.0007944914977997541,
-      "clip_ratio/high_mean": 0.0007944914977997541,
-      "clip_ratio/low_mean": 0.005214237666223198,
-      "clip_ratio/low_min": 0.005214237666223198,
-      "clip_ratio/region_mean": 0.006008729164022952,
-      "completions/clipped_ratio": 0.046875,
-      "completions/max_length": 79.0,
-      "completions/mean_length": 11.1953125,
-      "completions/min_length": 2.0,
-      "epoch": 1.0332409972299168,
-      "grad_norm": 1.128328936062302,
-      "kl": 2.6099998716963455,
-      "learning_rate": 0.00010275543423681621,
-      "loss": 0.01443527452647686,
-      "memory(GiB)": 80.03,
-      "reward": 0.5703125,
-      "reward_std": 0.34169840812683105,
-      "rewards/Response_no_think_reward/mean": 0.5703125,
-      "rewards/Response_no_think_reward/std": 1.26543128490448,
+      "train_speed(iter/s)": 0.021325
+    },
+    {
+      "epoch": 0.18783135571825296,
+      "grad_norm": 7.307145595550537,
+      "learning_rate": 0.00019896913125648955,
+      "logits/chosen": -0.8246339559555054,
+      "logits/rejected": -0.8683527708053589,
+      "logps/chosen": -6.234779357910156,
+      "logps/rejected": -18.237667083740234,
+      "loss": 1.4174689054489136,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7513330578804016,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1854678839445114,
+      "rewards/margins": 0.6419495940208435,
+      "rewards/rejected": -0.8274174332618713,
       "step": 93,
-      "train_speed(iter/s)": 0.00223
-    },
-    {
-      "clip_ratio/high_max": 0.024202606233302504,
-      "clip_ratio/high_mean": 0.024202606233302504,
-      "clip_ratio/low_mean": 0.017650849069468677,
-      "clip_ratio/low_min": 0.017650849069468677,
-      "clip_ratio/region_mean": 0.04185345536097884,
-      "epoch": 1.0443213296398892,
-      "grad_norm": 0.5979565041663649,
-      "kl": 1.9247902451315895,
-      "learning_rate": 0.00010091858142447265,
-      "loss": 0.005481676664203405,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021328
+    },
+    {
+      "epoch": 0.18985104771522343,
+      "grad_norm": 5.936891078948975,
+      "learning_rate": 0.00019892071378116376,
+      "logits/chosen": -0.7434009313583374,
+      "logits/rejected": -0.8614495396614075,
+      "logps/chosen": -4.436880111694336,
+      "logps/rejected": -27.014469146728516,
+      "loss": 0.9648631811141968,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.44619184732437134,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.08920584619045258,
+      "rewards/margins": 1.2521498203277588,
+      "rewards/rejected": -1.341355562210083,
       "step": 94,
-      "train_speed(iter/s)": 0.002243
-    },
-    {
-      "clip_ratio/high_max": 0.002601411077193916,
-      "clip_ratio/high_mean": 0.002601411077193916,
-      "clip_ratio/low_mean": 0.005088170932140201,
-      "clip_ratio/low_min": 0.005088170932140201,
-      "clip_ratio/region_mean": 0.007689581951126456,
-      "completions/clipped_ratio": 0.09375,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 18.25,
-      "completions/min_length": 4.0,
-      "epoch": 1.0554016620498614,
-      "grad_norm": 2.0648308983451917,
-      "kl": 1.713204285595566,
-      "learning_rate": 9.908141857552737e-05,
-      "loss": 0.015094820410013199,
-      "memory(GiB)": 80.03,
-      "reward": 0.8046875,
-      "reward_std": 0.47176384925842285,
-      "rewards/Response_no_think_reward/mean": 0.8046875,
-      "rewards/Response_no_think_reward/std": 1.1708977222442627,
+      "train_speed(iter/s)": 0.021331
+    },
+    {
+      "epoch": 0.1918707397121939,
+      "grad_norm": 11.764037132263184,
+      "learning_rate": 0.0001988711913857346,
+      "logits/chosen": -0.5709604024887085,
+      "logits/rejected": -0.7604628801345825,
+      "logps/chosen": -5.328193187713623,
+      "logps/rejected": -44.60304260253906,
+      "loss": 0.8271011114120483,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5297277569770813,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.0564165934920311,
+      "rewards/margins": 2.2518301010131836,
+      "rewards/rejected": -2.3082468509674072,
       "step": 95,
-      "train_speed(iter/s)": 0.002221
-    },
-    {
-      "clip_ratio/high_max": 0.01881888063508086,
-      "clip_ratio/high_mean": 0.01881888063508086,
-      "clip_ratio/low_mean": 0.026617751631420106,
-      "clip_ratio/low_min": 0.026617751631420106,
-      "clip_ratio/region_mean": 0.0454366315389052,
-      "epoch": 1.0664819944598338,
-      "grad_norm": 1.5544793796965792,
-      "kl": 1.6157679219031706,
-      "learning_rate": 9.724456576318381e-05,
-      "loss": 0.012739625759422779,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021329
+    },
+    {
+      "epoch": 0.19389043170916434,
+      "grad_norm": 4.440858840942383,
+      "learning_rate": 0.00019882056462335512,
+      "logits/chosen": -0.716915488243103,
+      "logits/rejected": -0.8559072017669678,
+      "logps/chosen": -4.923259258270264,
+      "logps/rejected": -22.751483917236328,
+      "loss": 1.0631147623062134,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5847352743148804,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.04070784151554108,
+      "rewards/margins": 0.9810720086097717,
+      "rewards/rejected": -0.9403641223907471,
       "step": 96,
-      "train_speed(iter/s)": 0.002234
-    },
-    {
-      "clip_ratio/high_max": 0.0020559211261570454,
-      "clip_ratio/high_mean": 0.0020559211261570454,
-      "clip_ratio/low_mean": 0.0043267360888421535,
-      "clip_ratio/low_min": 0.0043267360888421535,
-      "clip_ratio/region_mean": 0.006382657098583877,
-      "completions/clipped_ratio": 0.015625,
-      "completions/max_length": 41.0,
-      "completions/mean_length": 11.046875,
-      "completions/min_length": 3.0,
-      "epoch": 1.077562326869806,
-      "grad_norm": 2.5531547462941115,
-      "kl": 1.4817758100107312,
-      "learning_rate": 9.540864295600283e-05,
-      "loss": 0.016522858291864395,
-      "memory(GiB)": 80.03,
-      "reward": 0.9453125,
-      "reward_std": 0.4848037362098694,
-      "rewards/Response_no_think_reward/mean": 0.9453125,
-      "rewards/Response_no_think_reward/std": 1.1454023122787476,
+      "train_speed(iter/s)": 0.021331
+    },
+    {
+      "epoch": 0.1959101237061348,
+      "grad_norm": 4.534213542938232,
+      "learning_rate": 0.00019876883405951377,
+      "logits/chosen": -0.7735016345977783,
+      "logits/rejected": -0.8521608114242554,
+      "logps/chosen": -2.242138147354126,
+      "logps/rejected": -16.696674346923828,
+      "loss": 0.8221019506454468,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.30803507566452026,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.23840653896331787,
+      "rewards/margins": 0.7663496732711792,
+      "rewards/rejected": -0.5279431939125061,
       "step": 97,
-      "train_speed(iter/s)": 0.00222
-    },
-    {
-      "clip_ratio/high_max": 0.01358306163456291,
-      "clip_ratio/high_mean": 0.01358306163456291,
-      "clip_ratio/low_mean": 0.019928007503040135,
-      "clip_ratio/low_min": 0.019928007503040135,
-      "clip_ratio/region_mean": 0.03351106960326433,
-      "epoch": 1.0886426592797784,
-      "grad_norm": 1.0799722223140724,
-      "kl": 1.3886900492943823,
-      "learning_rate": 9.357426980865301e-05,
-      "loss": -0.0007159767556004226,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021332
+    },
+    {
+      "epoch": 0.19792981570310528,
+      "grad_norm": 6.823293685913086,
+      "learning_rate": 0.0001987160002720283,
+      "logits/chosen": -0.7845535278320312,
+      "logits/rejected": -0.8421120643615723,
+      "logps/chosen": -8.860337257385254,
+      "logps/rejected": -18.79197883605957,
+      "loss": 0.9576016068458557,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.476947546005249,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1753305345773697,
+      "rewards/margins": 1.0159931182861328,
+      "rewards/rejected": -0.8406625986099243,
       "step": 98,
-      "train_speed(iter/s)": 0.002234
-    },
-    {
-      "clip_ratio/high_max": 0.001544758939417079,
-      "clip_ratio/high_mean": 0.001544758939417079,
-      "clip_ratio/low_mean": 0.0037943847200949676,
-      "clip_ratio/low_min": 0.0037943847200949676,
-      "clip_ratio/region_mean": 0.005339143652236089,
-      "completions/clipped_ratio": 0.0546875,
-      "completions/max_length": 101.0,
-      "completions/mean_length": 18.546875,
-      "completions/min_length": 3.0,
-      "epoch": 1.0997229916897506,
-      "grad_norm": 0.943375664401446,
-      "kl": 1.7392279328778386,
-      "learning_rate": 9.174206545276677e-05,
-      "loss": 0.014223725534975529,
-      "memory(GiB)": 80.03,
-      "reward": 1.1015625,
-      "reward_std": 0.39179152250289917,
-      "rewards/Response_no_think_reward/mean": 1.1015625,
-      "rewards/Response_no_think_reward/std": 1.2221051454544067,
+      "train_speed(iter/s)": 0.021334
+    },
+    {
+      "epoch": 0.19994950770007575,
+      "grad_norm": 3.7400314807891846,
+      "learning_rate": 0.00019866206385103915,
+      "logits/chosen": -0.708592414855957,
+      "logits/rejected": -0.7664063572883606,
+      "logps/chosen": -5.081974983215332,
+      "logps/rejected": -16.826196670532227,
+      "loss": 1.022936224937439,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4172855615615845,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.049844808876514435,
+      "rewards/margins": 0.2858249545097351,
+      "rewards/rejected": -0.23598016798496246,
       "step": 99,
-      "train_speed(iter/s)": 0.002219
-    },
-    {
-      "clip_ratio/high_max": 0.01209607784403488,
-      "clip_ratio/high_mean": 0.01209607784403488,
-      "clip_ratio/low_mean": 0.011367922488716431,
-      "clip_ratio/low_min": 0.011367922488716431,
-      "clip_ratio/region_mean": 0.023464000318199396,
-      "epoch": 1.110803324099723,
-      "grad_norm": 0.8253425068817393,
-      "kl": 1.7898913251701742,
-      "learning_rate": 8.991264828797319e-05,
-      "loss": 0.006411677226424217,
-      "memory(GiB)": 80.03,
+      "train_speed(iter/s)": 0.021335
+    },
+    {
+      "epoch": 0.2019691996970462,
+      "grad_norm": 4.397147178649902,
+      "learning_rate": 0.00019860702539900287,
+      "logits/chosen": -0.5574980974197388,
+      "logits/rejected": -0.7396320700645447,
+      "logps/chosen": -2.61395001411438,
+      "logps/rejected": -33.31013107299805,
+      "loss": 0.8842970132827759,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3539983034133911,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.06448046863079071,
+      "rewards/margins": 0.9230259656906128,
+      "rewards/rejected": -0.8585454225540161,
       "step": 100,
-      "train_speed(iter/s)": 0.002232
-    },
-    {
-      "clip_ratio/high_max": 0.0038593837525695562,
-      "clip_ratio/high_mean": 0.0038593837525695562,
-      "clip_ratio/low_mean": 0.007897521747509018,
-      "clip_ratio/low_min": 0.007897521747509018,
-      "clip_ratio/region_mean": 0.011756905500078574,
-      "completions/clipped_ratio": 0.140625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 18.0390625,
-      "completions/min_length": 2.0,
-      "epoch": 1.1218836565096952,
-      "grad_norm": 2.1091313211648033,
-      "kl": 4.433779507409781,
-      "learning_rate": 8.808663577317764e-05,
-      "loss": 0.026584401726722717,
-      "memory(GiB)": 77.36,
-      "reward": 0.234375,
-      "reward_std": 0.353938490152359,
-      "rewards/Response_no_think_reward_1/mean": 0.234375,
-      "rewards/Response_no_think_reward_1/std": 0.8275223970413208,
+      "train_speed(iter/s)": 0.021334
+    },
+    {
+      "epoch": 0.20398889169401666,
+      "grad_norm": 3.6620795726776123,
+      "learning_rate": 0.0001985508855306855,
+      "logits/chosen": -0.7978264093399048,
+      "logits/rejected": -0.8318206667900085,
+      "logps/chosen": -3.0011239051818848,
+      "logps/rejected": -13.4129638671875,
+      "loss": 0.869354784488678,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.28029945492744446,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.17673368752002716,
+      "rewards/margins": 0.44924402236938477,
+      "rewards/rejected": -0.2725103795528412,
       "step": 101,
-      "train_speed(iter/s)": 0.112917
-    },
-    {
-      "clip_ratio/high_max": 0.0257808348396793,
-      "clip_ratio/high_mean": 0.0257808348396793,
-      "clip_ratio/low_mean": 0.013182859780499712,
-      "clip_ratio/low_min": 0.013182859780499712,
-      "clip_ratio/region_mean": 0.038963694794801995,
-      "epoch": 1.1329639889196677,
-      "grad_norm": 0.8069647208167382,
-      "kl": 2.7774715912528336,
-      "learning_rate": 8.626464421815919e-05,
-      "loss": 0.008937789127230644,
-      "memory(GiB)": 77.36,
+      "train_speed(iter/s)": 0.021264
+    },
+    {
+      "epoch": 0.20600858369098712,
+      "grad_norm": 2.9251015186309814,
+      "learning_rate": 0.00019849364487315558,
+      "logits/chosen": -0.6999006271362305,
+      "logits/rejected": -0.7932964563369751,
+      "logps/chosen": -4.888404369354248,
+      "logps/rejected": -15.926616668701172,
+      "loss": 0.9600812792778015,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4099733531475067,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.16860802471637726,
+      "rewards/margins": 0.5318610668182373,
+      "rewards/rejected": -0.36325308680534363,
       "step": 102,
-      "train_speed(iter/s)": 0.09385
-    },
-    {
-      "clip_ratio/high_max": 0.00028669723542407155,
-      "clip_ratio/high_mean": 0.00028669723542407155,
-      "clip_ratio/low_mean": 0.0008802816737443209,
-      "clip_ratio/low_min": 0.0008802816737443209,
-      "clip_ratio/region_mean": 0.0011669789091683924,
-      "completions/clipped_ratio": 0.109375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 18.140625,
-      "completions/min_length": 3.0,
-      "epoch": 1.1440443213296398,
-      "grad_norm": 3.4014714828505634,
-      "kl": 1.8993340344168246,
-      "learning_rate": 8.444728857555572e-05,
-      "loss": 0.029605647549033165,
-      "memory(GiB)": 77.47,
-      "reward": 0.390625,
-      "reward_std": 0.1173202246427536,
-      "rewards/Response_no_think_reward_1/mean": 0.390625,
-      "rewards/Response_no_think_reward_1/std": 0.8440096974372864,
+      "train_speed(iter/s)": 0.021266
+    },
+    {
+      "epoch": 0.2080282756879576,
+      "grad_norm": 3.172440767288208,
+      "learning_rate": 0.00019843530406577723,
+      "logits/chosen": -0.7246884107589722,
+      "logits/rejected": -0.7787467837333679,
+      "logps/chosen": -3.1632981300354004,
+      "logps/rejected": -21.545318603515625,
+      "loss": 0.7317727208137512,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3409346342086792,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1461772322654724,
+      "rewards/margins": 1.200048565864563,
+      "rewards/rejected": -1.0538712739944458,
       "step": 103,
-      "train_speed(iter/s)": 0.053798
-    },
-    {
-      "clip_ratio/high_max": 0.0013847328373230994,
-      "clip_ratio/high_mean": 0.0013847328373230994,
-      "clip_ratio/low_mean": 0.004156515002250671,
-      "clip_ratio/low_min": 0.004156515002250671,
-      "clip_ratio/region_mean": 0.00554124778136611,
-      "epoch": 1.1551246537396123,
-      "grad_norm": 0.6574286317590833,
-      "kl": 0.8905834904871881,
-      "learning_rate": 8.263518223330697e-05,
-      "loss": 0.007299074437469244,
-      "memory(GiB)": 77.47,
+      "train_speed(iter/s)": 0.021268
+    },
+    {
+      "epoch": 0.21004796768492806,
+      "grad_norm": 3.71928334236145,
+      "learning_rate": 0.00019837586376020294,
+      "logits/chosen": -0.6723949313163757,
+      "logits/rejected": -0.7337789535522461,
+      "logps/chosen": -9.011453628540039,
+      "logps/rejected": -24.695667266845703,
+      "loss": 0.8767037987709045,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.40366095304489136,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.017453588545322418,
+      "rewards/margins": 1.022116780281067,
+      "rewards/rejected": -1.0395703315734863,
       "step": 104,
-      "train_speed(iter/s)": 0.049358
-    },
-    {
-      "clip_ratio/high_max": 0.00036549707874655724,
-      "clip_ratio/high_mean": 0.00036549707874655724,
-      "clip_ratio/low_mean": 0.003542276710504666,
-      "clip_ratio/low_min": 0.003542276710504666,
-      "clip_ratio/region_mean": 0.003907773789251223,
-      "completions/clipped_ratio": 0.0546875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 12.1015625,
-      "completions/min_length": 2.0,
-      "epoch": 1.1662049861495845,
-      "grad_norm": 1.418255621232663,
-      "kl": 1.0295969531871378,
-      "learning_rate": 8.082893680762619e-05,
-      "loss": 0.008070322684943676,
-      "memory(GiB)": 77.47,
-      "reward": 0.5625,
-      "reward_std": 0.283821702003479,
-      "rewards/Response_no_think_reward_1/mean": 0.5625,
-      "rewards/Response_no_think_reward_1/std": 0.6728714108467102,
+      "train_speed(iter/s)": 0.02127
+    },
+    {
+      "epoch": 0.2120676596818985,
+      "grad_norm": 2.393298625946045,
+      "learning_rate": 0.00019831532462036636,
+      "logits/chosen": -0.5787035226821899,
+      "logits/rejected": -0.7247171998023987,
+      "logps/chosen": -3.2338626384735107,
+      "logps/rejected": -30.85213851928711,
+      "loss": 0.5790133476257324,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23591873049736023,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.273136168718338,
+      "rewards/margins": 1.4806981086730957,
+      "rewards/rejected": -1.2075618505477905,
       "step": 105,
-      "train_speed(iter/s)": 0.035507
-    },
-    {
-      "clip_ratio/high_max": 0.015719514689408243,
-      "clip_ratio/high_mean": 0.015719514689408243,
-      "clip_ratio/low_mean": 0.010580109432339668,
-      "clip_ratio/low_min": 0.010580109432339668,
-      "clip_ratio/region_mean": 0.02629962412174791,
-      "epoch": 1.1772853185595569,
-      "grad_norm": 0.9124403050854863,
-      "kl": 0.933376073371619,
-      "learning_rate": 7.902916193656898e-05,
-      "loss": -0.002763347467407584,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.02127
+    },
+    {
+      "epoch": 0.21408735167886897,
+      "grad_norm": 3.9807705879211426,
+      "learning_rate": 0.0001982536873224748,
+      "logits/chosen": -0.6726570129394531,
+      "logits/rejected": -0.7844520807266235,
+      "logps/chosen": -11.104076385498047,
+      "logps/rejected": -29.71591567993164,
+      "loss": 0.5819936990737915,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.16406820714473724,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.07211651653051376,
+      "rewards/margins": 1.3368288278579712,
+      "rewards/rejected": -1.2647120952606201,
       "step": 106,
-      "train_speed(iter/s)": 0.033633
-    },
-    {
-      "clip_ratio/high_max": 0.0023960003745742142,
-      "clip_ratio/high_mean": 0.0023960003745742142,
-      "clip_ratio/low_mean": 0.002585217938758433,
-      "clip_ratio/low_min": 0.002585217938758433,
-      "clip_ratio/region_mean": 0.004981218371540308,
-      "completions/clipped_ratio": 0.0625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 13.6328125,
-      "completions/min_length": 2.0,
-      "epoch": 1.188365650969529,
-      "grad_norm": 2.0082146476980807,
-      "kl": 1.3074679019264295,
-      "learning_rate": 7.72364650742707e-05,
-      "loss": 0.011914699338376522,
-      "memory(GiB)": 79.43,
-      "reward": 0.328125,
-      "reward_std": 0.18394747376441956,
-      "rewards/Response_no_think_reward_1/mean": 0.328125,
-      "rewards/Response_no_think_reward_1/std": 0.7110973596572876,
+      "train_speed(iter/s)": 0.02127
+    },
+    {
+      "epoch": 0.21610704367583944,
+      "grad_norm": 14.505188941955566,
+      "learning_rate": 0.00019819095255500178,
+      "logits/chosen": -0.6649254560470581,
+      "logits/rejected": -0.7490218877792358,
+      "logps/chosen": -6.695674896240234,
+      "logps/rejected": -36.64976501464844,
+      "loss": 0.9944093227386475,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5400701761245728,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.20841172337532043,
+      "rewards/margins": 1.6346428394317627,
+      "rewards/rejected": -1.4262311458587646,
       "step": 107,
-      "train_speed(iter/s)": 0.027655
-    },
-    {
-      "clip_ratio/high_max": 0.00717889575753361,
-      "clip_ratio/high_mean": 0.00717889575753361,
-      "clip_ratio/low_mean": 0.008150914101861417,
-      "clip_ratio/low_min": 0.008150914101861417,
-      "clip_ratio/region_mean": 0.015329810208640993,
-      "epoch": 1.1994459833795015,
-      "grad_norm": 0.5916941576021421,
-      "kl": 1.0833495813399168,
-      "learning_rate": 7.54514512859201e-05,
-      "loss": 0.0037035662680864334,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.02127
+    },
+    {
+      "epoch": 0.2181267356728099,
+      "grad_norm": 5.73347282409668,
+      "learning_rate": 0.00019812712101867922,
+      "logits/chosen": -0.7590409517288208,
+      "logits/rejected": -0.8766330480575562,
+      "logps/chosen": -1.7668696641921997,
+      "logps/rejected": -23.773542404174805,
+      "loss": 0.6851338744163513,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.25430163741111755,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.15534955263137817,
+      "rewards/margins": 1.3926160335540771,
+      "rewards/rejected": -1.2372663021087646,
       "step": 108,
-      "train_speed(iter/s)": 0.026591
-    },
-    {
-      "clip_ratio/high_max": 0.004251995822414756,
-      "clip_ratio/high_mean": 0.004251995822414756,
-      "clip_ratio/low_mean": 0.008190131688024849,
-      "clip_ratio/low_min": 0.008190131688024849,
-      "clip_ratio/region_mean": 0.012442127510439605,
-      "completions/clipped_ratio": 0.09375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 12.234375,
-      "completions/min_length": 4.0,
-      "epoch": 1.2105263157894737,
-      "grad_norm": 2.6396182071410137,
-      "kl": 2.141993957106024,
-      "learning_rate": 7.36747230435401e-05,
-      "loss": 0.011151588521897793,
-      "memory(GiB)": 79.43,
-      "reward": -0.2265625,
-      "reward_std": 0.3407740592956543,
-      "rewards/Response_no_think_reward_1/mean": -0.2265625,
-      "rewards/Response_no_think_reward_1/std": 0.6425201892852783,
+      "train_speed(iter/s)": 0.021273
+    },
+    {
+      "epoch": 0.22014642766978035,
+      "grad_norm": 2.351529121398926,
+      "learning_rate": 0.00019806219342648977,
+      "logits/chosen": -0.7287034392356873,
+      "logits/rejected": -0.8862197995185852,
+      "logps/chosen": -0.5065869688987732,
+      "logps/rejected": -29.2410945892334,
+      "loss": 0.35556384921073914,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10009891539812088,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.37776732444763184,
+      "rewards/margins": 1.7389698028564453,
+      "rewards/rejected": -1.361202597618103,
       "step": 109,
-      "train_speed(iter/s)": 0.022574
-    },
-    {
-      "clip_ratio/high_max": 0.004035328107420355,
-      "clip_ratio/high_mean": 0.004035328107420355,
-      "clip_ratio/low_mean": 0.08847818686626852,
-      "clip_ratio/low_min": 0.08847818686626852,
-      "clip_ratio/region_mean": 0.0925135153811425,
-      "epoch": 1.221606648199446,
-      "grad_norm": 1.4171762109533736,
-      "kl": 3.1860878374427557,
-      "learning_rate": 7.190688002264308e-05,
-      "loss": 0.0037668771110475063,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021275
+    },
+    {
+      "epoch": 0.22216611966675082,
+      "grad_norm": 4.292262077331543,
+      "learning_rate": 0.0001979961705036587,
+      "logits/chosen": -0.7477302551269531,
+      "logits/rejected": -0.8113406300544739,
+      "logps/chosen": -4.461059093475342,
+      "logps/rejected": -25.79521942138672,
+      "loss": 0.9549956321716309,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4869251251220703,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.08557192981243134,
+      "rewards/margins": 1.1444835662841797,
+      "rewards/rejected": -1.2300554513931274,
       "step": 110,
-      "train_speed(iter/s)": 0.021907
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.002568919211626053,
-      "clip_ratio/low_min": 0.002568919211626053,
-      "clip_ratio/region_mean": 0.002568919211626053,
-      "completions/clipped_ratio": 0.046875,
-      "completions/max_length": 48.0,
-      "completions/mean_length": 11.7734375,
-      "completions/min_length": 5.0,
-      "epoch": 1.2326869806094183,
-      "grad_norm": 1.02177770841713,
-      "kl": 1.6274185269139707,
-      "learning_rate": 7.014851889983057e-05,
-      "loss": 0.010478168725967407,
-      "memory(GiB)": 79.43,
-      "reward": 0.453125,
-      "reward_std": 0.3291260004043579,
-      "rewards/Response_no_think_reward_1/mean": 0.453125,
-      "rewards/Response_no_think_reward_1/std": 0.685730516910553,
+      "train_speed(iter/s)": 0.021277
+    },
+    {
+      "epoch": 0.22418581166372128,
+      "grad_norm": 3.231773853302002,
+      "learning_rate": 0.0001979290529876458,
+      "logits/chosen": -0.6231730580329895,
+      "logits/rejected": -0.8174745440483093,
+      "logps/chosen": -0.6967498064041138,
+      "logps/rejected": -34.312564849853516,
+      "loss": 0.4151938259601593,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14568573236465454,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3004146218299866,
+      "rewards/margins": 1.8394732475280762,
+      "rewards/rejected": -1.5390586853027344,
       "step": 111,
-      "train_speed(iter/s)": 0.019127
-    },
-    {
-      "clip_ratio/high_max": 0.013096909533487633,
-      "clip_ratio/high_mean": 0.013096909533487633,
-      "clip_ratio/low_mean": 0.016107605304569006,
-      "clip_ratio/low_min": 0.016107605304569006,
-      "clip_ratio/region_mean": 0.029204514692537487,
-      "epoch": 1.2437673130193905,
-      "grad_norm": 0.79999454502468,
-      "kl": 1.5544351362623274,
-      "learning_rate": 6.840023315140475e-05,
-      "loss": 0.0022247314918786287,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021277
+    },
+    {
+      "epoch": 0.22620550366069175,
+      "grad_norm": 17.476547241210938,
+      "learning_rate": 0.00019786084162813733,
+      "logits/chosen": -0.7515086531639099,
+      "logits/rejected": -0.8278781771659851,
+      "logps/chosen": -8.48164176940918,
+      "logps/rejected": -27.406028747558594,
+      "loss": 1.484164834022522,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.8838226795196533,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.3509487807750702,
+      "rewards/margins": 1.2049919366836548,
+      "rewards/rejected": -1.5559407472610474,
       "step": 112,
-      "train_speed(iter/s)": 0.018681
-    },
-    {
-      "clip_ratio/high_max": 0.0008340688509633765,
-      "clip_ratio/high_mean": 0.0008340688509633765,
-      "clip_ratio/low_mean": 0.007082634954713285,
-      "clip_ratio/low_min": 0.007082634954713285,
-      "clip_ratio/region_mean": 0.007916703820228577,
-      "completions/clipped_ratio": 0.1171875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 13.53125,
-      "completions/min_length": 3.0,
-      "epoch": 1.254847645429363,
-      "grad_norm": 1.6246389101003882,
-      "kl": 1.6449745513964444,
-      "learning_rate": 6.666261285306047e-05,
-      "loss": 0.014917208813130856,
-      "memory(GiB)": 79.43,
-      "reward": 0.1953125,
-      "reward_std": 0.2754020392894745,
-      "rewards/Response_no_think_reward_1/mean": 0.1953125,
-      "rewards/Response_no_think_reward_1/std": 0.869958758354187,
+      "train_speed(iter/s)": 0.021279
+    },
+    {
+      "epoch": 0.2282251956576622,
+      "grad_norm": 4.023671627044678,
+      "learning_rate": 0.00019779153718703745,
+      "logits/chosen": -0.6617270708084106,
+      "logits/rejected": -0.8029566407203674,
+      "logps/chosen": -3.8847668170928955,
+      "logps/rejected": -29.473770141601562,
+      "loss": 0.8206814527511597,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.33473947644233704,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1446269452571869,
+      "rewards/margins": 1.4900355339050293,
+      "rewards/rejected": -1.3454086780548096,
       "step": 113,
-      "train_speed(iter/s)": 0.016592
-    },
-    {
-      "clip_ratio/high_max": 0.006006928611896001,
-      "clip_ratio/high_mean": 0.006006928611896001,
-      "clip_ratio/low_mean": 0.039706501411274076,
-      "clip_ratio/low_min": 0.039706501411274076,
-      "clip_ratio/region_mean": 0.045713430270552635,
-      "epoch": 1.2659279778393353,
-      "grad_norm": 0.940807595373605,
-      "kl": 1.8106578167062253,
-      "learning_rate": 6.493624448072457e-05,
-      "loss": 0.006363555323332548,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.02128
+    },
+    {
+      "epoch": 0.23024488765463266,
+      "grad_norm": 1.5781970024108887,
+      "learning_rate": 0.00019772114043845965,
+      "logits/chosen": -0.7092657089233398,
+      "logits/rejected": -0.8382660150527954,
+      "logps/chosen": -0.47141486406326294,
+      "logps/rejected": -40.95867919921875,
+      "loss": 0.3395133912563324,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10217627137899399,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.24633201956748962,
+      "rewards/margins": 2.3744595050811768,
+      "rewards/rejected": -2.1281275749206543,
       "step": 114,
-      "train_speed(iter/s)": 0.016278
-    },
-    {
-      "clip_ratio/high_max": 0.0052188277477398515,
-      "clip_ratio/high_mean": 0.0052188277477398515,
-      "clip_ratio/low_mean": 0.004236701555782929,
-      "clip_ratio/low_min": 0.004236701555782929,
-      "clip_ratio/region_mean": 0.009455529390834272,
-      "completions/clipped_ratio": 0.1796875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 14.3671875,
-      "completions/min_length": 2.0,
-      "epoch": 1.2770083102493075,
-      "grad_norm": 2.8040352101197437,
-      "kl": 5.506896490347572,
-      "learning_rate": 6.322171071261071e-05,
-      "loss": 0.03229736536741257,
-      "memory(GiB)": 79.43,
-      "reward": 0.296875,
-      "reward_std": 0.43980443477630615,
-      "rewards/Response_no_think_reward_1/mean": 0.296875,
-      "rewards/Response_no_think_reward_1/std": 0.8905397057533264,
+      "train_speed(iter/s)": 0.021281
+    },
+    {
+      "epoch": 0.23226457965160313,
+      "grad_norm": 3.4185638427734375,
+      "learning_rate": 0.00019764965216871846,
+      "logits/chosen": -0.6823985576629639,
+      "logits/rejected": -0.8108518123626709,
+      "logps/chosen": -3.519054412841797,
+      "logps/rejected": -34.14986038208008,
+      "loss": 0.6295965909957886,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.34174439311027527,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1087992712855339,
+      "rewards/margins": 1.8799338340759277,
+      "rewards/rejected": -1.7711347341537476,
       "step": 115,
-      "train_speed(iter/s)": 0.014636
-    },
-    {
-      "clip_ratio/high_max": 0.01615347486222163,
-      "clip_ratio/high_mean": 0.01615347486222163,
-      "clip_ratio/low_mean": 0.015428576618432999,
-      "clip_ratio/low_min": 0.015428576618432999,
-      "clip_ratio/region_mean": 0.031582050723955035,
-      "epoch": 1.2880886426592797,
-      "grad_norm": 1.8650893219420759,
-      "kl": 4.258469146443531,
-      "learning_rate": 6.151959023255545e-05,
-      "loss": 0.020270783454179764,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021281
+    },
+    {
+      "epoch": 0.2342842716485736,
+      "grad_norm": 5.202569961547852,
+      "learning_rate": 0.00019757707317632028,
+      "logits/chosen": -0.7702198624610901,
+      "logits/rejected": -0.7985497117042542,
+      "logps/chosen": -8.110039710998535,
+      "logps/rejected": -23.981826782226562,
+      "loss": 0.9200264811515808,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.40714696049690247,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.011667863465845585,
+      "rewards/margins": 1.2929402589797974,
+      "rewards/rejected": -1.2812724113464355,
       "step": 116,
-      "train_speed(iter/s)": 0.014407
-    },
-    {
-      "clip_ratio/high_max": 0.005145640461705625,
-      "clip_ratio/high_mean": 0.005145640461705625,
-      "clip_ratio/low_mean": 0.008849590638419613,
-      "clip_ratio/low_min": 0.008849590638419613,
-      "clip_ratio/region_mean": 0.013995230983709916,
-      "completions/clipped_ratio": 0.1640625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 16.3125,
-      "completions/min_length": 2.0,
-      "epoch": 1.299168975069252,
-      "grad_norm": 2.4581301952944186,
-      "kl": 3.745300827547908,
-      "learning_rate": 5.983045753470308e-05,
-      "loss": 0.024165078997612,
-      "memory(GiB)": 79.43,
-      "reward": 0.453125,
-      "reward_std": 0.5826787948608398,
-      "rewards/Response_no_think_reward_1/mean": 0.453125,
-      "rewards/Response_no_think_reward_1/std": 0.8405039310455322,
+      "train_speed(iter/s)": 0.021282
+    },
+    {
+      "epoch": 0.23630396364554407,
+      "grad_norm": 3.570646286010742,
+      "learning_rate": 0.0001975034042719546,
+      "logits/chosen": -0.6833267211914062,
+      "logits/rejected": -0.832187294960022,
+      "logps/chosen": -5.206616401672363,
+      "logps/rejected": -43.44145965576172,
+      "loss": 0.7497320175170898,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4067853093147278,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1555595099925995,
+      "rewards/margins": 2.0963921546936035,
+      "rewards/rejected": -1.940832495689392,
       "step": 117,
-      "train_speed(iter/s)": 0.013208
-    },
-    {
-      "clip_ratio/high_max": 0.04523819196037948,
-      "clip_ratio/high_mean": 0.04523819196037948,
-      "clip_ratio/low_mean": 0.01510127488290891,
-      "clip_ratio/low_min": 0.01510127488290891,
-      "clip_ratio/region_mean": 0.060339466377627105,
-      "epoch": 1.3102493074792243,
-      "grad_norm": 2.131099568305027,
-      "kl": 2.847630614414811,
-      "learning_rate": 5.8154882729603876e-05,
-      "loss": 0.01335166022181511,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021283
+    },
+    {
+      "epoch": 0.2383236556425145,
+      "grad_norm": 2.559943914413452,
+      "learning_rate": 0.0001974286462784851,
+      "logits/chosen": -0.7291467785835266,
+      "logits/rejected": -0.8261964321136475,
+      "logps/chosen": -1.4934219121932983,
+      "logps/rejected": -20.029233932495117,
+      "loss": 0.575518786907196,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24626033008098602,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.35854265093803406,
+      "rewards/margins": 1.5817933082580566,
+      "rewards/rejected": -1.2232506275177002,
       "step": 118,
-      "train_speed(iter/s)": 0.013037
-    },
-    {
-      "clip_ratio/high_max": 0.002125054510543123,
-      "clip_ratio/high_mean": 0.002125054510543123,
-      "clip_ratio/low_mean": 0.0127813016588334,
-      "clip_ratio/low_min": 0.0127813016588334,
-      "clip_ratio/region_mean": 0.0149063560529612,
-      "completions/clipped_ratio": 0.203125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 19.59375,
-      "completions/min_length": 3.0,
-      "epoch": 1.3213296398891967,
-      "grad_norm": 2.690385409772627,
-      "kl": 2.0104650848079473,
-      "learning_rate": 5.64934313517927e-05,
-      "loss": 0.01620793715119362,
-      "memory(GiB)": 79.43,
-      "reward": 0.2578125,
-      "reward_std": 0.41504764556884766,
-      "rewards/Response_no_think_reward_1/mean": 0.2578125,
-      "rewards/Response_no_think_reward_1/std": 0.8625734448432922,
+      "train_speed(iter/s)": 0.021285
+    },
+    {
+      "epoch": 0.24034334763948498,
+      "grad_norm": 8.765168190002441,
+      "learning_rate": 0.00019735280003094015,
+      "logits/chosen": -0.7664045095443726,
+      "logits/rejected": -0.8301156759262085,
+      "logps/chosen": -11.605274200439453,
+      "logps/rejected": -33.71162414550781,
+      "loss": 1.43560791015625,
+      "memory(GiB)": 45.64,
+      "nll_loss": 1.0074396133422852,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.5399270057678223,
+      "rewards/margins": 1.4373902082443237,
+      "rewards/rejected": -1.9773170948028564,
       "step": 119,
-      "train_speed(iter/s)": 0.011541
-    },
-    {
-      "clip_ratio/high_max": 0.0026382115320302546,
-      "clip_ratio/high_mean": 0.0026382115320302546,
-      "clip_ratio/low_mean": 0.07074824426672421,
-      "clip_ratio/low_min": 0.07074824426672421,
-      "clip_ratio/region_mean": 0.07338645646814257,
-      "epoch": 1.332409972299169,
-      "grad_norm": 1.3611711034111147,
-      "kl": 2.366683575557545,
-      "learning_rate": 5.484666416891109e-05,
-      "loss": 0.00427972199395299,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021287
+    },
+    {
+      "epoch": 0.24236303963645545,
+      "grad_norm": 4.5908522605896,
+      "learning_rate": 0.00019727586637650373,
+      "logits/chosen": -0.6932565569877625,
+      "logits/rejected": -0.7234666347503662,
+      "logps/chosen": -17.431007385253906,
+      "logps/rejected": -34.0074462890625,
+      "loss": 1.0241637229919434,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6038533449172974,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.17532162368297577,
+      "rewards/margins": 1.6745084524154663,
+      "rewards/rejected": -1.849830150604248,
       "step": 120,
-      "train_speed(iter/s)": 0.011424
-    },
-    {
-      "clip_ratio/high_max": 0.00034340660204179585,
-      "clip_ratio/high_mean": 0.00034340660204179585,
-      "clip_ratio/low_mean": 0.0057576168910600245,
-      "clip_ratio/low_min": 0.0057576168910600245,
-      "clip_ratio/region_mean": 0.00610102349310182,
-      "completions/clipped_ratio": 0.109375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 14.1640625,
-      "completions/min_length": 3.0,
-      "epoch": 1.3434903047091413,
-      "grad_norm": 1.7730415769710828,
-      "kl": 2.253111455589533,
-      "learning_rate": 5.321513699243924e-05,
-      "loss": 0.015828199684619904,
-      "memory(GiB)": 79.43,
-      "reward": 0.2578125,
-      "reward_std": 0.2504267692565918,
-      "rewards/Response_no_think_reward_1/mean": 0.2578125,
-      "rewards/Response_no_think_reward_1/std": 0.8533961176872253,
+      "train_speed(iter/s)": 0.021287
+    },
+    {
+      "epoch": 0.24438273163342591,
+      "grad_norm": 4.291138648986816,
+      "learning_rate": 0.00019719784617450593,
+      "logits/chosen": -0.655267596244812,
+      "logits/rejected": -0.7929190993309021,
+      "logps/chosen": -6.5861992835998535,
+      "logps/rejected": -43.83182907104492,
+      "loss": 0.696444571018219,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4287887215614319,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.015995100140571594,
+      "rewards/margins": 2.3263349533081055,
+      "rewards/rejected": -2.342329978942871,
       "step": 121,
-      "train_speed(iter/s)": 0.010746
-    },
-    {
-      "clip_ratio/high_max": 0.002522797236451879,
-      "clip_ratio/high_mean": 0.002522797236451879,
-      "clip_ratio/low_mean": 0.03892370511312038,
-      "clip_ratio/low_min": 0.03892370511312038,
-      "clip_ratio/region_mean": 0.041446502320468426,
-      "epoch": 1.3545706371191135,
-      "grad_norm": 0.9270755733265906,
-      "kl": 2.852388353087008,
-      "learning_rate": 5.159940049010015e-05,
-      "loss": 0.00514911487698555,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021287
+    },
+    {
+      "epoch": 0.24640242363039636,
+      "grad_norm": 3.7837073802948,
+      "learning_rate": 0.0001971187402964132,
+      "logits/chosen": -0.6419543027877808,
+      "logits/rejected": -0.7936818599700928,
+      "logps/chosen": -5.325220108032227,
+      "logps/rejected": -35.03059768676758,
+      "loss": 0.760798990726471,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4128337502479553,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.004157397896051407,
+      "rewards/margins": 1.6647117137908936,
+      "rewards/rejected": -1.6688693761825562,
       "step": 122,
-      "train_speed(iter/s)": 0.010652
-    },
-    {
-      "clip_ratio/high_max": 0.0008753835718380287,
-      "clip_ratio/high_mean": 0.0008753835718380287,
-      "clip_ratio/low_mean": 0.0012499999720603228,
-      "clip_ratio/low_min": 0.0012499999720603228,
-      "clip_ratio/region_mean": 0.0021253835438983515,
-      "completions/clipped_ratio": 0.1171875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 15.8515625,
-      "completions/min_length": 4.0,
-      "epoch": 1.365650969529086,
-      "grad_norm": 24.163527329310554,
-      "kl": 24.496757203305606,
-      "learning_rate": 5.000000000000002e-05,
-      "loss": 0.23566541075706482,
-      "memory(GiB)": 79.43,
-      "reward": 0.3984375,
-      "reward_std": 0.3253360986709595,
-      "rewards/Response_no_think_reward_1/mean": 0.3984375,
-      "rewards/Response_no_think_reward_1/std": 0.6912255883216858,
+      "train_speed(iter/s)": 0.021287
+    },
+    {
+      "epoch": 0.24842211562736682,
+      "grad_norm": 3.332064390182495,
+      "learning_rate": 0.00019703854962581884,
+      "logits/chosen": -0.767221987247467,
+      "logits/rejected": -0.8504694700241089,
+      "logps/chosen": -1.6606477499008179,
+      "logps/rejected": -30.287626266479492,
+      "loss": 0.5448625087738037,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1951584815979004,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.18560823798179626,
+      "rewards/margins": 1.3272532224655151,
+      "rewards/rejected": -1.1416449546813965,
       "step": 123,
-      "train_speed(iter/s)": 0.010181
-    },
-    {
-      "clip_ratio/high_max": 0.0009191176941385493,
-      "clip_ratio/high_mean": 0.0009191176941385493,
-      "clip_ratio/low_mean": 0.025084137567318976,
-      "clip_ratio/low_min": 0.025084137567318976,
-      "clip_ratio/region_mean": 0.026003255392424762,
-      "epoch": 1.3767313019390581,
-      "grad_norm": 4.61798364138243,
-      "kl": 5.482510100933723,
-      "learning_rate": 4.841747534656763e-05,
-      "loss": 0.03564080968499184,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021289
+    },
+    {
+      "epoch": 0.25044180762433726,
+      "grad_norm": 2.538031816482544,
+      "learning_rate": 0.00019695727505843297,
+      "logits/chosen": -0.8040828704833984,
+      "logits/rejected": -0.8591914176940918,
+      "logps/chosen": -2.396242618560791,
+      "logps/rejected": -17.942195892333984,
+      "loss": 0.6009309887886047,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1990574449300766,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.31234806776046753,
+      "rewards/margins": 0.9391263723373413,
+      "rewards/rejected": -0.6267784237861633,
       "step": 124,
-      "train_speed(iter/s)": 0.010103
-    },
-    {
-      "clip_ratio/high_max": 0.0004921259824186563,
-      "clip_ratio/high_mean": 0.0004921259824186563,
-      "clip_ratio/low_mean": 0.0009282178361900151,
-      "clip_ratio/low_min": 0.0009282178361900151,
-      "clip_ratio/region_mean": 0.0014203438186086714,
-      "completions/clipped_ratio": 0.140625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 16.671875,
-      "completions/min_length": 3.0,
-      "epoch": 1.3878116343490305,
-      "grad_norm": 1.7230796893481761,
-      "kl": 1.7308420957997441,
-      "learning_rate": 4.685236065835443e-05,
-      "loss": 0.01634293608367443,
-      "memory(GiB)": 79.43,
-      "reward": 0.171875,
-      "reward_std": 0.13258251547813416,
-      "rewards/Response_no_think_reward_1/mean": 0.171875,
-      "rewards/Response_no_think_reward_1/std": 0.8971465826034546,
+      "train_speed(iter/s)": 0.021291
+    },
+    {
+      "epoch": 0.25246149962130776,
+      "grad_norm": 2.4364805221557617,
+      "learning_rate": 0.00019687491750207254,
+      "logits/chosen": -0.8258935213088989,
+      "logits/rejected": -0.8962618112564087,
+      "logps/chosen": -1.0966453552246094,
+      "logps/rejected": -16.684877395629883,
+      "loss": 0.6394660472869873,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15617281198501587,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.2913311719894409,
+      "rewards/margins": 0.7776257395744324,
+      "rewards/rejected": -0.48629459738731384,
       "step": 125,
-      "train_speed(iter/s)": 0.009672
-    },
-    {
-      "clip_ratio/high_max": 0.0003094059356953949,
-      "clip_ratio/high_mean": 0.0003094059356953949,
-      "clip_ratio/low_mean": 0.02542525064200163,
-      "clip_ratio/low_min": 0.02542525064200163,
-      "clip_ratio/region_mean": 0.025734656490385532,
-      "epoch": 1.3988919667590027,
-      "grad_norm": 0.6685767852950313,
-      "kl": 2.058195663616061,
-      "learning_rate": 4.530518418775733e-05,
-      "loss": 0.008037411607801914,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021292
+    },
+    {
+      "epoch": 0.2544811916182782,
+      "grad_norm": 4.309374809265137,
+      "learning_rate": 0.00019679147787665126,
+      "logits/chosen": -0.7291491627693176,
+      "logits/rejected": -0.8647027611732483,
+      "logps/chosen": -2.546858549118042,
+      "logps/rejected": -25.51129150390625,
+      "loss": 0.7438904047012329,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.29447755217552185,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.24194566905498505,
+      "rewards/margins": 0.7109313607215881,
+      "rewards/rejected": -0.4689857065677643,
       "step": 126,
-      "train_speed(iter/s)": 0.009605
-    },
-    {
-      "clip_ratio/high_max": 0.0035992932971566916,
-      "clip_ratio/high_mean": 0.0035992932971566916,
-      "clip_ratio/low_mean": 0.0026174120721407235,
-      "clip_ratio/low_min": 0.0026174120721407235,
-      "clip_ratio/region_mean": 0.006216705543920398,
-      "completions/clipped_ratio": 0.078125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 13.1015625,
-      "completions/min_length": 3.0,
-      "epoch": 1.4099722991689752,
-      "grad_norm": 0.898899557372063,
-      "kl": 1.8244512832025066,
-      "learning_rate": 4.3776468132724604e-05,
-      "loss": 0.013885595835745335,
-      "memory(GiB)": 79.43,
-      "reward": 0.703125,
-      "reward_std": 0.1841355264186859,
-      "rewards/Response_no_think_reward_1/mean": 0.703125,
-      "rewards/Response_no_think_reward_1/std": 0.552152156829834,
+      "train_speed(iter/s)": 0.021292
+    },
+    {
+      "epoch": 0.2565008836152487,
+      "grad_norm": 3.316589117050171,
+      "learning_rate": 0.00019670695711416928,
+      "logits/chosen": -0.7973051071166992,
+      "logits/rejected": -0.9037990570068359,
+      "logps/chosen": -1.6116214990615845,
+      "logps/rejected": -24.904531478881836,
+      "loss": 0.4576399624347687,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17375069856643677,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.22464421391487122,
+      "rewards/margins": 1.4565354585647583,
+      "rewards/rejected": -1.2318912744522095,
       "step": 127,
-      "train_speed(iter/s)": 0.009212
-    },
-    {
-      "clip_ratio/high_max": 0.00787301326636225,
-      "clip_ratio/high_mean": 0.00787301326636225,
-      "clip_ratio/low_mean": 0.003833794384263456,
-      "clip_ratio/low_min": 0.003833794384263456,
-      "clip_ratio/region_mean": 0.011706807999871671,
-      "epoch": 1.4210526315789473,
-      "grad_norm": 0.6061431338619399,
-      "kl": 1.8278243900567759,
-      "learning_rate": 4.2266728460505375e-05,
-      "loss": 0.009479128755629063,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021294
+    },
+    {
+      "epoch": 0.25852057561221914,
+      "grad_norm": 4.841390132904053,
+      "learning_rate": 0.00019662135615870275,
+      "logits/chosen": -0.7546025514602661,
+      "logits/rejected": -0.9116957187652588,
+      "logps/chosen": -2.7441368103027344,
+      "logps/rejected": -25.96759796142578,
+      "loss": 0.8130377531051636,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3844357430934906,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.1914481222629547,
+      "rewards/margins": 1.4247140884399414,
+      "rewards/rejected": -1.2332658767700195,
       "step": 128,
-      "train_speed(iter/s)": 0.009157
-    },
-    {
-      "clip_ratio/high_max": 0.0016208597226068377,
-      "clip_ratio/high_mean": 0.0016208597226068377,
-      "clip_ratio/low_mean": 0.004291799967177212,
-      "clip_ratio/low_min": 0.004291799967177212,
-      "clip_ratio/region_mean": 0.00591265968978405,
-      "completions/clipped_ratio": 0.2421875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 13.8359375,
-      "completions/min_length": 3.0,
-      "epoch": 1.4321329639889195,
-      "grad_norm": 4.256081630918958,
-      "kl": 8.371784689603373,
-      "learning_rate": 4.077647473350201e-05,
-      "loss": 0.07109890133142471,
-      "memory(GiB)": 79.43,
-      "reward": 0.1640625,
-      "reward_std": 0.12844271957874298,
-      "rewards/Response_no_think_reward_1/mean": 0.1640625,
-      "rewards/Response_no_think_reward_1/std": 0.9029901623725891,
+      "train_speed(iter/s)": 0.021295
+    },
+    {
+      "epoch": 0.2605402676091896,
+      "grad_norm": 6.839901447296143,
+      "learning_rate": 0.0001965346759663934,
+      "logits/chosen": -0.6810730695724487,
+      "logits/rejected": -0.8468779921531677,
+      "logps/chosen": -5.32491397857666,
+      "logps/rejected": -42.42945861816406,
+      "loss": 1.0299216508865356,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6279267072677612,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.009131789207458496,
+      "rewards/margins": 1.819998025894165,
+      "rewards/rejected": -1.829129934310913,
       "step": 129,
-      "train_speed(iter/s)": 0.008828
-    },
-    {
-      "clip_ratio/high_max": 0.00940803368575871,
-      "clip_ratio/high_mean": 0.00940803368575871,
-      "clip_ratio/low_mean": 0.006534474319778383,
-      "clip_ratio/low_min": 0.006534474319778383,
-      "clip_ratio/region_mean": 0.01594250788912177,
-      "epoch": 1.443213296398892,
-      "grad_norm": 2.0975168570419678,
-      "kl": 6.735093111405149,
-      "learning_rate": 3.9306209937284346e-05,
-      "loss": 0.05932926759123802,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021294
+    },
+    {
+      "epoch": 0.2625599596061601,
+      "grad_norm": 2.0061724185943604,
+      "learning_rate": 0.00019644691750543767,
+      "logits/chosen": -0.8441457152366638,
+      "logits/rejected": -0.9423956274986267,
+      "logps/chosen": -0.5530110597610474,
+      "logps/rejected": -32.842193603515625,
+      "loss": 0.25913411378860474,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.05609491094946861,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.27188676595687866,
+      "rewards/margins": 2.330637216567993,
+      "rewards/rejected": -2.0587503910064697,
       "step": 130,
-      "train_speed(iter/s)": 0.008781
-    },
-    {
-      "clip_ratio/high_max": 0.004439248572452925,
-      "clip_ratio/high_mean": 0.004439248572452925,
-      "clip_ratio/low_mean": 0.006494191708043218,
-      "clip_ratio/low_min": 0.006494191708043218,
-      "clip_ratio/region_mean": 0.01093344046967104,
-      "completions/clipped_ratio": 0.1484375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 13.9140625,
-      "completions/min_length": 3.0,
-      "epoch": 1.4542936288088644,
-      "grad_norm": 1.7347303322446779,
-      "kl": 3.3224903107620776,
-      "learning_rate": 3.7856430310823545e-05,
-      "loss": 0.028562916442751884,
-      "memory(GiB)": 79.43,
-      "reward": 0.2578125,
-      "reward_std": 0.4904649257659912,
-      "rewards/Response_no_think_reward_1/mean": 0.2578125,
-      "rewards/Response_no_think_reward_1/std": 0.8441190123558044,
+      "train_speed(iter/s)": 0.021295
+    },
+    {
+      "epoch": 0.2645796516031305,
+      "grad_norm": 4.784903526306152,
+      "learning_rate": 0.00019635808175607605,
+      "logits/chosen": -0.8365444540977478,
+      "logits/rejected": -0.9138449430465698,
+      "logps/chosen": -3.4112894535064697,
+      "logps/rejected": -23.938806533813477,
+      "loss": 0.7469592094421387,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.31692469120025635,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.09105291962623596,
+      "rewards/margins": 1.37967050075531,
+      "rewards/rejected": -1.288617730140686,
       "step": 131,
-      "train_speed(iter/s)": 0.008479
-    },
-    {
-      "clip_ratio/high_max": 0.03524596616625786,
-      "clip_ratio/high_mean": 0.03524596616625786,
-      "clip_ratio/low_mean": 0.00640316259523388,
-      "clip_ratio/low_min": 0.00640316259523388,
-      "clip_ratio/region_mean": 0.04164912860142067,
-      "epoch": 1.4653739612188366,
-      "grad_norm": 1.8414978565783118,
-      "kl": 2.55354578839615,
-      "learning_rate": 3.642762517900322e-05,
-      "loss": 0.016005922108888626,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021296
+    },
+    {
+      "epoch": 0.26659934360010096,
+      "grad_norm": 9.284527778625488,
+      "learning_rate": 0.00019626816971058205,
+      "logits/chosen": -0.8304141759872437,
+      "logits/rejected": -0.9100163578987122,
+      "logps/chosen": -3.7839527130126953,
+      "logps/rejected": -18.869930267333984,
+      "loss": 1.1523946523666382,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6580032706260681,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.240356907248497,
+      "rewards/margins": 1.1313059329986572,
+      "rewards/rejected": -0.890949010848999,
       "step": 132,
-      "train_speed(iter/s)": 0.008439
-    },
-    {
-      "clip_ratio/high_max": 0.00021331057359930128,
-      "clip_ratio/high_mean": 0.00021331057359930128,
-      "clip_ratio/low_mean": 0.007616169808898121,
-      "clip_ratio/low_min": 0.007616169808898121,
-      "clip_ratio/region_mean": 0.007829480382497422,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 16.9296875,
-      "completions/min_length": 4.0,
-      "epoch": 1.4764542936288088,
-      "grad_norm": 2.995948442993347,
-      "kl": 2.835317355929874,
-      "learning_rate": 3.5020276787464056e-05,
-      "loss": 0.020519524812698364,
-      "memory(GiB)": 79.43,
-      "reward": -0.109375,
-      "reward_std": 0.39537471532821655,
-      "rewards/Response_no_think_reward_1/mean": -0.109375,
-      "rewards/Response_no_think_reward_1/std": 0.7860434055328369,
+      "train_speed(iter/s)": 0.021298
+    },
+    {
+      "epoch": 0.26861903559707145,
+      "grad_norm": 4.299342632293701,
+      "learning_rate": 0.00019617718237325115,
+      "logits/chosen": -0.6347696781158447,
+      "logits/rejected": -0.7560770511627197,
+      "logps/chosen": -3.5856218338012695,
+      "logps/rejected": -30.4008846282959,
+      "loss": 0.739059567451477,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.38467657566070557,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.13314950466156006,
+      "rewards/margins": 1.3657082319259644,
+      "rewards/rejected": -1.2325587272644043,
       "step": 133,
-      "train_speed(iter/s)": 0.008214
-    },
-    {
-      "clip_ratio/high_max": 0.0014211501111276448,
-      "clip_ratio/high_mean": 0.0014211501111276448,
-      "clip_ratio/low_mean": 0.017294615099672228,
-      "clip_ratio/low_min": 0.017294615099672228,
-      "clip_ratio/region_mean": 0.01871576503617689,
-      "epoch": 1.4875346260387812,
-      "grad_norm": 2.3860638079880268,
-      "kl": 2.9496174114756286,
-      "learning_rate": 3.363486013983788e-05,
-      "loss": 0.015305472537875175,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021298
+    },
+    {
+      "epoch": 0.2706387275940419,
+      "grad_norm": 4.246809482574463,
+      "learning_rate": 0.00019608512076038962,
+      "logits/chosen": -0.7022749781608582,
+      "logits/rejected": -0.7915350794792175,
+      "logps/chosen": -4.822379112243652,
+      "logps/rejected": -24.07721710205078,
+      "loss": 0.8055979609489441,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3307011127471924,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.11167091131210327,
+      "rewards/margins": 1.3118420839309692,
+      "rewards/rejected": -1.2001712322235107,
       "step": 134,
-      "train_speed(iter/s)": 0.008178
-    },
-    {
-      "clip_ratio/high_max": 0.004125965555431321,
-      "clip_ratio/high_mean": 0.004125965555431321,
-      "clip_ratio/low_mean": 0.01121757403598167,
-      "clip_ratio/low_min": 0.01121757403598167,
-      "clip_ratio/region_mean": 0.01534353947499767,
-      "completions/clipped_ratio": 0.2421875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 17.59375,
-      "completions/min_length": 5.0,
-      "epoch": 1.4986149584487536,
-      "grad_norm": 1.9611447116444427,
-      "kl": 4.596401881426573,
-      "learning_rate": 3.227184283742591e-05,
-      "loss": 0.037781622260808945,
-      "memory(GiB)": 79.43,
-      "reward": 0.140625,
-      "reward_std": 0.3686816394329071,
-      "rewards/Response_no_think_reward_1/mean": 0.140625,
-      "rewards/Response_no_think_reward_1/std": 0.9698962569236755,
+      "train_speed(iter/s)": 0.021298
+    },
+    {
+      "epoch": 0.2726584195910124,
+      "grad_norm": 2.9997761249542236,
+      "learning_rate": 0.0001959919859003031,
+      "logits/chosen": -0.6153374910354614,
+      "logits/rejected": -0.7747887969017029,
+      "logps/chosen": -0.7465253472328186,
+      "logps/rejected": -28.333953857421875,
+      "loss": 0.5413554906845093,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14097237586975098,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.33766934275627136,
+      "rewards/margins": 0.9080523252487183,
+      "rewards/rejected": -0.5703830718994141,
       "step": 135,
-      "train_speed(iter/s)": 0.007948
-    },
-    {
-      "clip_ratio/high_max": 0.003497250087093562,
-      "clip_ratio/high_mean": 0.003497250087093562,
-      "clip_ratio/low_mean": 0.0877209399768617,
-      "clip_ratio/low_min": 0.0877209399768617,
-      "clip_ratio/region_mean": 0.09121819020947441,
-      "epoch": 1.5096952908587258,
-      "grad_norm": 1.7237744503611014,
-      "kl": 4.757093018852174,
-      "learning_rate": 3.093168492137557e-05,
-      "loss": 0.02426687255501747,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021296
+    },
+    {
+      "epoch": 0.27467811158798283,
+      "grad_norm": 3.5653076171875,
+      "learning_rate": 0.00019589777883328505,
+      "logits/chosen": -0.7418694496154785,
+      "logits/rejected": -0.8762567043304443,
+      "logps/chosen": -1.4205451011657715,
+      "logps/rejected": -21.64703941345215,
+      "loss": 0.6491340398788452,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.16818463802337646,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.2140020728111267,
+      "rewards/margins": 0.9494506120681763,
+      "rewards/rejected": -0.7354484796524048,
       "step": 136,
-      "train_speed(iter/s)": 0.007917
-    },
-    {
-      "clip_ratio/high_max": 0.0014806788822170347,
-      "clip_ratio/high_mean": 0.0014806788822170347,
-      "clip_ratio/low_mean": 0.008024309470783919,
-      "clip_ratio/low_min": 0.008024309470783919,
-      "clip_ratio/region_mean": 0.009504988382104784,
-      "completions/clipped_ratio": 0.2265625,
-      "completions/max_length": 49.0,
-      "completions/mean_length": 15.859375,
-      "completions/min_length": 4.0,
-      "epoch": 1.520775623268698,
-      "grad_norm": 2.1836739109067973,
-      "kl": 5.872505620121956,
-      "learning_rate": 2.9614838717408867e-05,
-      "loss": 0.05089029669761658,
-      "memory(GiB)": 79.43,
-      "reward": 0.1640625,
-      "reward_std": 0.48505210876464844,
-      "rewards/Response_no_think_reward_1/mean": 0.1640625,
-      "rewards/Response_no_think_reward_1/std": 0.8303053379058838,
+      "train_speed(iter/s)": 0.021298
+    },
+    {
+      "epoch": 0.2766978035849533,
+      "grad_norm": 2.6075761318206787,
+      "learning_rate": 0.00019580250061160539,
+      "logits/chosen": -0.5225172638893127,
+      "logits/rejected": -0.7319562435150146,
+      "logps/chosen": -2.6468346118927,
+      "logps/rejected": -41.52344512939453,
+      "loss": 0.5949529409408569,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.19340890645980835,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2633662819862366,
+      "rewards/margins": 1.1732457876205444,
+      "rewards/rejected": -0.9098795056343079,
       "step": 137,
-      "train_speed(iter/s)": 0.00773
-    },
-    {
-      "clip_ratio/high_max": 0.009889701497741044,
-      "clip_ratio/high_mean": 0.009889701497741044,
-      "clip_ratio/low_mean": 0.03272932127583772,
-      "clip_ratio/low_min": 0.03272932127583772,
-      "clip_ratio/region_mean": 0.04261902256985195,
-      "epoch": 1.5318559556786704,
-      "grad_norm": 1.660171536889304,
-      "kl": 5.461771305650473,
-      "learning_rate": 2.8321748683154893e-05,
-      "loss": 0.03914007171988487,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021297
+    },
+    {
+      "epoch": 0.27871749558192377,
+      "grad_norm": 3.9614176750183105,
+      "learning_rate": 0.00019570615229949842,
+      "logits/chosen": -0.636820375919342,
+      "logits/rejected": -0.7707249522209167,
+      "logps/chosen": -3.3777596950531006,
+      "logps/rejected": -26.16946029663086,
+      "loss": 0.758621335029602,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3399200439453125,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.15843479335308075,
+      "rewards/margins": 0.947920024394989,
+      "rewards/rejected": -0.7894852161407471,
       "step": 138,
-      "train_speed(iter/s)": 0.007702
-    },
-    {
-      "clip_ratio/high_max": 0.0019176136120222509,
-      "clip_ratio/high_mean": 0.0019176136120222509,
-      "clip_ratio/low_mean": 0.007726973562967032,
-      "clip_ratio/low_min": 0.007726973562967032,
-      "clip_ratio/region_mean": 0.009644587058573961,
-      "completions/clipped_ratio": 0.265625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 15.359375,
-      "completions/min_length": 4.0,
-      "epoch": 1.5429362880886428,
-      "grad_norm": 3.7132212896004106,
-      "kl": 4.60741166153457,
-      "learning_rate": 2.7052851258137935e-05,
-      "loss": 0.039852242916822433,
-      "memory(GiB)": 79.43,
-      "reward": 0.0078125,
-      "reward_std": 0.3704521656036377,
-      "rewards/Response_no_think_reward_1/mean": 0.0078125,
-      "rewards/Response_no_think_reward_1/std": 0.7985823154449463,
+      "train_speed(iter/s)": 0.021297
+    },
+    {
+      "epoch": 0.2807371875788942,
+      "grad_norm": 2.576108455657959,
+      "learning_rate": 0.00019560873497315116,
+      "logits/chosen": -0.6899356842041016,
+      "logits/rejected": -0.7887363433837891,
+      "logps/chosen": -1.9815049171447754,
+      "logps/rejected": -15.912079811096191,
+      "loss": 0.7393437623977661,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24023714661598206,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.33446478843688965,
+      "rewards/margins": 0.6615203619003296,
+      "rewards/rejected": -0.32705551385879517,
       "step": 139,
-      "train_speed(iter/s)": 0.007537
-    },
-    {
-      "clip_ratio/high_max": 0.0055117253214120865,
-      "clip_ratio/high_mean": 0.0055117253214120865,
-      "clip_ratio/low_mean": 0.016618184628896415,
-      "clip_ratio/low_min": 0.016618184628896415,
-      "clip_ratio/region_mean": 0.022129910183139145,
-      "epoch": 1.554016620498615,
-      "grad_norm": 2.5249297141615665,
-      "kl": 4.39259727431272,
-      "learning_rate": 2.5808574716471856e-05,
-      "loss": 0.03180728852748871,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021298
+    },
+    {
+      "epoch": 0.2827568795758647,
+      "grad_norm": 3.3134891986846924,
+      "learning_rate": 0.00019551024972069126,
+      "logits/chosen": -0.8353362679481506,
+      "logits/rejected": -0.8483090996742249,
+      "logps/chosen": -1.7245616912841797,
+      "logps/rejected": -15.235477447509766,
+      "loss": 0.5436122417449951,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1701793223619461,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3035600185394287,
+      "rewards/margins": 1.1392890214920044,
+      "rewards/rejected": -0.8357290029525757,
       "step": 140,
-      "train_speed(iter/s)": 0.007513
-    },
-    {
-      "clip_ratio/high_max": 0.001243597303982824,
-      "clip_ratio/high_mean": 0.001243597303982824,
-      "clip_ratio/low_mean": 0.009276433673221618,
-      "clip_ratio/low_min": 0.009276433673221618,
-      "clip_ratio/region_mean": 0.010520030977204442,
-      "completions/clipped_ratio": 0.1640625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 13.2578125,
-      "completions/min_length": 4.0,
-      "epoch": 1.5650969529085872,
-      "grad_norm": 2.0040824091240594,
-      "kl": 6.036298241931945,
-      "learning_rate": 2.4589339022310386e-05,
-      "loss": 0.044014573097229004,
-      "memory(GiB)": 79.43,
-      "reward": 0.2109375,
-      "reward_std": 0.3419404625892639,
-      "rewards/Response_no_think_reward_1/mean": 0.2109375,
-      "rewards/Response_no_think_reward_1/std": 0.7902191877365112,
+      "train_speed(iter/s)": 0.021301
+    },
+    {
+      "epoch": 0.28477657157283515,
+      "grad_norm": 4.701112270355225,
+      "learning_rate": 0.0001954106976421748,
+      "logits/chosen": -0.7467123866081238,
+      "logits/rejected": -0.742473840713501,
+      "logps/chosen": -7.344741344451904,
+      "logps/rejected": -13.826343536376953,
+      "loss": 1.0974572896957397,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.45263901352882385,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.08538863807916641,
+      "rewards/margins": 0.4929981231689453,
+      "rewards/rejected": -0.5783867835998535,
       "step": 141,
-      "train_speed(iter/s)": 0.007347
-    },
-    {
-      "clip_ratio/high_max": 0.008325316943228245,
-      "clip_ratio/high_mean": 0.008325316943228245,
-      "clip_ratio/low_mean": 0.024790967552689835,
-      "clip_ratio/low_min": 0.024790967552689835,
-      "clip_ratio/region_mean": 0.0331162846123334,
-      "epoch": 1.5761772853185596,
-      "grad_norm": 1.8697222079345355,
-      "kl": 6.215300239622593,
-      "learning_rate": 2.339555568810221e-05,
-      "loss": 0.03644668310880661,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021302
+    },
+    {
+      "epoch": 0.2867962635698056,
+      "grad_norm": 2.704679012298584,
+      "learning_rate": 0.00019531007984957408,
+      "logits/chosen": -0.5686406493186951,
+      "logits/rejected": -0.7427109479904175,
+      "logps/chosen": -2.140307664871216,
+      "logps/rejected": -37.99428939819336,
+      "loss": 0.5559962391853333,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15430670976638794,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3728140592575073,
+      "rewards/margins": 2.170938730239868,
+      "rewards/rejected": -1.7981246709823608,
       "step": 142,
-      "train_speed(iter/s)": 0.007326
-    },
-    {
-      "clip_ratio/high_max": 0.0029302738257683814,
-      "clip_ratio/high_mean": 0.0029302738257683814,
-      "clip_ratio/low_mean": 0.007880916586145759,
-      "clip_ratio/low_min": 0.007880916586145759,
-      "clip_ratio/region_mean": 0.01081119041191414,
-      "completions/clipped_ratio": 0.1953125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 16.0546875,
-      "completions/min_length": 5.0,
-      "epoch": 1.587257617728532,
-      "grad_norm": 2.558849178134003,
-      "kl": 3.834877057670383,
-      "learning_rate": 2.222762763569862e-05,
-      "loss": 0.029992224648594856,
-      "memory(GiB)": 79.43,
-      "reward": 0.1875,
-      "reward_std": 0.35771697759628296,
-      "rewards/Response_no_think_reward_1/mean": 0.1875,
-      "rewards/Response_no_think_reward_1/std": 0.8010819554328918,
+      "train_speed(iter/s)": 0.021302
+    },
+    {
+      "epoch": 0.2888159555667761,
+      "grad_norm": 6.6994123458862305,
+      "learning_rate": 0.0001952083974667652,
+      "logits/chosen": -0.6588355898857117,
+      "logits/rejected": -0.7316503524780273,
+      "logps/chosen": -7.539848804473877,
+      "logps/rejected": -26.308818817138672,
+      "loss": 1.176750659942627,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5222424268722534,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.17204269766807556,
+      "rewards/margins": 0.8352524042129517,
+      "rewards/rejected": -1.0072951316833496,
       "step": 143,
-      "train_speed(iter/s)": 0.007139
-    },
-    {
-      "clip_ratio/high_max": 0.003368975827470422,
-      "clip_ratio/high_mean": 0.003368975827470422,
-      "clip_ratio/low_mean": 0.02965959811990615,
-      "clip_ratio/low_min": 0.02965959811990615,
-      "clip_ratio/region_mean": 0.033028574180207215,
-      "epoch": 1.5983379501385042,
-      "grad_norm": 1.5351560901882282,
-      "kl": 4.278483287169365,
-      "learning_rate": 2.1085949060360654e-05,
-      "loss": 0.022547291591763496,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021302
+    },
+    {
+      "epoch": 0.2908356475637465,
+      "grad_norm": 5.0174384117126465,
+      "learning_rate": 0.00019510565162951537,
+      "logits/chosen": -0.720901608467102,
+      "logits/rejected": -0.7480478882789612,
+      "logps/chosen": -6.40681266784668,
+      "logps/rejected": -27.90705680847168,
+      "loss": 0.6913059949874878,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3720777928829193,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.31874871253967285,
+      "rewards/margins": 2.0500223636627197,
+      "rewards/rejected": -1.7312736511230469,
       "step": 144,
-      "train_speed(iter/s)": 0.007121
-    },
-    {
-      "clip_ratio/high_max": 0.002464834105921909,
-      "clip_ratio/high_mean": 0.002464834105921909,
-      "clip_ratio/low_mean": 0.01630687521537766,
-      "clip_ratio/low_min": 0.01630687521537766,
-      "clip_ratio/region_mean": 0.018771709233988076,
-      "completions/clipped_ratio": 0.1953125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 15.2265625,
-      "completions/min_length": 6.0,
-      "epoch": 1.6094182825484764,
-      "grad_norm": 2.2342359741828877,
-      "kl": 5.610230388585478,
-      "learning_rate": 1.9970905297711606e-05,
-      "loss": 0.04032519459724426,
-      "memory(GiB)": 79.43,
-      "reward": 0.15625,
-      "reward_std": 0.3392276465892792,
-      "rewards/Response_no_think_reward_1/mean": 0.15625,
-      "rewards/Response_no_think_reward_1/std": 0.8175004720687866,
+      "train_speed(iter/s)": 0.021303
+    },
+    {
+      "epoch": 0.29285533956071697,
+      "grad_norm": 6.825158596038818,
+      "learning_rate": 0.00019500184348547042,
+      "logits/chosen": -0.7821186780929565,
+      "logits/rejected": -0.8765726685523987,
+      "logps/chosen": -3.37898588180542,
+      "logps/rejected": -19.76290512084961,
+      "loss": 0.8391829133033752,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.339937299489975,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.13619036972522736,
+      "rewards/margins": 1.1103405952453613,
+      "rewards/rejected": -0.9741502404212952,
       "step": 145,
-      "train_speed(iter/s)": 0.006942
-    },
-    {
-      "clip_ratio/high_max": 0.004107788117835298,
-      "clip_ratio/high_mean": 0.004107788117835298,
-      "clip_ratio/low_mean": 0.07009978080168366,
-      "clip_ratio/low_min": 0.07009978080168366,
-      "clip_ratio/region_mean": 0.07420756877399981,
-      "epoch": 1.6204986149584486,
-      "grad_norm": 1.7237867651621368,
-      "kl": 6.150614712154493,
-      "learning_rate": 1.888287269367979e-05,
-      "loss": 0.03451818227767944,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021304
+    },
+    {
+      "epoch": 0.29487503155768746,
+      "grad_norm": 2.3948497772216797,
+      "learning_rate": 0.00019489697419414182,
+      "logits/chosen": -0.8977736830711365,
+      "logits/rejected": -0.954646110534668,
+      "logps/chosen": -1.5781641006469727,
+      "logps/rejected": -27.0587100982666,
+      "loss": 0.4295532703399658,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1162162497639656,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.24644073843955994,
+      "rewards/margins": 1.8821758031845093,
+      "rewards/rejected": -1.6357351541519165,
       "step": 146,
-      "train_speed(iter/s)": 0.006925
-    },
-    {
-      "clip_ratio/high_max": 0.0020879992516711354,
-      "clip_ratio/high_mean": 0.0020879992516711354,
-      "clip_ratio/low_mean": 0.0016714749799575657,
-      "clip_ratio/low_min": 0.0016714749799575657,
-      "clip_ratio/region_mean": 0.0037594741152133793,
-      "completions/clipped_ratio": 0.0390625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 12.09375,
-      "completions/min_length": 5.0,
-      "epoch": 1.631578947368421,
-      "grad_norm": 1.8301432866457292,
-      "kl": 1.3922554631717503,
-      "learning_rate": 1.7822218477475494e-05,
-      "loss": 0.015794314444065094,
-      "memory(GiB)": 79.43,
-      "reward": 0.5234375,
-      "reward_std": 0.20214977860450745,
-      "rewards/Response_no_think_reward_1/mean": 0.5234375,
-      "rewards/Response_no_think_reward_1/std": 0.6143282055854797,
+      "train_speed(iter/s)": 0.021306
+    },
+    {
+      "epoch": 0.2968947235546579,
+      "grad_norm": 6.068948268890381,
+      "learning_rate": 0.00019479104492689384,
+      "logits/chosen": -0.7972388863563538,
+      "logits/rejected": -0.9000754356384277,
+      "logps/chosen": -8.90436840057373,
+      "logps/rejected": -43.9642448425293,
+      "loss": 1.004248023033142,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7405007481575012,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.2410261034965515,
+      "rewards/margins": 2.683554172515869,
+      "rewards/rejected": -2.9245800971984863,
       "step": 147,
-      "train_speed(iter/s)": 0.006704
-    },
-    {
-      "clip_ratio/high_max": 0.004581842658808455,
-      "clip_ratio/high_mean": 0.004581842658808455,
-      "clip_ratio/low_mean": 0.00751026114448905,
-      "clip_ratio/low_min": 0.00751026114448905,
-      "clip_ratio/region_mean": 0.012092103715986013,
-      "epoch": 1.6426592797783934,
-      "grad_norm": 2.0798983048360977,
-      "kl": 1.5519673400558531,
-      "learning_rate": 1.6789300637645e-05,
-      "loss": 0.010683290660381317,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021307
+    },
+    {
+      "epoch": 0.2989144155516284,
+      "grad_norm": 6.354671478271484,
+      "learning_rate": 0.00019468405686693044,
+      "logits/chosen": -0.8239361643791199,
+      "logits/rejected": -0.8870259523391724,
+      "logps/chosen": -3.0636796951293945,
+      "logps/rejected": -32.0543327331543,
+      "loss": 0.6938296556472778,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.32726508378982544,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.2392302304506302,
+      "rewards/margins": 2.2771260738372803,
+      "rewards/rejected": -2.037895917892456,
       "step": 148,
-      "train_speed(iter/s)": 0.006691
-    },
-    {
-      "clip_ratio/high_max": 0.0013595109921880066,
-      "clip_ratio/high_mean": 0.0013595109921880066,
-      "clip_ratio/low_mean": 0.004456432332517579,
-      "clip_ratio/low_min": 0.004456432332517579,
-      "clip_ratio/region_mean": 0.005815943295601755,
-      "completions/clipped_ratio": 0.140625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 13.4453125,
-      "completions/min_length": 6.0,
-      "epoch": 1.6537396121883656,
-      "grad_norm": 2.124340745472962,
-      "kl": 4.468803564086556,
-      "learning_rate": 1.578446780124344e-05,
-      "loss": 0.04191342741250992,
-      "memory(GiB)": 79.43,
-      "reward": 0.09375,
-      "reward_std": 0.2688094973564148,
-      "rewards/Response_no_think_reward_1/mean": 0.09375,
-      "rewards/Response_no_think_reward_1/std": 0.7036183476448059,
+      "train_speed(iter/s)": 0.021309
+    },
+    {
+      "epoch": 0.30093410754859884,
+      "grad_norm": 2.746241331100464,
+      "learning_rate": 0.00019457601120928194,
+      "logits/chosen": -0.7442300915718079,
+      "logits/rejected": -0.8740307092666626,
+      "logps/chosen": -0.39770960807800293,
+      "logps/rejected": -24.435047149658203,
+      "loss": 0.323159396648407,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.05953114107251167,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.28615039587020874,
+      "rewards/margins": 1.8557114601135254,
+      "rewards/rejected": -1.5695610046386719,
       "step": 149,
-      "train_speed(iter/s)": 0.006516
-    },
-    {
-      "clip_ratio/high_max": 0.004650130198569968,
-      "clip_ratio/high_mean": 0.004650130198569968,
-      "clip_ratio/low_mean": 0.015789811441209167,
-      "clip_ratio/low_min": 0.015789811441209167,
-      "clip_ratio/region_mean": 0.020439941552467644,
-      "epoch": 1.6648199445983378,
-      "grad_norm": 1.3890201997929057,
-      "kl": 4.818290303461254,
-      "learning_rate": 1.4808059116167305e-05,
-      "loss": 0.03111656755208969,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.02131
+    },
+    {
+      "epoch": 0.3029537995455693,
+      "grad_norm": 7.339154243469238,
+      "learning_rate": 0.0001944669091607919,
+      "logits/chosen": -0.7669613361358643,
+      "logits/rejected": -0.8887062072753906,
+      "logps/chosen": -6.112335205078125,
+      "logps/rejected": -46.467247009277344,
+      "loss": 0.7917571663856506,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.44829899072647095,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.07194096595048904,
+      "rewards/margins": 2.9925551414489746,
+      "rewards/rejected": -3.0644962787628174,
       "step": 150,
-      "train_speed(iter/s)": 0.006505
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0031864915508776903,
-      "clip_ratio/low_min": 0.0031864915508776903,
-      "clip_ratio/region_mean": 0.0031864915508776903,
-      "completions/clipped_ratio": 0.1484375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 15.96875,
-      "completions/min_length": 3.0,
-      "epoch": 1.6759002770083102,
-      "grad_norm": 1.3263437432774172,
-      "kl": 3.4037277391953467,
-      "learning_rate": 1.3860404136686411e-05,
-      "loss": 0.02711915224790573,
-      "memory(GiB)": 79.43,
-      "reward": 0.2890625,
-      "reward_std": 0.20480823516845703,
-      "rewards/Response_no_think_reward_1/mean": 0.2890625,
-      "rewards/Response_no_think_reward_1/std": 0.7852212190628052,
+      "train_speed(iter/s)": 0.02131
+    },
+    {
+      "epoch": 0.3049734915425398,
+      "grad_norm": 13.051281929016113,
+      "learning_rate": 0.00019435675194010336,
+      "logits/chosen": -0.8439881801605225,
+      "logits/rejected": -0.93268883228302,
+      "logps/chosen": -4.412451267242432,
+      "logps/rejected": -46.751976013183594,
+      "loss": 0.8531545400619507,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.44560468196868896,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.06836838275194168,
+      "rewards/margins": 3.4782376289367676,
+      "rewards/rejected": -3.5466065406799316,
       "step": 151,
-      "train_speed(iter/s)": 0.006367
-    },
-    {
-      "clip_ratio/high_max": 0.002190002123825252,
-      "clip_ratio/high_mean": 0.002190002123825252,
-      "clip_ratio/low_mean": 0.010535595705732703,
-      "clip_ratio/low_min": 0.010535595705732703,
-      "clip_ratio/region_mean": 0.012725598004180938,
-      "epoch": 1.6869806094182827,
-      "grad_norm": 1.0936459304561021,
-      "kl": 3.5067162624327466,
-      "learning_rate": 1.294182271221377e-05,
-      "loss": 0.02425944060087204,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021262
+    },
+    {
+      "epoch": 0.3069931835395102,
+      "grad_norm": 3.5028750896453857,
+      "learning_rate": 0.00019424554077764546,
+      "logits/chosen": -0.7861919403076172,
+      "logits/rejected": -0.915184497833252,
+      "logps/chosen": -2.617560386657715,
+      "logps/rejected": -56.570960998535156,
+      "loss": 0.4112832546234131,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2496473491191864,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.11519647389650345,
+      "rewards/margins": 3.7134275436401367,
+      "rewards/rejected": -3.598231315612793,
       "step": 152,
-      "train_speed(iter/s)": 0.006357
-    },
-    {
-      "clip_ratio/high_max": 0.004777037829626352,
-      "clip_ratio/high_mean": 0.004777037829626352,
-      "clip_ratio/low_mean": 0.0015179030015133321,
-      "clip_ratio/low_min": 0.0015179030015133321,
-      "clip_ratio/region_mean": 0.006294940831139684,
-      "completions/clipped_ratio": 0.0625,
-      "completions/max_length": 47.0,
-      "completions/mean_length": 13.6796875,
-      "completions/min_length": 3.0,
-      "epoch": 1.6980609418282548,
-      "grad_norm": 3.294412855573239,
-      "kl": 4.274040638643783,
-      "learning_rate": 1.2052624879351104e-05,
-      "loss": 0.03787969425320625,
-      "memory(GiB)": 79.43,
-      "reward": 0.6015625,
-      "reward_std": 0.17859892547130585,
-      "rewards/Response_no_think_reward_1/mean": 0.6015625,
-      "rewards/Response_no_think_reward_1/std": 0.5931345224380493,
+      "train_speed(iter/s)": 0.021263
+    },
+    {
+      "epoch": 0.3090128755364807,
+      "grad_norm": 6.6451520919799805,
+      "learning_rate": 0.00019413327691561967,
+      "logits/chosen": -0.7669755816459656,
+      "logits/rejected": -0.8034001588821411,
+      "logps/chosen": -9.02087688446045,
+      "logps/rejected": -36.971290588378906,
+      "loss": 0.8952721357345581,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.42217180132865906,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.07903412729501724,
+      "rewards/margins": 2.4241135120391846,
+      "rewards/rejected": -2.503147602081299,
       "step": 153,
-      "train_speed(iter/s)": 0.006206
-    },
-    {
-      "clip_ratio/high_max": 0.005947966914391145,
-      "clip_ratio/high_mean": 0.005947966914391145,
-      "clip_ratio/low_mean": 0.004764656303450465,
-      "clip_ratio/low_min": 0.004764656303450465,
-      "clip_ratio/region_mean": 0.010712623450672254,
-      "epoch": 1.709141274238227,
-      "grad_norm": 2.9610057087874977,
-      "kl": 3.866908519703429,
-      "learning_rate": 1.119311075724625e-05,
-      "loss": 0.033227190375328064,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021263
+    },
+    {
+      "epoch": 0.31103256753345115,
+      "grad_norm": 6.9827470779418945,
+      "learning_rate": 0.00019401996160798573,
+      "logits/chosen": -0.7891846895217896,
+      "logits/rejected": -0.8431411981582642,
+      "logps/chosen": -12.274852752685547,
+      "logps/rejected": -40.9384765625,
+      "loss": 1.0033327341079712,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5167030692100525,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.11504307389259338,
+      "rewards/margins": 2.439614772796631,
+      "rewards/rejected": -2.5546579360961914,
       "step": 154,
-      "train_speed(iter/s)": 0.006199
-    },
-    {
-      "clip_ratio/high_max": 0.0032051282469183207,
-      "clip_ratio/high_mean": 0.0032051282469183207,
-      "clip_ratio/low_mean": 0.00683464459143579,
-      "clip_ratio/low_min": 0.00683464459143579,
-      "clip_ratio/region_mean": 0.01003977283835411,
-      "completions/clipped_ratio": 0.1875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 16.2265625,
-      "completions/min_length": 5.0,
-      "epoch": 1.7202216066481995,
-      "grad_norm": 3.0112138436872624,
-      "kl": 4.548647504067048,
-      "learning_rate": 1.0363570446297999e-05,
-      "loss": 0.03970736265182495,
-      "memory(GiB)": 79.43,
-      "reward": 0.328125,
-      "reward_std": 0.32412126660346985,
-      "rewards/Response_no_think_reward_1/mean": 0.328125,
-      "rewards/Response_no_think_reward_1/std": 0.743574857711792,
+      "train_speed(iter/s)": 0.021263
+    },
+    {
+      "epoch": 0.3130522595304216,
+      "grad_norm": 5.614060878753662,
+      "learning_rate": 0.0001939055961204478,
+      "logits/chosen": -0.8796770572662354,
+      "logits/rejected": -0.9147617816925049,
+      "logps/chosen": -7.191051959991455,
+      "logps/rejected": -29.8262882232666,
+      "loss": 0.8995785117149353,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.49978020787239075,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.055766861885786057,
+      "rewards/margins": 1.4225391149520874,
+      "rewards/rejected": -1.4783059358596802,
       "step": 155,
-      "train_speed(iter/s)": 0.006071
-    },
-    {
-      "clip_ratio/high_max": 0.0029493323527276516,
-      "clip_ratio/high_mean": 0.0029493323527276516,
-      "clip_ratio/low_mean": 0.008986421715235338,
-      "clip_ratio/low_min": 0.008986421715235338,
-      "clip_ratio/region_mean": 0.011935754009755328,
-      "epoch": 1.7313019390581719,
-      "grad_norm": 3.1784321577702666,
-      "kl": 4.521617598744342,
-      "learning_rate": 9.564283930242257e-06,
-      "loss": 0.03974776715040207,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021264
+    },
+    {
+      "epoch": 0.3150719515273921,
+      "grad_norm": 5.3426737785339355,
+      "learning_rate": 0.00019379018173044037,
+      "logits/chosen": -0.8801127076148987,
+      "logits/rejected": -0.9478501081466675,
+      "logps/chosen": -5.170099258422852,
+      "logps/rejected": -29.119112014770508,
+      "loss": 0.7686877846717834,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.36578065156936646,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.10118754208087921,
+      "rewards/margins": 1.4974370002746582,
+      "rewards/rejected": -1.3962492942810059,
       "step": 156,
-      "train_speed(iter/s)": 0.006064
-    },
-    {
-      "clip_ratio/high_max": 0.005233740259427577,
-      "clip_ratio/high_mean": 0.005233740259427577,
-      "clip_ratio/low_mean": 0.013618246564874426,
-      "clip_ratio/low_min": 0.013618246564874426,
-      "clip_ratio/region_mean": 0.018851986795198172,
-      "completions/clipped_ratio": 0.140625,
-      "completions/max_length": 46.0,
-      "completions/mean_length": 12.71875,
-      "completions/min_length": 2.0,
-      "epoch": 1.742382271468144,
-      "grad_norm": 3.107280979259554,
-      "kl": 7.697673750575632,
-      "learning_rate": 8.795520981652961e-06,
-      "loss": 0.04705927148461342,
-      "memory(GiB)": 79.43,
-      "reward": 0.171875,
-      "reward_std": 0.49000412225723267,
-      "rewards/Response_no_think_reward_1/mean": 0.171875,
-      "rewards/Response_no_think_reward_1/std": 0.8143339157104492,
+      "train_speed(iter/s)": 0.021265
+    },
+    {
+      "epoch": 0.31709164352436253,
+      "grad_norm": 6.185426235198975,
+      "learning_rate": 0.00019367371972711385,
+      "logits/chosen": -0.883526623249054,
+      "logits/rejected": -0.9433069229125977,
+      "logps/chosen": -2.170651435852051,
+      "logps/rejected": -28.505441665649414,
+      "loss": 0.7070222496986389,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.333578884601593,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.17996083199977875,
+      "rewards/margins": 1.5758326053619385,
+      "rewards/rejected": -1.395871639251709,
       "step": 157,
-      "train_speed(iter/s)": 0.005969
-    },
-    {
-      "clip_ratio/high_max": 0.018029920000117272,
-      "clip_ratio/high_mean": 0.018029920000117272,
-      "clip_ratio/low_mean": 0.011481558292871341,
-      "clip_ratio/low_min": 0.011481558292871341,
-      "clip_ratio/region_mean": 0.029511478263884783,
-      "epoch": 1.7534626038781163,
-      "grad_norm": 2.4164221842161395,
-      "kl": 7.264460427570157,
-      "learning_rate": 8.05754107088923e-06,
-      "loss": 0.0432392843067646,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021265
+    },
+    {
+      "epoch": 0.319111335521333,
+      "grad_norm": 4.604299068450928,
+      "learning_rate": 0.0001935562114113202,
+      "logits/chosen": -0.874811053276062,
+      "logits/rejected": -0.9069212079048157,
+      "logps/chosen": -6.411377906799316,
+      "logps/rejected": -28.813356399536133,
+      "loss": 0.8389774560928345,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4572408199310303,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1476302295923233,
+      "rewards/margins": 1.0184513330459595,
+      "rewards/rejected": -0.8708210587501526,
       "step": 158,
-      "train_speed(iter/s)": 0.005963
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.003813613730017096,
-      "clip_ratio/low_min": 0.003813613730017096,
-      "clip_ratio/region_mean": 0.003813613730017096,
-      "completions/clipped_ratio": 0.0859375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 14.8359375,
-      "completions/min_length": 4.0,
-      "epoch": 1.7645429362880887,
-      "grad_norm": 1.905247433106981,
-      "kl": 1.1771136652678251,
-      "learning_rate": 7.350593278519824e-06,
-      "loss": 0.00986653845757246,
-      "memory(GiB)": 79.43,
-      "reward": 0.359375,
-      "reward_std": 0.27169257402420044,
-      "rewards/Response_no_think_reward_1/mean": 0.359375,
-      "rewards/Response_no_think_reward_1/std": 0.7605879902839661,
+      "train_speed(iter/s)": 0.021266
+    },
+    {
+      "epoch": 0.32113102751830347,
+      "grad_norm": 2.668325424194336,
+      "learning_rate": 0.00019343765809559852,
+      "logits/chosen": -0.9191151261329651,
+      "logits/rejected": -0.9647189378738403,
+      "logps/chosen": -2.182382106781006,
+      "logps/rejected": -27.029888153076172,
+      "loss": 0.6324246525764465,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3407551646232605,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.22473560273647308,
+      "rewards/margins": 1.382723093032837,
+      "rewards/rejected": -1.1579875946044922,
       "step": 159,
-      "train_speed(iter/s)": 0.005878
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.003413711878238246,
-      "clip_ratio/low_min": 0.003413711878238246,
-      "clip_ratio/region_mean": 0.003413711878238246,
-      "epoch": 1.775623268698061,
-      "grad_norm": 1.9755946085728415,
-      "kl": 1.1371133443899453,
-      "learning_rate": 6.674916211254289e-06,
-      "loss": 0.009400583803653717,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021266
+    },
+    {
+      "epoch": 0.3231507195152739,
+      "grad_norm": 3.9123880863189697,
+      "learning_rate": 0.00019331806110416027,
+      "logits/chosen": -0.9419438242912292,
+      "logits/rejected": -0.9589441418647766,
+      "logps/chosen": -2.685596466064453,
+      "logps/rejected": -16.038766860961914,
+      "loss": 0.8650633692741394,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3469831943511963,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.12647470831871033,
+      "rewards/margins": 0.6830577254295349,
+      "rewards/rejected": -0.5565829873085022,
       "step": 160,
-      "train_speed(iter/s)": 0.005873
-    },
-    {
-      "clip_ratio/high_max": 0.0028935185400769114,
-      "clip_ratio/high_mean": 0.0028935185400769114,
-      "clip_ratio/low_mean": 0.007061597076244652,
-      "clip_ratio/low_min": 0.007061597076244652,
-      "clip_ratio/region_mean": 0.009955115616321564,
-      "completions/clipped_ratio": 0.1015625,
-      "completions/max_length": 50.0,
-      "completions/mean_length": 10.5625,
-      "completions/min_length": 2.0,
-      "epoch": 1.7867036011080333,
-      "grad_norm": 2.0459547550897166,
-      "kl": 3.880769203417003,
-      "learning_rate": 7.689418917193289e-05,
-      "loss": 0.024239888414740562,
-      "memory(GiB)": 77.36,
-      "reward": 0.234375,
-      "reward_std": 0.31300365924835205,
-      "rewards/Response_no_think_reward_1/mean": 0.234375,
-      "rewards/Response_no_think_reward_1/std": 0.7475352883338928,
+      "train_speed(iter/s)": 0.021268
+    },
+    {
+      "epoch": 0.3251704115122444,
+      "grad_norm": 3.562811851501465,
+      "learning_rate": 0.00019319742177287448,
+      "logits/chosen": -0.895343542098999,
+      "logits/rejected": -0.9661368727684021,
+      "logps/chosen": -3.5327184200286865,
+      "logps/rejected": -20.103837966918945,
+      "loss": 0.8163770437240601,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.42449474334716797,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.2701226472854614,
+      "rewards/margins": 1.0340347290039062,
+      "rewards/rejected": -0.7639120221138,
       "step": 161,
-      "train_speed(iter/s)": 0.21784
-    },
-    {
-      "clip_ratio/high_max": 0.005787037080153823,
-      "clip_ratio/high_mean": 0.005787037080153823,
-      "clip_ratio/low_mean": 0.011741390626411885,
-      "clip_ratio/low_min": 0.011741390626411885,
-      "clip_ratio/region_mean": 0.017528427706565708,
-      "epoch": 1.7977839335180055,
-      "grad_norm": 1.6284449438133224,
-      "kl": 3.9539552154019475,
-      "learning_rate": 7.570198200967362e-05,
-      "loss": 0.026077013462781906,
-      "memory(GiB)": 77.36,
+      "train_speed(iter/s)": 0.02127
+    },
+    {
+      "epoch": 0.32719010350921485,
+      "grad_norm": 6.563948631286621,
+      "learning_rate": 0.00019307574144925287,
+      "logits/chosen": -0.9778874516487122,
+      "logits/rejected": -1.0073670148849487,
+      "logps/chosen": -2.233975887298584,
+      "logps/rejected": -14.128082275390625,
+      "loss": 0.8808709979057312,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3910011947154999,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.16543619334697723,
+      "rewards/margins": 0.6482046246528625,
+      "rewards/rejected": -0.48276838660240173,
       "step": 162,
-      "train_speed(iter/s)": 0.173972
-    },
-    {
-      "clip_ratio/high_max": 0.0007267441833391786,
-      "clip_ratio/high_mean": 0.0007267441833391786,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0007267441833391786,
-      "completions/clipped_ratio": 0.078125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 15.1875,
-      "completions/min_length": 6.0,
-      "epoch": 1.8088642659279779,
-      "grad_norm": 6.190200790245519,
-      "kl": 2.1185005172155797,
-      "learning_rate": 7.451343403954856e-05,
-      "loss": 0.015710504725575447,
-      "memory(GiB)": 77.47,
-      "reward": 0.46875,
-      "reward_std": 0.11279275268316269,
-      "rewards/Response_no_think_reward_1/mean": 0.46875,
-      "rewards/Response_no_think_reward_1/std": 0.7310600280761719,
+      "train_speed(iter/s)": 0.021272
+    },
+    {
+      "epoch": 0.3292097955061853,
+      "grad_norm": 2.812866449356079,
+      "learning_rate": 0.0001929530214924348,
+      "logits/chosen": -0.8719493746757507,
+      "logits/rejected": -0.9090234041213989,
+      "logps/chosen": -2.135589361190796,
+      "logps/rejected": -34.29349136352539,
+      "loss": 0.6473661661148071,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23261404037475586,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.12609758973121643,
+      "rewards/margins": 1.2335658073425293,
+      "rewards/rejected": -1.1074683666229248,
       "step": 163,
-      "train_speed(iter/s)": 0.105292
-    },
-    {
-      "clip_ratio/high_max": 0.010034247010480613,
-      "clip_ratio/high_mean": 0.010034247010480613,
-      "clip_ratio/low_mean": 0.0050670221680775285,
-      "clip_ratio/low_min": 0.0050670221680775285,
-      "clip_ratio/region_mean": 0.015101269178558141,
-      "epoch": 1.8199445983379503,
-      "grad_norm": 1.5020471792427759,
-      "kl": 1.848701277282089,
-      "learning_rate": 7.332872425251018e-05,
-      "loss": 0.011468175798654556,
-      "memory(GiB)": 77.47,
+      "train_speed(iter/s)": 0.021271
+    },
+    {
+      "epoch": 0.3312294875031558,
+      "grad_norm": 6.056075096130371,
+      "learning_rate": 0.0001928292632731721,
+      "logits/chosen": -0.927330732345581,
+      "logits/rejected": -0.9583979845046997,
+      "logps/chosen": -2.9721577167510986,
+      "logps/rejected": -37.22073745727539,
+      "loss": 0.5903728604316711,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22814247012138367,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.24338555335998535,
+      "rewards/margins": 1.3612616062164307,
+      "rewards/rejected": -1.1178761720657349,
       "step": 164,
-      "train_speed(iter/s)": 0.094243
-    },
-    {
-      "clip_ratio/high_max": 0.0031305171723943204,
-      "clip_ratio/high_mean": 0.0031305171723943204,
-      "clip_ratio/low_mean": 0.006101353617850691,
-      "clip_ratio/low_min": 0.006101353617850691,
-      "clip_ratio/region_mean": 0.00923187073203735,
-      "completions/clipped_ratio": 0.0859375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 13.578125,
-      "completions/min_length": 4.0,
-      "epoch": 1.8310249307479225,
-      "grad_norm": 1.5859082442887886,
-      "kl": 3.1664798953570426,
-      "learning_rate": 7.21480310614947e-05,
-      "loss": 0.024385500699281693,
-      "memory(GiB)": 77.47,
-      "reward": 0.34375,
-      "reward_std": 0.3683975338935852,
-      "rewards/Response_no_think_reward_1/mean": 0.34375,
-      "rewards/Response_no_think_reward_1/std": 0.692337155342102,
+      "train_speed(iter/s)": 0.021271
+    },
+    {
+      "epoch": 0.3332491795001262,
+      "grad_norm": 3.5847744941711426,
+      "learning_rate": 0.00019270446817381377,
+      "logits/chosen": -0.9705857038497925,
+      "logits/rejected": -0.9790812134742737,
+      "logps/chosen": -2.228187084197998,
+      "logps/rejected": -22.54074478149414,
+      "loss": 0.582669198513031,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3219188451766968,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.31063607335090637,
+      "rewards/margins": 1.6680660247802734,
+      "rewards/rejected": -1.3574302196502686,
       "step": 165,
-      "train_speed(iter/s)": 0.069824
-    },
-    {
-      "clip_ratio/high_max": 0.019868265371769667,
-      "clip_ratio/high_mean": 0.019868265371769667,
-      "clip_ratio/low_mean": 0.011233113938942552,
-      "clip_ratio/low_min": 0.011233113938942552,
-      "clip_ratio/region_mean": 0.031101379543542862,
-      "epoch": 1.8421052631578947,
-      "grad_norm": 1.8023731350310037,
-      "kl": 2.8112484337762,
-      "learning_rate": 7.097153227455379e-05,
-      "loss": 0.013743140734732151,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021273
+    },
+    {
+      "epoch": 0.33526887149709667,
+      "grad_norm": 3.5586040019989014,
+      "learning_rate": 0.00019257863758829035,
+      "logits/chosen": -0.9236764311790466,
+      "logits/rejected": -0.9713916182518005,
+      "logps/chosen": -2.841557264328003,
+      "logps/rejected": -41.60185241699219,
+      "loss": 0.667453408241272,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.45156461000442505,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1021651178598404,
+      "rewards/margins": 2.591975450515747,
+      "rewards/rejected": -2.4898104667663574,
       "step": 166,
-      "train_speed(iter/s)": 0.064907
-    },
-    {
-      "clip_ratio/high_max": 0.0011916428920812905,
-      "clip_ratio/high_mean": 0.0011916428920812905,
-      "clip_ratio/low_mean": 0.0034684768179431558,
-      "clip_ratio/low_min": 0.0034684768179431558,
-      "clip_ratio/region_mean": 0.004660119710024446,
-      "completions/clipped_ratio": 0.046875,
-      "completions/max_length": 31.0,
-      "completions/mean_length": 9.9765625,
-      "completions/min_length": 3.0,
-      "epoch": 1.8531855955678669,
-      "grad_norm": 2.973635940137616,
-      "kl": 2.157054567243904,
-      "learning_rate": 6.97994050680772e-05,
-      "loss": 0.017193181440234184,
-      "memory(GiB)": 79.43,
-      "reward": 0.484375,
-      "reward_std": 0.22119547426700592,
-      "rewards/Response_no_think_reward_1/mean": 0.484375,
-      "rewards/Response_no_think_reward_1/std": 0.7527835369110107,
+      "train_speed(iter/s)": 0.021274
+    },
+    {
+      "epoch": 0.33728856349406716,
+      "grad_norm": 11.787897109985352,
+      "learning_rate": 0.00019245177292209867,
+      "logits/chosen": -0.9395866394042969,
+      "logits/rejected": -0.9611519575119019,
+      "logps/chosen": -5.94741153717041,
+      "logps/rejected": -48.858680725097656,
+      "loss": 1.042246699333191,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6328973174095154,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.07254321873188019,
+      "rewards/margins": 2.8825278282165527,
+      "rewards/rejected": -2.955070734024048,
       "step": 167,
-      "train_speed(iter/s)": 0.052203
-    },
-    {
-      "clip_ratio/high_max": 0.006835455656982958,
-      "clip_ratio/high_mean": 0.006835455656982958,
-      "clip_ratio/low_mean": 0.046273494604974985,
-      "clip_ratio/low_min": 0.046273494604974985,
-      "clip_ratio/region_mean": 0.05310894921422005,
-      "epoch": 1.8642659279778393,
-      "grad_norm": 1.4603345730407995,
-      "kl": 3.1518404465168715,
-      "learning_rate": 6.863182596011087e-05,
-      "loss": 0.009599600918591022,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021274
+    },
+    {
+      "epoch": 0.3393082554910376,
+      "grad_norm": 2.1068975925445557,
+      "learning_rate": 0.00019232387559228587,
+      "logits/chosen": -0.9090099930763245,
+      "logits/rejected": -0.9743850827217102,
+      "logps/chosen": -3.3460116386413574,
+      "logps/rejected": -49.71737289428711,
+      "loss": 0.3520747423171997,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20601019263267517,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.022673189640045166,
+      "rewards/margins": 3.1425955295562744,
+      "rewards/rejected": -3.165269136428833,
       "step": 168,
-      "train_speed(iter/s)": 0.04954
-    },
-    {
-      "clip_ratio/high_max": 0.0006965868233237416,
-      "clip_ratio/high_mean": 0.0006965868233237416,
-      "clip_ratio/low_mean": 0.002629449882078916,
-      "clip_ratio/low_min": 0.002629449882078916,
-      "clip_ratio/region_mean": 0.003326036676298827,
-      "completions/clipped_ratio": 0.0859375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 13.03125,
-      "completions/min_length": 3.0,
-      "epoch": 1.8753462603878117,
-      "grad_norm": 1.302354063269936,
-      "kl": 1.0980142642074497,
-      "learning_rate": 6.746897078377372e-05,
-      "loss": 0.01619286648929119,
-      "memory(GiB)": 79.43,
-      "reward": 0.359375,
-      "reward_std": 0.31949180364608765,
-      "rewards/Response_no_think_reward_1/mean": 0.359375,
-      "rewards/Response_no_think_reward_1/std": 0.7810186743736267,
+      "train_speed(iter/s)": 0.021274
+    },
+    {
+      "epoch": 0.3413279474880081,
+      "grad_norm": 13.362913131713867,
+      "learning_rate": 0.0001921949470274338,
+      "logits/chosen": -0.9718740582466125,
+      "logits/rejected": -0.9933693408966064,
+      "logps/chosen": -16.02916717529297,
+      "logps/rejected": -40.69719696044922,
+      "loss": 1.3985366821289062,
+      "memory(GiB)": 45.64,
+      "nll_loss": 1.129913330078125,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.9396551847457886,
+      "rewards/margins": 1.5676053762435913,
+      "rewards/rejected": -2.507260799407959,
       "step": 169,
-      "train_speed(iter/s)": 0.041791
-    },
-    {
-      "clip_ratio/high_max": 0.0037332845386117697,
-      "clip_ratio/high_mean": 0.0037332845386117697,
-      "clip_ratio/low_mean": 0.013076976756565273,
-      "clip_ratio/low_min": 0.013076976756565273,
-      "clip_ratio/region_mean": 0.016810261004138738,
-      "epoch": 1.886426592797784,
-      "grad_norm": 0.6775107653086064,
-      "kl": 0.8065175649971934,
-      "learning_rate": 6.6311014660778e-05,
-      "loss": 0.005674303974956274,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021275
+    },
+    {
+      "epoch": 0.34334763948497854,
+      "grad_norm": 12.149360656738281,
+      "learning_rate": 0.00019206498866764288,
+      "logits/chosen": -0.9391348361968994,
+      "logits/rejected": -0.9958175420761108,
+      "logps/chosen": -4.999204635620117,
+      "logps/rejected": -34.91378402709961,
+      "loss": 1.1959863901138306,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.683914303779602,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1860208809375763,
+      "rewards/margins": 2.0848278999328613,
+      "rewards/rejected": -2.2708487510681152,
       "step": 170,
-      "train_speed(iter/s)": 0.040129
-    },
-    {
-      "clip_ratio/high_max": 0.0017115735390689224,
-      "clip_ratio/high_mean": 0.0017115735390689224,
-      "clip_ratio/low_mean": 0.004471359890885651,
-      "clip_ratio/low_min": 0.004471359890885651,
-      "clip_ratio/region_mean": 0.006182933488162234,
-      "completions/clipped_ratio": 0.078125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 10.8359375,
-      "completions/min_length": 3.0,
-      "epoch": 1.897506925207756,
-      "grad_norm": 1.6323523762740895,
-      "kl": 1.2415385083295405,
-      "learning_rate": 6.515813197505656e-05,
-      "loss": 0.01243288628757,
-      "memory(GiB)": 79.43,
-      "reward": 0.265625,
-      "reward_std": 0.25462424755096436,
-      "rewards/Response_no_think_reward_1/mean": 0.265625,
-      "rewards/Response_no_think_reward_1/std": 0.7475352883338928,
+      "train_speed(iter/s)": 0.021276
+    },
+    {
+      "epoch": 0.345367331481949,
+      "grad_norm": 4.593865871429443,
+      "learning_rate": 0.0001919340019645161,
+      "logits/chosen": -0.8601446151733398,
+      "logits/rejected": -0.9445533752441406,
+      "logps/chosen": -6.576286792755127,
+      "logps/rejected": -35.74972152709961,
+      "loss": 0.8745996952056885,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.48976749181747437,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.1266278177499771,
+      "rewards/margins": 1.351570963859558,
+      "rewards/rejected": -1.478198766708374,
       "step": 171,
-      "train_speed(iter/s)": 0.035141
-    },
-    {
-      "clip_ratio/high_max": 0.015582295309286565,
-      "clip_ratio/high_mean": 0.015582295309286565,
-      "clip_ratio/low_mean": 0.010968676884658635,
-      "clip_ratio/low_min": 0.010968676884658635,
-      "clip_ratio/region_mean": 0.026550972019322217,
-      "epoch": 1.9085872576177285,
-      "grad_norm": 0.8010680473291589,
-      "kl": 1.0659651298337849,
-      "learning_rate": 6.401049634650118e-05,
-      "loss": 4.9868394853547215e-05,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021277
+    },
+    {
+      "epoch": 0.3473870234789195,
+      "grad_norm": 6.634700775146484,
+      "learning_rate": 0.00019180198838114282,
+      "logits/chosen": -0.9056757688522339,
+      "logits/rejected": -0.9392012357711792,
+      "logps/chosen": -1.7248777151107788,
+      "logps/rejected": -23.6937255859375,
+      "loss": 0.8178808093070984,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.37585797905921936,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.18669246137142181,
+      "rewards/margins": 1.2759226560592651,
+      "rewards/rejected": -1.0892301797866821,
       "step": 172,
-      "train_speed(iter/s)": 0.034005
-    },
-    {
-      "clip_ratio/high_max": 0.0024205285299103707,
-      "clip_ratio/high_mean": 0.0024205285299103707,
-      "clip_ratio/low_mean": 0.006205722645972855,
-      "clip_ratio/low_min": 0.006205722645972855,
-      "clip_ratio/region_mean": 0.008626251190435141,
-      "completions/clipped_ratio": 0.078125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 10.5,
-      "completions/min_length": 2.0,
-      "epoch": 1.919667590027701,
-      "grad_norm": 2.2598530338882874,
-      "kl": 3.693746126416954,
-      "learning_rate": 6.286828060481626e-05,
-      "loss": 0.02687670849263668,
-      "memory(GiB)": 79.43,
-      "reward": 0.328125,
-      "reward_std": 0.3272990882396698,
-      "rewards/Response_no_think_reward_1/mean": 0.328125,
-      "rewards/Response_no_think_reward_1/std": 0.7220855355262756,
+      "train_speed(iter/s)": 0.021279
+    },
+    {
+      "epoch": 0.3494067154758899,
+      "grad_norm": 6.2399725914001465,
+      "learning_rate": 0.00019166894939208237,
+      "logits/chosen": -0.8763759732246399,
+      "logits/rejected": -0.8875811100006104,
+      "logps/chosen": -4.987262725830078,
+      "logps/rejected": -23.384920120239258,
+      "loss": 0.953944742679596,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5001811385154724,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.11620373278856277,
+      "rewards/margins": 1.522448182106018,
+      "rewards/rejected": -1.6386518478393555,
       "step": 173,
-      "train_speed(iter/s)": 0.030698
-    },
-    {
-      "clip_ratio/high_max": 0.01826251167221926,
-      "clip_ratio/high_mean": 0.01826251167221926,
-      "clip_ratio/low_mean": 0.027282171708066016,
-      "clip_ratio/low_min": 0.027282171708066016,
-      "clip_ratio/region_mean": 0.04554468300193548,
-      "epoch": 1.9307479224376731,
-      "grad_norm": 2.095627271198394,
-      "kl": 2.8586582938878564,
-      "learning_rate": 6.173165676349103e-05,
-      "loss": 0.017260747030377388,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.02128
+    },
+    {
+      "epoch": 0.3514264074728604,
+      "grad_norm": 2.378427743911743,
+      "learning_rate": 0.0001915348864833476,
+      "logits/chosen": -0.8616642951965332,
+      "logits/rejected": -0.9031955003738403,
+      "logps/chosen": -2.75161075592041,
+      "logps/rejected": -17.232093811035156,
+      "loss": 0.7261631488800049,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3087785542011261,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.24855753779411316,
+      "rewards/margins": 0.8990098237991333,
+      "rewards/rejected": -0.650452196598053,
       "step": 174,
-      "train_speed(iter/s)": 0.029861
-    },
-    {
-      "clip_ratio/high_max": 0.001936484724865295,
-      "clip_ratio/high_mean": 0.001936484724865295,
-      "clip_ratio/low_mean": 0.006831976061221212,
-      "clip_ratio/low_min": 0.006831976061221212,
-      "clip_ratio/region_mean": 0.008768460800638422,
-      "completions/clipped_ratio": 0.1640625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 15.4453125,
-      "completions/min_length": 4.0,
-      "epoch": 1.9418282548476453,
-      "grad_norm": 3.0827807424330964,
-      "kl": 1.9612275282852352,
-      "learning_rate": 6.060079599389521e-05,
-      "loss": 0.020160475745797157,
-      "memory(GiB)": 79.43,
-      "reward": 0.3125,
-      "reward_std": 0.29061752557754517,
-      "rewards/Response_no_think_reward_1/mean": 0.3125,
-      "rewards/Response_no_think_reward_1/std": 0.6728714108467102,
+      "train_speed(iter/s)": 0.021282
+    },
+    {
+      "epoch": 0.35344609946983085,
+      "grad_norm": 2.4263904094696045,
+      "learning_rate": 0.00019139980115238827,
+      "logits/chosen": -0.8389776945114136,
+      "logits/rejected": -0.8697970509529114,
+      "logps/chosen": -4.028109073638916,
+      "logps/rejected": -18.706117630004883,
+      "loss": 0.780457615852356,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2751180827617645,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.006659353151917458,
+      "rewards/margins": 0.7851797342300415,
+      "rewards/rejected": -0.791839063167572,
       "step": 175,
-      "train_speed(iter/s)": 0.027244
-    },
-    {
-      "clip_ratio/high_max": 0.002366035128943622,
-      "clip_ratio/high_mean": 0.002366035128943622,
-      "clip_ratio/low_mean": 0.02325177617603913,
-      "clip_ratio/low_min": 0.02325177617603913,
-      "clip_ratio/region_mean": 0.02561781130498275,
-      "epoch": 1.9529085872576177,
-      "grad_norm": 1.4280426957771801,
-      "kl": 2.292961670376826,
-      "learning_rate": 5.947586859950103e-05,
-      "loss": 0.010125662200152874,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021283
+    },
+    {
+      "epoch": 0.3554657914668013,
+      "grad_norm": 2.074300765991211,
+      "learning_rate": 0.00019126369490807447,
+      "logits/chosen": -0.8048940896987915,
+      "logits/rejected": -0.884000301361084,
+      "logps/chosen": -1.4055310487747192,
+      "logps/rejected": -25.309282302856445,
+      "loss": 0.4804037809371948,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14255806803703308,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.40084120631217957,
+      "rewards/margins": 1.3346383571624756,
+      "rewards/rejected": -0.9337972402572632,
       "step": 176,
-      "train_speed(iter/s)": 0.026595
-    },
-    {
-      "clip_ratio/high_max": 0.0025192692410200834,
-      "clip_ratio/high_mean": 0.0025192692410200834,
-      "clip_ratio/low_mean": 0.008224956574849784,
-      "clip_ratio/low_min": 0.008224956574849784,
-      "clip_ratio/region_mean": 0.01074422593228519,
-      "completions/clipped_ratio": 0.1328125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 14.640625,
-      "completions/min_length": 2.0,
-      "epoch": 1.9639889196675901,
-      "grad_norm": 1.6671459210295219,
-      "kl": 2.1587753768544644,
-      "learning_rate": 5.83570439902363e-05,
-      "loss": 0.01727483607828617,
-      "memory(GiB)": 79.43,
-      "reward": 0.3515625,
-      "reward_std": 0.40715324878692627,
-      "rewards/Response_no_think_reward_1/mean": 0.3515625,
-      "rewards/Response_no_think_reward_1/std": 0.7273059487342834,
+      "train_speed(iter/s)": 0.021284
+    },
+    {
+      "epoch": 0.3574854834637718,
+      "grad_norm": 3.744903802871704,
+      "learning_rate": 0.00019112656927067955,
+      "logits/chosen": -0.8155094981193542,
+      "logits/rejected": -0.8395469784736633,
+      "logps/chosen": -11.99010944366455,
+      "logps/rejected": -26.437166213989258,
+      "loss": 0.8095741271972656,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.34381529688835144,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.11786942183971405,
+      "rewards/margins": 1.111412763595581,
+      "rewards/rejected": -1.2292821407318115,
       "step": 177,
-      "train_speed(iter/s)": 0.024303
-    },
-    {
-      "clip_ratio/high_max": 0.010156267147976905,
-      "clip_ratio/high_mean": 0.010156267147976905,
-      "clip_ratio/low_mean": 0.044217503629624844,
-      "clip_ratio/low_min": 0.044217503629624844,
-      "clip_ratio/region_mean": 0.05437377095222473,
-      "epoch": 1.9750692520775623,
-      "grad_norm": 1.0768089968910826,
-      "kl": 2.5309184610377997,
-      "learning_rate": 5.7244490656971815e-05,
-      "loss": 0.004669432528316975,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021284
+    },
+    {
+      "epoch": 0.35950517546074223,
+      "grad_norm": 2.4447503089904785,
+      "learning_rate": 0.00019098842577186314,
+      "logits/chosen": -0.8492040634155273,
+      "logits/rejected": -0.9052773714065552,
+      "logps/chosen": -2.7093420028686523,
+      "logps/rejected": -21.28760528564453,
+      "loss": 0.6568049192428589,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.21920911967754364,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.19283369183540344,
+      "rewards/margins": 1.1686737537384033,
+      "rewards/rejected": -0.9758400321006775,
       "step": 178,
-      "train_speed(iter/s)": 0.02381
-    },
-    {
-      "clip_ratio/high_max": 0.004732780216727406,
-      "clip_ratio/high_mean": 0.004732780216727406,
-      "clip_ratio/low_mean": 0.008979557853308506,
-      "clip_ratio/low_min": 0.008979557853308506,
-      "clip_ratio/region_mean": 0.013712337939068675,
-      "completions/clipped_ratio": 0.171875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 10.34375,
-      "completions/min_length": 3.0,
-      "epoch": 1.9861495844875345,
-      "grad_norm": 5.022277357841641,
-      "kl": 8.668162640184164,
-      "learning_rate": 5.613837614614727e-05,
-      "loss": 0.07592084258794785,
-      "memory(GiB)": 79.43,
-      "reward": 0.359375,
-      "reward_std": 0.43547919392585754,
-      "rewards/Response_no_think_reward_1/mean": 0.359375,
-      "rewards/Response_no_think_reward_1/std": 0.7395931482315063,
+      "train_speed(iter/s)": 0.021285
+    },
+    {
+      "epoch": 0.3615248674577127,
+      "grad_norm": 2.275618314743042,
+      "learning_rate": 0.0001908492659546543,
+      "logits/chosen": -0.7995213866233826,
+      "logits/rejected": -0.9124236106872559,
+      "logps/chosen": -2.1147406101226807,
+      "logps/rejected": -30.72578239440918,
+      "loss": 0.5593333840370178,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1950627565383911,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.16707345843315125,
+      "rewards/margins": 1.2975778579711914,
+      "rewards/rejected": -1.1305043697357178,
       "step": 179,
-      "train_speed(iter/s)": 0.021893
-    },
-    {
-      "clip_ratio/high_max": 0.012752994283800945,
-      "clip_ratio/high_mean": 0.012752994283800945,
-      "clip_ratio/low_mean": 0.013070159242488444,
-      "clip_ratio/low_min": 0.013070159242488444,
-      "clip_ratio/region_mean": 0.025823153846431524,
-      "epoch": 1.997229916897507,
-      "grad_norm": 3.8459944315903036,
-      "kl": 5.999185686931014,
-      "learning_rate": 5.503886703453933e-05,
-      "loss": 0.05208824574947357,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021285
+    },
+    {
+      "epoch": 0.36354455945468317,
+      "grad_norm": 3.495725631713867,
+      "learning_rate": 0.00019070909137343408,
+      "logits/chosen": -0.9321005940437317,
+      "logits/rejected": -0.9968794584274292,
+      "logps/chosen": -3.873347759246826,
+      "logps/rejected": -28.580020904541016,
+      "loss": 0.6998752951622009,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.46146360039711,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.157605841755867,
+      "rewards/margins": 1.7749366760253906,
+      "rewards/rejected": -1.6173310279846191,
       "step": 180,
-      "train_speed(iter/s)": 0.02151
-    },
-    {
-      "clip_ratio/high_max": 0.003978091655881144,
-      "clip_ratio/high_mean": 0.003978091655881144,
-      "clip_ratio/low_mean": 0.010502228324185126,
-      "clip_ratio/low_min": 0.010502228324185126,
-      "clip_ratio/region_mean": 0.0144803200091701,
-      "completions/clipped_ratio": 0.21875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 18.1328125,
-      "completions/min_length": 3.0,
-      "epoch": 2.011080332409972,
-      "grad_norm": 2.078473476427749,
-      "kl": 4.435361886702594,
-      "learning_rate": 5.3946128904176e-05,
-      "loss": 0.04282068833708763,
-      "memory(GiB)": 79.43,
-      "reward": 0.3828125,
-      "reward_std": 0.39536070823669434,
-      "rewards/Response_no_think_reward_1/mean": 0.3828125,
-      "rewards/Response_no_think_reward_1/std": 0.7542122602462769,
+      "train_speed(iter/s)": 0.021286
+    },
+    {
+      "epoch": 0.3655642514516536,
+      "grad_norm": 2.6631152629852295,
+      "learning_rate": 0.0001905679035939181,
+      "logits/chosen": -0.9386301636695862,
+      "logits/rejected": -1.0215297937393188,
+      "logps/chosen": -0.5281402468681335,
+      "logps/rejected": -32.908546447753906,
+      "loss": 0.3208782374858856,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11616937071084976,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.27659526467323303,
+      "rewards/margins": 2.29314923286438,
+      "rewards/rejected": -2.016554117202759,
       "step": 181,
-      "train_speed(iter/s)": 0.020048
-    },
-    {
-      "clip_ratio/high_max": 0.008797917238553055,
-      "clip_ratio/high_mean": 0.008797917238553055,
-      "clip_ratio/low_mean": 0.032597673358395696,
-      "clip_ratio/low_min": 0.032597673358395696,
-      "clip_ratio/region_mean": 0.041395590524189174,
-      "epoch": 2.0221606648199444,
-      "grad_norm": 2.4243106079822865,
-      "kl": 4.070879372608033,
-      "learning_rate": 5.286032631740023e-05,
-      "loss": 0.031270649284124374,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021288
+    },
+    {
+      "epoch": 0.3675839434486241,
+      "grad_norm": 10.830909729003906,
+      "learning_rate": 0.00019042570419313925,
+      "logits/chosen": -0.970279335975647,
+      "logits/rejected": -1.0291110277175903,
+      "logps/chosen": -8.81902027130127,
+      "logps/rejected": -42.53478240966797,
+      "loss": 1.1662126779556274,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6375398635864258,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.26809704303741455,
+      "rewards/margins": 2.575526237487793,
+      "rewards/rejected": -2.843623161315918,
       "step": 182,
-      "train_speed(iter/s)": 0.019738
-    },
-    {
-      "clip_ratio/high_max": 0.0034528043179307133,
-      "clip_ratio/high_mean": 0.0034528043179307133,
-      "clip_ratio/low_mean": 0.0063797495095059276,
-      "clip_ratio/low_min": 0.0063797495095059276,
-      "clip_ratio/region_mean": 0.00983255379833281,
-      "completions/clipped_ratio": 0.078125,
-      "completions/max_length": 41.0,
-      "completions/mean_length": 11.1171875,
-      "completions/min_length": 2.0,
-      "epoch": 2.033240997229917,
-      "grad_norm": 2.23823245032825,
-      "kl": 6.291976309614256,
-      "learning_rate": 5.1781622792087735e-05,
-      "loss": 0.035929106175899506,
-      "memory(GiB)": 79.43,
-      "reward": 0.3515625,
-      "reward_std": 0.25183364748954773,
-      "rewards/Response_no_think_reward_1/mean": 0.3515625,
-      "rewards/Response_no_think_reward_1/std": 0.7895961403846741,
+      "train_speed(iter/s)": 0.021289
+    },
+    {
+      "epoch": 0.36960363544559455,
+      "grad_norm": 9.357040405273438,
+      "learning_rate": 0.0001902824947594299,
+      "logits/chosen": -0.9984275102615356,
+      "logits/rejected": -1.0901621580123901,
+      "logps/chosen": -4.183664321899414,
+      "logps/rejected": -42.764862060546875,
+      "loss": 0.8461188077926636,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.662725031375885,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.08130670338869095,
+      "rewards/margins": 3.4639174938201904,
+      "rewards/rejected": -3.3826112747192383,
       "step": 183,
-      "train_speed(iter/s)": 0.018597
-    },
-    {
-      "clip_ratio/high_max": 0.005401399568654597,
-      "clip_ratio/high_mean": 0.005401399568654597,
-      "clip_ratio/low_mean": 0.017693198169581592,
-      "clip_ratio/low_min": 0.017693198169581592,
-      "clip_ratio/region_mean": 0.023094597971066833,
-      "epoch": 2.044321329639889,
-      "grad_norm": 1.8820731345647987,
-      "kl": 5.607178944861516,
-      "learning_rate": 5.071018077702161e-05,
-      "loss": 0.026268446817994118,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.02129
+    },
+    {
+      "epoch": 0.371623327442565,
+      "grad_norm": 22.538454055786133,
+      "learning_rate": 0.00019013827689240436,
+      "logits/chosen": -0.9899927973747253,
+      "logits/rejected": -1.0915980339050293,
+      "logps/chosen": -5.971269130706787,
+      "logps/rejected": -41.92043685913086,
+      "loss": 1.3363139629364014,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7483702301979065,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.24273402988910675,
+      "rewards/margins": 2.6009511947631836,
+      "rewards/rejected": -2.8436849117279053,
       "step": 184,
-      "train_speed(iter/s)": 0.018339
-    },
-    {
-      "clip_ratio/high_max": 0.0011341095669195056,
-      "clip_ratio/high_mean": 0.0011341095669195056,
-      "clip_ratio/low_mean": 0.002640133607201278,
-      "clip_ratio/low_min": 0.002640133607201278,
-      "clip_ratio/region_mean": 0.003774243174120784,
-      "completions/clipped_ratio": 0.1171875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 12.90625,
-      "completions/min_length": 4.0,
-      "epoch": 2.0554016620498614,
-      "grad_norm": 1.162828426123081,
-      "kl": 2.032533585326746,
-      "learning_rate": 4.964616162742826e-05,
-      "loss": 0.018750667572021484,
-      "memory(GiB)": 79.43,
-      "reward": 0.4765625,
-      "reward_std": 0.2650260925292969,
-      "rewards/Response_no_think_reward_1/mean": 0.4765625,
-      "rewards/Response_no_think_reward_1/std": 0.7205077409744263,
+      "train_speed(iter/s)": 0.021291
+    },
+    {
+      "epoch": 0.3736430194395355,
+      "grad_norm": 5.44999361038208,
+      "learning_rate": 0.0001899930522029408,
+      "logits/chosen": -0.9289810061454773,
+      "logits/rejected": -1.0461653470993042,
+      "logps/chosen": -13.561850547790527,
+      "logps/rejected": -50.344093322753906,
+      "loss": 1.021875262260437,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7987765669822693,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.4458293318748474,
+      "rewards/margins": 2.7672858238220215,
+      "rewards/rejected": -3.2131152153015137,
       "step": 185,
-      "train_speed(iter/s)": 0.017419
-    },
-    {
-      "clip_ratio/high_max": 0.023568872129544616,
-      "clip_ratio/high_mean": 0.023568872129544616,
-      "clip_ratio/low_mean": 0.003940592927392572,
-      "clip_ratio/low_min": 0.003940592927392572,
-      "clip_ratio/region_mean": 0.02750946453306824,
-      "epoch": 2.0664819944598336,
-      "grad_norm": 1.5607380960763384,
-      "kl": 1.8544287201948464,
-      "learning_rate": 4.8589725580677835e-05,
-      "loss": 0.017856435850262642,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021291
+    },
+    {
+      "epoch": 0.3756627114365059,
+      "grad_norm": 9.218350410461426,
+      "learning_rate": 0.00018984682231316333,
+      "logits/chosen": -0.9281139373779297,
+      "logits/rejected": -1.0329409837722778,
+      "logps/chosen": -3.859086036682129,
+      "logps/rejected": -30.334747314453125,
+      "loss": 1.411182165145874,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.8451413512229919,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.08102743327617645,
+      "rewards/margins": 1.733138084411621,
+      "rewards/rejected": -1.6521105766296387,
       "step": 186,
-      "train_speed(iter/s)": 0.017196
-    },
-    {
-      "clip_ratio/high_max": 0.0021306818234734237,
-      "clip_ratio/high_mean": 0.0021306818234734237,
-      "clip_ratio/low_mean": 0.009969204489607364,
-      "clip_ratio/low_min": 0.009969204489607364,
-      "clip_ratio/region_mean": 0.01209988642949611,
-      "completions/clipped_ratio": 0.1015625,
-      "completions/max_length": 47.0,
-      "completions/mean_length": 10.5859375,
-      "completions/min_length": 3.0,
-      "epoch": 2.0775623268698062,
-      "grad_norm": 2.2199233392807804,
-      "kl": 4.046690948307514,
-      "learning_rate": 4.754103173215313e-05,
-      "loss": 0.02398597076535225,
-      "memory(GiB)": 79.43,
-      "reward": 0.6171875,
-      "reward_std": 0.23733052611351013,
-      "rewards/Response_no_think_reward_1/mean": 0.6171875,
-      "rewards/Response_no_think_reward_1/std": 0.6413702368736267,
+      "train_speed(iter/s)": 0.021292
+    },
+    {
+      "epoch": 0.3776824034334764,
+      "grad_norm": 9.405385971069336,
+      "learning_rate": 0.00018969958885642398,
+      "logits/chosen": -0.9063903093338013,
+      "logits/rejected": -1.0287251472473145,
+      "logps/chosen": -5.265182971954346,
+      "logps/rejected": -53.523624420166016,
+      "loss": 0.5908830165863037,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.38780495524406433,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.0466960184276104,
+      "rewards/margins": 3.7127225399017334,
+      "rewards/rejected": -3.6660263538360596,
       "step": 187,
-      "train_speed(iter/s)": 0.016352
-    },
-    {
-      "clip_ratio/high_max": 0.012073863414116204,
-      "clip_ratio/high_mean": 0.012073863414116204,
-      "clip_ratio/low_mean": 0.029048353899270296,
-      "clip_ratio/low_min": 0.029048353899270296,
-      "clip_ratio/region_mean": 0.041122217662632465,
-      "epoch": 2.0886426592797784,
-      "grad_norm": 0.9728732612498153,
-      "kl": 4.168077672366053,
-      "learning_rate": 4.6500238011290295e-05,
-      "loss": 0.013498052954673767,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021292
+    },
+    {
+      "epoch": 0.37970209543044686,
+      "grad_norm": 2.527911901473999,
+      "learning_rate": 0.00018955135347728432,
+      "logits/chosen": -0.749373733997345,
+      "logits/rejected": -0.9382051229476929,
+      "logps/chosen": -1.4075103998184204,
+      "logps/rejected": -48.75277328491211,
+      "loss": 0.32720643281936646,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15030686557292938,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2724253535270691,
+      "rewards/margins": 2.877960205078125,
+      "rewards/rejected": -2.605534791946411,
       "step": 188,
-      "train_speed(iter/s)": 0.016166
-    },
-    {
-      "clip_ratio/high_max": 0.0011479975655674934,
-      "clip_ratio/high_mean": 0.0011479975655674934,
-      "clip_ratio/low_mean": 0.01187260658480227,
-      "clip_ratio/low_min": 0.01187260658480227,
-      "clip_ratio/region_mean": 0.013020604150369763,
-      "completions/clipped_ratio": 0.140625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 13.515625,
-      "completions/min_length": 2.0,
-      "epoch": 2.0997229916897506,
-      "grad_norm": 2.9644587346763975,
-      "kl": 2.9032904393970966,
-      "learning_rate": 4.546750115779538e-05,
-      "loss": 0.025254033505916595,
-      "memory(GiB)": 79.43,
-      "reward": 0.390625,
-      "reward_std": 0.3936568796634674,
-      "rewards/Response_no_think_reward_1/mean": 0.390625,
-      "rewards/Response_no_think_reward_1/std": 0.815541684627533,
+      "train_speed(iter/s)": 0.021292
+    },
+    {
+      "epoch": 0.3817217874274173,
+      "grad_norm": 3.514798164367676,
+      "learning_rate": 0.00018940211783149722,
+      "logits/chosen": -0.9332194924354553,
+      "logits/rejected": -0.9634183049201965,
+      "logps/chosen": -3.9122090339660645,
+      "logps/rejected": -23.81725311279297,
+      "loss": 0.5536522269248962,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2619181275367737,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2673307955265045,
+      "rewards/margins": 1.8769687414169312,
+      "rewards/rejected": -1.609637975692749,
       "step": 189,
-      "train_speed(iter/s)": 0.015408
-    },
-    {
-      "clip_ratio/high_max": 0.002348073641769588,
-      "clip_ratio/high_mean": 0.002348073641769588,
-      "clip_ratio/low_mean": 0.03768142650369555,
-      "clip_ratio/low_min": 0.03768142650369555,
-      "clip_ratio/region_mean": 0.04002950026188046,
-      "epoch": 2.110803324099723,
-      "grad_norm": 1.6848916127813878,
-      "kl": 3.357306843623519,
-      "learning_rate": 4.444297669803981e-05,
-      "loss": 0.011942790821194649,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021293
+    },
+    {
+      "epoch": 0.3837414794243878,
+      "grad_norm": 3.0047051906585693,
+      "learning_rate": 0.00018925188358598813,
+      "logits/chosen": -0.874540388584137,
+      "logits/rejected": -0.9682676792144775,
+      "logps/chosen": -1.5275218486785889,
+      "logps/rejected": -36.20308303833008,
+      "loss": 0.476752907037735,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.12829114496707916,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1873767375946045,
+      "rewards/margins": 2.56374454498291,
+      "rewards/rejected": -2.3763678073883057,
       "step": 190,
-      "train_speed(iter/s)": 0.01525
-    },
-    {
-      "clip_ratio/high_max": 0.0038575184298679233,
-      "clip_ratio/high_mean": 0.0038575184298679233,
-      "clip_ratio/low_mean": 0.0018564801721367985,
-      "clip_ratio/low_min": 0.0018564801721367985,
-      "clip_ratio/region_mean": 0.005713998660212383,
-      "completions/clipped_ratio": 0.1640625,
-      "completions/max_length": 49.0,
-      "completions/mean_length": 12.40625,
-      "completions/min_length": 2.0,
-      "epoch": 2.1218836565096955,
-      "grad_norm": 2.185626617314445,
-      "kl": 4.500719710253179,
-      "learning_rate": 4.342681892163868e-05,
-      "loss": 0.04343116655945778,
-      "memory(GiB)": 79.43,
-      "reward": 0.3125,
-      "reward_std": 0.3249424993991852,
-      "rewards/Response_no_think_reward_1/mean": 0.3125,
-      "rewards/Response_no_think_reward_1/std": 0.8108515739440918,
+      "train_speed(iter/s)": 0.021294
+    },
+    {
+      "epoch": 0.38576117142135824,
+      "grad_norm": 5.604814052581787,
+      "learning_rate": 0.0001891006524188368,
+      "logits/chosen": -0.8748272657394409,
+      "logits/rejected": -0.8527596592903137,
+      "logps/chosen": -22.366464614868164,
+      "logps/rejected": -31.544967651367188,
+      "loss": 0.844088077545166,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.34909212589263916,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.017623694613575935,
+      "rewards/margins": 1.6229170560836792,
+      "rewards/rejected": -1.6052933931350708,
       "step": 191,
-      "train_speed(iter/s)": 0.014605
-    },
-    {
-      "clip_ratio/high_max": 0.005653464584611356,
-      "clip_ratio/high_mean": 0.005653464584611356,
-      "clip_ratio/low_mean": 0.014596144377719611,
-      "clip_ratio/low_min": 0.014596144377719611,
-      "clip_ratio/region_mean": 0.020249608729500324,
-      "epoch": 2.1329639889196677,
-      "grad_norm": 1.3831255673808591,
-      "kl": 3.3093000794760883,
-      "learning_rate": 4.241918085821547e-05,
-      "loss": 0.025410175323486328,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021293
+    },
+    {
+      "epoch": 0.3877808634183287,
+      "grad_norm": 3.902900457382202,
+      "learning_rate": 0.0001889484260192582,
+      "logits/chosen": -0.8291831612586975,
+      "logits/rejected": -0.9175705909729004,
+      "logps/chosen": -3.4213364124298096,
+      "logps/rejected": -28.334596633911133,
+      "loss": 0.6280531883239746,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2871948778629303,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.20435035228729248,
+      "rewards/margins": 1.6512333154678345,
+      "rewards/rejected": -1.446882963180542,
       "step": 192,
-      "train_speed(iter/s)": 0.014469
-    },
-    {
-      "clip_ratio/high_max": 0.0007621950935572386,
-      "clip_ratio/high_mean": 0.0007621950935572386,
-      "clip_ratio/low_mean": 0.016430354735348374,
-      "clip_ratio/low_min": 0.016430354735348374,
-      "clip_ratio/region_mean": 0.017192550061736256,
-      "completions/clipped_ratio": 0.1875,
-      "completions/max_length": 52.0,
-      "completions/mean_length": 13.7578125,
-      "completions/min_length": 3.0,
-      "epoch": 2.14404432132964,
-      "grad_norm": 4.027535318826777,
-      "kl": 5.601793970912695,
-      "learning_rate": 4.142021425435612e-05,
-      "loss": 0.06872062385082245,
-      "memory(GiB)": 79.43,
-      "reward": 0.171875,
-      "reward_std": 0.3266732692718506,
-      "rewards/Response_no_think_reward_1/mean": 0.171875,
-      "rewards/Response_no_think_reward_1/std": 0.8334481716156006,
+      "train_speed(iter/s)": 0.021293
+    },
+    {
+      "epoch": 0.3898005554152992,
+      "grad_norm": 3.3589909076690674,
+      "learning_rate": 0.00018879520608758394,
+      "logits/chosen": -0.7861786484718323,
+      "logits/rejected": -0.9745673537254333,
+      "logps/chosen": -2.2813668251037598,
+      "logps/rejected": -30.406993865966797,
+      "loss": 0.46809083223342896,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24172116816043854,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2660973072052002,
+      "rewards/margins": 1.8777257204055786,
+      "rewards/rejected": -1.6116284132003784,
       "step": 193,
-      "train_speed(iter/s)": 0.013947
-    },
-    {
-      "clip_ratio/high_max": 0.0032193634542636573,
-      "clip_ratio/high_mean": 0.0032193634542636573,
-      "clip_ratio/low_mean": 0.0076680079801008105,
-      "clip_ratio/low_min": 0.0076680079801008105,
-      "clip_ratio/region_mean": 0.010887371434364468,
-      "epoch": 2.155124653739612,
-      "grad_norm": 4.385106696987604,
-      "kl": 3.2986946790479124,
-      "learning_rate": 4.0430069550756665e-05,
-      "loss": 0.057711075991392136,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021294
+    },
+    {
+      "epoch": 0.3918202474122696,
+      "grad_norm": 3.7471225261688232,
+      "learning_rate": 0.000188640994335243,
+      "logits/chosen": -0.8293952941894531,
+      "logits/rejected": -0.9208282232284546,
+      "logps/chosen": -2.756782293319702,
+      "logps/rejected": -28.273818969726562,
+      "loss": 0.7544887065887451,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3577630817890167,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.12236475944519043,
+      "rewards/margins": 1.8170613050460815,
+      "rewards/rejected": -1.6946964263916016,
       "step": 194,
-      "train_speed(iter/s)": 0.013826
-    },
-    {
-      "clip_ratio/high_max": 0.0006905972259119153,
-      "clip_ratio/high_mean": 0.0006905972259119153,
-      "clip_ratio/low_mean": 0.0036786437267437577,
-      "clip_ratio/low_min": 0.0036786437267437577,
-      "clip_ratio/region_mean": 0.004369240894448012,
-      "completions/clipped_ratio": 0.046875,
-      "completions/max_length": 49.0,
-      "completions/mean_length": 12.3359375,
-      "completions/min_length": 6.0,
-      "epoch": 2.1662049861495847,
-      "grad_norm": 1.606761596776354,
-      "kl": 1.8307960720267147,
-      "learning_rate": 3.944889585956746e-05,
-      "loss": 0.01529858261346817,
-      "memory(GiB)": 79.43,
-      "reward": 0.53125,
-      "reward_std": 0.13994136452674866,
-      "rewards/Response_no_think_reward_1/mean": 0.53125,
-      "rewards/Response_no_think_reward_1/std": 0.6010162234306335,
+      "train_speed(iter/s)": 0.021295
+    },
+    {
+      "epoch": 0.3938399394092401,
+      "grad_norm": 3.194539785385132,
+      "learning_rate": 0.00018848579248474288,
+      "logits/chosen": -0.7125469446182251,
+      "logits/rejected": -0.888473391532898,
+      "logps/chosen": -4.232546329498291,
+      "logps/rejected": -34.417152404785156,
+      "loss": 0.634448230266571,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23650826513767242,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.3275882303714752,
+      "rewards/margins": 1.5964200496673584,
+      "rewards/rejected": -1.2688318490982056,
       "step": 195,
-      "train_speed(iter/s)": 0.01337
-    },
-    {
-      "clip_ratio/high_max": 0.0014173414674587548,
-      "clip_ratio/high_mean": 0.0014173414674587548,
-      "clip_ratio/low_mean": 0.01785696716979146,
-      "clip_ratio/low_min": 0.01785696716979146,
-      "clip_ratio/region_mean": 0.019274308579042554,
-      "epoch": 2.177285318559557,
-      "grad_norm": 0.6605318527442696,
-      "kl": 1.4683757405728102,
-      "learning_rate": 3.847684094193733e-05,
-      "loss": 0.006466372404247522,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021294
+    },
+    {
+      "epoch": 0.39585963140621055,
+      "grad_norm": 7.319854736328125,
+      "learning_rate": 0.00018832960226965008,
+      "logits/chosen": -0.8826249837875366,
+      "logits/rejected": -0.9558060169219971,
+      "logps/chosen": -2.2309622764587402,
+      "logps/rejected": -23.310834884643555,
+      "loss": 0.7857663035392761,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.31681591272354126,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.3184216618537903,
+      "rewards/margins": 1.3100265264511108,
+      "rewards/rejected": -0.9916048049926758,
       "step": 196,
-      "train_speed(iter/s)": 0.013262
-    },
-    {
-      "clip_ratio/high_max": 0.0005787037080153823,
-      "clip_ratio/high_mean": 0.0005787037080153823,
-      "clip_ratio/low_mean": 0.00706394191365689,
-      "clip_ratio/low_min": 0.00706394191365689,
-      "clip_ratio/region_mean": 0.007642645505256951,
-      "completions/clipped_ratio": 0.1171875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 12.6640625,
-      "completions/min_length": 4.0,
-      "epoch": 2.188365650969529,
-      "grad_norm": 2.268325311129086,
-      "kl": 1.9495291512284894,
-      "learning_rate": 3.751405118576138e-05,
-      "loss": 0.021440906450152397,
-      "memory(GiB)": 79.43,
-      "reward": 0.40625,
-      "reward_std": 0.33038538694381714,
-      "rewards/Response_no_think_reward_1/mean": 0.40625,
-      "rewards/Response_no_think_reward_1/std": 0.692337155342102,
+      "train_speed(iter/s)": 0.021296
+    },
+    {
+      "epoch": 0.397879323403181,
+      "grad_norm": 4.467423439025879,
+      "learning_rate": 0.00018817242543457108,
+      "logits/chosen": -0.7410279512405396,
+      "logits/rejected": -0.8783193826675415,
+      "logps/chosen": -6.2165632247924805,
+      "logps/rejected": -40.68191909790039,
+      "loss": 0.8568143844604492,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4732620418071747,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.06536143273115158,
+      "rewards/margins": 1.7412124872207642,
+      "rewards/rejected": -1.6758511066436768,
       "step": 197,
-      "train_speed(iter/s)": 0.012815
-    },
-    {
-      "clip_ratio/high_max": 0.0015153939457377419,
-      "clip_ratio/high_mean": 0.0015153939457377419,
-      "clip_ratio/low_mean": 0.03491706313798204,
-      "clip_ratio/low_min": 0.03491706313798204,
-      "clip_ratio/region_mean": 0.03643245715647936,
-      "epoch": 2.1994459833795013,
-      "grad_norm": 1.065092801753265,
-      "kl": 2.4326230198785197,
-      "learning_rate": 3.6560671583635467e-05,
-      "loss": 0.014199762605130672,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021296
+    },
+    {
+      "epoch": 0.3998990154001515,
+      "grad_norm": 3.227818250656128,
+      "learning_rate": 0.0001880142637351325,
+      "logits/chosen": -0.8976326584815979,
+      "logits/rejected": -0.9664545059204102,
+      "logps/chosen": -2.204209804534912,
+      "logps/rejected": -19.951866149902344,
+      "loss": 0.5582164525985718,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18982845544815063,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.3193460702896118,
+      "rewards/margins": 1.213867425918579,
+      "rewards/rejected": -0.8945214748382568,
       "step": 198,
-      "train_speed(iter/s)": 0.012721
-    },
-    {
-      "clip_ratio/high_max": 0.002581158187240362,
-      "clip_ratio/high_mean": 0.002581158187240362,
-      "clip_ratio/low_mean": 0.0033617604058235884,
-      "clip_ratio/low_min": 0.0033617604058235884,
-      "clip_ratio/region_mean": 0.0059429185930639505,
-      "completions/clipped_ratio": 0.09375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 11.40625,
-      "completions/min_length": 2.0,
-      "epoch": 2.2105263157894735,
-      "grad_norm": 4.969169060239286,
-      "kl": 2.6644327271496877,
-      "learning_rate": 3.561684571102087e-05,
-      "loss": 0.04970448464155197,
-      "memory(GiB)": 79.43,
-      "reward": 0.609375,
-      "reward_std": 0.189372718334198,
-      "rewards/Response_no_think_reward_1/mean": 0.609375,
-      "rewards/Response_no_think_reward_1/std": 0.617773175239563,
+      "train_speed(iter/s)": 0.021297
+    },
+    {
+      "epoch": 0.40191870739712193,
+      "grad_norm": 5.091889381408691,
+      "learning_rate": 0.00018785511893796176,
+      "logits/chosen": -0.813483476638794,
+      "logits/rejected": -0.9725134968757629,
+      "logps/chosen": -1.0746464729309082,
+      "logps/rejected": -31.801054000854492,
+      "loss": 0.38803631067276,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18849264085292816,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.29135608673095703,
+      "rewards/margins": 1.9811937808990479,
+      "rewards/rejected": -1.6898375749588013,
       "step": 199,
-      "train_speed(iter/s)": 0.012318
-    },
-    {
-      "clip_ratio/high_max": 0.007839692523702979,
-      "clip_ratio/high_mean": 0.007839692523702979,
-      "clip_ratio/low_mean": 0.007207914255559444,
-      "clip_ratio/low_min": 0.007207914255559444,
-      "clip_ratio/region_mean": 0.015047606895677745,
-      "epoch": 2.221606648199446,
-      "grad_norm": 1.5017699084351217,
-      "kl": 1.7442057720618322,
-      "learning_rate": 3.468271570462235e-05,
-      "loss": 0.03236105293035507,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021298
+    },
+    {
+      "epoch": 0.4039383993940924,
+      "grad_norm": 3.733981132507324,
+      "learning_rate": 0.00018769499282066717,
+      "logits/chosen": -0.8599748015403748,
+      "logits/rejected": -0.9630089998245239,
+      "logps/chosen": -2.635427236557007,
+      "logps/rejected": -24.271150588989258,
+      "loss": 0.6127274036407471,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2767713665962219,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2570668160915375,
+      "rewards/margins": 1.4302881956100464,
+      "rewards/rejected": -1.1732213497161865,
       "step": 200,
-      "train_speed(iter/s)": 0.012235
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.011193181620910764,
-      "clip_ratio/low_min": 0.011193181620910764,
-      "clip_ratio/region_mean": 0.011193181620910764,
-      "completions/clipped_ratio": 0.109375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 14.9140625,
-      "completions/min_length": 3.0,
-      "epoch": 2.2326869806094183,
-      "grad_norm": 4.931712878754855,
-      "kl": 5.507446703501046,
-      "learning_rate": 3.375842224098281e-05,
-      "loss": 0.04869800806045532,
-      "memory(GiB)": 79.43,
-      "reward": 0.0546875,
-      "reward_std": 0.13610614836215973,
-      "rewards/Response_no_think_reward_1/mean": 0.0546875,
-      "rewards/Response_no_think_reward_1/std": 0.9076108336448669,
+      "train_speed(iter/s)": 0.021299
+    },
+    {
+      "epoch": 0.40595809139106287,
+      "grad_norm": 4.356893062591553,
+      "learning_rate": 0.0001875338871718182,
+      "logits/chosen": -0.8239525556564331,
+      "logits/rejected": -0.9406594634056091,
+      "logps/chosen": -3.5920815467834473,
+      "logps/rejected": -33.701141357421875,
+      "loss": 0.7486278414726257,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.34808939695358276,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.06076638400554657,
+      "rewards/margins": 1.9439024925231934,
+      "rewards/rejected": -1.8831360340118408,
       "step": 201,
-      "train_speed(iter/s)": 0.011859
-    },
-    {
-      "clip_ratio/high_max": 0.0006249999860301614,
-      "clip_ratio/high_mean": 0.0006249999860301614,
-      "clip_ratio/low_mean": 0.01068181823939085,
-      "clip_ratio/low_min": 0.01068181823939085,
-      "clip_ratio/region_mean": 0.011306818574666977,
-      "epoch": 2.2437673130193905,
-      "grad_norm": 1.976139653305045,
-      "kl": 2.772067047073506,
-      "learning_rate": 3.2844104515298155e-05,
-      "loss": 0.02979501150548458,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021262
+    },
+    {
+      "epoch": 0.4079777833880333,
+      "grad_norm": 3.3695449829101562,
+      "learning_rate": 0.00018737180379092537,
+      "logits/chosen": -0.7839801907539368,
+      "logits/rejected": -0.8861851096153259,
+      "logps/chosen": -8.617507934570312,
+      "logps/rejected": -47.99607849121094,
+      "loss": 0.5823696851730347,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3239917457103729,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.02249305695295334,
+      "rewards/margins": 3.1341774463653564,
+      "rewards/rejected": -3.1116843223571777,
       "step": 202,
-      "train_speed(iter/s)": 0.011784
-    },
-    {
-      "clip_ratio/high_max": 0.0021551724057644606,
-      "clip_ratio/high_mean": 0.0021551724057644606,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0021551724057644606,
-      "completions/clipped_ratio": 0.0625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 15.109375,
-      "completions/min_length": 2.0,
-      "epoch": 2.254847645429363,
-      "grad_norm": 1.5562382605817529,
-      "kl": 0.8431436920873239,
-      "learning_rate": 3.19399002204547e-05,
-      "loss": 0.007927711121737957,
-      "memory(GiB)": 79.43,
-      "reward": 0.5546875,
-      "reward_std": 0.1965562254190445,
-      "rewards/Response_no_think_reward_1/mean": 0.5546875,
-      "rewards/Response_no_think_reward_1/std": 0.612322211265564,
+      "train_speed(iter/s)": 0.021262
+    },
+    {
+      "epoch": 0.4099974753850038,
+      "grad_norm": 2.5396463871002197,
+      "learning_rate": 0.00018720874448842034,
+      "logits/chosen": -0.8206271529197693,
+      "logits/rejected": -1.0252444744110107,
+      "logps/chosen": -1.0570971965789795,
+      "logps/rejected": -47.1510124206543,
+      "loss": 0.324438214302063,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11744873970746994,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.36073753237724304,
+      "rewards/margins": 3.558206558227539,
+      "rewards/rejected": -3.1974687576293945,
       "step": 203,
-      "train_speed(iter/s)": 0.011458
-    },
-    {
-      "clip_ratio/high_max": 0.01512533612549305,
-      "clip_ratio/high_mean": 0.01512533612549305,
-      "clip_ratio/low_mean": 0.011428028345108032,
-      "clip_ratio/low_min": 0.011428028345108032,
-      "clip_ratio/region_mean": 0.026553363888524473,
-      "epoch": 2.2659279778393353,
-      "grad_norm": 0.7974321717950155,
-      "kl": 0.8073496993965819,
-      "learning_rate": 3.104594552629331e-05,
-      "loss": -0.0036839484237134457,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021263
+    },
+    {
+      "epoch": 0.41201716738197425,
+      "grad_norm": 3.693103551864624,
+      "learning_rate": 0.00018704471108563548,
+      "logits/chosen": -0.8473316431045532,
+      "logits/rejected": -0.9990904927253723,
+      "logps/chosen": -4.564356803894043,
+      "logps/rejected": -40.657447814941406,
+      "loss": 0.4926657974720001,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2881479859352112,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.39431607723236084,
+      "rewards/margins": 2.6521053314208984,
+      "rewards/rejected": -2.257789373397827,
       "step": 204,
-      "train_speed(iter/s)": 0.011391
-    },
-    {
-      "clip_ratio/high_max": 0.0038362154737114906,
-      "clip_ratio/high_mean": 0.0038362154737114906,
-      "clip_ratio/low_mean": 0.006098126468714327,
-      "clip_ratio/low_min": 0.006098126468714327,
-      "clip_ratio/region_mean": 0.009934341884218156,
-      "completions/clipped_ratio": 0.0625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 9.859375,
-      "completions/min_length": 3.0,
-      "epoch": 2.2770083102493075,
-      "grad_norm": 1.6738122176232313,
-      "kl": 2.2672509452095255,
-      "learning_rate": 3.016237505910272e-05,
-      "loss": 0.019055092707276344,
-      "memory(GiB)": 79.43,
-      "reward": 0.3828125,
-      "reward_std": 0.24184317886829376,
-      "rewards/Response_no_think_reward_1/mean": 0.3828125,
-      "rewards/Response_no_think_reward_1/std": 0.6289736032485962,
+      "train_speed(iter/s)": 0.021263
+    },
+    {
+      "epoch": 0.4140368593789447,
+      "grad_norm": 5.1197896003723145,
+      "learning_rate": 0.00018687970541478364,
+      "logits/chosen": -0.8696497082710266,
+      "logits/rejected": -1.0545570850372314,
+      "logps/chosen": -1.4695444107055664,
+      "logps/rejected": -46.127044677734375,
+      "loss": 0.34414029121398926,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.16013705730438232,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.33041495084762573,
+      "rewards/margins": 3.691737174987793,
+      "rewards/rejected": -3.3613224029541016,
       "step": 205,
-      "train_speed(iter/s)": 0.011117
-    },
-    {
-      "clip_ratio/high_max": 0.008882887894287705,
-      "clip_ratio/high_mean": 0.008882887894287705,
-      "clip_ratio/low_mean": 0.0127694628899917,
-      "clip_ratio/low_min": 0.0127694628899917,
-      "clip_ratio/region_mean": 0.021652350667864084,
-      "epoch": 2.2880886426592797,
-      "grad_norm": 1.7770705331591035,
-      "kl": 2.6307186615886167,
-      "learning_rate": 2.9289321881345254e-05,
-      "loss": 0.020140212029218674,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021264
+    },
+    {
+      "epoch": 0.4160565513759152,
+      "grad_norm": 11.712461471557617,
+      "learning_rate": 0.00018671372931893773,
+      "logits/chosen": -0.9976873993873596,
+      "logits/rejected": -1.0082725286483765,
+      "logps/chosen": -6.205466270446777,
+      "logps/rejected": -21.121868133544922,
+      "loss": 1.509117841720581,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.9143091440200806,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.13785380125045776,
+      "rewards/margins": 1.1608891487121582,
+      "rewards/rejected": -1.2987430095672607,
       "step": 206,
-      "train_speed(iter/s)": 0.011054
-    },
-    {
-      "clip_ratio/high_max": 0.0027895565144717693,
-      "clip_ratio/high_mean": 0.0027895565144717693,
-      "clip_ratio/low_mean": 0.004148252191953361,
-      "clip_ratio/low_min": 0.004148252191953361,
-      "clip_ratio/region_mean": 0.0069378085900098085,
-      "completions/clipped_ratio": 0.09375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 11.28125,
-      "completions/min_length": 2.0,
-      "epoch": 2.299168975069252,
-      "grad_norm": 1.319228923221061,
-      "kl": 2.2385971848852932,
-      "learning_rate": 2.8426917471618144e-05,
-      "loss": 0.02266935259103775,
-      "memory(GiB)": 79.43,
-      "reward": 0.3359375,
-      "reward_std": 0.2283492535352707,
-      "rewards/Response_no_think_reward_1/mean": 0.3359375,
-      "rewards/Response_no_think_reward_1/std": 0.7018237709999084,
+      "train_speed(iter/s)": 0.021265
+    },
+    {
+      "epoch": 0.4180762433728856,
+      "grad_norm": 4.418537616729736,
+      "learning_rate": 0.00018654678465201,
+      "logits/chosen": -0.9154266119003296,
+      "logits/rejected": -0.981490969657898,
+      "logps/chosen": -9.175884246826172,
+      "logps/rejected": -32.25775146484375,
+      "loss": 0.7383416891098022,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3417145013809204,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.03129662945866585,
+      "rewards/margins": 2.172858476638794,
+      "rewards/rejected": -2.14156174659729,
       "step": 207,
-      "train_speed(iter/s)": 0.010791
-    },
-    {
-      "clip_ratio/high_max": 0.011985209857812151,
-      "clip_ratio/high_mean": 0.011985209857812151,
-      "clip_ratio/low_mean": 0.007749008131213486,
-      "clip_ratio/low_min": 0.007749008131213486,
-      "clip_ratio/region_mean": 0.019734218367375433,
-      "epoch": 2.3102493074792245,
-      "grad_norm": 1.4392087091443533,
-      "kl": 2.1376722818240523,
-      "learning_rate": 2.7575291704853323e-05,
-      "loss": 0.019175298511981964,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021266
+    },
+    {
+      "epoch": 0.4200959353698561,
+      "grad_norm": 4.682573318481445,
+      "learning_rate": 0.0001863788732787314,
+      "logits/chosen": -0.845721423625946,
+      "logits/rejected": -1.0325840711593628,
+      "logps/chosen": -3.112144708633423,
+      "logps/rejected": -48.1558952331543,
+      "loss": 0.4932224452495575,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3019799590110779,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16389404237270355,
+      "rewards/margins": 3.949674367904663,
+      "rewards/rejected": -3.785780429840088,
       "step": 208,
-      "train_speed(iter/s)": 0.010735
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.005309505155310035,
-      "clip_ratio/low_min": 0.005309505155310035,
-      "clip_ratio/region_mean": 0.005309505155310035,
-      "completions/clipped_ratio": 0.1484375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 13.78125,
-      "completions/min_length": 2.0,
-      "epoch": 2.3213296398891967,
-      "grad_norm": 3.9723735950660957,
-      "kl": 5.278170272591524,
-      "learning_rate": 2.673457283275873e-05,
-      "loss": 0.035180311650037766,
-      "memory(GiB)": 79.43,
-      "reward": 0.40625,
-      "reward_std": 0.13098980486392975,
-      "rewards/Response_no_think_reward_1/mean": 0.40625,
-      "rewards/Response_no_think_reward_1/std": 0.6692044734954834,
+      "train_speed(iter/s)": 0.021267
+    },
+    {
+      "epoch": 0.42211562736682656,
+      "grad_norm": 6.930049419403076,
+      "learning_rate": 0.0001862099970746308,
+      "logits/chosen": -0.8911486268043518,
+      "logits/rejected": -1.048769474029541,
+      "logps/chosen": -3.7173314094543457,
+      "logps/rejected": -37.73887252807617,
+      "loss": 0.6971174478530884,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3776175379753113,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.14115534722805023,
+      "rewards/margins": 2.630204200744629,
+      "rewards/rejected": -2.489048719406128,
       "step": 209,
-      "train_speed(iter/s)": 0.010491
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.005584693106357008,
-      "clip_ratio/low_min": 0.005584693106357008,
-      "clip_ratio/region_mean": 0.005584693106357008,
-      "epoch": 2.332409972299169,
-      "grad_norm": 1.3665368093756183,
-      "kl": 2.833973544766195,
-      "learning_rate": 2.5904887464504114e-05,
-      "loss": 0.021212518215179443,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021268
+    },
+    {
+      "epoch": 0.424135319363797,
+      "grad_norm": 3.0014336109161377,
+      "learning_rate": 0.00018604015792601396,
+      "logits/chosen": -0.7730808854103088,
+      "logits/rejected": -0.9816968441009521,
+      "logps/chosen": -0.3805132210254669,
+      "logps/rejected": -53.63943099975586,
+      "loss": 0.21242156624794006,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.05388123169541359,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3490655720233917,
+      "rewards/margins": 3.7549197673797607,
+      "rewards/rejected": -3.4058542251586914,
       "step": 210,
-      "train_speed(iter/s)": 0.010441
-    },
-    {
-      "clip_ratio/high_max": 0.0037202382227405906,
-      "clip_ratio/high_mean": 0.0037202382227405906,
-      "clip_ratio/low_mean": 0.0047081211232580245,
-      "clip_ratio/low_min": 0.0047081211232580245,
-      "clip_ratio/region_mean": 0.008428359229583293,
-      "completions/clipped_ratio": 0.109375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 13.515625,
-      "completions/min_length": 2.0,
-      "epoch": 2.343490304709141,
-      "grad_norm": 13.071565563919009,
-      "kl": 3.0507688876241446,
-      "learning_rate": 2.5086360547654087e-05,
-      "loss": 0.03715192899107933,
-      "memory(GiB)": 79.43,
-      "reward": 0.3515625,
-      "reward_std": 0.23045912384986877,
-      "rewards/Response_no_think_reward_1/mean": 0.3515625,
-      "rewards/Response_no_think_reward_1/std": 0.8189665675163269,
+      "train_speed(iter/s)": 0.021268
+    },
+    {
+      "epoch": 0.4261550113607675,
+      "grad_norm": 6.788722991943359,
+      "learning_rate": 0.00018586935772994245,
+      "logits/chosen": -0.7963772416114807,
+      "logits/rejected": -1.0568972826004028,
+      "logps/chosen": -1.2283453941345215,
+      "logps/rejected": -59.22148132324219,
+      "loss": 0.18988274037837982,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.12394578754901886,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4420156478881836,
+      "rewards/margins": 4.728057384490967,
+      "rewards/rejected": -4.286042213439941,
       "step": 211,
-      "train_speed(iter/s)": 0.010202
-    },
-    {
-      "clip_ratio/high_max": 0.003481080086203292,
-      "clip_ratio/high_mean": 0.003481080086203292,
-      "clip_ratio/low_mean": 0.006368864676915109,
-      "clip_ratio/low_min": 0.006368864676915109,
-      "clip_ratio/region_mean": 0.009849944588495418,
-      "epoch": 2.3545706371191137,
-      "grad_norm": 3.097490825276586,
-      "kl": 2.710134948603809,
-      "learning_rate": 2.4279115349351543e-05,
-      "loss": 0.022687038406729698,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021269
+    },
+    {
+      "epoch": 0.42817470335773794,
+      "grad_norm": 2.8118438720703125,
+      "learning_rate": 0.00018569759839421265,
+      "logits/chosen": -0.819808840751648,
+      "logits/rejected": -0.9025678038597107,
+      "logps/chosen": -2.9665253162384033,
+      "logps/rejected": -40.64404296875,
+      "loss": 0.33937186002731323,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11274304240942001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4729611575603485,
+      "rewards/margins": 3.319221019744873,
+      "rewards/rejected": -2.8462600708007812,
       "step": 212,
-      "train_speed(iter/s)": 0.010156
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.002016128972172737,
-      "clip_ratio/low_min": 0.002016128972172737,
-      "clip_ratio/region_mean": 0.002016128972172737,
-      "completions/clipped_ratio": 0.046875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 10.421875,
-      "completions/min_length": 2.0,
-      "epoch": 2.365650969529086,
-      "grad_norm": 1.136594776957956,
-      "kl": 1.1303195441141725,
-      "learning_rate": 2.3483273437754107e-05,
-      "loss": 0.009659601375460625,
-      "memory(GiB)": 79.43,
-      "reward": 0.390625,
-      "reward_std": 0.10205793380737305,
-      "rewards/Response_no_think_reward_1/mean": 0.390625,
-      "rewards/Response_no_think_reward_1/std": 0.7342506647109985,
+      "train_speed(iter/s)": 0.021269
+    },
+    {
+      "epoch": 0.4301943953547084,
+      "grad_norm": 4.191134452819824,
+      "learning_rate": 0.00018552488183733412,
+      "logits/chosen": -0.8641024231910706,
+      "logits/rejected": -0.9831883907318115,
+      "logps/chosen": -7.590506553649902,
+      "logps/rejected": -40.67595672607422,
+      "loss": 0.6653822064399719,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3946245610713959,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.004725143313407898,
+      "rewards/margins": 2.8683340549468994,
+      "rewards/rejected": -2.8730592727661133,
       "step": 213,
-      "train_speed(iter/s)": 0.009943
-    },
-    {
-      "clip_ratio/high_max": 0.006048386916518211,
-      "clip_ratio/high_mean": 0.006048386916518211,
-      "clip_ratio/low_mean": 0.00942321156617254,
-      "clip_ratio/low_min": 0.00942321156617254,
-      "clip_ratio/region_mean": 0.015471598482690752,
-      "epoch": 2.376731301939058,
-      "grad_norm": 0.825911363622962,
-      "kl": 1.1256521647737827,
-      "learning_rate": 2.26989546637263e-05,
-      "loss": 0.005634521599858999,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021269
+    },
+    {
+      "epoch": 0.4322140873516789,
+      "grad_norm": 7.408899307250977,
+      "learning_rate": 0.00018535120998850848,
+      "logits/chosen": -0.9360211491584778,
+      "logits/rejected": -1.0136916637420654,
+      "logps/chosen": -5.061653137207031,
+      "logps/rejected": -31.533058166503906,
+      "loss": 0.8134801983833313,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.38820868730545044,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.2650642693042755,
+      "rewards/margins": 2.2353947162628174,
+      "rewards/rejected": -1.9703304767608643,
       "step": 214,
-      "train_speed(iter/s)": 0.009901
-    },
-    {
-      "clip_ratio/high_max": 0.001342517207376659,
-      "clip_ratio/high_mean": 0.001342517207376659,
-      "clip_ratio/low_mean": 0.0013586956774815917,
-      "clip_ratio/low_min": 0.0013586956774815917,
-      "clip_ratio/region_mean": 0.0027012128848582506,
-      "completions/clipped_ratio": 0.1328125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 15.6484375,
-      "completions/min_length": 4.0,
-      "epoch": 2.3878116343490303,
-      "grad_norm": 1.6607743819090053,
-      "kl": 1.4292651428841054,
-      "learning_rate": 2.1926277142790552e-05,
-      "loss": 0.01813752017915249,
-      "memory(GiB)": 79.43,
-      "reward": 0.609375,
-      "reward_std": 0.31577742099761963,
-      "rewards/Response_no_think_reward_1/mean": 0.609375,
-      "rewards/Response_no_think_reward_1/std": 0.6668102145195007,
+      "train_speed(iter/s)": 0.021271
+    },
+    {
+      "epoch": 0.4342337793486493,
+      "grad_norm": 6.260317802429199,
+      "learning_rate": 0.0001851765847876076,
+      "logits/chosen": -0.9736087322235107,
+      "logits/rejected": -1.0754610300064087,
+      "logps/chosen": -6.311574935913086,
+      "logps/rejected": -41.57005310058594,
+      "loss": 1.0864337682724,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6948180794715881,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.11255162954330444,
+      "rewards/margins": 2.995018482208252,
+      "rewards/rejected": -3.107570171356201,
       "step": 215,
-      "train_speed(iter/s)": 0.009685
-    },
-    {
-      "clip_ratio/high_max": 0.0026189450873062015,
-      "clip_ratio/high_mean": 0.0026189450873062015,
-      "clip_ratio/low_mean": 0.0053210301557555795,
-      "clip_ratio/low_min": 0.0053210301557555795,
-      "clip_ratio/region_mean": 0.007939975359477103,
-      "epoch": 2.398891966759003,
-      "grad_norm": 1.3712211159888634,
-      "kl": 1.414744115434587,
-      "learning_rate": 2.116535723733938e-05,
-      "loss": 0.01291576400399208,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021272
+    },
+    {
+      "epoch": 0.4362534713456198,
+      "grad_norm": 13.104352951049805,
+      "learning_rate": 0.00018500100818515222,
+      "logits/chosen": -0.6429583430290222,
+      "logits/rejected": -0.9696972966194153,
+      "logps/chosen": -2.051022529602051,
+      "logps/rejected": -83.3814697265625,
+      "loss": 0.6120116710662842,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.312576562166214,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.121861532330513,
+      "rewards/margins": 5.671273708343506,
+      "rewards/rejected": -5.549411773681641,
       "step": 216,
-      "train_speed(iter/s)": 0.009645
-    },
-    {
-      "clip_ratio/high_max": 0.002932063478510827,
-      "clip_ratio/high_mean": 0.002932063478510827,
-      "clip_ratio/low_mean": 0.0015625000232830644,
-      "clip_ratio/low_min": 0.0015625000232830644,
-      "clip_ratio/region_mean": 0.004494563501793891,
-      "completions/clipped_ratio": 0.0703125,
-      "completions/max_length": 49.0,
-      "completions/mean_length": 10.9296875,
-      "completions/min_length": 2.0,
-      "epoch": 2.409972299168975,
-      "grad_norm": 6.046911872213327,
-      "kl": 3.113054335270135,
-      "learning_rate": 2.0416309539111654e-05,
-      "loss": 0.050445497035980225,
-      "memory(GiB)": 79.43,
-      "reward": 0.484375,
-      "reward_std": 0.2109457552433014,
-      "rewards/Response_no_think_reward_1/mean": 0.484375,
-      "rewards/Response_no_think_reward_1/std": 0.8416741490364075,
+      "train_speed(iter/s)": 0.021271
+    },
+    {
+      "epoch": 0.43827316334259026,
+      "grad_norm": 7.3358564376831055,
+      "learning_rate": 0.0001848244821422899,
+      "logits/chosen": -1.0172042846679688,
+      "logits/rejected": -1.0574982166290283,
+      "logps/chosen": -3.74454927444458,
+      "logps/rejected": -25.982912063598633,
+      "loss": 0.8323017358779907,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.30521970987319946,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1191268116235733,
+      "rewards/margins": 1.8004627227783203,
+      "rewards/rejected": -1.6813361644744873,
       "step": 217,
-      "train_speed(iter/s)": 0.009417
-    },
-    {
-      "clip_ratio/high_max": 0.0037691170582547784,
-      "clip_ratio/high_mean": 0.0037691170582547784,
-      "clip_ratio/low_mean": 0.0015625000232830644,
-      "clip_ratio/low_min": 0.0015625000232830644,
-      "clip_ratio/region_mean": 0.005331617081537843,
-      "epoch": 2.4210526315789473,
-      "grad_norm": 5.501927716874391,
-      "kl": 2.3150660254395916,
-      "learning_rate": 1.967924685193552e-05,
-      "loss": 0.030173499137163162,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021273
+    },
+    {
+      "epoch": 0.4402928553395607,
+      "grad_norm": 2.6182119846343994,
+      "learning_rate": 0.00018464700863077312,
+      "logits/chosen": -0.995885968208313,
+      "logits/rejected": -1.102242350578308,
+      "logps/chosen": -1.5725170373916626,
+      "logps/rejected": -46.24748229980469,
+      "loss": 0.3959670662879944,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1685146987438202,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.18964146077632904,
+      "rewards/margins": 3.7616095542907715,
+      "rewards/rejected": -3.5719680786132812,
       "step": 218,
-      "train_speed(iter/s)": 0.009382
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0234375,
-      "completions/max_length": 47.0,
-      "completions/mean_length": 7.734375,
-      "completions/min_length": 2.0,
-      "epoch": 2.4321329639889195,
-      "grad_norm": 1.8000862094404895,
-      "kl": 1.0117176891799318,
-      "learning_rate": 1.8954280174740537e-05,
-      "loss": 0.009762465953826904,
-      "memory(GiB)": 79.43,
-      "reward": 0.7265625,
-      "reward_std": 0.19728107750415802,
-      "rewards/Response_no_think_reward_1/mean": 0.7265625,
-      "rewards/Response_no_think_reward_1/std": 0.5130554437637329,
+      "train_speed(iter/s)": 0.021274
+    },
+    {
+      "epoch": 0.4423125473365312,
+      "grad_norm": 5.691047191619873,
+      "learning_rate": 0.00018446858963293763,
+      "logits/chosen": -0.9241669178009033,
+      "logits/rejected": -1.0488460063934326,
+      "logps/chosen": -2.7383437156677246,
+      "logps/rejected": -53.106632232666016,
+      "loss": 0.46598395705223083,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.28670936822891235,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.08014581352472305,
+      "rewards/margins": 3.8416402339935303,
+      "rewards/rejected": -3.7614941596984863,
       "step": 219,
-      "train_speed(iter/s)": 0.009197
-    },
-    {
-      "clip_ratio/high_max": 0.012120938044972718,
-      "clip_ratio/high_mean": 0.012120938044972718,
-      "clip_ratio/low_mean": 0.008152684022206813,
-      "clip_ratio/low_min": 0.008152684022206813,
-      "clip_ratio/region_mean": 0.020273622183594853,
-      "epoch": 2.443213296398892,
-      "grad_norm": 1.1304227715929336,
-      "kl": 1.0098483412293717,
-      "learning_rate": 1.824151868484164e-05,
-      "loss": 0.002491909544914961,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021275
+    },
+    {
+      "epoch": 0.44433223933350163,
+      "grad_norm": 6.792243957519531,
+      "learning_rate": 0.0001842892271416797,
+      "logits/chosen": -0.867421567440033,
+      "logits/rejected": -1.0047706365585327,
+      "logps/chosen": -3.9948360919952393,
+      "logps/rejected": -40.925697326660156,
+      "loss": 0.559948205947876,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2816776633262634,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.27128446102142334,
+      "rewards/margins": 2.5824544429779053,
+      "rewards/rejected": -2.3111701011657715,
       "step": 220,
-      "train_speed(iter/s)": 0.009165
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0390625,
-      "completions/max_length": 38.0,
-      "completions/mean_length": 10.0546875,
-      "completions/min_length": 2.0,
-      "epoch": 2.4542936288088644,
-      "grad_norm": 0.7487543034262131,
-      "kl": 2.248408433049917,
-      "learning_rate": 1.7541069721497493e-05,
-      "loss": 0.020080924034118652,
-      "memory(GiB)": 79.43,
-      "reward": 0.5234375,
-      "reward_std": 0.0765409916639328,
-      "rewards/Response_no_think_reward_1/mean": 0.5234375,
-      "rewards/Response_no_think_reward_1/std": 0.8320815563201904,
+      "train_speed(iter/s)": 0.021275
+    },
+    {
+      "epoch": 0.44635193133047213,
+      "grad_norm": 2.8233304023742676,
+      "learning_rate": 0.00018410892316043448,
+      "logits/chosen": -0.7455387711524963,
+      "logits/rejected": -1.0043134689331055,
+      "logps/chosen": -2.28206205368042,
+      "logps/rejected": -50.90422821044922,
+      "loss": 0.4489433765411377,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17719773948192596,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.17336970567703247,
+      "rewards/margins": 3.006608009338379,
+      "rewards/rejected": -2.833238363265991,
       "step": 221,
-      "train_speed(iter/s)": 0.008978
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.002016128972172737,
-      "clip_ratio/low_min": 0.002016128972172737,
-      "clip_ratio/region_mean": 0.002016128972172737,
-      "epoch": 2.4653739612188366,
-      "grad_norm": 0.5471388079740459,
-      "kl": 2.286591000854969,
-      "learning_rate": 1.6853038769745467e-05,
-      "loss": 0.020101318135857582,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021276
+    },
+    {
+      "epoch": 0.44837162332744257,
+      "grad_norm": 3.7978217601776123,
+      "learning_rate": 0.00018392767970315313,
+      "logits/chosen": -0.9136677980422974,
+      "logits/rejected": -1.06996488571167,
+      "logps/chosen": -1.5614748001098633,
+      "logps/rejected": -34.510162353515625,
+      "loss": 0.3262411057949066,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1898116022348404,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3149145543575287,
+      "rewards/margins": 2.7444238662719727,
+      "rewards/rejected": -2.429509162902832,
       "step": 222,
-      "train_speed(iter/s)": 0.008949
-    },
-    {
-      "clip_ratio/high_max": 0.00021186440426390618,
-      "clip_ratio/high_mean": 0.00021186440426390618,
-      "clip_ratio/low_mean": 0.008871822035871446,
-      "clip_ratio/low_min": 0.008871822035871446,
-      "clip_ratio/region_mean": 0.009083686396479607,
-      "completions/clipped_ratio": 0.09375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 13.7109375,
-      "completions/min_length": 2.0,
-      "epoch": 2.4764542936288088,
-      "grad_norm": 1.362062642699485,
-      "kl": 2.5377135479357094,
-      "learning_rate": 1.6177529444516194e-05,
-      "loss": 0.02206542156636715,
-      "memory(GiB)": 79.43,
-      "reward": 0.296875,
-      "reward_std": 0.19568344950675964,
-      "rewards/Response_no_think_reward_1/mean": 0.296875,
-      "rewards/Response_no_think_reward_1/std": 0.8993381261825562,
+      "train_speed(iter/s)": 0.021277
+    },
+    {
+      "epoch": 0.450391315324413,
+      "grad_norm": 6.886748313903809,
+      "learning_rate": 0.00018374549879428062,
+      "logits/chosen": -0.9338952302932739,
+      "logits/rejected": -1.0804352760314941,
+      "logps/chosen": -1.0283788442611694,
+      "logps/rejected": -38.30894088745117,
+      "loss": 0.3192234933376312,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11882957816123962,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.24301038682460785,
+      "rewards/margins": 2.798785924911499,
+      "rewards/rejected": -2.5557756423950195,
       "step": 223,
-      "train_speed(iter/s)": 0.008782
-    },
-    {
-      "clip_ratio/high_max": 0.0016772599192336202,
-      "clip_ratio/high_mean": 0.0016772599192336202,
-      "clip_ratio/low_mean": 0.006158006319310516,
-      "clip_ratio/low_min": 0.006158006319310516,
-      "clip_ratio/region_mean": 0.007835266180336475,
-      "epoch": 2.487534626038781,
-      "grad_norm": 1.146370544556826,
-      "kl": 2.2770726842572913,
-      "learning_rate": 1.551464347502929e-05,
-      "loss": 0.019708380103111267,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021278
+    },
+    {
+      "epoch": 0.4524110073213835,
+      "grad_norm": 6.402444362640381,
+      "learning_rate": 0.000183562382468733,
+      "logits/chosen": -0.9219646453857422,
+      "logits/rejected": -1.0332969427108765,
+      "logps/chosen": -4.296561241149902,
+      "logps/rejected": -32.83451843261719,
+      "loss": 1.1228792667388916,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.8876572251319885,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.020390525460243225,
+      "rewards/margins": 2.142768383026123,
+      "rewards/rejected": -2.163159132003784,
       "step": 224,
-      "train_speed(iter/s)": 0.008755
-    },
-    {
-      "clip_ratio/high_max": 0.0006412798393284902,
-      "clip_ratio/high_mean": 0.0006412798393284902,
-      "clip_ratio/low_mean": 0.005606794758932665,
-      "clip_ratio/low_min": 0.005606794758932665,
-      "clip_ratio/region_mean": 0.006248074598261155,
-      "completions/clipped_ratio": 0.0390625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 11.546875,
-      "completions/min_length": 3.0,
-      "epoch": 2.4986149584487536,
-      "grad_norm": 15.413339320982411,
-      "kl": 8.634065281590665,
-      "learning_rate": 1.486448068947348e-05,
-      "loss": 0.12368878722190857,
-      "memory(GiB)": 79.43,
-      "reward": 0.3828125,
-      "reward_std": 0.24082913994789124,
-      "rewards/Response_no_think_reward_1/mean": 0.3828125,
-      "rewards/Response_no_think_reward_1/std": 0.7000685334205627,
+      "train_speed(iter/s)": 0.021279
+    },
+    {
+      "epoch": 0.45443069931835395,
+      "grad_norm": 9.758877754211426,
+      "learning_rate": 0.00018337833277187472,
+      "logits/chosen": -0.8922550082206726,
+      "logits/rejected": -0.9859142303466797,
+      "logps/chosen": -7.753718376159668,
+      "logps/rejected": -28.561172485351562,
+      "loss": 1.4200571775436401,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.8696328997612,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.07261030375957489,
+      "rewards/margins": 1.1861965656280518,
+      "rewards/rejected": -1.2588069438934326,
       "step": 225,
-      "train_speed(iter/s)": 0.008606
-    },
-    {
-      "clip_ratio/high_max": 0.002233636550954543,
-      "clip_ratio/high_mean": 0.002233636550954543,
-      "clip_ratio/low_mean": 0.0067390410986263305,
-      "clip_ratio/low_min": 0.0067390410986263305,
-      "clip_ratio/region_mean": 0.008972677649580874,
-      "epoch": 2.509695290858726,
-      "grad_norm": 9.203465486887826,
-      "kl": 6.063290272684753,
-      "learning_rate": 1.42271389999728e-05,
-      "loss": 0.08513150364160538,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.02128
+    },
+    {
+      "epoch": 0.4564503913153244,
+      "grad_norm": 7.866222858428955,
+      "learning_rate": 0.0001831933517594957,
+      "logits/chosen": -0.7848281860351562,
+      "logits/rejected": -1.0049529075622559,
+      "logps/chosen": -2.8492534160614014,
+      "logps/rejected": -53.778289794921875,
+      "loss": 0.520083487033844,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3156394362449646,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.19083034992218018,
+      "rewards/margins": 3.20526123046875,
+      "rewards/rejected": -3.0144309997558594,
       "step": 226,
-      "train_speed(iter/s)": 0.008581
-    },
-    {
-      "clip_ratio/high_max": 0.0003799392143264413,
-      "clip_ratio/high_mean": 0.0003799392143264413,
-      "clip_ratio/low_mean": 0.0011398176429793239,
-      "clip_ratio/low_min": 0.0011398176429793239,
-      "clip_ratio/region_mean": 0.0015197568573057652,
-      "completions/clipped_ratio": 0.0859375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 10.0390625,
-      "completions/min_length": 3.0,
-      "epoch": 2.520775623268698,
-      "grad_norm": 2.120226005232822,
-      "kl": 0.9155051370617002,
-      "learning_rate": 1.360271438784133e-05,
-      "loss": 0.010180685669183731,
-      "memory(GiB)": 79.43,
-      "reward": 0.1484375,
-      "reward_std": 0.1979907602071762,
-      "rewards/Response_no_think_reward_1/mean": 0.1484375,
-      "rewards/Response_no_think_reward_1/std": 0.7326990962028503,
+      "train_speed(iter/s)": 0.02128
+    },
+    {
+      "epoch": 0.4584700833122949,
+      "grad_norm": 4.694428443908691,
+      "learning_rate": 0.00018300744149778853,
+      "logits/chosen": -0.9140058755874634,
+      "logits/rejected": -1.035470724105835,
+      "logps/chosen": -0.7746766805648804,
+      "logps/rejected": -31.849538803100586,
+      "loss": 0.41979673504829407,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1669030785560608,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.34211465716362,
+      "rewards/margins": 2.157820701599121,
+      "rewards/rejected": -1.8157060146331787,
       "step": 227,
-      "train_speed(iter/s)": 0.008446
-    },
-    {
-      "clip_ratio/high_max": 0.0009498480358161032,
-      "clip_ratio/high_mean": 0.0009498480358161032,
-      "clip_ratio/low_mean": 0.009764322196133435,
-      "clip_ratio/low_min": 0.009764322196133435,
-      "clip_ratio/region_mean": 0.010714170290157199,
-      "epoch": 2.5318559556786706,
-      "grad_norm": 1.4078021108161598,
-      "kl": 0.9818148800404742,
-      "learning_rate": 1.2991300889128866e-05,
-      "loss": 0.004585812333971262,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021281
+    },
+    {
+      "epoch": 0.4604897753092653,
+      "grad_norm": 4.092323303222656,
+      "learning_rate": 0.00018282060406332512,
+      "logits/chosen": -0.8949604034423828,
+      "logits/rejected": -1.0626131296157837,
+      "logps/chosen": -2.425996780395508,
+      "logps/rejected": -28.149372100830078,
+      "loss": 0.4806259274482727,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22964058816432953,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2916340231895447,
+      "rewards/margins": 1.6805919408798218,
+      "rewards/rejected": -1.3889580965042114,
       "step": 228,
-      "train_speed(iter/s)": 0.008423
-    },
-    {
-      "clip_ratio/high_max": 0.0011812333250418305,
-      "clip_ratio/high_mean": 0.0011812333250418305,
-      "clip_ratio/low_mean": 0.0039037082460708916,
-      "clip_ratio/low_min": 0.0039037082460708916,
-      "clip_ratio/region_mean": 0.005084941629320383,
-      "completions/clipped_ratio": 0.0390625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 8.265625,
-      "completions/min_length": 2.0,
-      "epoch": 2.542936288088643,
-      "grad_norm": 5.0825431269611965,
-      "kl": 1.4881244308780879,
-      "learning_rate": 1.2392990580459352e-05,
-      "loss": 0.01816009357571602,
-      "memory(GiB)": 79.43,
-      "reward": 0.265625,
-      "reward_std": 0.19226360321044922,
-      "rewards/Response_no_think_reward_1/mean": 0.265625,
-      "rewards/Response_no_think_reward_1/std": 0.9004318714141846,
+      "train_speed(iter/s)": 0.021281
+    },
+    {
+      "epoch": 0.4625094673062358,
+      "grad_norm": 4.152270317077637,
+      "learning_rate": 0.0001826328415430339,
+      "logits/chosen": -1.0330709218978882,
+      "logits/rejected": -1.1299700736999512,
+      "logps/chosen": -2.0911142826080322,
+      "logps/rejected": -25.8432674407959,
+      "loss": 0.47937411069869995,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24223287403583527,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.22816987335681915,
+      "rewards/margins": 1.9592314958572388,
+      "rewards/rejected": -1.7310616970062256,
       "step": 229,
-      "train_speed(iter/s)": 0.008282
-    },
-    {
-      "clip_ratio/high_max": 0.0009686482953839004,
-      "clip_ratio/high_mean": 0.0009686482953839004,
-      "clip_ratio/low_mean": 0.00444972823606804,
-      "clip_ratio/low_min": 0.00444972823606804,
-      "clip_ratio/region_mean": 0.0054183765314519405,
-      "epoch": 2.554016620498615,
-      "grad_norm": 5.947762469687869,
-      "kl": 1.5637177263852209,
-      "learning_rate": 1.1807873565164506e-05,
-      "loss": 0.014085257425904274,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021282
+    },
+    {
+      "epoch": 0.46452915930320626,
+      "grad_norm": 3.1440939903259277,
+      "learning_rate": 0.00018244415603417603,
+      "logits/chosen": -0.9888077974319458,
+      "logits/rejected": -1.087904930114746,
+      "logps/chosen": -1.7838736772537231,
+      "logps/rejected": -31.170108795166016,
+      "loss": 0.5677547454833984,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2625994384288788,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.36328813433647156,
+      "rewards/margins": 2.194064140319824,
+      "rewards/rejected": -1.8307762145996094,
       "step": 230,
-      "train_speed(iter/s)": 0.008261
-    },
-    {
-      "clip_ratio/high_max": 0.001238734505022876,
-      "clip_ratio/high_mean": 0.001238734505022876,
-      "clip_ratio/low_mean": 0.0045786225236952305,
-      "clip_ratio/low_min": 0.0045786225236952305,
-      "clip_ratio/region_mean": 0.0058173570287181064,
-      "completions/clipped_ratio": 0.1640625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 15.609375,
-      "completions/min_length": 3.0,
-      "epoch": 2.565096952908587,
-      "grad_norm": 2.991093002864201,
-      "kl": 2.183058348018676,
-      "learning_rate": 1.1236037959714618e-05,
-      "loss": 0.021889833733439445,
-      "memory(GiB)": 79.43,
-      "reward": 0.4609375,
-      "reward_std": 0.3135277330875397,
-      "rewards/Response_no_think_reward_1/mean": 0.4609375,
-      "rewards/Response_no_think_reward_1/std": 0.7827102541923523,
+      "train_speed(iter/s)": 0.021284
+    },
+    {
+      "epoch": 0.4665488513001767,
+      "grad_norm": 3.9995431900024414,
+      "learning_rate": 0.00018225454964432248,
+      "logits/chosen": -0.8009510040283203,
+      "logits/rejected": -0.9875888824462891,
+      "logps/chosen": -4.417139530181885,
+      "logps/rejected": -39.54981231689453,
+      "loss": 0.5815356969833374,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4281938374042511,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.21771493554115295,
+      "rewards/margins": 2.3999886512756348,
+      "rewards/rejected": -2.1822738647460938,
       "step": 231,
-      "train_speed(iter/s)": 0.008131
-    },
-    {
-      "clip_ratio/high_max": 0.0028425113996490836,
-      "clip_ratio/high_mean": 0.0028425113996490836,
-      "clip_ratio/low_mean": 0.009468507007113658,
-      "clip_ratio/low_min": 0.009468507007113658,
-      "clip_ratio/region_mean": 0.012311018639593385,
-      "epoch": 2.5761772853185594,
-      "grad_norm": 2.565607083550086,
-      "kl": 1.9908064976334572,
-      "learning_rate": 1.067756988044848e-05,
-      "loss": 0.016938552260398865,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021284
+    },
+    {
+      "epoch": 0.4685685432971472,
+      "grad_norm": 5.97980260848999,
+      "learning_rate": 0.00018206402449132995,
+      "logits/chosen": -0.8073972463607788,
+      "logits/rejected": -1.005049705505371,
+      "logps/chosen": -3.3927805423736572,
+      "logps/rejected": -40.57596206665039,
+      "loss": 0.6254691481590271,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.36383697390556335,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1796584278345108,
+      "rewards/margins": 1.9802054166793823,
+      "rewards/rejected": -1.8005468845367432,
       "step": 232,
-      "train_speed(iter/s)": 0.008111
-    },
-    {
-      "clip_ratio/high_max": 0.0014164648891892284,
-      "clip_ratio/high_mean": 0.0014164648891892284,
-      "clip_ratio/low_mean": 0.0031525760132353753,
-      "clip_ratio/low_min": 0.0031525760132353753,
-      "clip_ratio/region_mean": 0.004569040902424604,
-      "completions/clipped_ratio": 0.1015625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 9.75,
-      "completions/min_length": 2.0,
-      "epoch": 2.587257617728532,
-      "grad_norm": 5.0565474550432326,
-      "kl": 5.047875659758574,
-      "learning_rate": 1.0132553430604608e-05,
-      "loss": 0.07161970436573029,
-      "memory(GiB)": 79.43,
-      "reward": 0.2265625,
-      "reward_std": 0.17859892547130585,
-      "rewards/Response_no_think_reward_1/mean": 0.2265625,
-      "rewards/Response_no_think_reward_1/std": 0.7446908354759216,
+      "train_speed(iter/s)": 0.021284
+    },
+    {
+      "epoch": 0.47058823529411764,
+      "grad_norm": 3.2046210765838623,
+      "learning_rate": 0.00018187258270331783,
+      "logits/chosen": -0.843147873878479,
+      "logits/rejected": -0.9698415398597717,
+      "logps/chosen": -6.980717182159424,
+      "logps/rejected": -47.44036865234375,
+      "loss": 0.42867493629455566,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22475841641426086,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.42296847701072693,
+      "rewards/margins": 3.3137428760528564,
+      "rewards/rejected": -2.8907744884490967,
       "step": 233,
-      "train_speed(iter/s)": 0.008004
-    },
-    {
-      "clip_ratio/high_max": 0.006010896002408117,
-      "clip_ratio/high_mean": 0.006010896002408117,
-      "clip_ratio/low_mean": 0.006369341921526939,
-      "clip_ratio/low_min": 0.006369341921526939,
-      "clip_ratio/region_mean": 0.01238023757468909,
-      "epoch": 2.598337950138504,
-      "grad_norm": 3.9632773907657626,
-      "kl": 4.599423869600287,
-      "learning_rate": 9.601070687655667e-06,
-      "loss": 0.062261972576379776,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021283
+    },
+    {
+      "epoch": 0.47260792729108814,
+      "grad_norm": 5.650546550750732,
+      "learning_rate": 0.00018168022641864377,
+      "logits/chosen": -0.9458031058311462,
+      "logits/rejected": -1.022731900215149,
+      "logps/chosen": -5.699041366577148,
+      "logps/rejected": -27.91295051574707,
+      "loss": 0.6052364706993103,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.25157076120376587,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.17266854643821716,
+      "rewards/margins": 2.01995587348938,
+      "rewards/rejected": -1.8472874164581299,
       "step": 234,
-      "train_speed(iter/s)": 0.007986
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.001923076924867928,
-      "clip_ratio/low_min": 0.001923076924867928,
-      "clip_ratio/region_mean": 0.001923076924867928,
-      "completions/clipped_ratio": 0.1015625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 9.453125,
-      "completions/min_length": 2.0,
-      "epoch": 2.6094182825484764,
-      "grad_norm": 5.621375964647713,
-      "kl": 4.620725775370374,
-      "learning_rate": 9.083201690947763e-06,
-      "loss": 0.0421258881688118,
-      "memory(GiB)": 79.43,
-      "reward": 0.28125,
-      "reward_std": 0.1468440443277359,
-      "rewards/Response_no_think_reward_1/mean": 0.28125,
-      "rewards/Response_no_think_reward_1/std": 0.7830638289451599,
+      "train_speed(iter/s)": 0.021283
+    },
+    {
+      "epoch": 0.4746276192880586,
+      "grad_norm": 7.562004089355469,
+      "learning_rate": 0.00018148695778588033,
+      "logits/chosen": -0.8491840362548828,
+      "logits/rejected": -0.9722462296485901,
+      "logps/chosen": -8.168458938598633,
+      "logps/rejected": -39.646427154541016,
+      "loss": 1.076995849609375,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6254736185073853,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.23536130785942078,
+      "rewards/margins": 2.3698079586029053,
+      "rewards/rejected": -2.134446859359741,
       "step": 235,
-      "train_speed(iter/s)": 0.007861
-    },
-    {
-      "clip_ratio/high_max": 0.010077075916342437,
-      "clip_ratio/high_mean": 0.010077075916342437,
-      "clip_ratio/low_mean": 0.000961538462433964,
-      "clip_ratio/low_min": 0.000961538462433964,
-      "clip_ratio/region_mean": 0.01103861432056874,
-      "epoch": 2.6204986149584486,
-      "grad_norm": 4.170803291082545,
-      "kl": 4.307271543191746,
-      "learning_rate": 8.579024429646932e-06,
-      "loss": 0.03107025846838951,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021283
+    },
+    {
+      "epoch": 0.476647311285029,
+      "grad_norm": 2.8358967304229736,
+      "learning_rate": 0.00018129277896379077,
+      "logits/chosen": -0.8474506139755249,
+      "logits/rejected": -0.9761014580726624,
+      "logps/chosen": -1.0130198001861572,
+      "logps/rejected": -34.24652099609375,
+      "loss": 0.3748179078102112,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11387713998556137,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.42450040578842163,
+      "rewards/margins": 2.2273645401000977,
+      "rewards/rejected": -1.8028637170791626,
       "step": 236,
-      "train_speed(iter/s)": 0.007844
-    },
-    {
-      "clip_ratio/high_max": 0.00024319066142197698,
-      "clip_ratio/high_mean": 0.00024319066142197698,
-      "clip_ratio/low_mean": 0.0016447368543595076,
-      "clip_ratio/low_min": 0.0016447368543595076,
-      "clip_ratio/region_mean": 0.0018879275157814845,
-      "completions/clipped_ratio": 0.1015625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 12.21875,
-      "completions/min_length": 2.0,
-      "epoch": 2.6315789473684212,
-      "grad_norm": 1.3493712018234787,
-      "kl": 1.6652233960921876,
-      "learning_rate": 8.088614830994223e-06,
-      "loss": 0.015416830778121948,
-      "memory(GiB)": 79.43,
-      "reward": 0.1640625,
-      "reward_std": 0.21537472307682037,
-      "rewards/Response_no_think_reward_1/mean": 0.1640625,
-      "rewards/Response_no_think_reward_1/std": 0.7713097929954529,
+      "train_speed(iter/s)": 0.021283
+    },
+    {
+      "epoch": 0.4786670032819995,
+      "grad_norm": 24.549625396728516,
+      "learning_rate": 0.00018109769212130487,
+      "logits/chosen": -0.7827628254890442,
+      "logits/rejected": -0.899127721786499,
+      "logps/chosen": -12.455778121948242,
+      "logps/rejected": -39.55864715576172,
+      "loss": 1.0212831497192383,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6375862956047058,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.05945993959903717,
+      "rewards/margins": 2.1959457397460938,
+      "rewards/rejected": -2.2554056644439697,
       "step": 237,
-      "train_speed(iter/s)": 0.007735
-    },
-    {
-      "clip_ratio/high_max": 0.0005482456181198359,
-      "clip_ratio/high_mean": 0.0005482456181198359,
-      "clip_ratio/low_mean": 0.0018879275157814845,
-      "clip_ratio/low_min": 0.0018879275157814845,
-      "clip_ratio/region_mean": 0.0024361731339013204,
-      "epoch": 2.6426592797783934,
-      "grad_norm": 1.3201384353784884,
-      "kl": 1.5714624499087222,
-      "learning_rate": 7.612046748871327e-06,
-      "loss": 0.01534443162381649,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021283
+    },
+    {
+      "epoch": 0.48068669527896996,
+      "grad_norm": 5.986259460449219,
+      "learning_rate": 0.00018090169943749476,
+      "logits/chosen": -0.9809411764144897,
+      "logits/rejected": -1.0568145513534546,
+      "logps/chosen": -2.8511006832122803,
+      "logps/rejected": -30.153661727905273,
+      "loss": 0.6427941918373108,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.31609585881233215,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.20257410407066345,
+      "rewards/margins": 2.3083198070526123,
+      "rewards/rejected": -2.105745553970337,
       "step": 238,
-      "train_speed(iter/s)": 0.00772
-    },
-    {
-      "clip_ratio/high_max": 0.00204707685043104,
-      "clip_ratio/high_mean": 0.00204707685043104,
-      "clip_ratio/low_mean": 0.007919068011688069,
-      "clip_ratio/low_min": 0.007919068011688069,
-      "clip_ratio/region_mean": 0.009966144862119108,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 36.0,
-      "completions/mean_length": 10.8515625,
-      "completions/min_length": 2.0,
-      "epoch": 2.6537396121883656,
-      "grad_norm": 2.827545573836865,
-      "kl": 1.8283915198408067,
-      "learning_rate": 7.149391952678452e-06,
-      "loss": 0.017831573262810707,
-      "memory(GiB)": 79.43,
-      "reward": 0.2890625,
-      "reward_std": 0.3461739718914032,
-      "rewards/Response_no_think_reward_1/mean": 0.2890625,
-      "rewards/Response_no_think_reward_1/std": 0.7852212190628052,
+      "train_speed(iter/s)": 0.021285
+    },
+    {
+      "epoch": 0.4827063872759404,
+      "grad_norm": 6.145867824554443,
+      "learning_rate": 0.00018070480310155066,
+      "logits/chosen": -0.8342568874359131,
+      "logits/rejected": -0.9062321782112122,
+      "logps/chosen": -7.222947120666504,
+      "logps/rejected": -37.760414123535156,
+      "loss": 0.8602092862129211,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.47236353158950806,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.14982852339744568,
+      "rewards/margins": 2.0299620628356934,
+      "rewards/rejected": -1.8801336288452148,
       "step": 239,
-      "train_speed(iter/s)": 0.00762
-    },
-    {
-      "clip_ratio/high_max": 0.005028763844165951,
-      "clip_ratio/high_mean": 0.005028763844165951,
-      "clip_ratio/low_mean": 0.008577116299420595,
-      "clip_ratio/low_min": 0.008577116299420595,
-      "clip_ratio/region_mean": 0.013605880201794207,
-      "epoch": 2.664819944598338,
-      "grad_norm": 3.1585454235161925,
-      "kl": 1.771346734254621,
-      "learning_rate": 6.700720116526116e-06,
-      "loss": 0.01860269345343113,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021285
+    },
+    {
+      "epoch": 0.4847260792729109,
+      "grad_norm": 5.842182159423828,
+      "learning_rate": 0.0001805070053127563,
+      "logits/chosen": -0.8298416137695312,
+      "logits/rejected": -0.9391087889671326,
+      "logps/chosen": -4.0047993659973145,
+      "logps/rejected": -34.49217224121094,
+      "loss": 0.6390224695205688,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.391327440738678,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.296629935503006,
+      "rewards/margins": 2.591277837753296,
+      "rewards/rejected": -2.294647693634033,
       "step": 240,
-      "train_speed(iter/s)": 0.007606
-    },
-    {
-      "clip_ratio/high_max": 0.0017265193164348602,
-      "clip_ratio/high_mean": 0.0017265193164348602,
-      "clip_ratio/low_mean": 0.0015245491813402623,
-      "clip_ratio/low_min": 0.0015245491813402623,
-      "clip_ratio/region_mean": 0.003251068526878953,
-      "completions/clipped_ratio": 0.0390625,
-      "completions/max_length": 46.0,
-      "completions/mean_length": 8.765625,
-      "completions/min_length": 2.0,
-      "epoch": 2.67590027700831,
-      "grad_norm": 2.4712156808930987,
-      "kl": 3.0323174638469936,
-      "learning_rate": 6.266098808742516e-06,
-      "loss": 0.027942461892962456,
-      "memory(GiB)": 79.43,
-      "reward": 0.2734375,
-      "reward_std": 0.09069566428661346,
-      "rewards/Response_no_think_reward_1/mean": 0.2734375,
-      "rewards/Response_no_think_reward_1/std": 0.7908416986465454,
+      "train_speed(iter/s)": 0.021285
+    },
+    {
+      "epoch": 0.48674577126988133,
+      "grad_norm": 3.0892484188079834,
+      "learning_rate": 0.0001803083082804645,
+      "logits/chosen": -0.6407896876335144,
+      "logits/rejected": -0.8386159539222717,
+      "logps/chosen": -2.6044552326202393,
+      "logps/rejected": -43.223602294921875,
+      "loss": 0.5524685978889465,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23806366324424744,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.17818674445152283,
+      "rewards/margins": 2.3168139457702637,
+      "rewards/rejected": -2.138627052307129,
       "step": 241,
-      "train_speed(iter/s)": 0.007496
-    },
-    {
-      "clip_ratio/high_max": 0.0027624310459941626,
-      "clip_ratio/high_mean": 0.0027624310459941626,
-      "clip_ratio/low_mean": 0.006610044481931254,
-      "clip_ratio/low_min": 0.006610044481931254,
-      "clip_ratio/region_mean": 0.009372475324198604,
-      "epoch": 2.6869806094182827,
-      "grad_norm": 1.4174226924918587,
-      "kl": 2.842063266711193,
-      "learning_rate": 5.8455934816979305e-06,
-      "loss": 0.025462469086050987,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021284
+    },
+    {
+      "epoch": 0.48876546326685183,
+      "grad_norm": 2.5642099380493164,
+      "learning_rate": 0.00018010871422407236,
+      "logits/chosen": -0.5903664231300354,
+      "logits/rejected": -0.817259669303894,
+      "logps/chosen": -1.1348421573638916,
+      "logps/rejected": -54.072288513183594,
+      "loss": 0.390775203704834,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18238955736160278,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.31289827823638916,
+      "rewards/margins": 3.0836544036865234,
+      "rewards/rejected": -2.7707560062408447,
       "step": 242,
-      "train_speed(iter/s)": 0.007483
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0027961052255705,
-      "clip_ratio/low_min": 0.0027961052255705,
-      "clip_ratio/region_mean": 0.0027961052255705,
-      "completions/clipped_ratio": 0.0546875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 10.0625,
-      "completions/min_length": 2.0,
-      "epoch": 2.698060941828255,
-      "grad_norm": 2.4284263318080272,
-      "kl": 1.1247594757005572,
-      "learning_rate": 5.439267461947883e-06,
-      "loss": 0.013277001678943634,
-      "memory(GiB)": 79.43,
-      "reward": 0.4765625,
-      "reward_std": 0.2019251585006714,
-      "rewards/Response_no_think_reward_1/mean": 0.4765625,
-      "rewards/Response_no_think_reward_1/std": 0.8031909465789795,
+      "train_speed(iter/s)": 0.021284
+    },
+    {
+      "epoch": 0.49078515526382227,
+      "grad_norm": 4.531945705413818,
+      "learning_rate": 0.00017990822537299647,
+      "logits/chosen": -0.7170388698577881,
+      "logits/rejected": -0.8259367942810059,
+      "logps/chosen": -6.029257774353027,
+      "logps/rejected": -34.971519470214844,
+      "loss": 0.7846373915672302,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.31968259811401367,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.002697870135307312,
+      "rewards/margins": 1.6093158721923828,
+      "rewards/rejected": -1.612013816833496,
       "step": 243,
-      "train_speed(iter/s)": 0.007378
-    },
-    {
-      "clip_ratio/high_max": 0.00271630787756294,
-      "clip_ratio/high_mean": 0.00271630787756294,
-      "clip_ratio/low_mean": 0.003980028559453785,
-      "clip_ratio/low_min": 0.003980028559453785,
-      "clip_ratio/region_mean": 0.0066963364370167255,
-      "epoch": 2.709141274238227,
-      "grad_norm": 1.999946631431668,
-      "kl": 1.150102037936449,
-      "learning_rate": 5.047181940696333e-06,
-      "loss": 0.011227283626794815,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021284
+    },
+    {
+      "epoch": 0.4928048472607927,
+      "grad_norm": 3.57047176361084,
+      "learning_rate": 0.00017970684396664813,
+      "logits/chosen": -0.7720434069633484,
+      "logits/rejected": -0.9465057253837585,
+      "logps/chosen": -3.873821258544922,
+      "logps/rejected": -37.18175506591797,
+      "loss": 0.8110936284065247,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.44118532538414,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.19488100707530975,
+      "rewards/margins": 1.9362576007843018,
+      "rewards/rejected": -1.741376519203186,
       "step": 244,
-      "train_speed(iter/s)": 0.007365
-    },
-    {
-      "clip_ratio/high_max": 0.003839633078314364,
-      "clip_ratio/high_mean": 0.003839633078314364,
-      "clip_ratio/low_mean": 0.002922048792243004,
-      "clip_ratio/low_min": 0.002922048792243004,
-      "clip_ratio/region_mean": 0.006761681521311402,
-      "completions/clipped_ratio": 0.0390625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 10.1875,
-      "completions/min_length": 2.0,
-      "epoch": 2.7202216066481997,
-      "grad_norm": 114.02408735149653,
-      "kl": 4.607287279635784,
-      "learning_rate": 4.669395964580614e-06,
-      "loss": 0.09313861280679703,
-      "memory(GiB)": 79.43,
-      "reward": 0.265625,
-      "reward_std": 0.22673699259757996,
-      "rewards/Response_no_think_reward_1/mean": 0.265625,
-      "rewards/Response_no_think_reward_1/std": 0.7475352883338928,
+      "train_speed(iter/s)": 0.021284
+    },
+    {
+      "epoch": 0.4948245392577632,
+      "grad_norm": 5.007617473602295,
+      "learning_rate": 0.0001795045722544083,
+      "logits/chosen": -0.7931674122810364,
+      "logits/rejected": -0.8741809129714966,
+      "logps/chosen": -1.8064591884613037,
+      "logps/rejected": -22.11163330078125,
+      "loss": 0.6888466477394104,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.31985488533973694,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.28564998507499695,
+      "rewards/margins": 1.2898226976394653,
+      "rewards/rejected": -1.0041728019714355,
       "step": 245,
-      "train_speed(iter/s)": 0.00727
-    },
-    {
-      "clip_ratio/high_max": 0.004987254389561713,
-      "clip_ratio/high_mean": 0.004987254389561713,
-      "clip_ratio/low_mean": 0.005028058891184628,
-      "clip_ratio/low_min": 0.005028058891184628,
-      "clip_ratio/region_mean": 0.01001531328074634,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 32.0,
-      "completions/mean_length": 9.0390625,
-      "completions/min_length": 2.0,
-      "epoch": 1.6472060050041701,
-      "grad_norm": 7.885151261057047,
-      "kl": 4.317316887667403,
-      "learning_rate": 0.0001363371892680654,
-      "loss": 0.06505110114812851,
-      "memory(GiB)": 77.36,
-      "reward": 0.390625,
-      "reward_std": 0.2467075139284134,
-      "rewards/Response_no_think_reward_1/mean": 0.390625,
-      "rewards/Response_no_think_reward_1/std": 0.7124801278114319,
+      "train_speed(iter/s)": 0.021285
+    },
+    {
+      "epoch": 0.49684423125473365,
+      "grad_norm": 6.91418981552124,
+      "learning_rate": 0.00017930141249560233,
+      "logits/chosen": -0.8298520445823669,
+      "logits/rejected": -0.9262869358062744,
+      "logps/chosen": -1.8928561210632324,
+      "logps/rejected": -22.42644691467285,
+      "loss": 0.5054484605789185,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17920702695846558,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.19144940376281738,
+      "rewards/margins": 1.3705973625183105,
+      "rewards/rejected": -1.1791479587554932,
       "step": 246,
-      "train_speed(iter/s)": 0.212467
-    },
-    {
-      "clip_ratio/high_max": 0.008637543302029371,
-      "clip_ratio/high_mean": 0.008637543302029371,
-      "clip_ratio/low_mean": 0.007979447836987674,
-      "clip_ratio/low_min": 0.007979447836987674,
-      "clip_ratio/region_mean": 0.016616990906186402,
-      "epoch": 1.6538782318598833,
-      "grad_norm": 9.765727829411468,
-      "kl": 5.302235448267311,
-      "learning_rate": 0.00013581952149432303,
-      "loss": 0.07505911588668823,
-      "memory(GiB)": 77.36,
+      "train_speed(iter/s)": 0.021286
+    },
+    {
+      "epoch": 0.4988639232517041,
+      "grad_norm": 5.468963146209717,
+      "learning_rate": 0.00017909736695947485,
+      "logits/chosen": -0.8126583695411682,
+      "logits/rejected": -0.9218447804450989,
+      "logps/chosen": -5.573441982269287,
+      "logps/rejected": -31.411596298217773,
+      "loss": 0.640342652797699,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2689912021160126,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.249068945646286,
+      "rewards/margins": 1.9487626552581787,
+      "rewards/rejected": -1.6996936798095703,
       "step": 247,
-      "train_speed(iter/s)": 0.182988
-    },
-    {
-      "clip_ratio/high_max": 0.00024900399148464203,
-      "clip_ratio/high_mean": 0.00024900399148464203,
-      "clip_ratio/low_mean": 0.006602977169677615,
-      "clip_ratio/low_min": 0.006602977169677615,
-      "clip_ratio/region_mean": 0.006851981161162257,
-      "completions/clipped_ratio": 0.1484375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 13.0625,
-      "completions/min_length": 2.0,
-      "epoch": 1.6605504587155964,
-      "grad_norm": 5.732775568700692,
-      "kl": 7.248534864047542,
-      "learning_rate": 0.0001353007501870137,
-      "loss": 0.07064881920814514,
-      "memory(GiB)": 77.47,
-      "reward": 0.0234375,
-      "reward_std": 0.21814784407615662,
-      "rewards/Response_no_think_reward_1/mean": 0.0234375,
-      "rewards/Response_no_think_reward_1/std": 0.8080777525901794,
+      "train_speed(iter/s)": 0.021287
+    },
+    {
+      "epoch": 0.5008836152486745,
+      "grad_norm": 5.13163423538208,
+      "learning_rate": 0.0001788924379251645,
+      "logits/chosen": -0.6676142811775208,
+      "logits/rejected": -0.7854112982749939,
+      "logps/chosen": -5.0913262367248535,
+      "logps/rejected": -33.67715835571289,
+      "loss": 0.6214959621429443,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.30665862560272217,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.09342917054891586,
+      "rewards/margins": 2.036255121231079,
+      "rewards/rejected": -1.9428261518478394,
       "step": 248,
-      "train_speed(iter/s)": 0.101239
-    },
-    {
-      "clip_ratio/high_max": 0.009486032678978518,
-      "clip_ratio/high_mean": 0.009486032678978518,
-      "clip_ratio/low_mean": 0.01039744314039126,
-      "clip_ratio/low_min": 0.01039744314039126,
-      "clip_ratio/region_mean": 0.019883474858943373,
-      "epoch": 1.6672226855713093,
-      "grad_norm": 6.377381125669229,
-      "kl": 2.9445294842589647,
-      "learning_rate": 0.00013478089132852716,
-      "loss": 0.08124249428510666,
-      "memory(GiB)": 77.47,
+      "train_speed(iter/s)": 0.021287
+    },
+    {
+      "epoch": 0.502903307245645,
+      "grad_norm": 5.3085246086120605,
+      "learning_rate": 0.00017868662768167828,
+      "logits/chosen": -0.8283058404922485,
+      "logits/rejected": -0.8981807827949524,
+      "logps/chosen": -3.5498528480529785,
+      "logps/rejected": -25.92713737487793,
+      "loss": 0.8023664355278015,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3884091079235077,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.14035727083683014,
+      "rewards/margins": 1.476125717163086,
+      "rewards/rejected": -1.335768461227417,
       "step": 249,
-      "train_speed(iter/s)": 0.094255
-    },
-    {
-      "clip_ratio/high_max": 0.0010964912362396717,
-      "clip_ratio/high_mean": 0.0010964912362396717,
-      "clip_ratio/low_mean": 0.003289473708719015,
-      "clip_ratio/low_min": 0.003289473708719015,
-      "clip_ratio/region_mean": 0.004385964944958687,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 7.9140625,
-      "completions/min_length": 2.0,
-      "epoch": 1.6738949124270226,
-      "grad_norm": 1.96116793666507,
-      "kl": 2.669485174041256,
-      "learning_rate": 0.0001342599609347585,
-      "loss": 0.02923566661775112,
-      "memory(GiB)": 77.47,
-      "reward": -0.0625,
-      "reward_std": 0.09127141535282135,
-      "rewards/Response_no_think_reward_1/mean": -0.0625,
-      "rewards/Response_no_think_reward_1/std": 0.9200308322906494,
+      "train_speed(iter/s)": 0.021287
+    },
+    {
+      "epoch": 0.5049229992426155,
+      "grad_norm": 3.970473051071167,
+      "learning_rate": 0.0001784799385278661,
+      "logits/chosen": -0.7480765581130981,
+      "logits/rejected": -0.850541889667511,
+      "logps/chosen": -2.1585793495178223,
+      "logps/rejected": -25.883459091186523,
+      "loss": 0.5557476878166199,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23389576375484467,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.27245235443115234,
+      "rewards/margins": 1.5237624645233154,
+      "rewards/rejected": -1.2513102293014526,
       "step": 250,
-      "train_speed(iter/s)": 0.065352
-    },
-    {
-      "clip_ratio/high_max": 0.005629746010527015,
-      "clip_ratio/high_mean": 0.005629746010527015,
-      "clip_ratio/low_mean": 0.0412575276568532,
-      "clip_ratio/low_min": 0.0412575276568532,
-      "clip_ratio/region_mean": 0.04688727576285601,
-      "epoch": 1.6805671392827355,
-      "grad_norm": 1.933812668241347,
-      "kl": 3.5041955505930673,
-      "learning_rate": 0.000133737975054615,
-      "loss": 0.024076396599411964,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021288
+    },
+    {
+      "epoch": 0.506942691239586,
+      "grad_norm": 2.7458653450012207,
+      "learning_rate": 0.00017827237277239514,
+      "logits/chosen": -0.7085766792297363,
+      "logits/rejected": -0.8612480163574219,
+      "logps/chosen": -1.5254360437393188,
+      "logps/rejected": -41.573387145996094,
+      "loss": 0.4261833429336548,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20671844482421875,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.23911702632904053,
+      "rewards/margins": 2.4618849754333496,
+      "rewards/rejected": -2.2227678298950195,
       "step": 251,
-      "train_speed(iter/s)": 0.062446
-    },
-    {
-      "clip_ratio/high_max": 0.00119566879584454,
-      "clip_ratio/high_mean": 0.00119566879584454,
-      "clip_ratio/low_mean": 0.0004687021573772654,
-      "clip_ratio/low_min": 0.0004687021573772654,
-      "clip_ratio/region_mean": 0.0016643709386698902,
-      "completions/clipped_ratio": 0.140625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 12.6953125,
-      "completions/min_length": 2.0,
-      "epoch": 1.6872393661384488,
-      "grad_norm": 2.4412702069582983,
-      "kl": 3.9371446361765265,
-      "learning_rate": 0.00013321494976952132,
-      "loss": 0.03714549541473389,
-      "memory(GiB)": 79.43,
-      "reward": 0.1796875,
-      "reward_std": 0.13782459497451782,
-      "rewards/Response_no_think_reward_1/mean": 0.1796875,
-      "rewards/Response_no_think_reward_1/std": 0.9258628487586975,
+      "train_speed(iter/s)": 0.021259
+    },
+    {
+      "epoch": 0.5089623832365564,
+      "grad_norm": 4.110019207000732,
+      "learning_rate": 0.00017806393273372395,
+      "logits/chosen": -0.6960774064064026,
+      "logits/rejected": -0.773073673248291,
+      "logps/chosen": -5.731773376464844,
+      "logps/rejected": -40.48484420776367,
+      "loss": 0.6885837912559509,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.38311967253685,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.13538384437561035,
+      "rewards/margins": 2.754824638366699,
+      "rewards/rejected": -2.619440793991089,
       "step": 252,
-      "train_speed(iter/s)": 0.047385
-    },
-    {
-      "clip_ratio/high_max": 0.007317493204027414,
-      "clip_ratio/high_mean": 0.007317493204027414,
-      "clip_ratio/low_mean": 0.005186333088204265,
-      "clip_ratio/low_min": 0.005186333088204265,
-      "clip_ratio/region_mean": 0.012503825942985713,
-      "epoch": 1.6939115929941617,
-      "grad_norm": 4.580377931462122,
-      "kl": 1.7039022920653224,
-      "learning_rate": 0.00013269090119292444,
-      "loss": 0.021267667412757874,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021259
+    },
+    {
+      "epoch": 0.5109820752335269,
+      "grad_norm": 2.490933656692505,
+      "learning_rate": 0.0001778546207400766,
+      "logits/chosen": -0.7139323353767395,
+      "logits/rejected": -0.8478928208351135,
+      "logps/chosen": -2.643089771270752,
+      "logps/rejected": -49.735321044921875,
+      "loss": 0.4857577383518219,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20252788066864014,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2547016441822052,
+      "rewards/margins": 3.3733713626861572,
+      "rewards/rejected": -3.1186697483062744,
       "step": 253,
-      "train_speed(iter/s)": 0.045915
-    },
-    {
-      "clip_ratio/high_max": 0.0008462663099635392,
-      "clip_ratio/high_mean": 0.0008462663099635392,
-      "clip_ratio/low_mean": 0.005740528285969049,
-      "clip_ratio/low_min": 0.005740528285969049,
-      "clip_ratio/region_mean": 0.006586794595932588,
-      "completions/clipped_ratio": 0.1328125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 13.03125,
-      "completions/min_length": 2.0,
-      "epoch": 1.700583819849875,
-      "grad_norm": 2.9913603629745245,
-      "kl": 1.2227803394198418,
-      "learning_rate": 0.00013216584546979702,
-      "loss": 0.01559203490614891,
-      "memory(GiB)": 79.43,
-      "reward": 0.015625,
-      "reward_std": 0.27663108706474304,
-      "rewards/Response_no_think_reward_1/mean": 0.015625,
-      "rewards/Response_no_think_reward_1/std": 0.9555834531784058,
+      "train_speed(iter/s)": 0.021258
+    },
+    {
+      "epoch": 0.5130017672304974,
+      "grad_norm": 1.8276517391204834,
+      "learning_rate": 0.00017764443912941672,
+      "logits/chosen": -0.8284695148468018,
+      "logits/rejected": -0.9083284735679626,
+      "logps/chosen": -0.5445702075958252,
+      "logps/rejected": -36.77948760986328,
+      "loss": 0.3219444751739502,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10567262023687363,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.28718090057373047,
+      "rewards/margins": 2.473656415939331,
+      "rewards/rejected": -2.1864752769470215,
       "step": 254,
-      "train_speed(iter/s)": 0.037104
-    },
-    {
-      "clip_ratio/high_max": 0.007223855180200189,
-      "clip_ratio/high_mean": 0.007223855180200189,
-      "clip_ratio/low_mean": 0.10893735339050181,
-      "clip_ratio/low_min": 0.10893735339050181,
-      "clip_ratio/region_mean": 0.11616120918188244,
-      "epoch": 1.707256046705588,
-      "grad_norm": 1.9655670074579659,
-      "kl": 2.0845303861424327,
-      "learning_rate": 0.00013163979877614,
-      "loss": 0.011529060080647469,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021259
+    },
+    {
+      "epoch": 0.5150214592274678,
+      "grad_norm": 1.7103385925292969,
+      "learning_rate": 0.00017743339024942135,
+      "logits/chosen": -0.8364546298980713,
+      "logits/rejected": -0.9355967044830322,
+      "logps/chosen": -1.8402678966522217,
+      "logps/rejected": -41.97530746459961,
+      "loss": 0.3561093807220459,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15152454376220703,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3776871860027313,
+      "rewards/margins": 3.4742887020111084,
+      "rewards/rejected": -3.096601963043213,
       "step": 255,
-      "train_speed(iter/s)": 0.036232
-    },
-    {
-      "clip_ratio/high_max": 0.006948165129870176,
-      "clip_ratio/high_mean": 0.006948165129870176,
-      "clip_ratio/low_mean": 0.00925130266114138,
-      "clip_ratio/low_min": 0.00925130266114138,
-      "clip_ratio/region_mean": 0.01619946787832305,
-      "completions/clipped_ratio": 0.359375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 14.140625,
-      "completions/min_length": 2.0,
-      "epoch": 1.713928273561301,
-      "grad_norm": 4.730855130110655,
-      "kl": 5.062177191488445,
-      "learning_rate": 0.00013111277731848442,
-      "loss": 0.05786250904202461,
-      "memory(GiB)": 79.43,
-      "reward": -0.0859375,
-      "reward_std": 0.410529226064682,
-      "rewards/Response_no_think_reward_1/mean": -0.0859375,
-      "rewards/Response_no_think_reward_1/std": 0.869675874710083,
+      "train_speed(iter/s)": 0.02126
+    },
+    {
+      "epoch": 0.5170411512244383,
+      "grad_norm": 5.862149238586426,
+      "learning_rate": 0.00017722147645745468,
+      "logits/chosen": -0.8007981777191162,
+      "logits/rejected": -0.9000130891799927,
+      "logps/chosen": -4.51971435546875,
+      "logps/rejected": -49.85738754272461,
+      "loss": 0.6844031810760498,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23998194932937622,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.11539852619171143,
+      "rewards/margins": 3.546253204345703,
+      "rewards/rejected": -3.661651849746704,
       "step": 256,
-      "train_speed(iter/s)": 0.033266
-    },
-    {
-      "clip_ratio/high_max": 0.014393029327038676,
-      "clip_ratio/high_mean": 0.014393029327038676,
-      "clip_ratio/low_mean": 0.1303235706873238,
-      "clip_ratio/low_min": 0.1303235706873238,
-      "clip_ratio/region_mean": 0.14471660286653787,
-      "epoch": 1.7206005004170142,
-      "grad_norm": 6.110613692874304,
-      "kl": 6.920241659507155,
-      "learning_rate": 0.00013058479733339185,
-      "loss": 0.0579487606883049,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021261
+    },
+    {
+      "epoch": 0.5190608432214088,
+      "grad_norm": 23.92522621154785,
+      "learning_rate": 0.0001770087001205418,
+      "logits/chosen": -0.7442535161972046,
+      "logits/rejected": -0.8944082260131836,
+      "logps/chosen": -5.961298942565918,
+      "logps/rejected": -71.44552612304688,
+      "loss": 1.19035804271698,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7366040945053101,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.2813420295715332,
+      "rewards/margins": 4.797372817993164,
+      "rewards/rejected": -5.078714847564697,
       "step": 257,
-      "train_speed(iter/s)": 0.032582
-    },
-    {
-      "clip_ratio/high_max": 0.0021560846944339573,
-      "clip_ratio/high_mean": 0.0021560846944339573,
-      "clip_ratio/low_mean": 0.027111168601550162,
-      "clip_ratio/low_min": 0.027111168601550162,
-      "clip_ratio/region_mean": 0.029267253237776458,
-      "completions/clipped_ratio": 0.296875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 9.203125,
-      "completions/min_length": 2.0,
-      "epoch": 1.7272727272727273,
-      "grad_norm": 13.319715864612775,
-      "kl": 13.24667002509068,
-      "learning_rate": 0.00013005587508695444,
-      "loss": 0.12780021131038666,
-      "memory(GiB)": 79.43,
-      "reward": 0.109375,
-      "reward_std": 0.3405221104621887,
-      "rewards/Response_no_think_reward_1/mean": 0.109375,
-      "rewards/Response_no_think_reward_1/std": 0.8982430100440979,
+      "train_speed(iter/s)": 0.021261
+    },
+    {
+      "epoch": 0.5210805352183792,
+      "grad_norm": 9.221707344055176,
+      "learning_rate": 0.00017679506361534215,
+      "logits/chosen": -0.8309729099273682,
+      "logits/rejected": -0.9244940280914307,
+      "logps/chosen": -3.6046130657196045,
+      "logps/rejected": -38.451324462890625,
+      "loss": 1.0726348161697388,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5696147084236145,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.056192442774772644,
+      "rewards/margins": 2.7530393600463867,
+      "rewards/rejected": -2.6968469619750977,
       "step": 258,
-      "train_speed(iter/s)": 0.030335
-    },
-    {
-      "clip_ratio/high_max": 0.025854239764157683,
-      "clip_ratio/high_mean": 0.025854239764157683,
-      "clip_ratio/low_mean": 0.02940721553750336,
-      "clip_ratio/low_min": 0.02940721553750336,
-      "clip_ratio/region_mean": 0.055261454777792096,
-      "epoch": 1.7339449541284404,
-      "grad_norm": 7.873376409391432,
-      "kl": 3.398330974082228,
-      "learning_rate": 0.00012952602687429362,
-      "loss": 0.08237586915493011,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021262
+    },
+    {
+      "epoch": 0.5231002272153497,
+      "grad_norm": 3.6904711723327637,
+      "learning_rate": 0.0001765805693281231,
+      "logits/chosen": -0.7574508786201477,
+      "logits/rejected": -0.8763286471366882,
+      "logps/chosen": -1.6263139247894287,
+      "logps/rejected": -46.19388198852539,
+      "loss": 0.5760003328323364,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.32455456256866455,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.25203612446784973,
+      "rewards/margins": 3.3660078048706055,
+      "rewards/rejected": -3.1139719486236572,
       "step": 259,
-      "train_speed(iter/s)": 0.029779
-    },
-    {
-      "clip_ratio/high_max": 0.0010593220358714461,
-      "clip_ratio/high_mean": 0.0010593220358714461,
-      "clip_ratio/low_mean": 0.025134949712082744,
-      "clip_ratio/low_min": 0.025134949712082744,
-      "clip_ratio/region_mean": 0.02619427174795419,
-      "completions/clipped_ratio": 0.3828125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 11.0390625,
-      "completions/min_length": 2.0,
-      "epoch": 1.7406171809841533,
-      "grad_norm": 5.622914024348242,
-      "kl": 8.146455611102283,
-      "learning_rate": 0.00012899526901905822,
-      "loss": 0.09567078202962875,
-      "memory(GiB)": 79.43,
-      "reward": 0.046875,
-      "reward_std": 0.30238547921180725,
-      "rewards/Response_no_think_reward_1/mean": 0.046875,
-      "rewards/Response_no_think_reward_1/std": 0.8681537508964539,
+      "train_speed(iter/s)": 0.021263
+    },
+    {
+      "epoch": 0.5251199192123202,
+      "grad_norm": 6.571902751922607,
+      "learning_rate": 0.00017636521965473323,
+      "logits/chosen": -0.7622162103652954,
+      "logits/rejected": -0.8849962949752808,
+      "logps/chosen": -3.6163456439971924,
+      "logps/rejected": -57.187339782714844,
+      "loss": 0.513578474521637,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.28239667415618896,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.26628056168556213,
+      "rewards/margins": 4.149145603179932,
+      "rewards/rejected": -3.8828649520874023,
       "step": 260,
-      "train_speed(iter/s)": 0.028068
-    },
-    {
-      "clip_ratio/high_max": 0.01715171878458932,
-      "clip_ratio/high_mean": 0.01715171878458932,
-      "clip_ratio/low_mean": 0.0735002284636721,
-      "clip_ratio/low_min": 0.0735002284636721,
-      "clip_ratio/region_mean": 0.0906519484706223,
-      "epoch": 1.7472894078398666,
-      "grad_norm": 8.808673951478834,
-      "kl": 5.425452136900276,
-      "learning_rate": 0.00012846361787292136,
-      "loss": 0.08871767669916153,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021263
+    },
+    {
+      "epoch": 0.5271396112092905,
+      "grad_norm": 3.009442090988159,
+      "learning_rate": 0.00017614901700057552,
+      "logits/chosen": -0.7535040974617004,
+      "logits/rejected": -0.8197404146194458,
+      "logps/chosen": -1.611877679824829,
+      "logps/rejected": -36.75214767456055,
+      "loss": 0.4263669550418854,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1808186173439026,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.355543851852417,
+      "rewards/margins": 2.128063678741455,
+      "rewards/rejected": -1.7725193500518799,
       "step": 261,
-      "train_speed(iter/s)": 0.027598
-    },
-    {
-      "clip_ratio/high_max": 0.000811688310932368,
-      "clip_ratio/high_mean": 0.000811688310932368,
-      "clip_ratio/low_mean": 0.009334841219242662,
-      "clip_ratio/low_min": 0.009334841219242662,
-      "clip_ratio/region_mean": 0.01014652947196737,
-      "completions/clipped_ratio": 0.3984375,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 10.890625,
-      "completions/min_length": 2.0,
-      "epoch": 1.7539616346955795,
-      "grad_norm": 9.615584503884596,
-      "kl": 8.083851436153054,
-      "learning_rate": 0.00012793108981507694,
-      "loss": 0.11578933149576187,
-      "memory(GiB)": 79.43,
-      "reward": -0.0859375,
-      "reward_std": 0.41268494725227356,
-      "rewards/Response_no_think_reward_1/mean": -0.0859375,
-      "rewards/Response_no_think_reward_1/std": 0.8605742454528809,
+      "train_speed(iter/s)": 0.021264
+    },
+    {
+      "epoch": 0.529159303206261,
+      "grad_norm": 10.684004783630371,
+      "learning_rate": 0.0001759319637805806,
+      "logits/chosen": -0.7372271418571472,
+      "logits/rejected": -0.8281789422035217,
+      "logps/chosen": -1.6989682912826538,
+      "logps/rejected": -33.11771011352539,
+      "loss": 0.5836552381515503,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24641528725624084,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.42360562086105347,
+      "rewards/margins": 2.2778561115264893,
+      "rewards/rejected": -1.8542505502700806,
       "step": 262,
-      "train_speed(iter/s)": 0.02625
-    },
-    {
-      "clip_ratio/high_max": 0.020523510698694736,
-      "clip_ratio/high_mean": 0.020523510698694736,
-      "clip_ratio/low_mean": 0.04563906986732036,
-      "clip_ratio/low_min": 0.04563906986732036,
-      "clip_ratio/region_mean": 0.0661625819047913,
-      "epoch": 1.7606338615512929,
-      "grad_norm": 30.924872115013905,
-      "kl": 3.7772719897329807,
-      "learning_rate": 0.0001273977012517348,
-      "loss": 0.21410413086414337,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021265
+    },
+    {
+      "epoch": 0.5311789952032315,
+      "grad_norm": 4.7525248527526855,
+      "learning_rate": 0.00017571406241917968,
+      "logits/chosen": -0.7032569646835327,
+      "logits/rejected": -0.8048897385597229,
+      "logps/chosen": -3.0269763469696045,
+      "logps/rejected": -29.489608764648438,
+      "loss": 0.5058411359786987,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.19841410219669342,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.27225935459136963,
+      "rewards/margins": 2.0580902099609375,
+      "rewards/rejected": -1.7858312129974365,
       "step": 263,
-      "train_speed(iter/s)": 0.025852
-    },
-    {
-      "clip_ratio/high_max": 0.0010416667209938169,
-      "clip_ratio/high_mean": 0.0010416667209938169,
-      "clip_ratio/low_mean": 0.013979690615087748,
-      "clip_ratio/low_min": 0.013979690615087748,
-      "clip_ratio/region_mean": 0.015021357336081564,
-      "completions/clipped_ratio": 0.140625,
-      "completions/max_length": 41.0,
-      "completions/mean_length": 7.9609375,
-      "completions/min_length": 2.0,
-      "epoch": 1.7673060884070058,
-      "grad_norm": 2.801515232228881,
-      "kl": 1.5485301897861063,
-      "learning_rate": 0.00012686346861561537,
-      "loss": 0.014268179424107075,
-      "memory(GiB)": 79.43,
-      "reward": 0.015625,
-      "reward_std": 0.2223242074251175,
-      "rewards/Response_no_think_reward_1/mean": 0.015625,
-      "rewards/Response_no_think_reward_1/std": 0.8227510452270508,
+      "train_speed(iter/s)": 0.021265
+    },
+    {
+      "epoch": 0.5331986872002019,
+      "grad_norm": 2.464600086212158,
+      "learning_rate": 0.0001754953153502775,
+      "logits/chosen": -0.6921654939651489,
+      "logits/rejected": -0.8245174288749695,
+      "logps/chosen": -1.2894872426986694,
+      "logps/rejected": -41.8453369140625,
+      "loss": 0.36821237206459045,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10888063907623291,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.26967915892601013,
+      "rewards/margins": 2.722353458404541,
+      "rewards/rejected": -2.452674150466919,
       "step": 264,
-      "train_speed(iter/s)": 0.024697
-    },
-    {
-      "clip_ratio/high_max": 0.0018305982230231166,
-      "clip_ratio/high_mean": 0.0018305982230231166,
-      "clip_ratio/low_mean": 0.0754491506377235,
-      "clip_ratio/low_min": 0.0754491506377235,
-      "clip_ratio/region_mean": 0.07727974897716194,
-      "epoch": 1.773978315262719,
-      "grad_norm": 5.020289308786792,
-      "kl": 4.038693455513567,
-      "learning_rate": 0.0001263284083654435,
-      "loss": 0.035946328192949295,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021265
+    },
+    {
+      "epoch": 0.5352183791971724,
+      "grad_norm": 3.666675329208374,
+      "learning_rate": 0.00017527572501722512,
+      "logits/chosen": -0.6526811718940735,
+      "logits/rejected": -0.8120133876800537,
+      "logps/chosen": -5.671820640563965,
+      "logps/rejected": -57.84872055053711,
+      "loss": 0.6365722417831421,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.44796761870384216,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.04257254675030708,
+      "rewards/margins": 3.3321990966796875,
+      "rewards/rejected": -3.289626121520996,
       "step": 265,
-      "train_speed(iter/s)": 0.024353
-    },
-    {
-      "clip_ratio/high_max": 0.0015625000232830644,
-      "clip_ratio/high_mean": 0.0015625000232830644,
-      "clip_ratio/low_mean": 0.011535812984220684,
-      "clip_ratio/low_min": 0.011535812984220684,
-      "clip_ratio/region_mean": 0.013098313007503748,
-      "completions/clipped_ratio": 0.2421875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 7.0546875,
-      "completions/min_length": 2.0,
-      "epoch": 1.780650542118432,
-      "grad_norm": 4.823867965705569,
-      "kl": 5.8212087393421825,
-      "learning_rate": 0.00012579253698544125,
-      "loss": 0.07973380386829376,
-      "memory(GiB)": 79.43,
-      "reward": 0.0078125,
-      "reward_std": 0.30061954259872437,
-      "rewards/Response_no_think_reward_1/mean": 0.0078125,
-      "rewards/Response_no_think_reward_1/std": 0.6459577083587646,
+      "train_speed(iter/s)": 0.021266
+    },
+    {
+      "epoch": 0.5372380711941429,
+      "grad_norm": 5.50975227355957,
+      "learning_rate": 0.00017505529387279277,
+      "logits/chosen": -0.7191404104232788,
+      "logits/rejected": -0.7549193501472473,
+      "logps/chosen": -11.20957088470459,
+      "logps/rejected": -24.620988845825195,
+      "loss": 0.8978201746940613,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3969779908657074,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3098965585231781,
+      "rewards/margins": 1.0103328227996826,
+      "rewards/rejected": -1.320229411125183,
       "step": 266,
-      "train_speed(iter/s)": 0.023359
-    },
-    {
-      "clip_ratio/high_max": 0.0031250000465661287,
-      "clip_ratio/high_mean": 0.0031250000465661287,
-      "clip_ratio/low_mean": 0.11317632021382451,
-      "clip_ratio/low_min": 0.11317632021382451,
-      "clip_ratio/region_mean": 0.11630132002756,
-      "epoch": 1.787322768974145,
-      "grad_norm": 11.546843380663908,
-      "kl": 10.476074979640543,
-      "learning_rate": 0.00012525587098482005,
-      "loss": 0.09422159940004349,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021266
+    },
+    {
+      "epoch": 0.5392577631911134,
+      "grad_norm": 9.796467781066895,
+      "learning_rate": 0.0001748340243791422,
+      "logits/chosen": -0.6914748549461365,
+      "logits/rejected": -0.8334357738494873,
+      "logps/chosen": -5.802427291870117,
+      "logps/rejected": -41.96017074584961,
+      "loss": 0.6360026597976685,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3448495864868164,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2543778419494629,
+      "rewards/margins": 2.576064348220825,
+      "rewards/rejected": -2.321686267852783,
       "step": 267,
-      "train_speed(iter/s)": 0.023058
-    },
-    {
-      "clip_ratio/high_max": 0.000811688310932368,
-      "clip_ratio/high_mean": 0.000811688310932368,
-      "clip_ratio/low_mean": 0.006867786491056904,
-      "clip_ratio/low_min": 0.006867786491056904,
-      "clip_ratio/region_mean": 0.007679474772885442,
-      "completions/clipped_ratio": 0.1328125,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 9.8515625,
-      "completions/min_length": 3.0,
-      "epoch": 1.7939949958298582,
-      "grad_norm": 48.055640901825676,
-      "kl": 22.36742792606401,
-      "learning_rate": 0.0001247184268972722,
-      "loss": 0.4101108908653259,
-      "memory(GiB)": 79.43,
-      "reward": 0.125,
-      "reward_std": 0.2585597634315491,
-      "rewards/Response_no_think_reward_1/mean": 0.125,
-      "rewards/Response_no_think_reward_1/std": 0.6274557709693909,
+      "train_speed(iter/s)": 0.021266
+    },
+    {
+      "epoch": 0.5412774551880838,
+      "grad_norm": 4.284933567047119,
+      "learning_rate": 0.00017461191900779936,
+      "logits/chosen": -0.713287353515625,
+      "logits/rejected": -0.7692936658859253,
+      "logps/chosen": -4.46176290512085,
+      "logps/rejected": -30.945892333984375,
+      "loss": 0.5797103047370911,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3343329429626465,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1580563336610794,
+      "rewards/margins": 2.0286142826080322,
+      "rewards/rejected": -1.8705579042434692,
       "step": 268,
-      "train_speed(iter/s)": 0.022062
-    },
-    {
-      "clip_ratio/high_max": 0.00529984722379595,
-      "clip_ratio/high_mean": 0.00529984722379595,
-      "clip_ratio/low_mean": 0.01420470466837287,
-      "clip_ratio/low_min": 0.01420470466837287,
-      "clip_ratio/region_mean": 0.01950455189216882,
-      "epoch": 1.8006672226855713,
-      "grad_norm": 10.145223773494124,
-      "kl": 7.905034697459087,
-      "learning_rate": 0.00012418022128046142,
-      "loss": 0.22370362281799316,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021267
+    },
+    {
+      "epoch": 0.5432971471850543,
+      "grad_norm": 3.172943353652954,
+      "learning_rate": 0.00017438898023962679,
+      "logits/chosen": -0.7907764911651611,
+      "logits/rejected": -0.8711207509040833,
+      "logps/chosen": -1.7791467905044556,
+      "logps/rejected": -29.88024139404297,
+      "loss": 0.33940574526786804,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14780378341674805,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2693939208984375,
+      "rewards/margins": 2.3542535305023193,
+      "rewards/rejected": -2.0848593711853027,
       "step": 269,
-      "train_speed(iter/s)": 0.021799
-    },
-    {
-      "clip_ratio/high_max": 0.0020380434580147266,
-      "clip_ratio/high_mean": 0.0020380434580147266,
-      "clip_ratio/low_mean": 0.004399414756335318,
-      "clip_ratio/low_min": 0.004399414756335318,
-      "clip_ratio/region_mean": 0.006437458097934723,
-      "completions/clipped_ratio": 0.1640625,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 9.84375,
-      "completions/min_length": 3.0,
-      "epoch": 1.8073394495412844,
-      "grad_norm": 2.925526379332068,
-      "kl": 1.9621957493945956,
-      "learning_rate": 0.0001236412707155127,
-      "loss": 0.02435774728655815,
-      "memory(GiB)": 79.43,
-      "reward": 0.34375,
-      "reward_std": 0.22380223870277405,
-      "rewards/Response_no_think_reward_1/mean": 0.34375,
-      "rewards/Response_no_think_reward_1/std": 0.6329222321510315,
+      "train_speed(iter/s)": 0.021268
+    },
+    {
+      "epoch": 0.5453168391820248,
+      "grad_norm": 5.463645935058594,
+      "learning_rate": 0.00017416521056479577,
+      "logits/chosen": -0.7378922700881958,
+      "logits/rejected": -0.8519225120544434,
+      "logps/chosen": -5.347389221191406,
+      "logps/rejected": -35.428524017333984,
+      "loss": 0.6419250965118408,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.49513840675354004,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.09582029283046722,
+      "rewards/margins": 2.3689239025115967,
+      "rewards/rejected": -2.2731034755706787,
       "step": 270,
-      "train_speed(iter/s)": 0.021038
-    },
-    {
-      "clip_ratio/high_max": 0.027234246022999287,
-      "clip_ratio/high_mean": 0.027234246022999287,
-      "clip_ratio/low_mean": 0.014790827757678926,
-      "clip_ratio/low_min": 0.014790827757678926,
-      "clip_ratio/region_mean": 0.04202507343143225,
-      "epoch": 1.8140116763969973,
-      "grad_norm": 7.85150802599005,
-      "kl": 1.2954727746546268,
-      "learning_rate": 0.0001231015918065016,
-      "loss": 0.046614713966846466,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021269
+    },
+    {
+      "epoch": 0.5473365311789952,
+      "grad_norm": 2.791165828704834,
+      "learning_rate": 0.00017394061248275872,
+      "logits/chosen": -0.7984417080879211,
+      "logits/rejected": -0.8422114849090576,
+      "logps/chosen": -6.475491046905518,
+      "logps/rejected": -26.260494232177734,
+      "loss": 0.47203221917152405,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23484748601913452,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4524388909339905,
+      "rewards/margins": 1.7292096614837646,
+      "rewards/rejected": -1.2767709493637085,
       "step": 271,
-      "train_speed(iter/s)": 0.020801
-    },
-    {
-      "clip_ratio/high_max": 0.0007812500116415322,
-      "clip_ratio/high_mean": 0.0007812500116415322,
-      "clip_ratio/low_mean": 0.008673313190229237,
-      "clip_ratio/low_min": 0.008673313190229237,
-      "clip_ratio/region_mean": 0.00945456320187077,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 23.0,
-      "completions/mean_length": 8.578125,
-      "completions/min_length": 2.0,
-      "epoch": 1.8206839032527107,
-      "grad_norm": 5.663376365839488,
-      "kl": 3.0526226649526507,
-      "learning_rate": 0.00012256120117994245,
-      "loss": 0.029910365119576454,
-      "memory(GiB)": 79.43,
-      "reward": 0.2578125,
-      "reward_std": 0.2626354694366455,
-      "rewards/Response_no_think_reward_1/mean": 0.2578125,
-      "rewards/Response_no_think_reward_1/std": 0.70147305727005,
+      "train_speed(iter/s)": 0.021269
+    },
+    {
+      "epoch": 0.5493562231759657,
+      "grad_norm": 2.7601311206817627,
+      "learning_rate": 0.00017371518850222112,
+      "logits/chosen": -0.7166031002998352,
+      "logits/rejected": -0.8156049251556396,
+      "logps/chosen": -1.3833684921264648,
+      "logps/rejected": -45.072750091552734,
+      "loss": 0.4653846323490143,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.185410737991333,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1616918444633484,
+      "rewards/margins": 2.9714224338531494,
+      "rewards/rejected": -2.8097305297851562,
       "step": 272,
-      "train_speed(iter/s)": 0.020032
-    },
-    {
-      "clip_ratio/high_max": 0.003040286072064191,
-      "clip_ratio/high_mean": 0.003040286072064191,
-      "clip_ratio/low_mean": 0.06708688009530306,
-      "clip_ratio/low_min": 0.06708688009530306,
-      "clip_ratio/region_mean": 0.07012716663302854,
-      "epoch": 1.8273561301084236,
-      "grad_norm": 4.421434755408977,
-      "kl": 5.31667997315526,
-      "learning_rate": 0.0001220201154842765,
-      "loss": 0.03301015496253967,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021269
+    },
+    {
+      "epoch": 0.5513759151729362,
+      "grad_norm": 4.9853434562683105,
+      "learning_rate": 0.00017348894114111344,
+      "logits/chosen": -0.7632589340209961,
+      "logits/rejected": -0.7834702730178833,
+      "logps/chosen": -5.120803356170654,
+      "logps/rejected": -20.066801071166992,
+      "loss": 0.7621564865112305,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.39412054419517517,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.09666721522808075,
+      "rewards/margins": 1.2335360050201416,
+      "rewards/rejected": -1.1368687152862549,
       "step": 273,
-      "train_speed(iter/s)": 0.019826
-    },
-    {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.007332374923862517,
-      "clip_ratio/low_min": 0.007332374923862517,
-      "clip_ratio/region_mean": 0.007332374923862517,
-      "completions/clipped_ratio": 0.1796875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 11.125,
-      "completions/min_length": 2.0,
-      "epoch": 1.834028356964137,
-      "grad_norm": 1.9255495711888,
-      "kl": 2.3658070964738727,
-      "learning_rate": 0.00012147835138935868,
-      "loss": 0.040864668786525726,
-      "memory(GiB)": 79.43,
-      "reward": -0.3203125,
-      "reward_std": 0.22869102656841278,
-      "rewards/Response_no_think_reward_1/mean": -0.3203125,
-      "rewards/Response_no_think_reward_1/std": 0.7091482281684875,
+      "train_speed(iter/s)": 0.02127
+    },
+    {
+      "epoch": 0.5533956071699065,
+      "grad_norm": 6.144747734069824,
+      "learning_rate": 0.00017326187292656333,
+      "logits/chosen": -0.7423402070999146,
+      "logits/rejected": -0.8353608846664429,
+      "logps/chosen": -3.019026756286621,
+      "logps/rejected": -38.725494384765625,
+      "loss": 0.4919193685054779,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.29635170102119446,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.20840522646903992,
+      "rewards/margins": 2.5846292972564697,
+      "rewards/rejected": -2.3762240409851074,
       "step": 274,
-      "train_speed(iter/s)": 0.019168
-    },
-    {
-      "clip_ratio/high_max": 0.014975278521887958,
-      "clip_ratio/high_mean": 0.014975278521887958,
-      "clip_ratio/low_mean": 0.056431527715176344,
-      "clip_ratio/low_min": 0.056431527715176344,
-      "clip_ratio/region_mean": 0.07140680588781834,
-      "epoch": 1.8407005838198498,
-      "grad_norm": 1.7607705528260904,
-      "kl": 2.3127799107460305,
-      "learning_rate": 0.00012093592558594416,
-      "loss": 0.021540286019444466,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021271
+    },
+    {
+      "epoch": 0.555415299166877,
+      "grad_norm": 5.072936058044434,
+      "learning_rate": 0.00017303398639486695,
+      "logits/chosen": -0.8050971627235413,
+      "logits/rejected": -0.8979619741439819,
+      "logps/chosen": -3.597996473312378,
+      "logps/rejected": -38.68907165527344,
+      "loss": 0.6185693144798279,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.36338648200035095,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.04303528368473053,
+      "rewards/margins": 2.610353946685791,
+      "rewards/rejected": -2.567318916320801,
       "step": 275,
-      "train_speed(iter/s)": 0.018983
-    },
-    {
-      "clip_ratio/high_max": 0.00027173911803402007,
-      "clip_ratio/high_mean": 0.00027173911803402007,
-      "clip_ratio/low_mean": 0.0030319468351081014,
-      "clip_ratio/low_min": 0.0030319468351081014,
-      "clip_ratio/region_mean": 0.003303685924038291,
-      "completions/clipped_ratio": 0.1171875,
-      "completions/max_length": 51.0,
-      "completions/mean_length": 10.21875,
-      "completions/min_length": 5.0,
-      "epoch": 1.847372810675563,
-      "grad_norm": 1.3845096214977168,
-      "kl": 0.5768661912588868,
-      "learning_rate": 0.00012039285478517417,
-      "loss": 0.005703174974769354,
-      "memory(GiB)": 79.43,
-      "reward": 0.296875,
-      "reward_std": 0.31093141436576843,
-      "rewards/Response_no_think_reward_1/mean": 0.296875,
-      "rewards/Response_no_think_reward_1/std": 0.6563964486122131,
+      "train_speed(iter/s)": 0.021272
+    },
+    {
+      "epoch": 0.5574349911638475,
+      "grad_norm": 4.154740810394287,
+      "learning_rate": 0.00017280528409146094,
+      "logits/chosen": -0.8580950498580933,
+      "logits/rejected": -0.9338577389717102,
+      "logps/chosen": -1.9045888185501099,
+      "logps/rejected": -29.787948608398438,
+      "loss": 0.6412729024887085,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2939704358577728,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.12071260064840317,
+      "rewards/margins": 2.376711130142212,
+      "rewards/rejected": -2.255998373031616,
       "step": 276,
-      "train_speed(iter/s)": 0.018355
-    },
-    {
-      "clip_ratio/high_max": 0.02822316304082051,
-      "clip_ratio/high_mean": 0.02822316304082051,
-      "clip_ratio/low_mean": 0.023227162193506956,
-      "clip_ratio/low_min": 0.023227162193506956,
-      "clip_ratio/region_mean": 0.05145032424479723,
-      "epoch": 1.854045037531276,
-      "grad_norm": 2.277051850874954,
-      "kl": 1.444888403537334,
-      "learning_rate": 0.00011984915571806108,
-      "loss": 0.004672436509281397,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021272
+    },
+    {
+      "epoch": 0.5594546831608179,
+      "grad_norm": 4.429795265197754,
+      "learning_rate": 0.00017257576857089397,
+      "logits/chosen": -0.7823255658149719,
+      "logits/rejected": -0.8295686841011047,
+      "logps/chosen": -6.783672332763672,
+      "logps/rejected": -42.67618942260742,
+      "loss": 0.5918665528297424,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3899191915988922,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.04123719036579132,
+      "rewards/margins": 2.868664503097534,
+      "rewards/rejected": -2.827427387237549,
       "step": 277,
-      "train_speed(iter/s)": 0.018189
-    },
-    {
-      "clip_ratio/high_max": 0.006991013826336712,
-      "clip_ratio/high_mean": 0.006991013826336712,
-      "clip_ratio/low_mean": 0.009722222341224551,
-      "clip_ratio/low_min": 0.009722222341224551,
-      "clip_ratio/region_mean": 0.016713236400391906,
-      "completions/clipped_ratio": 0.0703125,
-      "completions/max_length": 25.0,
-      "completions/mean_length": 6.9765625,
-      "completions/min_length": 2.0,
-      "epoch": 1.8607172643869891,
-      "grad_norm": 35.77457650983362,
-      "kl": 31.698298294795677,
-      "learning_rate": 0.000119304845134973,
-      "loss": 0.27273401618003845,
-      "memory(GiB)": 79.43,
-      "reward": 0.1015625,
-      "reward_std": 0.17464229464530945,
-      "rewards/Response_no_think_reward_1/mean": 0.1015625,
-      "rewards/Response_no_think_reward_1/std": 0.6621350049972534,
+      "train_speed(iter/s)": 0.021273
+    },
+    {
+      "epoch": 0.5614743751577884,
+      "grad_norm": 3.5302765369415283,
+      "learning_rate": 0.00017234544239679806,
+      "logits/chosen": -0.7818064093589783,
+      "logits/rejected": -0.8903212547302246,
+      "logps/chosen": -1.2885193824768066,
+      "logps/rejected": -25.56497573852539,
+      "loss": 0.5648685693740845,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20888982713222504,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.252044677734375,
+      "rewards/margins": 1.763919472694397,
+      "rewards/rejected": -1.5118746757507324,
       "step": 278,
-      "train_speed(iter/s)": 0.017652
-    },
-    {
-      "clip_ratio/high_max": 0.009891506866551936,
-      "clip_ratio/high_mean": 0.009891506866551936,
-      "clip_ratio/low_mean": 0.0069444444961845875,
-      "clip_ratio/low_min": 0.0069444444961845875,
-      "clip_ratio/region_mean": 0.016835951362736523,
-      "epoch": 1.8673894912427023,
-      "grad_norm": 4.542577039630815,
-      "kl": 3.7718499208567664,
-      "learning_rate": 0.00011875993980511773,
-      "loss": 0.07798631489276886,
-      "memory(GiB)": 79.43,
+      "train_speed(iter/s)": 0.021274
+    },
+    {
+      "epoch": 0.5634940671547589,
+      "grad_norm": 1.8682705163955688,
+      "learning_rate": 0.0001721143081418601,
+      "logits/chosen": -0.6173131465911865,
+      "logits/rejected": -0.7885771989822388,
+      "logps/chosen": -0.348890095949173,
+      "logps/rejected": -61.137290954589844,
+      "loss": 0.2171340137720108,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.0689445212483406,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4386233389377594,
+      "rewards/margins": 4.034773826599121,
+      "rewards/rejected": -3.5961508750915527,
       "step": 279,
-      "train_speed(iter/s)": 0.017502
-    },
-    {
-      "clip_ratio/high_max": 0.002536526066251099,
-      "clip_ratio/high_mean": 0.002536526066251099,
-      "clip_ratio/low_mean": 0.0024038462433964014,
-      "clip_ratio/low_min": 0.0024038462433964014,
-      "clip_ratio/region_mean": 0.0049403723096475005,
-      "completions/clipped_ratio": 0.0859375,
-      "completions/max_length": 27.0,
-      "completions/mean_length": 8.1484375,
-      "completions/min_length": 2.0,
-      "epoch": 1.8740617180984154,
-      "grad_norm": 2.8417572027389006,
-      "kl": 3.5999494855445846,
-      "learning_rate": 0.00011821445651602616,
-      "loss": 0.038278549909591675,
-      "memory(GiB)": 79.43,
-      "reward": 0.078125,
-      "reward_std": 0.18506525456905365,
-      "rewards/Response_no_think_reward_1/mean": 0.078125,
-      "rewards/Response_no_think_reward_1/std": 0.7797574400901794,
+      "train_speed(iter/s)": 0.021274
+    },
+    {
+      "epoch": 0.5655137591517294,
+      "grad_norm": 1.9534425735473633,
+      "learning_rate": 0.00017188236838779295,
+      "logits/chosen": -0.6642014384269714,
+      "logits/rejected": -0.8017529249191284,
+      "logps/chosen": -1.7374509572982788,
+      "logps/rejected": -56.20750427246094,
+      "loss": 0.22880004346370697,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09635397791862488,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3913995027542114,
+      "rewards/margins": 3.391270160675049,
+      "rewards/rejected": -2.999870777130127,
       "step": 280,
-      "train_speed(iter/s)": 0.016994
+      "train_speed(iter/s)": 0.021273
+    },
+    {
+      "epoch": 0.5675334511486998,
+      "grad_norm": 6.31340217590332,
+      "learning_rate": 0.0001716496257253068,
+      "logits/chosen": -0.7913269996643066,
+      "logits/rejected": -0.9220376014709473,
+      "logps/chosen": -3.3653981685638428,
+      "logps/rejected": -47.17043685913086,
+      "loss": 0.7093809843063354,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.40024468302726746,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.22013141214847565,
+      "rewards/margins": 3.3141214847564697,
+      "rewards/rejected": -3.0939903259277344,
+      "step": 281,
+      "train_speed(iter/s)": 0.021274
+    },
+    {
+      "epoch": 0.5695531431456703,
+      "grad_norm": 2.4428505897521973,
+      "learning_rate": 0.00017141608275408006,
+      "logits/chosen": -0.8131429553031921,
+      "logits/rejected": -0.9399922490119934,
+      "logps/chosen": -1.98673415184021,
+      "logps/rejected": -42.24494552612305,
+      "loss": 0.4994111657142639,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2407137006521225,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.18601495027542114,
+      "rewards/margins": 2.6695024967193604,
+      "rewards/rejected": -2.483487367630005,
+      "step": 282,
+      "train_speed(iter/s)": 0.021274
+    },
+    {
+      "epoch": 0.5715728351426408,
+      "grad_norm": 1.5563158988952637,
+      "learning_rate": 0.0001711817420827305,
+      "logits/chosen": -0.6783189177513123,
+      "logits/rejected": -0.9088720083236694,
+      "logps/chosen": -0.37805676460266113,
+      "logps/rejected": -64.65815734863281,
+      "loss": 0.18766579031944275,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.08086632192134857,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3040025234222412,
+      "rewards/margins": 4.409641265869141,
+      "rewards/rejected": -4.10563850402832,
+      "step": 283,
+      "train_speed(iter/s)": 0.021275
+    },
+    {
+      "epoch": 0.5735925271396112,
+      "grad_norm": 7.082139015197754,
+      "learning_rate": 0.00017094660632878582,
+      "logits/chosen": -0.7040260434150696,
+      "logits/rejected": -0.7696897387504578,
+      "logps/chosen": -8.167627334594727,
+      "logps/rejected": -38.17253112792969,
+      "loss": 0.9717364311218262,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5700907111167908,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.012362787500023842,
+      "rewards/margins": 2.3398733139038086,
+      "rewards/rejected": -2.327510356903076,
+      "step": 284,
+      "train_speed(iter/s)": 0.021275
+    },
+    {
+      "epoch": 0.5756122191365817,
+      "grad_norm": 5.067822456359863,
+      "learning_rate": 0.00017071067811865476,
+      "logits/chosen": -0.802982747554779,
+      "logits/rejected": -0.8972690105438232,
+      "logps/chosen": -2.8758363723754883,
+      "logps/rejected": -46.713172912597656,
+      "loss": 0.5463993549346924,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.284506618976593,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16443192958831787,
+      "rewards/margins": 3.5550990104675293,
+      "rewards/rejected": -3.3906667232513428,
+      "step": 285,
+      "train_speed(iter/s)": 0.021275
+    },
+    {
+      "epoch": 0.5776319111335522,
+      "grad_norm": 4.077809810638428,
+      "learning_rate": 0.00017047396008759754,
+      "logits/chosen": -0.8403116464614868,
+      "logits/rejected": -0.8808307647705078,
+      "logps/chosen": -6.7314581871032715,
+      "logps/rejected": -30.9017333984375,
+      "loss": 0.6438797116279602,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3068145215511322,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.03903551399707794,
+      "rewards/margins": 1.8479769229888916,
+      "rewards/rejected": -1.887012243270874,
+      "step": 286,
+      "train_speed(iter/s)": 0.021276
+    },
+    {
+      "epoch": 0.5796516031305226,
+      "grad_norm": 2.9939074516296387,
+      "learning_rate": 0.00017023645487969645,
+      "logits/chosen": -0.7674691081047058,
+      "logits/rejected": -0.8917283415794373,
+      "logps/chosen": -3.4815213680267334,
+      "logps/rejected": -56.125099182128906,
+      "loss": 0.29649537801742554,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15909157693386078,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16410662233829498,
+      "rewards/margins": 4.206541061401367,
+      "rewards/rejected": -4.042433738708496,
+      "step": 287,
+      "train_speed(iter/s)": 0.021276
+    },
+    {
+      "epoch": 0.581671295127493,
+      "grad_norm": 12.105512619018555,
+      "learning_rate": 0.00016999816514782647,
+      "logits/chosen": -0.7554559707641602,
+      "logits/rejected": -0.8941792845726013,
+      "logps/chosen": -6.372040271759033,
+      "logps/rejected": -39.95315170288086,
+      "loss": 1.1656122207641602,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6600521206855774,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.05390087515115738,
+      "rewards/margins": 2.4357922077178955,
+      "rewards/rejected": -2.4896934032440186,
+      "step": 288,
+      "train_speed(iter/s)": 0.021277
+    },
+    {
+      "epoch": 0.5836909871244635,
+      "grad_norm": 5.215820789337158,
+      "learning_rate": 0.0001697590935536254,
+      "logits/chosen": -0.8481535911560059,
+      "logits/rejected": -0.9250099062919617,
+      "logps/chosen": -2.248663902282715,
+      "logps/rejected": -45.463008880615234,
+      "loss": 0.4629695415496826,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22682401537895203,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.0752679854631424,
+      "rewards/margins": 3.593411922454834,
+      "rewards/rejected": -3.5181431770324707,
+      "step": 289,
+      "train_speed(iter/s)": 0.021278
+    },
+    {
+      "epoch": 0.5857106791214339,
+      "grad_norm": 7.402690887451172,
+      "learning_rate": 0.00016951924276746425,
+      "logits/chosen": -0.5944700241088867,
+      "logits/rejected": -0.7739753723144531,
+      "logps/chosen": -8.305325508117676,
+      "logps/rejected": -56.55354690551758,
+      "loss": 1.023478627204895,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5631112456321716,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.024727948009967804,
+      "rewards/margins": 3.671443462371826,
+      "rewards/rejected": -3.696171522140503,
+      "step": 290,
+      "train_speed(iter/s)": 0.021277
+    },
+    {
+      "epoch": 0.5877303711184044,
+      "grad_norm": 9.028773307800293,
+      "learning_rate": 0.00016927861546841747,
+      "logits/chosen": -0.827506959438324,
+      "logits/rejected": -0.8843482136726379,
+      "logps/chosen": -3.57808256149292,
+      "logps/rejected": -27.147445678710938,
+      "loss": 0.9430906772613525,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.36645135283470154,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.07185473293066025,
+      "rewards/margins": 1.882130742073059,
+      "rewards/rejected": -1.810275912284851,
+      "step": 291,
+      "train_speed(iter/s)": 0.021278
+    },
+    {
+      "epoch": 0.5897500631153749,
+      "grad_norm": 3.0350229740142822,
+      "learning_rate": 0.00016903721434423306,
+      "logits/chosen": -0.9094471335411072,
+      "logits/rejected": -0.9361112713813782,
+      "logps/chosen": -2.4875197410583496,
+      "logps/rejected": -24.18358612060547,
+      "loss": 0.4315643012523651,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24477659165859222,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3049478530883789,
+      "rewards/margins": 2.156770706176758,
+      "rewards/rejected": -1.8518224954605103,
+      "step": 292,
+      "train_speed(iter/s)": 0.021279
+    },
+    {
+      "epoch": 0.5917697551123454,
+      "grad_norm": 4.291833877563477,
+      "learning_rate": 0.0001687950420913022,
+      "logits/chosen": -0.7475178241729736,
+      "logits/rejected": -0.8380208015441895,
+      "logps/chosen": -3.275129795074463,
+      "logps/rejected": -32.63922882080078,
+      "loss": 0.6617104411125183,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4119285047054291,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08241842687129974,
+      "rewards/margins": 2.206611394882202,
+      "rewards/rejected": -2.124192953109741,
+      "step": 293,
+      "train_speed(iter/s)": 0.02128
+    },
+    {
+      "epoch": 0.5937894471093158,
+      "grad_norm": 4.568887710571289,
+      "learning_rate": 0.00016855210141462963,
+      "logits/chosen": -0.7708112001419067,
+      "logits/rejected": -0.8631462454795837,
+      "logps/chosen": -2.731691837310791,
+      "logps/rejected": -34.596675872802734,
+      "loss": 0.5602784156799316,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3268437683582306,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.27219781279563904,
+      "rewards/margins": 2.2604424953460693,
+      "rewards/rejected": -1.9882445335388184,
+      "step": 294,
+      "train_speed(iter/s)": 0.02128
+    },
+    {
+      "epoch": 0.5958091391062863,
+      "grad_norm": 4.01273250579834,
+      "learning_rate": 0.0001683083950278031,
+      "logits/chosen": -0.8505802750587463,
+      "logits/rejected": -0.9058212637901306,
+      "logps/chosen": -1.9776384830474854,
+      "logps/rejected": -29.04343605041504,
+      "loss": 0.5075977444648743,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.26625630259513855,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16404712200164795,
+      "rewards/margins": 2.069391965866089,
+      "rewards/rejected": -1.905344843864441,
+      "step": 295,
+      "train_speed(iter/s)": 0.021282
+    },
+    {
+      "epoch": 0.5978288311032568,
+      "grad_norm": 2.308389186859131,
+      "learning_rate": 0.00016806392565296311,
+      "logits/chosen": -0.7509005665779114,
+      "logits/rejected": -0.8670397996902466,
+      "logps/chosen": -2.6714932918548584,
+      "logps/rejected": -31.49539566040039,
+      "loss": 0.4104999899864197,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14024530351161957,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3330576419830322,
+      "rewards/margins": 1.9010692834854126,
+      "rewards/rejected": -1.5680116415023804,
+      "step": 296,
+      "train_speed(iter/s)": 0.021282
+    },
+    {
+      "epoch": 0.5998485231002272,
+      "grad_norm": 7.128021717071533,
+      "learning_rate": 0.00016781869602077264,
+      "logits/chosen": -0.7379200458526611,
+      "logits/rejected": -0.8878840804100037,
+      "logps/chosen": -3.651395320892334,
+      "logps/rejected": -39.45635986328125,
+      "loss": 0.8120747208595276,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.46895888447761536,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.07825376093387604,
+      "rewards/margins": 1.777091145515442,
+      "rewards/rejected": -1.698837399482727,
+      "step": 297,
+      "train_speed(iter/s)": 0.021282
+    },
+    {
+      "epoch": 0.6018682150971977,
+      "grad_norm": 3.926884651184082,
+      "learning_rate": 0.00016757270887038654,
+      "logits/chosen": -0.7688292264938354,
+      "logits/rejected": -0.9089186191558838,
+      "logps/chosen": -1.3693759441375732,
+      "logps/rejected": -37.80522918701172,
+      "loss": 0.4208237826824188,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17390652000904083,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1809903383255005,
+      "rewards/margins": 2.1318840980529785,
+      "rewards/rejected": -1.9508938789367676,
+      "step": 298,
+      "train_speed(iter/s)": 0.021282
+    },
+    {
+      "epoch": 0.6038879070941682,
+      "grad_norm": 3.3669636249542236,
+      "learning_rate": 0.00016732596694942083,
+      "logits/chosen": -0.74614417552948,
+      "logits/rejected": -0.8711810111999512,
+      "logps/chosen": -4.000098705291748,
+      "logps/rejected": -40.62786102294922,
+      "loss": 0.5480436682701111,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23701626062393188,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.11138585954904556,
+      "rewards/margins": 1.9405393600463867,
+      "rewards/rejected": -1.8291537761688232,
+      "step": 299,
+      "train_speed(iter/s)": 0.021282
+    },
+    {
+      "epoch": 0.6059075990911386,
+      "grad_norm": 2.457587480545044,
+      "learning_rate": 0.00016707847301392236,
+      "logits/chosen": -0.8123277425765991,
+      "logits/rejected": -0.9144627451896667,
+      "logps/chosen": -4.013862133026123,
+      "logps/rejected": -33.994991302490234,
+      "loss": 0.5886133909225464,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.31008380651474,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.23923330008983612,
+      "rewards/margins": 2.1191067695617676,
+      "rewards/rejected": -1.879873514175415,
+      "step": 300,
+      "train_speed(iter/s)": 0.021282
+    },
+    {
+      "epoch": 0.6059075990911386,
+      "eval_logits/chosen": -0.8244539499282837,
+      "eval_logits/rejected": -0.9301112294197083,
+      "eval_logps/chosen": -2.4614338874816895,
+      "eval_logps/rejected": -37.376708984375,
+      "eval_loss": 0.5653835535049438,
+      "eval_nll_loss": 0.26967209577560425,
+      "eval_rewards/accuracies": 0.862500011920929,
+      "eval_rewards/chosen": 0.19960978627204895,
+      "eval_rewards/margins": 2.2363221645355225,
+      "eval_rewards/rejected": -2.036712169647217,
+      "eval_runtime": 92.5823,
+      "eval_samples_per_second": 0.864,
+      "eval_steps_per_second": 0.432,
+      "step": 300
+    },
+    {
+      "epoch": 0.6079272910881091,
+      "grad_norm": 1.4425960779190063,
+      "learning_rate": 0.00016683022982833757,
+      "logits/chosen": -0.9167453050613403,
+      "logits/rejected": -0.9786884784698486,
+      "logps/chosen": -0.2554780840873718,
+      "logps/rejected": -30.395259857177734,
+      "loss": 0.28913912177085876,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.056439101696014404,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2730258107185364,
+      "rewards/margins": 2.289043664932251,
+      "rewards/rejected": -2.0160179138183594,
+      "step": 301,
+      "train_speed(iter/s)": 0.021119
+    },
+    {
+      "epoch": 0.6099469830850796,
+      "grad_norm": 2.9968175888061523,
+      "learning_rate": 0.00016658124016548197,
+      "logits/chosen": -0.9765443801879883,
+      "logits/rejected": -1.0245364904403687,
+      "logps/chosen": -1.0650570392608643,
+      "logps/rejected": -35.4031867980957,
+      "loss": 0.43358710408210754,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18546591699123383,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2088906615972519,
+      "rewards/margins": 2.796689748764038,
+      "rewards/rejected": -2.587799310684204,
+      "step": 302,
+      "train_speed(iter/s)": 0.021121
+    },
+    {
+      "epoch": 0.6119666750820499,
+      "grad_norm": 4.542461395263672,
+      "learning_rate": 0.000166331506806509,
+      "logits/chosen": -0.9329476356506348,
+      "logits/rejected": -1.0012348890304565,
+      "logps/chosen": -2.2945432662963867,
+      "logps/rejected": -31.613216400146484,
+      "loss": 0.5403211116790771,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2446451485157013,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2515539526939392,
+      "rewards/margins": 2.3975443840026855,
+      "rewards/rejected": -2.1459906101226807,
+      "step": 303,
+      "train_speed(iter/s)": 0.021123
+    },
+    {
+      "epoch": 0.6139863670790204,
+      "grad_norm": 3.4531707763671875,
+      "learning_rate": 0.00016608103254087906,
+      "logits/chosen": -0.7963448762893677,
+      "logits/rejected": -0.9296629428863525,
+      "logps/chosen": -3.21559476852417,
+      "logps/rejected": -49.27157211303711,
+      "loss": 0.5262452363967896,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2854577302932739,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08577384054660797,
+      "rewards/margins": 3.5433404445648193,
+      "rewards/rejected": -3.457566261291504,
+      "step": 304,
+      "train_speed(iter/s)": 0.021123
+    },
+    {
+      "epoch": 0.6160060590759909,
+      "grad_norm": 5.113759994506836,
+      "learning_rate": 0.00016582982016632818,
+      "logits/chosen": -0.7916707396507263,
+      "logits/rejected": -0.934760570526123,
+      "logps/chosen": -2.8785698413848877,
+      "logps/rejected": -56.01650619506836,
+      "loss": 0.6735590100288391,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.42455944418907166,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.24037745594978333,
+      "rewards/margins": 3.957566261291504,
+      "rewards/rejected": -3.717188835144043,
+      "step": 305,
+      "train_speed(iter/s)": 0.021124
+    },
+    {
+      "epoch": 0.6180257510729614,
+      "grad_norm": 1.8782538175582886,
+      "learning_rate": 0.00016557787248883696,
+      "logits/chosen": -0.8707424402236938,
+      "logits/rejected": -0.9912842512130737,
+      "logps/chosen": -1.2852058410644531,
+      "logps/rejected": -55.1501579284668,
+      "loss": 0.26773229241371155,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11378525197505951,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.32057979702949524,
+      "rewards/margins": 4.313370227813721,
+      "rewards/rejected": -3.992790460586548,
+      "step": 306,
+      "train_speed(iter/s)": 0.021124
+    },
+    {
+      "epoch": 0.6200454430699318,
+      "grad_norm": 12.83481216430664,
+      "learning_rate": 0.00016532519232259916,
+      "logits/chosen": -0.8531877398490906,
+      "logits/rejected": -0.9290578365325928,
+      "logps/chosen": -3.8104918003082275,
+      "logps/rejected": -58.618621826171875,
+      "loss": 0.3679554760456085,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2755557596683502,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2785298526287079,
+      "rewards/margins": 4.633764743804932,
+      "rewards/rejected": -4.355234622955322,
+      "step": 307,
+      "train_speed(iter/s)": 0.021125
+    },
+    {
+      "epoch": 0.6220651350669023,
+      "grad_norm": 3.644946336746216,
+      "learning_rate": 0.00016507178248999024,
+      "logits/chosen": -0.8718377947807312,
+      "logits/rejected": -1.0232672691345215,
+      "logps/chosen": -1.1764187812805176,
+      "logps/rejected": -66.40010070800781,
+      "loss": 0.4798681139945984,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2249489575624466,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.23639193177223206,
+      "rewards/margins": 4.824877738952637,
+      "rewards/rejected": -4.5884857177734375,
+      "step": 308,
+      "train_speed(iter/s)": 0.021126
+    },
+    {
+      "epoch": 0.6240848270638728,
+      "grad_norm": 3.6663687229156494,
+      "learning_rate": 0.00016481764582153586,
+      "logits/chosen": -0.8408634066581726,
+      "logits/rejected": -0.9267845153808594,
+      "logps/chosen": -4.433281898498535,
+      "logps/rejected": -32.701011657714844,
+      "loss": 0.47992053627967834,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.21022652089595795,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.5157691240310669,
+      "rewards/margins": 2.2275538444519043,
+      "rewards/rejected": -1.7117843627929688,
+      "step": 309,
+      "train_speed(iter/s)": 0.021127
+    },
+    {
+      "epoch": 0.6261045190608432,
+      "grad_norm": 8.538496017456055,
+      "learning_rate": 0.00016456278515588024,
+      "logits/chosen": -0.8827572464942932,
+      "logits/rejected": -0.9595881700515747,
+      "logps/chosen": -5.372483253479004,
+      "logps/rejected": -43.774436950683594,
+      "loss": 0.74099200963974,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4238053560256958,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08053889870643616,
+      "rewards/margins": 2.704007148742676,
+      "rewards/rejected": -2.6234686374664307,
+      "step": 310,
+      "train_speed(iter/s)": 0.021128
+    },
+    {
+      "epoch": 0.6281242110578137,
+      "grad_norm": 3.513566493988037,
+      "learning_rate": 0.00016430720333975452,
+      "logits/chosen": -0.8878616094589233,
+      "logits/rejected": -0.9629725217819214,
+      "logps/chosen": -2.4065513610839844,
+      "logps/rejected": -43.71923065185547,
+      "loss": 0.4641585052013397,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2343730330467224,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1930648535490036,
+      "rewards/margins": 3.418928623199463,
+      "rewards/rejected": -3.2258636951446533,
+      "step": 311,
+      "train_speed(iter/s)": 0.021129
+    },
+    {
+      "epoch": 0.6301439030547842,
+      "grad_norm": 6.264689922332764,
+      "learning_rate": 0.00016405090322794483,
+      "logits/chosen": -0.9043357968330383,
+      "logits/rejected": -0.9985758066177368,
+      "logps/chosen": -2.7129530906677246,
+      "logps/rejected": -41.665489196777344,
+      "loss": 0.6599389910697937,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3321758210659027,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.17053501307964325,
+      "rewards/margins": 2.564326047897339,
+      "rewards/rejected": -2.3937911987304688,
+      "step": 312,
+      "train_speed(iter/s)": 0.021131
+    },
+    {
+      "epoch": 0.6321635950517546,
+      "grad_norm": 22.02393913269043,
+      "learning_rate": 0.00016379388768326063,
+      "logits/chosen": -0.8338424563407898,
+      "logits/rejected": -0.9313662052154541,
+      "logps/chosen": -4.164543151855469,
+      "logps/rejected": -50.46995544433594,
+      "loss": 1.0124552249908447,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6529191732406616,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.016213715076446533,
+      "rewards/margins": 3.471205949783325,
+      "rewards/rejected": -3.4549922943115234,
+      "step": 313,
+      "train_speed(iter/s)": 0.021131
+    },
+    {
+      "epoch": 0.6341832870487251,
+      "grad_norm": 10.904000282287598,
+      "learning_rate": 0.00016353615957650236,
+      "logits/chosen": -0.7540192008018494,
+      "logits/rejected": -0.8433898687362671,
+      "logps/chosen": -10.264131546020508,
+      "logps/rejected": -41.65949630737305,
+      "loss": 0.5326629877090454,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.239348366856575,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.253826379776001,
+      "rewards/margins": 2.7431883811950684,
+      "rewards/rejected": -2.4893617630004883,
+      "step": 314,
+      "train_speed(iter/s)": 0.021131
+    },
+    {
+      "epoch": 0.6362029790456956,
+      "grad_norm": 3.1705944538116455,
+      "learning_rate": 0.00016327772178642986,
+      "logits/chosen": -0.9517133831977844,
+      "logits/rejected": -1.0139639377593994,
+      "logps/chosen": -0.973305344581604,
+      "logps/rejected": -38.540000915527344,
+      "loss": 0.3936949670314789,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1697213351726532,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.20970085263252258,
+      "rewards/margins": 3.3330564498901367,
+      "rewards/rejected": -3.1233556270599365,
+      "step": 315,
+      "train_speed(iter/s)": 0.021133
+    },
+    {
+      "epoch": 0.638222671042666,
+      "grad_norm": 5.624232769012451,
+      "learning_rate": 0.00016301857719972976,
+      "logits/chosen": -0.8001008629798889,
+      "logits/rejected": -0.8618478775024414,
+      "logps/chosen": -1.4323068857192993,
+      "logps/rejected": -35.56591033935547,
+      "loss": 0.4376400411128998,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14159713685512543,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.20484629273414612,
+      "rewards/margins": 2.43072247505188,
+      "rewards/rejected": -2.2258763313293457,
+      "step": 316,
+      "train_speed(iter/s)": 0.021134
+    },
+    {
+      "epoch": 0.6402423630396364,
+      "grad_norm": 7.583630561828613,
+      "learning_rate": 0.0001627587287109836,
+      "logits/chosen": -0.8299413919448853,
+      "logits/rejected": -0.9769677519798279,
+      "logps/chosen": -3.7215254306793213,
+      "logps/rejected": -60.84526062011719,
+      "loss": 0.5378049612045288,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.32689476013183594,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.025372333824634552,
+      "rewards/margins": 4.722233295440674,
+      "rewards/rejected": -4.6968607902526855,
+      "step": 317,
+      "train_speed(iter/s)": 0.021135
+    },
+    {
+      "epoch": 0.6422620550366069,
+      "grad_norm": 8.251932144165039,
+      "learning_rate": 0.00016249817922263517,
+      "logits/chosen": -0.8568125367164612,
+      "logits/rejected": -0.931611955165863,
+      "logps/chosen": -5.391039848327637,
+      "logps/rejected": -45.42852020263672,
+      "loss": 0.5377641916275024,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.29841428995132446,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.31022340059280396,
+      "rewards/margins": 3.579132080078125,
+      "rewards/rejected": -3.268908977508545,
+      "step": 318,
+      "train_speed(iter/s)": 0.021135
+    },
+    {
+      "epoch": 0.6442817470335774,
+      "grad_norm": 4.026428699493408,
+      "learning_rate": 0.00016223693164495835,
+      "logits/chosen": -0.77679044008255,
+      "logits/rejected": -0.8818118572235107,
+      "logps/chosen": -5.223626136779785,
+      "logps/rejected": -46.39476013183594,
+      "loss": 0.5932655930519104,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.30503058433532715,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.009123122319579124,
+      "rewards/margins": 3.3854122161865234,
+      "rewards/rejected": -3.3945353031158447,
+      "step": 319,
+      "train_speed(iter/s)": 0.021136
+    },
+    {
+      "epoch": 0.6463014390305478,
+      "grad_norm": 3.859316110610962,
+      "learning_rate": 0.00016197498889602448,
+      "logits/chosen": -0.9226962924003601,
+      "logits/rejected": -0.984259843826294,
+      "logps/chosen": -0.3310558497905731,
+      "logps/rejected": -39.15312576293945,
+      "loss": 0.21047677099704742,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.053509388118982315,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.37339526414871216,
+      "rewards/margins": 3.054718494415283,
+      "rewards/rejected": -2.681323289871216,
+      "step": 320,
+      "train_speed(iter/s)": 0.021137
+    },
+    {
+      "epoch": 0.6483211310275183,
+      "grad_norm": 4.3604888916015625,
+      "learning_rate": 0.00016171235390166985,
+      "logits/chosen": -0.9235320687294006,
+      "logits/rejected": -0.9902734160423279,
+      "logps/chosen": -2.538111448287964,
+      "logps/rejected": -48.80242156982422,
+      "loss": 0.5195378661155701,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23220032453536987,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.17571590840816498,
+      "rewards/margins": 3.9404807090759277,
+      "rewards/rejected": -3.7647647857666016,
+      "step": 321,
+      "train_speed(iter/s)": 0.021138
+    },
+    {
+      "epoch": 0.6503408230244888,
+      "grad_norm": 9.97143268585205,
+      "learning_rate": 0.00016144902959546286,
+      "logits/chosen": -0.9121385216712952,
+      "logits/rejected": -0.9658838510513306,
+      "logps/chosen": -5.540125846862793,
+      "logps/rejected": -45.613990783691406,
+      "loss": 0.8853093385696411,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.35688427090644836,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.05821647495031357,
+      "rewards/margins": 3.1649882793426514,
+      "rewards/rejected": -3.2232048511505127,
+      "step": 322,
+      "train_speed(iter/s)": 0.021139
+    },
+    {
+      "epoch": 0.6523605150214592,
+      "grad_norm": 4.8883819580078125,
+      "learning_rate": 0.0001611850189186714,
+      "logits/chosen": -0.9210085868835449,
+      "logits/rejected": -0.9733687043190002,
+      "logps/chosen": -4.037269592285156,
+      "logps/rejected": -41.21742630004883,
+      "loss": 0.5509124398231506,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.32746627926826477,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.21709240972995758,
+      "rewards/margins": 3.416368246078491,
+      "rewards/rejected": -3.199275493621826,
+      "step": 323,
+      "train_speed(iter/s)": 0.02114
+    },
+    {
+      "epoch": 0.6543802070184297,
+      "grad_norm": 2.4553871154785156,
+      "learning_rate": 0.00016092032482023,
+      "logits/chosen": -0.9412962794303894,
+      "logits/rejected": -1.0209475755691528,
+      "logps/chosen": -0.632290244102478,
+      "logps/rejected": -33.6468505859375,
+      "loss": 0.23645614087581635,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.07834430038928986,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.43455106019973755,
+      "rewards/margins": 3.022073745727539,
+      "rewards/rejected": -2.587522506713867,
+      "step": 324,
+      "train_speed(iter/s)": 0.021142
+    },
+    {
+      "epoch": 0.6563998990154002,
+      "grad_norm": 2.446058511734009,
+      "learning_rate": 0.00016065495025670675,
+      "logits/chosen": -0.8765738606452942,
+      "logits/rejected": -0.9390391111373901,
+      "logps/chosen": -5.492425918579102,
+      "logps/rejected": -34.9351692199707,
+      "loss": 0.5046104788780212,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.34279099106788635,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.22330977022647858,
+      "rewards/margins": 2.7527146339416504,
+      "rewards/rejected": -2.529404878616333,
+      "step": 325,
+      "train_speed(iter/s)": 0.021143
+    },
+    {
+      "epoch": 0.6584195910123706,
+      "grad_norm": 18.5092716217041,
+      "learning_rate": 0.00016038889819227045,
+      "logits/chosen": -0.8468933701515198,
+      "logits/rejected": -0.9487016201019287,
+      "logps/chosen": -1.3551874160766602,
+      "logps/rejected": -38.7204475402832,
+      "loss": 0.4489996135234833,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2548639476299286,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3463600277900696,
+      "rewards/margins": 2.8273019790649414,
+      "rewards/rejected": -2.4809420108795166,
+      "step": 326,
+      "train_speed(iter/s)": 0.021144
+    },
+    {
+      "epoch": 0.6604392830093411,
+      "grad_norm": 6.140425682067871,
+      "learning_rate": 0.00016012217159865739,
+      "logits/chosen": -0.8302001357078552,
+      "logits/rejected": -0.924152135848999,
+      "logps/chosen": -3.6062324047088623,
+      "logps/rejected": -56.65538787841797,
+      "loss": 0.784902811050415,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3868030309677124,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.006980091333389282,
+      "rewards/margins": 3.1850905418395996,
+      "rewards/rejected": -3.178110122680664,
+      "step": 327,
+      "train_speed(iter/s)": 0.021144
+    },
+    {
+      "epoch": 0.6624589750063116,
+      "grad_norm": 2.915184259414673,
+      "learning_rate": 0.00015985477345513817,
+      "logits/chosen": -0.7988994121551514,
+      "logits/rejected": -0.9357851147651672,
+      "logps/chosen": -5.732859134674072,
+      "logps/rejected": -39.42655944824219,
+      "loss": 0.347720742225647,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.16449037194252014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.23062925040721893,
+      "rewards/margins": 2.8876638412475586,
+      "rewards/rejected": -2.6570348739624023,
+      "step": 328,
+      "train_speed(iter/s)": 0.021145
+    },
+    {
+      "epoch": 0.664478667003282,
+      "grad_norm": 7.8771162033081055,
+      "learning_rate": 0.00015958670674848442,
+      "logits/chosen": -0.857587456703186,
+      "logits/rejected": -0.9888629913330078,
+      "logps/chosen": -6.550410270690918,
+      "logps/rejected": -41.466896057128906,
+      "loss": 0.709037184715271,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.40757060050964355,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.43693459033966064,
+      "rewards/margins": 2.189579486846924,
+      "rewards/rejected": -1.7526450157165527,
+      "step": 329,
+      "train_speed(iter/s)": 0.021145
+    },
+    {
+      "epoch": 0.6664983590002524,
+      "grad_norm": 19.00602149963379,
+      "learning_rate": 0.00015931797447293552,
+      "logits/chosen": -0.8882402181625366,
+      "logits/rejected": -0.9668765068054199,
+      "logps/chosen": -3.752054214477539,
+      "logps/rejected": -34.47447204589844,
+      "loss": 0.6838876605033875,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4321058392524719,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.22359441220760345,
+      "rewards/margins": 2.5547034740448,
+      "rewards/rejected": -2.331109046936035,
+      "step": 330,
+      "train_speed(iter/s)": 0.021146
+    },
+    {
+      "epoch": 0.668518050997223,
+      "grad_norm": 1.7641230821609497,
+      "learning_rate": 0.00015904857963016506,
+      "logits/chosen": -0.7825491428375244,
+      "logits/rejected": -0.9280312061309814,
+      "logps/chosen": -0.6239898204803467,
+      "logps/rejected": -43.15408706665039,
+      "loss": 0.24098820984363556,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09237208962440491,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2947159707546234,
+      "rewards/margins": 3.279745578765869,
+      "rewards/rejected": -2.985029697418213,
+      "step": 331,
+      "train_speed(iter/s)": 0.021146
+    },
+    {
+      "epoch": 0.6705377429941933,
+      "grad_norm": 2.2421019077301025,
+      "learning_rate": 0.00015877852522924732,
+      "logits/chosen": -0.8130238652229309,
+      "logits/rejected": -0.9327349662780762,
+      "logps/chosen": -0.8687487244606018,
+      "logps/rejected": -47.515663146972656,
+      "loss": 0.2235592156648636,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09912680834531784,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2986675500869751,
+      "rewards/margins": 3.4509923458099365,
+      "rewards/rejected": -3.152324914932251,
+      "step": 332,
+      "train_speed(iter/s)": 0.021147
+    },
+    {
+      "epoch": 0.6725574349911638,
+      "grad_norm": 8.463959693908691,
+      "learning_rate": 0.0001585078142866237,
+      "logits/chosen": -0.8485694527626038,
+      "logits/rejected": -0.9816790819168091,
+      "logps/chosen": -4.081976890563965,
+      "logps/rejected": -34.49391174316406,
+      "loss": 0.8111600279808044,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5827361941337585,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.010378856211900711,
+      "rewards/margins": 2.1897881031036377,
+      "rewards/rejected": -2.1794090270996094,
+      "step": 333,
+      "train_speed(iter/s)": 0.021149
+    },
+    {
+      "epoch": 0.6745771269881343,
+      "grad_norm": 3.4926505088806152,
+      "learning_rate": 0.00015823644982606905,
+      "logits/chosen": -0.9417222142219543,
+      "logits/rejected": -1.0309451818466187,
+      "logps/chosen": -0.7485335469245911,
+      "logps/rejected": -33.3504524230957,
+      "loss": 0.307941198348999,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09651970863342285,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.388710618019104,
+      "rewards/margins": 2.98738956451416,
+      "rewards/rejected": -2.5986788272857666,
+      "step": 334,
+      "train_speed(iter/s)": 0.02115
+    },
+    {
+      "epoch": 0.6765968189851048,
+      "grad_norm": 10.243365287780762,
+      "learning_rate": 0.00015796443487865776,
+      "logits/chosen": -0.909482479095459,
+      "logits/rejected": -0.9898862838745117,
+      "logps/chosen": -6.335696220397949,
+      "logps/rejected": -46.39136505126953,
+      "loss": 0.6261657476425171,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.43729183077812195,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.27775925397872925,
+      "rewards/margins": 3.944711446762085,
+      "rewards/rejected": -3.666951894760132,
+      "step": 335,
+      "train_speed(iter/s)": 0.021151
+    },
+    {
+      "epoch": 0.6786165109820752,
+      "grad_norm": 7.904644012451172,
+      "learning_rate": 0.00015769177248273008,
+      "logits/chosen": -0.9360239505767822,
+      "logits/rejected": -1.049852967262268,
+      "logps/chosen": -7.70269250869751,
+      "logps/rejected": -51.02676773071289,
+      "loss": 0.8781910538673401,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5348291397094727,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.07586444914340973,
+      "rewards/margins": 3.814580202102661,
+      "rewards/rejected": -3.738715648651123,
+      "step": 336,
+      "train_speed(iter/s)": 0.021152
+    },
+    {
+      "epoch": 0.6806362029790457,
+      "grad_norm": 16.91658592224121,
+      "learning_rate": 0.00015741846568385808,
+      "logits/chosen": -0.8461523652076721,
+      "logits/rejected": -1.0379836559295654,
+      "logps/chosen": -8.801255226135254,
+      "logps/rejected": -61.471771240234375,
+      "loss": 1.1453731060028076,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7435779571533203,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.463644415140152,
+      "rewards/margins": 4.270198345184326,
+      "rewards/rejected": -4.733842849731445,
+      "step": 337,
+      "train_speed(iter/s)": 0.021152
+    },
+    {
+      "epoch": 0.6826558949760162,
+      "grad_norm": 12.933732986450195,
+      "learning_rate": 0.00015714451753481168,
+      "logits/chosen": -0.6461415886878967,
+      "logits/rejected": -0.9560658931732178,
+      "logps/chosen": -6.069940567016602,
+      "logps/rejected": -53.62686538696289,
+      "loss": 1.2495336532592773,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.8185646533966064,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.3106077015399933,
+      "rewards/margins": 3.0313639640808105,
+      "rewards/rejected": -3.3419713973999023,
+      "step": 338,
+      "train_speed(iter/s)": 0.021152
+    },
+    {
+      "epoch": 0.6846755869729866,
+      "grad_norm": 6.386441230773926,
+      "learning_rate": 0.00015686993109552443,
+      "logits/chosen": -0.9001967906951904,
+      "logits/rejected": -1.0288763046264648,
+      "logps/chosen": -4.8095598220825195,
+      "logps/rejected": -63.7664909362793,
+      "loss": 0.6220850348472595,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.42656922340393066,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.07018661499023438,
+      "rewards/margins": 4.841952323913574,
+      "rewards/rejected": -4.77176570892334,
+      "step": 339,
+      "train_speed(iter/s)": 0.021153
+    },
+    {
+      "epoch": 0.6866952789699571,
+      "grad_norm": 5.94568395614624,
+      "learning_rate": 0.00015659470943305955,
+      "logits/chosen": -0.8113903999328613,
+      "logits/rejected": -1.0205042362213135,
+      "logps/chosen": -3.7077159881591797,
+      "logps/rejected": -56.63233947753906,
+      "loss": 0.6539210081100464,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4288255274295807,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.059340257197618484,
+      "rewards/margins": 4.157015800476074,
+      "rewards/rejected": -4.097675323486328,
+      "step": 340,
+      "train_speed(iter/s)": 0.021154
+    },
+    {
+      "epoch": 0.6887149709669276,
+      "grad_norm": 3.9790964126586914,
+      "learning_rate": 0.00015631885562157543,
+      "logits/chosen": -0.7372142672538757,
+      "logits/rejected": -0.9902328848838806,
+      "logps/chosen": -4.715322017669678,
+      "logps/rejected": -48.823387145996094,
+      "loss": 0.6435255408287048,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3544943630695343,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.05755168944597244,
+      "rewards/margins": 2.823580741882324,
+      "rewards/rejected": -2.766029119491577,
+      "step": 341,
+      "train_speed(iter/s)": 0.021153
+    },
+    {
+      "epoch": 0.690734662963898,
+      "grad_norm": 3.6011016368865967,
+      "learning_rate": 0.00015604237274229147,
+      "logits/chosen": -0.9505451321601868,
+      "logits/rejected": -1.0710150003433228,
+      "logps/chosen": -1.6771836280822754,
+      "logps/rejected": -42.753135681152344,
+      "loss": 0.3884910047054291,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2611246109008789,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.41318997740745544,
+      "rewards/margins": 3.458676338195801,
+      "rewards/rejected": -3.0454862117767334,
+      "step": 342,
+      "train_speed(iter/s)": 0.021155
+    },
+    {
+      "epoch": 0.6927543549608685,
+      "grad_norm": 10.275211334228516,
+      "learning_rate": 0.00015576526388345367,
+      "logits/chosen": -0.9035798907279968,
+      "logits/rejected": -1.0091338157653809,
+      "logps/chosen": -4.990538597106934,
+      "logps/rejected": -41.765995025634766,
+      "loss": 0.998170018196106,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7295805215835571,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.13218021392822266,
+      "rewards/margins": 3.011774778366089,
+      "rewards/rejected": -2.879594326019287,
+      "step": 343,
+      "train_speed(iter/s)": 0.021155
+    },
+    {
+      "epoch": 0.694774046957839,
+      "grad_norm": 8.156365394592285,
+      "learning_rate": 0.0001554875321402999,
+      "logits/chosen": -0.8031080961227417,
+      "logits/rejected": -0.9648129940032959,
+      "logps/chosen": -2.28568696975708,
+      "logps/rejected": -35.34866714477539,
+      "loss": 0.5425023436546326,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24677146971225739,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.23254749178886414,
+      "rewards/margins": 2.27335262298584,
+      "rewards/rejected": -2.0408051013946533,
+      "step": 344,
+      "train_speed(iter/s)": 0.021156
+    },
+    {
+      "epoch": 0.6967937389548093,
+      "grad_norm": 4.11509370803833,
+      "learning_rate": 0.00015520918061502569,
+      "logits/chosen": -0.8795959949493408,
+      "logits/rejected": -1.0246540307998657,
+      "logps/chosen": -1.0050594806671143,
+      "logps/rejected": -38.232444763183594,
+      "loss": 0.32429325580596924,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1644802689552307,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3108518719673157,
+      "rewards/margins": 2.897658109664917,
+      "rewards/rejected": -2.586806297302246,
+      "step": 345,
+      "train_speed(iter/s)": 0.021157
+    },
+    {
+      "epoch": 0.6988134309517798,
+      "grad_norm": 3.702918529510498,
+      "learning_rate": 0.00015493021241674918,
+      "logits/chosen": -0.8280074000358582,
+      "logits/rejected": -0.9851573705673218,
+      "logps/chosen": -1.3124903440475464,
+      "logps/rejected": -35.455692291259766,
+      "loss": 0.46161431074142456,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20427490770816803,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2279079556465149,
+      "rewards/margins": 2.2844018936157227,
+      "rewards/rejected": -2.0564939975738525,
+      "step": 346,
+      "train_speed(iter/s)": 0.021158
+    },
+    {
+      "epoch": 0.7008331229487503,
+      "grad_norm": 7.40757417678833,
+      "learning_rate": 0.0001546506306614768,
+      "logits/chosen": -0.8412176966667175,
+      "logits/rejected": -0.9579489231109619,
+      "logps/chosen": -6.378708839416504,
+      "logps/rejected": -44.914512634277344,
+      "loss": 0.6551076173782349,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3504900634288788,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.00600462406873703,
+      "rewards/margins": 3.1142947673797607,
+      "rewards/rejected": -3.1202995777130127,
+      "step": 347,
+      "train_speed(iter/s)": 0.021159
+    },
+    {
+      "epoch": 0.7028528149457208,
+      "grad_norm": 2.9458255767822266,
+      "learning_rate": 0.0001543704384720681,
+      "logits/chosen": -0.7899777889251709,
+      "logits/rejected": -0.9945221543312073,
+      "logps/chosen": -1.4169785976409912,
+      "logps/rejected": -50.57292556762695,
+      "loss": 0.3628111779689789,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15231113135814667,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.226759135723114,
+      "rewards/margins": 3.434785842895508,
+      "rewards/rejected": -3.208026647567749,
+      "step": 348,
+      "train_speed(iter/s)": 0.021159
+    },
+    {
+      "epoch": 0.7048725069426912,
+      "grad_norm": 4.447574615478516,
+      "learning_rate": 0.00015408963897820113,
+      "logits/chosen": -0.8848673105239868,
+      "logits/rejected": -0.8949815034866333,
+      "logps/chosen": -9.218939781188965,
+      "logps/rejected": -32.96113967895508,
+      "loss": 0.6976296901702881,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.31593263149261475,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1799672544002533,
+      "rewards/margins": 2.5064430236816406,
+      "rewards/rejected": -2.3264756202697754,
+      "step": 349,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 0.7068921989396617,
+      "grad_norm": 3.5722556114196777,
+      "learning_rate": 0.00015380823531633729,
+      "logits/chosen": -0.9537544250488281,
+      "logits/rejected": -1.0240473747253418,
+      "logps/chosen": -3.9818813800811768,
+      "logps/rejected": -31.99053382873535,
+      "loss": 0.6191223859786987,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.34284406900405884,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.09681667387485504,
+      "rewards/margins": 2.406569480895996,
+      "rewards/rejected": -2.309752941131592,
+      "step": 350,
+      "train_speed(iter/s)": 0.021161
+    },
+    {
+      "epoch": 0.7089118909366322,
+      "grad_norm": 3.7325596809387207,
+      "learning_rate": 0.00015352623062968648,
+      "logits/chosen": -0.922427773475647,
+      "logits/rejected": -0.9690065383911133,
+      "logps/chosen": -4.305528163909912,
+      "logps/rejected": -23.71919059753418,
+      "loss": 0.7693547010421753,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.33499932289123535,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.06479213386774063,
+      "rewards/margins": 1.370113730430603,
+      "rewards/rejected": -1.3053216934204102,
+      "step": 351,
+      "train_speed(iter/s)": 0.021139
+    },
+    {
+      "epoch": 0.7109315829336026,
+      "grad_norm": 16.7460880279541,
+      "learning_rate": 0.00015324362806817186,
+      "logits/chosen": -0.7807352542877197,
+      "logits/rejected": -0.9729360342025757,
+      "logps/chosen": -7.726840972900391,
+      "logps/rejected": -53.6373176574707,
+      "loss": 1.1061019897460938,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6384282112121582,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.23256784677505493,
+      "rewards/margins": 2.8845200538635254,
+      "rewards/rejected": -3.1170878410339355,
+      "step": 352,
+      "train_speed(iter/s)": 0.021139
+    },
+    {
+      "epoch": 0.7129512749305731,
+      "grad_norm": 2.80759596824646,
+      "learning_rate": 0.00015296043078839473,
+      "logits/chosen": -0.7530017495155334,
+      "logits/rejected": -0.9337269067764282,
+      "logps/chosen": -1.4654000997543335,
+      "logps/rejected": -55.76970672607422,
+      "loss": 0.254292368888855,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13821597397327423,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.28305867314338684,
+      "rewards/margins": 4.615407943725586,
+      "rewards/rejected": -4.3323493003845215,
+      "step": 353,
+      "train_speed(iter/s)": 0.02114
+    },
+    {
+      "epoch": 0.7149709669275436,
+      "grad_norm": 4.502577781677246,
+      "learning_rate": 0.00015267664195359917,
+      "logits/chosen": -0.9706122279167175,
+      "logits/rejected": -1.0545164346694946,
+      "logps/chosen": -1.6742087602615356,
+      "logps/rejected": -41.54838180541992,
+      "loss": 0.518170177936554,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2445138841867447,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.10661309212446213,
+      "rewards/margins": 3.0441462993621826,
+      "rewards/rejected": -2.937532901763916,
+      "step": 354,
+      "train_speed(iter/s)": 0.021141
+    },
+    {
+      "epoch": 0.716990658924514,
+      "grad_norm": 5.956605434417725,
+      "learning_rate": 0.00015239226473363687,
+      "logits/chosen": -0.9984610080718994,
+      "logits/rejected": -1.0658961534500122,
+      "logps/chosen": -2.667590379714966,
+      "logps/rejected": -30.01835823059082,
+      "loss": 0.6159235239028931,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.25425097346305847,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.13858197629451752,
+      "rewards/margins": 2.178638219833374,
+      "rewards/rejected": -2.0400562286376953,
+      "step": 355,
+      "train_speed(iter/s)": 0.021142
+    },
+    {
+      "epoch": 0.7190103509214845,
+      "grad_norm": 2.670485734939575,
+      "learning_rate": 0.00015210730230493162,
+      "logits/chosen": -0.8936488628387451,
+      "logits/rejected": -1.0093450546264648,
+      "logps/chosen": -3.5686304569244385,
+      "logps/rejected": -55.824180603027344,
+      "loss": 0.3414130210876465,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20404773950576782,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.14025713503360748,
+      "rewards/margins": 3.9673044681549072,
+      "rewards/rejected": -3.827047109603882,
+      "step": 356,
+      "train_speed(iter/s)": 0.021142
+    },
+    {
+      "epoch": 0.721030042918455,
+      "grad_norm": 4.567800521850586,
+      "learning_rate": 0.00015182175785044387,
+      "logits/chosen": -0.981977105140686,
+      "logits/rejected": -0.96744304895401,
+      "logps/chosen": -7.694738388061523,
+      "logps/rejected": -33.95967483520508,
+      "loss": 0.5043447613716125,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24890656769275665,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2846420407295227,
+      "rewards/margins": 2.429332733154297,
+      "rewards/rejected": -2.144690752029419,
+      "step": 357,
+      "train_speed(iter/s)": 0.021143
+    },
+    {
+      "epoch": 0.7230497349154253,
+      "grad_norm": 8.642942428588867,
+      "learning_rate": 0.00015153563455963499,
+      "logits/chosen": -0.9537326097488403,
+      "logits/rejected": -1.1215823888778687,
+      "logps/chosen": -2.9258816242218018,
+      "logps/rejected": -48.52583312988281,
+      "loss": 0.7397590279579163,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4563119411468506,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.18413014709949493,
+      "rewards/margins": 4.132811069488525,
+      "rewards/rejected": -3.9486806392669678,
+      "step": 358,
+      "train_speed(iter/s)": 0.021144
+    },
+    {
+      "epoch": 0.7250694269123958,
+      "grad_norm": 24.88861083984375,
+      "learning_rate": 0.00015124893562843208,
+      "logits/chosen": -0.9031403660774231,
+      "logits/rejected": -0.9935811758041382,
+      "logps/chosen": -8.023815155029297,
+      "logps/rejected": -40.717247009277344,
+      "loss": 0.8925535082817078,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5370101928710938,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3610239028930664,
+      "rewards/margins": 3.118933916091919,
+      "rewards/rejected": -2.7579100131988525,
+      "step": 359,
+      "train_speed(iter/s)": 0.021144
+    },
+    {
+      "epoch": 0.7270891189093663,
+      "grad_norm": 10.635350227355957,
+      "learning_rate": 0.00015096166425919175,
+      "logits/chosen": -0.9903375506401062,
+      "logits/rejected": -1.0812005996704102,
+      "logps/chosen": -3.357611656188965,
+      "logps/rejected": -47.11662292480469,
+      "loss": 0.7105602025985718,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3658413290977478,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.13873916864395142,
+      "rewards/margins": 3.40531063079834,
+      "rewards/rejected": -3.266571283340454,
+      "step": 360,
+      "train_speed(iter/s)": 0.021145
+    },
+    {
+      "epoch": 0.7291088109063368,
+      "grad_norm": 18.06879234313965,
+      "learning_rate": 0.0001506738236606648,
+      "logits/chosen": -1.099637508392334,
+      "logits/rejected": -1.1945291757583618,
+      "logps/chosen": -8.104093551635742,
+      "logps/rejected": -45.32279968261719,
+      "loss": 1.623255729675293,
+      "memory(GiB)": 45.64,
+      "nll_loss": 1.006649374961853,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.31839191913604736,
+      "rewards/margins": 3.014329433441162,
+      "rewards/rejected": -3.332721471786499,
+      "step": 361,
+      "train_speed(iter/s)": 0.021146
+    },
+    {
+      "epoch": 0.7311285029033072,
+      "grad_norm": 15.128271102905273,
+      "learning_rate": 0.00015038541704796003,
+      "logits/chosen": -1.1882942914962769,
+      "logits/rejected": -1.2618598937988281,
+      "logps/chosen": -1.8502236604690552,
+      "logps/rejected": -69.4446792602539,
+      "loss": 0.38569730520248413,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24810141324996948,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1845402717590332,
+      "rewards/margins": 6.129173278808594,
+      "rewards/rejected": -5.944633483886719,
+      "step": 362,
+      "train_speed(iter/s)": 0.021148
+    },
+    {
+      "epoch": 0.7331481949002777,
+      "grad_norm": 5.277956485748291,
+      "learning_rate": 0.00015009644764250863,
+      "logits/chosen": -1.163029432296753,
+      "logits/rejected": -1.2435169219970703,
+      "logps/chosen": -5.300413608551025,
+      "logps/rejected": -73.11473846435547,
+      "loss": 0.8010823130607605,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5916219353675842,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.007427714765071869,
+      "rewards/margins": 5.836453914642334,
+      "rewards/rejected": -5.829026222229004,
+      "step": 363,
+      "train_speed(iter/s)": 0.021148
+    },
+    {
+      "epoch": 0.7351678868972482,
+      "grad_norm": 75.7375717163086,
+      "learning_rate": 0.0001498069186720279,
+      "logits/chosen": -1.1564539670944214,
+      "logits/rejected": -1.1993024349212646,
+      "logps/chosen": -2.832427978515625,
+      "logps/rejected": -73.21650695800781,
+      "loss": 0.8798462748527527,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4809403121471405,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.17453742027282715,
+      "rewards/margins": 5.794814109802246,
+      "rewards/rejected": -5.620276927947998,
+      "step": 364,
+      "train_speed(iter/s)": 0.021149
+    },
+    {
+      "epoch": 0.7371875788942186,
+      "grad_norm": 2.991504669189453,
+      "learning_rate": 0.00014951683337048537,
+      "logits/chosen": -1.1292376518249512,
+      "logits/rejected": -1.1874810457229614,
+      "logps/chosen": -6.187487602233887,
+      "logps/rejected": -55.482765197753906,
+      "loss": 0.45134469866752625,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22427882254123688,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3069310486316681,
+      "rewards/margins": 4.149868488311768,
+      "rewards/rejected": -3.842937707901001,
+      "step": 365,
+      "train_speed(iter/s)": 0.02115
+    },
+    {
+      "epoch": 0.7392072708911891,
+      "grad_norm": 10.73784065246582,
+      "learning_rate": 0.00014922619497806277,
+      "logits/chosen": -1.167018175125122,
+      "logits/rejected": -1.2153444290161133,
+      "logps/chosen": -1.954573631286621,
+      "logps/rejected": -61.2001953125,
+      "loss": 0.5314586162567139,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2992059588432312,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.14029161632061005,
+      "rewards/margins": 4.6798810958862305,
+      "rewards/rejected": -4.539588928222656,
+      "step": 366,
+      "train_speed(iter/s)": 0.02115
+    },
+    {
+      "epoch": 0.7412269628881596,
+      "grad_norm": 5.413460731506348,
+      "learning_rate": 0.0001489350067411196,
+      "logits/chosen": -1.1709797382354736,
+      "logits/rejected": -1.1748571395874023,
+      "logps/chosen": -7.91160774230957,
+      "logps/rejected": -46.791404724121094,
+      "loss": 0.5436979532241821,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.30416107177734375,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3512665033340454,
+      "rewards/margins": 3.6528432369232178,
+      "rewards/rejected": -3.301577091217041,
+      "step": 367,
+      "train_speed(iter/s)": 0.021151
+    },
+    {
+      "epoch": 0.74324665488513,
+      "grad_norm": 7.643467903137207,
+      "learning_rate": 0.00014864327191215702,
+      "logits/chosen": -1.1821328401565552,
+      "logits/rejected": -1.2253855466842651,
+      "logps/chosen": -2.3498167991638184,
+      "logps/rejected": -73.46287536621094,
+      "loss": 0.664942741394043,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.454881489276886,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1444016695022583,
+      "rewards/margins": 5.324404716491699,
+      "rewards/rejected": -5.1800031661987305,
+      "step": 368,
+      "train_speed(iter/s)": 0.021151
+    },
+    {
+      "epoch": 0.7452663468821005,
+      "grad_norm": 3.926517963409424,
+      "learning_rate": 0.00014835099374978147,
+      "logits/chosen": -1.1555640697479248,
+      "logits/rejected": -1.2078311443328857,
+      "logps/chosen": -0.9639624953269958,
+      "logps/rejected": -67.16943359375,
+      "loss": 0.2394624501466751,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1322186291217804,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2654881775379181,
+      "rewards/margins": 6.110224723815918,
+      "rewards/rejected": -5.844736099243164,
+      "step": 369,
+      "train_speed(iter/s)": 0.021152
+    },
+    {
+      "epoch": 0.747286038879071,
+      "grad_norm": 4.563858985900879,
+      "learning_rate": 0.00014805817551866838,
+      "logits/chosen": -1.183912992477417,
+      "logits/rejected": -1.2156834602355957,
+      "logps/chosen": -1.4219800233840942,
+      "logps/rejected": -39.744171142578125,
+      "loss": 0.3572657108306885,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15966495871543884,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4396134316921234,
+      "rewards/margins": 3.298137903213501,
+      "rewards/rejected": -2.8585243225097656,
+      "step": 370,
+      "train_speed(iter/s)": 0.021153
+    },
+    {
+      "epoch": 0.7493057308760414,
+      "grad_norm": 4.50530481338501,
+      "learning_rate": 0.00014776482048952551,
+      "logits/chosen": -1.1194803714752197,
+      "logits/rejected": -1.1540480852127075,
+      "logps/chosen": -1.7459050416946411,
+      "logps/rejected": -64.54890441894531,
+      "loss": 0.2934742867946625,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15494152903556824,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2886925935745239,
+      "rewards/margins": 5.029353141784668,
+      "rewards/rejected": -4.740660190582275,
+      "step": 371,
+      "train_speed(iter/s)": 0.021153
+    },
+    {
+      "epoch": 0.7513254228730119,
+      "grad_norm": 7.041140556335449,
+      "learning_rate": 0.00014747093193905657,
+      "logits/chosen": -1.1590553522109985,
+      "logits/rejected": -1.1508252620697021,
+      "logps/chosen": -3.047581434249878,
+      "logps/rejected": -61.70812225341797,
+      "loss": 0.6074828505516052,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3978445827960968,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.13817714154720306,
+      "rewards/margins": 4.407755374908447,
+      "rewards/rejected": -4.269577980041504,
+      "step": 372,
+      "train_speed(iter/s)": 0.021154
+    },
+    {
+      "epoch": 0.7533451148699823,
+      "grad_norm": 4.780473709106445,
+      "learning_rate": 0.00014717651314992455,
+      "logits/chosen": -1.1245406866073608,
+      "logits/rejected": -1.1541649103164673,
+      "logps/chosen": -2.052983283996582,
+      "logps/rejected": -59.029022216796875,
+      "loss": 0.3612958490848541,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17177711427211761,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1790371835231781,
+      "rewards/margins": 4.619558811187744,
+      "rewards/rejected": -4.440521717071533,
+      "step": 373,
+      "train_speed(iter/s)": 0.021154
+    },
+    {
+      "epoch": 0.7553648068669528,
+      "grad_norm": 2.6618669033050537,
+      "learning_rate": 0.00014688156741071514,
+      "logits/chosen": -1.16391921043396,
+      "logits/rejected": -1.2101376056671143,
+      "logps/chosen": -2.5643341541290283,
+      "logps/rejected": -50.99317932128906,
+      "loss": 0.359174907207489,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14501263201236725,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2054893523454666,
+      "rewards/margins": 4.245234489440918,
+      "rewards/rejected": -4.039744853973389,
+      "step": 374,
+      "train_speed(iter/s)": 0.021155
+    },
+    {
+      "epoch": 0.7573844988639232,
+      "grad_norm": 2.183286666870117,
+      "learning_rate": 0.00014658609801589982,
+      "logits/chosen": -1.197989583015442,
+      "logits/rejected": -1.232681155204773,
+      "logps/chosen": -1.0838813781738281,
+      "logps/rejected": -53.68153762817383,
+      "loss": 0.27284619212150574,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11144410073757172,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3119613528251648,
+      "rewards/margins": 4.7982072830200195,
+      "rewards/rejected": -4.486246109008789,
+      "step": 375,
+      "train_speed(iter/s)": 0.021156
+    },
+    {
+      "epoch": 0.7594041908608937,
+      "grad_norm": 6.413065433502197,
+      "learning_rate": 0.00014629010826579928,
+      "logits/chosen": -1.217897891998291,
+      "logits/rejected": -1.279034972190857,
+      "logps/chosen": -2.029020071029663,
+      "logps/rejected": -67.19940948486328,
+      "loss": 0.3730330765247345,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2322002500295639,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2920472323894501,
+      "rewards/margins": 5.175812244415283,
+      "rewards/rejected": -4.883764266967773,
+      "step": 376,
+      "train_speed(iter/s)": 0.021157
+    },
+    {
+      "epoch": 0.7614238828578642,
+      "grad_norm": 3.451570987701416,
+      "learning_rate": 0.0001459936014665464,
+      "logits/chosen": -1.17360258102417,
+      "logits/rejected": -1.2139333486557007,
+      "logps/chosen": -4.350042819976807,
+      "logps/rejected": -56.49632263183594,
+      "loss": 0.5775706171989441,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.39697498083114624,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.23329809308052063,
+      "rewards/margins": 4.431912422180176,
+      "rewards/rejected": -4.198614597320557,
+      "step": 377,
+      "train_speed(iter/s)": 0.021158
+    },
+    {
+      "epoch": 0.7634435748548346,
+      "grad_norm": 5.291248798370361,
+      "learning_rate": 0.00014569658093004935,
+      "logits/chosen": -1.1762566566467285,
+      "logits/rejected": -1.2233586311340332,
+      "logps/chosen": -3.534919261932373,
+      "logps/rejected": -80.47354888916016,
+      "loss": 0.30777013301849365,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2154829502105713,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2291412353515625,
+      "rewards/margins": 6.180717468261719,
+      "rewards/rejected": -5.951576232910156,
+      "step": 378,
+      "train_speed(iter/s)": 0.021158
+    },
+    {
+      "epoch": 0.7654632668518051,
+      "grad_norm": 10.057852745056152,
+      "learning_rate": 0.00014539904997395468,
+      "logits/chosen": -1.1968324184417725,
+      "logits/rejected": -1.2274653911590576,
+      "logps/chosen": -3.546328544616699,
+      "logps/rejected": -69.02865600585938,
+      "loss": 0.6399840712547302,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4272352159023285,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.11763279885053635,
+      "rewards/margins": 5.601714611053467,
+      "rewards/rejected": -5.484081745147705,
+      "step": 379,
+      "train_speed(iter/s)": 0.021158
+    },
+    {
+      "epoch": 0.7674829588487756,
+      "grad_norm": 4.334756374359131,
+      "learning_rate": 0.00014510101192161018,
+      "logits/chosen": -1.2146495580673218,
+      "logits/rejected": -1.2545320987701416,
+      "logps/chosen": -6.085969924926758,
+      "logps/rejected": -41.06128692626953,
+      "loss": 0.776848316192627,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5693851709365845,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19634687900543213,
+      "rewards/margins": 3.313415050506592,
+      "rewards/rejected": -3.1170685291290283,
+      "step": 380,
+      "train_speed(iter/s)": 0.021159
+    },
+    {
+      "epoch": 0.769502650845746,
+      "grad_norm": 6.6673479080200195,
+      "learning_rate": 0.00014480247010202775,
+      "logits/chosen": -1.2620035409927368,
+      "logits/rejected": -1.2927439212799072,
+      "logps/chosen": -0.9638640880584717,
+      "logps/rejected": -58.5118293762207,
+      "loss": 0.3140888810157776,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13222388923168182,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.43909886479377747,
+      "rewards/margins": 5.030383110046387,
+      "rewards/rejected": -4.591284275054932,
+      "step": 381,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 0.7715223428427165,
+      "grad_norm": 1.8640215396881104,
+      "learning_rate": 0.00014450342784984633,
+      "logits/chosen": -1.2142149209976196,
+      "logits/rejected": -1.2449159622192383,
+      "logps/chosen": -3.2471883296966553,
+      "logps/rejected": -69.44312286376953,
+      "loss": 0.2647269666194916,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17502562701702118,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.32493484020233154,
+      "rewards/margins": 5.946244239807129,
+      "rewards/rejected": -5.621310234069824,
+      "step": 382,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 0.773542034839687,
+      "grad_norm": 4.235260963439941,
+      "learning_rate": 0.00014420388850529446,
+      "logits/chosen": -1.2407033443450928,
+      "logits/rejected": -1.280212163925171,
+      "logps/chosen": -1.4121766090393066,
+      "logps/rejected": -73.05975341796875,
+      "loss": 0.4667883813381195,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23148246109485626,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.42525410652160645,
+      "rewards/margins": 5.927689552307129,
+      "rewards/rejected": -5.502435207366943,
+      "step": 383,
+      "train_speed(iter/s)": 0.021161
+    },
+    {
+      "epoch": 0.7755617268366574,
+      "grad_norm": 11.74973201751709,
+      "learning_rate": 0.00014390385541415308,
+      "logits/chosen": -1.2475506067276,
+      "logits/rejected": -1.2707443237304688,
+      "logps/chosen": -2.454108953475952,
+      "logps/rejected": -42.124366760253906,
+      "loss": 0.6018331050872803,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3722446858882904,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.23790600895881653,
+      "rewards/margins": 3.314408779144287,
+      "rewards/rejected": -3.076502561569214,
+      "step": 384,
+      "train_speed(iter/s)": 0.021162
+    },
+    {
+      "epoch": 0.7775814188336279,
+      "grad_norm": 2.7534830570220947,
+      "learning_rate": 0.0001436033319277183,
+      "logits/chosen": -1.2355518341064453,
+      "logits/rejected": -1.2741280794143677,
+      "logps/chosen": -0.7196122407913208,
+      "logps/rejected": -107.31944274902344,
+      "loss": 0.1968921422958374,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1137460246682167,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2591235637664795,
+      "rewards/margins": 8.63725757598877,
+      "rewards/rejected": -8.378134727478027,
+      "step": 385,
+      "train_speed(iter/s)": 0.021162
+    },
+    {
+      "epoch": 0.7796011108305984,
+      "grad_norm": 5.864130020141602,
+      "learning_rate": 0.00014330232140276366,
+      "logits/chosen": -1.1965060234069824,
+      "logits/rejected": -1.2489956617355347,
+      "logps/chosen": -3.9303858280181885,
+      "logps/rejected": -66.87359619140625,
+      "loss": 0.6629573106765747,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5041288137435913,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.20213821530342102,
+      "rewards/margins": 5.429197311401367,
+      "rewards/rejected": -5.2270588874816895,
+      "step": 386,
+      "train_speed(iter/s)": 0.021162
+    },
+    {
+      "epoch": 0.7816208028275689,
+      "grad_norm": 0.9686940312385559,
+      "learning_rate": 0.0001430008272015029,
+      "logits/chosen": -1.168556571006775,
+      "logits/rejected": -1.2702208757400513,
+      "logps/chosen": -0.1927298754453659,
+      "logps/rejected": -86.75350189208984,
+      "loss": 0.07656482607126236,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.02479482628405094,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3893800973892212,
+      "rewards/margins": 6.8833112716674805,
+      "rewards/rejected": -6.493931770324707,
+      "step": 387,
+      "train_speed(iter/s)": 0.021162
+    },
+    {
+      "epoch": 0.7836404948245392,
+      "grad_norm": 6.69142484664917,
+      "learning_rate": 0.0001426988526915523,
+      "logits/chosen": -1.1132441759109497,
+      "logits/rejected": -1.2206928730010986,
+      "logps/chosen": -2.1822941303253174,
+      "logps/rejected": -58.27075958251953,
+      "loss": 0.4767617881298065,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2635282874107361,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.18697461485862732,
+      "rewards/margins": 4.062845706939697,
+      "rewards/rejected": -3.875871419906616,
+      "step": 388,
+      "train_speed(iter/s)": 0.021163
+    },
+    {
+      "epoch": 0.7856601868215097,
+      "grad_norm": 22.11477279663086,
+      "learning_rate": 0.00014239640124589302,
+      "logits/chosen": -1.1612333059310913,
+      "logits/rejected": -1.2209270000457764,
+      "logps/chosen": -5.149811267852783,
+      "logps/rejected": -56.67973327636719,
+      "loss": 0.9844444394111633,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.464616984128952,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.07719806581735611,
+      "rewards/margins": 3.927542209625244,
+      "rewards/rejected": -4.004740238189697,
+      "step": 389,
+      "train_speed(iter/s)": 0.021163
+    },
+    {
+      "epoch": 0.7876798788184802,
+      "grad_norm": 4.598104476928711,
+      "learning_rate": 0.0001420934762428335,
+      "logits/chosen": -1.191237211227417,
+      "logits/rejected": -1.2505682706832886,
+      "logps/chosen": -1.6766197681427002,
+      "logps/rejected": -68.58138275146484,
+      "loss": 0.37751591205596924,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1912289708852768,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2715514898300171,
+      "rewards/margins": 6.051302909851074,
+      "rewards/rejected": -5.779752731323242,
+      "step": 390,
+      "train_speed(iter/s)": 0.021164
+    },
+    {
+      "epoch": 0.7896995708154506,
+      "grad_norm": 5.1656084060668945,
+      "learning_rate": 0.00014179008106597173,
+      "logits/chosen": -1.1447128057479858,
+      "logits/rejected": -1.2311046123504639,
+      "logps/chosen": -1.3589807748794556,
+      "logps/rejected": -69.0888900756836,
+      "loss": 0.37555360794067383,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20962855219841003,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2835935950279236,
+      "rewards/margins": 6.014133453369141,
+      "rewards/rejected": -5.730539798736572,
+      "step": 391,
+      "train_speed(iter/s)": 0.021165
+    },
+    {
+      "epoch": 0.7917192628124211,
+      "grad_norm": 4.515217304229736,
+      "learning_rate": 0.0001414862191041574,
+      "logits/chosen": -1.1313319206237793,
+      "logits/rejected": -1.1735104322433472,
+      "logps/chosen": -1.4379080533981323,
+      "logps/rejected": -41.75006103515625,
+      "loss": 0.42747819423675537,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13288989663124084,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.21147757768630981,
+      "rewards/margins": 3.0256876945495605,
+      "rewards/rejected": -2.8142104148864746,
+      "step": 392,
+      "train_speed(iter/s)": 0.021166
+    },
+    {
+      "epoch": 0.7937389548093916,
+      "grad_norm": 4.281467437744141,
+      "learning_rate": 0.00014118189375145402,
+      "logits/chosen": -1.129544734954834,
+      "logits/rejected": -1.2433701753616333,
+      "logps/chosen": -1.0194271802902222,
+      "logps/rejected": -68.8100357055664,
+      "loss": 0.4009433388710022,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2110380381345749,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.42351850867271423,
+      "rewards/margins": 5.658205032348633,
+      "rewards/rejected": -5.234686374664307,
+      "step": 393,
+      "train_speed(iter/s)": 0.021166
+    },
+    {
+      "epoch": 0.795758646806362,
+      "grad_norm": 0.9730565547943115,
+      "learning_rate": 0.0001408771084071012,
+      "logits/chosen": -1.098219394683838,
+      "logits/rejected": -1.2555054426193237,
+      "logps/chosen": -0.3178718388080597,
+      "logps/rejected": -82.52906799316406,
+      "loss": 0.08471984416246414,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.025890445336699486,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.41851806640625,
+      "rewards/margins": 7.259293556213379,
+      "rewards/rejected": -6.840775489807129,
+      "step": 394,
+      "train_speed(iter/s)": 0.021167
+    },
+    {
+      "epoch": 0.7977783388033325,
+      "grad_norm": 6.422388076782227,
+      "learning_rate": 0.0001405718664754764,
+      "logits/chosen": -1.0757089853286743,
+      "logits/rejected": -1.2194029092788696,
+      "logps/chosen": -4.098117351531982,
+      "logps/rejected": -79.46504211425781,
+      "loss": 0.9334844946861267,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.670303463935852,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.09435828030109406,
+      "rewards/margins": 6.080230236053467,
+      "rewards/rejected": -6.174588680267334,
+      "step": 395,
+      "train_speed(iter/s)": 0.021168
+    },
+    {
+      "epoch": 0.799798030800303,
+      "grad_norm": 3.1609609127044678,
+      "learning_rate": 0.0001402661713660571,
+      "logits/chosen": -1.1343252658843994,
+      "logits/rejected": -1.2600665092468262,
+      "logps/chosen": -1.6706289052963257,
+      "logps/rejected": -69.32755279541016,
+      "loss": 0.22293509542942047,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17708872258663177,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.18604204058647156,
+      "rewards/margins": 6.022045612335205,
+      "rewards/rejected": -5.836003303527832,
+      "step": 396,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 0.8018177227972734,
+      "grad_norm": 3.5824060440063477,
+      "learning_rate": 0.0001399600264933827,
+      "logits/chosen": -1.1588817834854126,
+      "logits/rejected": -1.2686594724655151,
+      "logps/chosen": -2.480526924133301,
+      "logps/rejected": -77.53683471679688,
+      "loss": 0.400388240814209,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2272554636001587,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2708975374698639,
+      "rewards/margins": 6.85975980758667,
+      "rewards/rejected": -6.58886194229126,
+      "step": 397,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 0.8038374147942439,
+      "grad_norm": 8.915884017944336,
+      "learning_rate": 0.00013965343527701628,
+      "logits/chosen": -1.0931580066680908,
+      "logits/rejected": -1.2515380382537842,
+      "logps/chosen": -6.809703826904297,
+      "logps/rejected": -92.00086975097656,
+      "loss": 1.2307904958724976,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.950491189956665,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.23660066723823547,
+      "rewards/margins": 7.736156463623047,
+      "rewards/rejected": -7.9727559089660645,
+      "step": 398,
+      "train_speed(iter/s)": 0.02117
+    },
+    {
+      "epoch": 0.8058571067912144,
+      "grad_norm": 16.942861557006836,
+      "learning_rate": 0.00013934640114150656,
+      "logits/chosen": -1.1165294647216797,
+      "logits/rejected": -1.2258877754211426,
+      "logps/chosen": -5.151846885681152,
+      "logps/rejected": -50.20682144165039,
+      "loss": 1.4095838069915771,
+      "memory(GiB)": 45.64,
+      "nll_loss": 1.0055750608444214,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.08993971347808838,
+      "rewards/margins": 3.8719429969787598,
+      "rewards/rejected": -3.9618825912475586,
+      "step": 399,
+      "train_speed(iter/s)": 0.021171
+    },
+    {
+      "epoch": 0.8078767987881847,
+      "grad_norm": 12.716894149780273,
+      "learning_rate": 0.00013903892751634947,
+      "logits/chosen": -1.1147105693817139,
+      "logits/rejected": -1.2179588079452515,
+      "logps/chosen": -2.3593387603759766,
+      "logps/rejected": -60.22994613647461,
+      "loss": 0.3005334734916687,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17930257320404053,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2562834620475769,
+      "rewards/margins": 5.470202445983887,
+      "rewards/rejected": -5.213918685913086,
+      "step": 400,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 0.8098964907851552,
+      "grad_norm": 3.6407527923583984,
+      "learning_rate": 0.00013873101783594999,
+      "logits/chosen": -0.9179055690765381,
+      "logits/rejected": -1.0948255062103271,
+      "logps/chosen": -7.396864414215088,
+      "logps/rejected": -76.39356994628906,
+      "loss": 0.7279776334762573,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5141282081604004,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.07542237639427185,
+      "rewards/margins": 5.8221588134765625,
+      "rewards/rejected": -5.746736526489258,
+      "step": 401,
+      "train_speed(iter/s)": 0.021153
+    },
+    {
+      "epoch": 0.8119161827821257,
+      "grad_norm": 4.730077743530273,
+      "learning_rate": 0.00013842267553958371,
+      "logits/chosen": -1.0015758275985718,
+      "logits/rejected": -1.173001766204834,
+      "logps/chosen": -1.2833012342453003,
+      "logps/rejected": -65.50496673583984,
+      "loss": 0.31842923164367676,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15187285840511322,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3435157239437103,
+      "rewards/margins": 5.360050201416016,
+      "rewards/rejected": -5.016533851623535,
+      "step": 402,
+      "train_speed(iter/s)": 0.021154
+    },
+    {
+      "epoch": 0.8139358747790962,
+      "grad_norm": 6.681663513183594,
+      "learning_rate": 0.00013811390407135837,
+      "logits/chosen": -1.0345579385757446,
+      "logits/rejected": -1.1669750213623047,
+      "logps/chosen": -2.711009979248047,
+      "logps/rejected": -48.982208251953125,
+      "loss": 0.49727287888526917,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.25284403562545776,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.15987528860569,
+      "rewards/margins": 4.172056674957275,
+      "rewards/rejected": -4.012181282043457,
+      "step": 403,
+      "train_speed(iter/s)": 0.021155
+    },
+    {
+      "epoch": 0.8159555667760666,
+      "grad_norm": 3.184199571609497,
+      "learning_rate": 0.00013780470688017562,
+      "logits/chosen": -1.0638835430145264,
+      "logits/rejected": -1.1304998397827148,
+      "logps/chosen": -4.9558868408203125,
+      "logps/rejected": -44.02895736694336,
+      "loss": 0.6054225564002991,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3829798102378845,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.39171290397644043,
+      "rewards/margins": 3.7432100772857666,
+      "rewards/rejected": -3.3514974117279053,
+      "step": 404,
+      "train_speed(iter/s)": 0.021156
+    },
+    {
+      "epoch": 0.8179752587730371,
+      "grad_norm": 16.700468063354492,
+      "learning_rate": 0.00013749508741969213,
+      "logits/chosen": -0.9126327633857727,
+      "logits/rejected": -0.9981098771095276,
+      "logps/chosen": -8.324628829956055,
+      "logps/rejected": -54.67494201660156,
+      "loss": 1.088732361793518,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7484469413757324,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.14237897098064423,
+      "rewards/margins": 3.1626644134521484,
+      "rewards/rejected": -3.3050432205200195,
+      "step": 405,
+      "train_speed(iter/s)": 0.021156
+    },
+    {
+      "epoch": 0.8199949507700076,
+      "grad_norm": 4.572319507598877,
+      "learning_rate": 0.00013718504914828135,
+      "logits/chosen": -0.9639456272125244,
+      "logits/rejected": -1.0909277200698853,
+      "logps/chosen": -2.374218225479126,
+      "logps/rejected": -28.60363006591797,
+      "loss": 0.6309629082679749,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3386070728302002,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.38088110089302063,
+      "rewards/margins": 2.043933868408203,
+      "rewards/rejected": -1.6630527973175049,
+      "step": 406,
+      "train_speed(iter/s)": 0.021157
+    },
+    {
+      "epoch": 0.822014642766978,
+      "grad_norm": 7.175300598144531,
+      "learning_rate": 0.00013687459552899465,
+      "logits/chosen": -1.011370062828064,
+      "logits/rejected": -1.1552919149398804,
+      "logps/chosen": -0.751252293586731,
+      "logps/rejected": -58.353179931640625,
+      "loss": 0.3189142942428589,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1253315508365631,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2173149585723877,
+      "rewards/margins": 4.045991897583008,
+      "rewards/rejected": -3.828676700592041,
+      "step": 407,
+      "train_speed(iter/s)": 0.021157
+    },
+    {
+      "epoch": 0.8240343347639485,
+      "grad_norm": 3.638871908187866,
+      "learning_rate": 0.0001365637300295229,
+      "logits/chosen": -0.9884570240974426,
+      "logits/rejected": -1.1229445934295654,
+      "logps/chosen": -2.4416940212249756,
+      "logps/rejected": -49.12045669555664,
+      "loss": 0.5083186030387878,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.30594146251678467,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.28965339064598083,
+      "rewards/margins": 3.306330919265747,
+      "rewards/rejected": -3.0166776180267334,
+      "step": 408,
+      "train_speed(iter/s)": 0.021158
+    },
+    {
+      "epoch": 0.826054026760919,
+      "grad_norm": 1.9313740730285645,
+      "learning_rate": 0.0001362524561221574,
+      "logits/chosen": -1.0589873790740967,
+      "logits/rejected": -1.155687928199768,
+      "logps/chosen": -0.7494394183158875,
+      "logps/rejected": -61.899314880371094,
+      "loss": 0.16012489795684814,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.06837154924869537,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.422152578830719,
+      "rewards/margins": 5.1323323249816895,
+      "rewards/rejected": -4.710179805755615,
+      "step": 409,
+      "train_speed(iter/s)": 0.021159
+    },
+    {
+      "epoch": 0.8280737187578894,
+      "grad_norm": 2.8826475143432617,
+      "learning_rate": 0.00013594077728375128,
+      "logits/chosen": -1.046625018119812,
+      "logits/rejected": -1.1403248310089111,
+      "logps/chosen": -2.6342790126800537,
+      "logps/rejected": -47.86298370361328,
+      "loss": 0.3936201333999634,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.19051307439804077,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.13923531770706177,
+      "rewards/margins": 3.3216652870178223,
+      "rewards/rejected": -3.182429790496826,
+      "step": 410,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 0.8300934107548599,
+      "grad_norm": 7.247873306274414,
+      "learning_rate": 0.00013562869699568076,
+      "logits/chosen": -1.1259405612945557,
+      "logits/rejected": -1.1454825401306152,
+      "logps/chosen": -7.434700012207031,
+      "logps/rejected": -49.61088562011719,
+      "loss": 0.960975170135498,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7216768860816956,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.03636153042316437,
+      "rewards/margins": 3.5744729042053223,
+      "rewards/rejected": -3.6108345985412598,
+      "step": 411,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 0.8321131027518304,
+      "grad_norm": 2.7764785289764404,
+      "learning_rate": 0.00013531621874380613,
+      "logits/chosen": -0.9636606574058533,
+      "logits/rejected": -1.1189731359481812,
+      "logps/chosen": -3.8168373107910156,
+      "logps/rejected": -65.32124328613281,
+      "loss": 0.31505468487739563,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2553892135620117,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2123510092496872,
+      "rewards/margins": 5.209415912628174,
+      "rewards/rejected": -4.997064590454102,
+      "step": 412,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 0.8341327947488008,
+      "grad_norm": 4.009552001953125,
+      "learning_rate": 0.0001350033460184327,
+      "logits/chosen": -0.8576051592826843,
+      "logits/rejected": -1.1664515733718872,
+      "logps/chosen": -1.3953887224197388,
+      "logps/rejected": -94.75813293457031,
+      "loss": 0.1995779424905777,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09238594025373459,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.33278360962867737,
+      "rewards/margins": 6.894500255584717,
+      "rewards/rejected": -6.5617170333862305,
+      "step": 413,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 0.8361524867457713,
+      "grad_norm": 14.014391899108887,
+      "learning_rate": 0.00013469008231427207,
+      "logits/chosen": -1.1141268014907837,
+      "logits/rejected": -1.1793636083602905,
+      "logps/chosen": -2.8426718711853027,
+      "logps/rejected": -64.9253158569336,
+      "loss": 0.6592072248458862,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.44861316680908203,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.20249105989933014,
+      "rewards/margins": 5.202639579772949,
+      "rewards/rejected": -5.000148296356201,
+      "step": 414,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 0.8381721787427417,
+      "grad_norm": 6.667239189147949,
+      "learning_rate": 0.00013437643113040301,
+      "logits/chosen": -1.1405432224273682,
+      "logits/rejected": -1.205533504486084,
+      "logps/chosen": -4.049654960632324,
+      "logps/rejected": -65.5449447631836,
+      "loss": 0.955814003944397,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.7577894926071167,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.011894673109054565,
+      "rewards/margins": 5.261639595031738,
+      "rewards/rejected": -5.273534297943115,
+      "step": 415,
+      "train_speed(iter/s)": 0.021161
+    },
+    {
+      "epoch": 0.8401918707397122,
+      "grad_norm": 5.3052802085876465,
+      "learning_rate": 0.00013406239597023225,
+      "logits/chosen": -1.1984199285507202,
+      "logits/rejected": -1.2300504446029663,
+      "logps/chosen": -2.0026376247406006,
+      "logps/rejected": -44.80060958862305,
+      "loss": 0.32053616642951965,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1967550367116928,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.29741793870925903,
+      "rewards/margins": 3.937462329864502,
+      "rewards/rejected": -3.6400444507598877,
+      "step": 416,
+      "train_speed(iter/s)": 0.021162
+    },
+    {
+      "epoch": 0.8422115627366826,
+      "grad_norm": 4.6360578536987305,
+      "learning_rate": 0.0001337479803414555,
+      "logits/chosen": -1.1135053634643555,
+      "logits/rejected": -1.1990042924880981,
+      "logps/chosen": -1.2526822090148926,
+      "logps/rejected": -64.39313507080078,
+      "loss": 0.3324093520641327,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.19946074485778809,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.22650685906410217,
+      "rewards/margins": 5.059194087982178,
+      "rewards/rejected": -4.8326873779296875,
+      "step": 417,
+      "train_speed(iter/s)": 0.021162
+    },
+    {
+      "epoch": 0.8442312547336531,
+      "grad_norm": 5.64458703994751,
+      "learning_rate": 0.0001334331877560182,
+      "logits/chosen": -1.1316285133361816,
+      "logits/rejected": -1.1844420433044434,
+      "logps/chosen": -2.270009994506836,
+      "logps/rejected": -73.60192108154297,
+      "loss": 0.37290775775909424,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2037116289138794,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.22195757925510406,
+      "rewards/margins": 6.114259719848633,
+      "rewards/rejected": -5.892301559448242,
+      "step": 418,
+      "train_speed(iter/s)": 0.021163
+    },
+    {
+      "epoch": 0.8462509467306236,
+      "grad_norm": 1.5520920753479004,
+      "learning_rate": 0.00013311802173007626,
+      "logits/chosen": -1.1409199237823486,
+      "logits/rejected": -1.203282356262207,
+      "logps/chosen": -2.087766408920288,
+      "logps/rejected": -75.88261413574219,
+      "loss": 0.19689969718456268,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11977825313806534,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1527182012796402,
+      "rewards/margins": 6.459143161773682,
+      "rewards/rejected": -6.306424617767334,
+      "step": 419,
+      "train_speed(iter/s)": 0.021164
+    },
+    {
+      "epoch": 0.848270638727594,
+      "grad_norm": 16.449495315551758,
+      "learning_rate": 0.0001328024857839569,
+      "logits/chosen": -1.189044713973999,
+      "logits/rejected": -1.2236502170562744,
+      "logps/chosen": -4.713018894195557,
+      "logps/rejected": -54.21986389160156,
+      "loss": 0.7003884315490723,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4415695369243622,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.018294498324394226,
+      "rewards/margins": 4.57138204574585,
+      "rewards/rejected": -4.589676856994629,
+      "step": 420,
+      "train_speed(iter/s)": 0.021164
+    },
+    {
+      "epoch": 0.8502903307245645,
+      "grad_norm": 28.160564422607422,
+      "learning_rate": 0.00013248658344211926,
+      "logits/chosen": -1.0967679023742676,
+      "logits/rejected": -1.1361645460128784,
+      "logps/chosen": -6.815154552459717,
+      "logps/rejected": -83.51364135742188,
+      "loss": 1.2551347017288208,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.8729581832885742,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.03463226556777954,
+      "rewards/margins": 6.367086410522461,
+      "rewards/rejected": -6.401719093322754,
+      "step": 421,
+      "train_speed(iter/s)": 0.021164
+    },
+    {
+      "epoch": 0.852310022721535,
+      "grad_norm": 31.47573471069336,
+      "learning_rate": 0.00013217031823311488,
+      "logits/chosen": -1.1553871631622314,
+      "logits/rejected": -1.2064706087112427,
+      "logps/chosen": -5.438415050506592,
+      "logps/rejected": -65.18042755126953,
+      "loss": 0.8344112038612366,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5628060698509216,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.07962656021118164,
+      "rewards/margins": 5.513628959655762,
+      "rewards/rejected": -5.434001922607422,
+      "step": 422,
+      "train_speed(iter/s)": 0.021165
+    },
+    {
+      "epoch": 0.8543297147185054,
+      "grad_norm": 8.784391403198242,
+      "learning_rate": 0.0001318536936895487,
+      "logits/chosen": -1.123747706413269,
+      "logits/rejected": -1.1673023700714111,
+      "logps/chosen": -3.4688830375671387,
+      "logps/rejected": -54.17964172363281,
+      "loss": 0.8599854707717896,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5710347890853882,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.038751691579818726,
+      "rewards/margins": 4.241340637207031,
+      "rewards/rejected": -4.2025885581970215,
+      "step": 423,
+      "train_speed(iter/s)": 0.021166
+    },
+    {
+      "epoch": 0.8563494067154759,
+      "grad_norm": 22.89176368713379,
+      "learning_rate": 0.00013153671334803905,
+      "logits/chosen": -1.1089229583740234,
+      "logits/rejected": -1.1420047283172607,
+      "logps/chosen": -4.063858985900879,
+      "logps/rejected": -40.476722717285156,
+      "loss": 0.9364627599716187,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6597320437431335,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.02382955513894558,
+      "rewards/margins": 3.0120341777801514,
+      "rewards/rejected": -2.9882047176361084,
+      "step": 424,
+      "train_speed(iter/s)": 0.021166
+    },
+    {
+      "epoch": 0.8583690987124464,
+      "grad_norm": 4.699946880340576,
+      "learning_rate": 0.00013121938074917865,
+      "logits/chosen": -1.0667518377304077,
+      "logits/rejected": -1.1017130613327026,
+      "logps/chosen": -4.643237113952637,
+      "logps/rejected": -60.97563934326172,
+      "loss": 0.39269566535949707,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3167289197444916,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.22700132429599762,
+      "rewards/margins": 4.699875354766846,
+      "rewards/rejected": -4.472874641418457,
+      "step": 425,
+      "train_speed(iter/s)": 0.021167
+    },
+    {
+      "epoch": 0.8603887907094168,
+      "grad_norm": 2.7371695041656494,
+      "learning_rate": 0.00013090169943749476,
+      "logits/chosen": -1.076597809791565,
+      "logits/rejected": -1.0993525981903076,
+      "logps/chosen": -2.2984302043914795,
+      "logps/rejected": -54.213531494140625,
+      "loss": 0.3011862337589264,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17973017692565918,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19591867923736572,
+      "rewards/margins": 3.4975361824035645,
+      "rewards/rejected": -3.3016176223754883,
+      "step": 426,
+      "train_speed(iter/s)": 0.021168
+    },
+    {
+      "epoch": 0.8624084827063873,
+      "grad_norm": 17.75054168701172,
+      "learning_rate": 0.00013058367296140967,
+      "logits/chosen": -1.046842098236084,
+      "logits/rejected": -1.0758651494979858,
+      "logps/chosen": -4.21718692779541,
+      "logps/rejected": -39.91408920288086,
+      "loss": 0.9825822114944458,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6494057774543762,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.017109857872128487,
+      "rewards/margins": 2.6019086837768555,
+      "rewards/rejected": -2.584798812866211,
+      "step": 427,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 0.8644281747033578,
+      "grad_norm": 2.5992624759674072,
+      "learning_rate": 0.00013026530487320113,
+      "logits/chosen": -1.0623323917388916,
+      "logits/rejected": -1.1041500568389893,
+      "logps/chosen": -1.7210079431533813,
+      "logps/rejected": -52.09022521972656,
+      "loss": 0.3279334008693695,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.19231894612312317,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.27669844031333923,
+      "rewards/margins": 4.07616662979126,
+      "rewards/rejected": -3.7994682788848877,
+      "step": 428,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 0.8664478667003283,
+      "grad_norm": 3.9208383560180664,
+      "learning_rate": 0.00012994659872896257,
+      "logits/chosen": -1.0508674383163452,
+      "logits/rejected": -1.1102675199508667,
+      "logps/chosen": -3.953282594680786,
+      "logps/rejected": -63.04370880126953,
+      "loss": 0.7971250414848328,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5409887433052063,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.06726647913455963,
+      "rewards/margins": 4.291454792022705,
+      "rewards/rejected": -4.224188327789307,
+      "step": 429,
+      "train_speed(iter/s)": 0.02117
+    },
+    {
+      "epoch": 0.8684675586972986,
+      "grad_norm": 2.1997742652893066,
+      "learning_rate": 0.00012962755808856342,
+      "logits/chosen": -1.0617396831512451,
+      "logits/rejected": -1.0978208780288696,
+      "logps/chosen": -1.448944330215454,
+      "logps/rejected": -47.621673583984375,
+      "loss": 0.17363907396793365,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10171680897474289,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2539137005805969,
+      "rewards/margins": 3.7236616611480713,
+      "rewards/rejected": -3.4697484970092773,
+      "step": 430,
+      "train_speed(iter/s)": 0.02117
+    },
+    {
+      "epoch": 0.8704872506942691,
+      "grad_norm": 5.373311519622803,
+      "learning_rate": 0.00012930818651560934,
+      "logits/chosen": -1.015924096107483,
+      "logits/rejected": -1.0387259721755981,
+      "logps/chosen": -3.2455596923828125,
+      "logps/rejected": -39.7397575378418,
+      "loss": 0.8503936529159546,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.522122859954834,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.04042099043726921,
+      "rewards/margins": 2.7263174057006836,
+      "rewards/rejected": -2.7667384147644043,
+      "step": 431,
+      "train_speed(iter/s)": 0.021171
+    },
+    {
+      "epoch": 0.8725069426912396,
+      "grad_norm": 10.711043357849121,
+      "learning_rate": 0.00012898848757740246,
+      "logits/chosen": -0.9647800922393799,
+      "logits/rejected": -0.9854017496109009,
+      "logps/chosen": -9.754167556762695,
+      "logps/rejected": -35.71137619018555,
+      "loss": 1.1322778463363647,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6833858489990234,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.09812775254249573,
+      "rewards/margins": 1.88960862159729,
+      "rewards/rejected": -1.7914810180664062,
+      "step": 432,
+      "train_speed(iter/s)": 0.021171
+    },
+    {
+      "epoch": 0.87452663468821,
+      "grad_norm": 6.029493808746338,
+      "learning_rate": 0.00012866846484490147,
+      "logits/chosen": -1.05926513671875,
+      "logits/rejected": -1.0726540088653564,
+      "logps/chosen": -1.8859708309173584,
+      "logps/rejected": -38.563228607177734,
+      "loss": 0.3466527760028839,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2102343887090683,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.403474360704422,
+      "rewards/margins": 3.249852180480957,
+      "rewards/rejected": -2.8463780879974365,
+      "step": 433,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 0.8765463266851805,
+      "grad_norm": 9.18895149230957,
+      "learning_rate": 0.0001283481218926818,
+      "logits/chosen": -1.060423731803894,
+      "logits/rejected": -1.0363166332244873,
+      "logps/chosen": -4.131833553314209,
+      "logps/rejected": -75.67689514160156,
+      "loss": 0.7851709723472595,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5702386498451233,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1319551020860672,
+      "rewards/margins": 5.650537490844727,
+      "rewards/rejected": -5.518582820892334,
+      "step": 434,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 0.878566018682151,
+      "grad_norm": 6.691397666931152,
+      "learning_rate": 0.00012802746229889563,
+      "logits/chosen": -1.0681829452514648,
+      "logits/rejected": -1.0891364812850952,
+      "logps/chosen": -1.9405295848846436,
+      "logps/rejected": -50.287044525146484,
+      "loss": 0.4823431670665741,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2815513014793396,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.21010005474090576,
+      "rewards/margins": 3.8635170459747314,
+      "rewards/rejected": -3.653416633605957,
+      "step": 435,
+      "train_speed(iter/s)": 0.021173
+    },
+    {
+      "epoch": 0.8805857106791214,
+      "grad_norm": 4.134374141693115,
+      "learning_rate": 0.00012770648964523194,
+      "logits/chosen": -1.0445590019226074,
+      "logits/rejected": -1.0800678730010986,
+      "logps/chosen": -4.192701816558838,
+      "logps/rejected": -41.08876037597656,
+      "loss": 0.764896035194397,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6386605501174927,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.05937701463699341,
+      "rewards/margins": 3.2779440879821777,
+      "rewards/rejected": -3.21856689453125,
+      "step": 436,
+      "train_speed(iter/s)": 0.021174
+    },
+    {
+      "epoch": 0.8826054026760919,
+      "grad_norm": 5.5645976066589355,
+      "learning_rate": 0.0001273852075168765,
+      "logits/chosen": -1.052649736404419,
+      "logits/rejected": -1.0750157833099365,
+      "logps/chosen": -3.2202606201171875,
+      "logps/rejected": -39.1766471862793,
+      "loss": 0.7316163778305054,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.35654330253601074,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.04998503625392914,
+      "rewards/margins": 2.966275453567505,
+      "rewards/rejected": -2.916290521621704,
+      "step": 437,
+      "train_speed(iter/s)": 0.021175
+    },
+    {
+      "epoch": 0.8846250946730624,
+      "grad_norm": 3.430967330932617,
+      "learning_rate": 0.0001270636195024719,
+      "logits/chosen": -1.0656282901763916,
+      "logits/rejected": -1.0755031108856201,
+      "logps/chosen": -0.5838297605514526,
+      "logps/rejected": -53.800167083740234,
+      "loss": 0.23027637600898743,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11958169937133789,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.35133183002471924,
+      "rewards/margins": 4.607048034667969,
+      "rewards/rejected": -4.255716323852539,
+      "step": 438,
+      "train_speed(iter/s)": 0.021176
+    },
+    {
+      "epoch": 0.8866447866700328,
+      "grad_norm": 4.035180568695068,
+      "learning_rate": 0.00012674172919407734,
+      "logits/chosen": -1.035705327987671,
+      "logits/rejected": -1.046338677406311,
+      "logps/chosen": -1.762102484703064,
+      "logps/rejected": -41.274600982666016,
+      "loss": 0.44759538769721985,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17419928312301636,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2473682463169098,
+      "rewards/margins": 3.196018695831299,
+      "rewards/rejected": -2.948650360107422,
+      "step": 439,
+      "train_speed(iter/s)": 0.021177
+    },
+    {
+      "epoch": 0.8886644786670033,
+      "grad_norm": 5.988144397735596,
+      "learning_rate": 0.00012641954018712863,
+      "logits/chosen": -1.0401451587677002,
+      "logits/rejected": -1.0481927394866943,
+      "logps/chosen": -1.6200381517410278,
+      "logps/rejected": -40.854026794433594,
+      "loss": 0.4076470732688904,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.28579434752464294,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2611207365989685,
+      "rewards/margins": 3.813446044921875,
+      "rewards/rejected": -3.5523252487182617,
+      "step": 440,
+      "train_speed(iter/s)": 0.021178
+    },
+    {
+      "epoch": 0.8906841706639738,
+      "grad_norm": 4.463588237762451,
+      "learning_rate": 0.00012609705608039782,
+      "logits/chosen": -1.0440635681152344,
+      "logits/rejected": -1.038417935371399,
+      "logps/chosen": -1.1576311588287354,
+      "logps/rejected": -78.23765563964844,
+      "loss": 0.2654612958431244,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14011946320533752,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.31270933151245117,
+      "rewards/margins": 5.999755859375,
+      "rewards/rejected": -5.687046051025391,
+      "step": 441,
+      "train_speed(iter/s)": 0.021178
+    },
+    {
+      "epoch": 0.8927038626609443,
+      "grad_norm": 3.0731234550476074,
+      "learning_rate": 0.00012577428047595344,
+      "logits/chosen": -1.016157865524292,
+      "logits/rejected": -1.0280359983444214,
+      "logps/chosen": -3.0516724586486816,
+      "logps/rejected": -60.31180191040039,
+      "loss": 0.4081045091152191,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23720964789390564,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2602730989456177,
+      "rewards/margins": 4.627702236175537,
+      "rewards/rejected": -4.367428779602051,
+      "step": 442,
+      "train_speed(iter/s)": 0.021178
+    },
+    {
+      "epoch": 0.8947235546579146,
+      "grad_norm": 2.895439624786377,
+      "learning_rate": 0.00012545121697911962,
+      "logits/chosen": -1.0191925764083862,
+      "logits/rejected": -1.0200212001800537,
+      "logps/chosen": -2.8940157890319824,
+      "logps/rejected": -57.44503402709961,
+      "loss": 0.36436915397644043,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1371701955795288,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2842387855052948,
+      "rewards/margins": 4.40120792388916,
+      "rewards/rejected": -4.116969108581543,
+      "step": 443,
+      "train_speed(iter/s)": 0.021178
+    },
+    {
+      "epoch": 0.8967432466548851,
+      "grad_norm": 9.891191482543945,
+      "learning_rate": 0.00012512786919843648,
+      "logits/chosen": -1.0200165510177612,
+      "logits/rejected": -1.032355785369873,
+      "logps/chosen": -2.6891350746154785,
+      "logps/rejected": -47.72188949584961,
+      "loss": 0.7450097799301147,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5138753652572632,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2414613664150238,
+      "rewards/margins": 4.151015281677246,
+      "rewards/rejected": -3.9095540046691895,
+      "step": 444,
+      "train_speed(iter/s)": 0.021179
+    },
+    {
+      "epoch": 0.8987629386518556,
+      "grad_norm": 3.1112067699432373,
+      "learning_rate": 0.00012480424074561933,
+      "logits/chosen": -1.025837779045105,
+      "logits/rejected": -1.0225645303726196,
+      "logps/chosen": -1.7361624240875244,
+      "logps/rejected": -58.649635314941406,
+      "loss": 0.2522731125354767,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09138274937868118,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2967418134212494,
+      "rewards/margins": 4.663671970367432,
+      "rewards/rejected": -4.3669304847717285,
+      "step": 445,
+      "train_speed(iter/s)": 0.021179
+    },
+    {
+      "epoch": 0.900782630648826,
+      "grad_norm": 14.950603485107422,
+      "learning_rate": 0.00012448033523551865,
+      "logits/chosen": -1.0079376697540283,
+      "logits/rejected": -1.0010699033737183,
+      "logps/chosen": -7.840879440307617,
+      "logps/rejected": -26.104421615600586,
+      "loss": 1.1924169063568115,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4921947121620178,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.1946742981672287,
+      "rewards/margins": 1.5733067989349365,
+      "rewards/rejected": -1.7679810523986816,
+      "step": 446,
+      "train_speed(iter/s)": 0.02118
+    },
+    {
+      "epoch": 0.9028023226457965,
+      "grad_norm": 3.8511011600494385,
+      "learning_rate": 0.00012415615628607945,
+      "logits/chosen": -1.015897274017334,
+      "logits/rejected": -1.014409065246582,
+      "logps/chosen": -2.20813250541687,
+      "logps/rejected": -45.50457763671875,
+      "loss": 0.5370233058929443,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.34335654973983765,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.12667731940746307,
+      "rewards/margins": 3.747370958328247,
+      "rewards/rejected": -3.6206939220428467,
+      "step": 447,
+      "train_speed(iter/s)": 0.021181
+    },
+    {
+      "epoch": 0.904822014642767,
+      "grad_norm": 4.594604969024658,
+      "learning_rate": 0.0001238317075183011,
+      "logits/chosen": -1.0298758745193481,
+      "logits/rejected": -1.0120558738708496,
+      "logps/chosen": -0.9791274070739746,
+      "logps/rejected": -79.58103942871094,
+      "loss": 0.3356885612010956,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.19914807379245758,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.22014901041984558,
+      "rewards/margins": 5.962551116943359,
+      "rewards/rejected": -5.742402076721191,
+      "step": 448,
+      "train_speed(iter/s)": 0.021181
+    },
+    {
+      "epoch": 0.9068417066397374,
+      "grad_norm": 3.488224983215332,
+      "learning_rate": 0.00012350699255619677,
+      "logits/chosen": -1.0189076662063599,
+      "logits/rejected": -1.0452524423599243,
+      "logps/chosen": -1.572611689567566,
+      "logps/rejected": -41.083641052246094,
+      "loss": 0.36099478602409363,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.19940336048603058,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.20073141157627106,
+      "rewards/margins": 3.2094714641571045,
+      "rewards/rejected": -3.008740186691284,
+      "step": 449,
+      "train_speed(iter/s)": 0.021182
+    },
+    {
+      "epoch": 0.9088613986367079,
+      "grad_norm": 1.7710275650024414,
+      "learning_rate": 0.00012318201502675285,
+      "logits/chosen": -0.9990462064743042,
+      "logits/rejected": -1.00516939163208,
+      "logps/chosen": -1.445307731628418,
+      "logps/rejected": -54.3629035949707,
+      "loss": 0.2929742634296417,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14184969663619995,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3061780631542206,
+      "rewards/margins": 4.353916645050049,
+      "rewards/rejected": -4.047738552093506,
+      "step": 450,
+      "train_speed(iter/s)": 0.021182
+    },
+    {
+      "epoch": 0.9108810906336784,
+      "grad_norm": 5.618566989898682,
+      "learning_rate": 0.00012285677855988864,
+      "logits/chosen": -1.0002800226211548,
+      "logits/rejected": -1.0010613203048706,
+      "logps/chosen": -3.4435813426971436,
+      "logps/rejected": -65.43953704833984,
+      "loss": 0.5797629952430725,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2722090482711792,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.13838529586791992,
+      "rewards/margins": 4.899312973022461,
+      "rewards/rejected": -4.760928153991699,
+      "step": 451,
+      "train_speed(iter/s)": 0.021166
+    },
+    {
+      "epoch": 0.9129007826306488,
+      "grad_norm": 22.944427490234375,
+      "learning_rate": 0.00012253128678841568,
+      "logits/chosen": -1.011128544807434,
+      "logits/rejected": -1.0143555402755737,
+      "logps/chosen": -15.104331970214844,
+      "logps/rejected": -42.862266540527344,
+      "loss": 1.11042058467865,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3811737298965454,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3371947109699249,
+      "rewards/margins": 2.6465537548065186,
+      "rewards/rejected": -2.983748435974121,
+      "step": 452,
+      "train_speed(iter/s)": 0.021166
+    },
+    {
+      "epoch": 0.9149204746276193,
+      "grad_norm": 4.528737545013428,
+      "learning_rate": 0.0001222055433479972,
+      "logits/chosen": -1.0228255987167358,
+      "logits/rejected": -1.0586241483688354,
+      "logps/chosen": -1.2600595951080322,
+      "logps/rejected": -73.49124145507812,
+      "loss": 0.28426051139831543,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14906807243824005,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2577366530895233,
+      "rewards/margins": 5.7026872634887695,
+      "rewards/rejected": -5.444951057434082,
+      "step": 453,
+      "train_speed(iter/s)": 0.021167
+    },
+    {
+      "epoch": 0.9169401666245898,
+      "grad_norm": 4.978267669677734,
+      "learning_rate": 0.0001218795518771075,
+      "logits/chosen": -1.0025545358657837,
+      "logits/rejected": -1.0025382041931152,
+      "logps/chosen": -0.9946606755256653,
+      "logps/rejected": -37.86664581298828,
+      "loss": 0.34124690294265747,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09514570236206055,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.231367290019989,
+      "rewards/margins": 2.811678171157837,
+      "rewards/rejected": -2.580310821533203,
+      "step": 454,
+      "train_speed(iter/s)": 0.021168
+    },
+    {
+      "epoch": 0.9189598586215603,
+      "grad_norm": 5.106620788574219,
+      "learning_rate": 0.00012155331601699136,
+      "logits/chosen": -1.0310418605804443,
+      "logits/rejected": -1.0054916143417358,
+      "logps/chosen": -1.0205014944076538,
+      "logps/rejected": -85.00090026855469,
+      "loss": 0.18072457611560822,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1114395335316658,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.41535112261772156,
+      "rewards/margins": 6.101562023162842,
+      "rewards/rejected": -5.686211109161377,
+      "step": 455,
+      "train_speed(iter/s)": 0.021167
+    },
+    {
+      "epoch": 0.9209795506185307,
+      "grad_norm": 1.438368797302246,
+      "learning_rate": 0.0001212268394116233,
+      "logits/chosen": -1.0807647705078125,
+      "logits/rejected": -1.057398796081543,
+      "logps/chosen": -0.6302530765533447,
+      "logps/rejected": -55.822792053222656,
+      "loss": 0.25450506806373596,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09770296514034271,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2143504023551941,
+      "rewards/margins": 4.336926460266113,
+      "rewards/rejected": -4.1225762367248535,
+      "step": 456,
+      "train_speed(iter/s)": 0.021168
+    },
+    {
+      "epoch": 0.9229992426155011,
+      "grad_norm": 2.4940450191497803,
+      "learning_rate": 0.00012090012570766685,
+      "logits/chosen": -1.047285556793213,
+      "logits/rejected": -1.0609232187271118,
+      "logps/chosen": -3.7559874057769775,
+      "logps/rejected": -58.776737213134766,
+      "loss": 0.2771581709384918,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1527220904827118,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3420707583427429,
+      "rewards/margins": 5.054807662963867,
+      "rewards/rejected": -4.712737083435059,
+      "step": 457,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 0.9250189346124716,
+      "grad_norm": 16.446552276611328,
+      "learning_rate": 0.00012057317855443395,
+      "logits/chosen": -1.1021456718444824,
+      "logits/rejected": -1.1048663854599,
+      "logps/chosen": -5.987412452697754,
+      "logps/rejected": -61.24203872680664,
+      "loss": 0.6597545742988586,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.40991318225860596,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.06934911757707596,
+      "rewards/margins": 4.689109802246094,
+      "rewards/rejected": -4.758459091186523,
+      "step": 458,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 0.927038626609442,
+      "grad_norm": 2.537680149078369,
+      "learning_rate": 0.00012024600160384416,
+      "logits/chosen": -1.1070061922073364,
+      "logits/rejected": -1.1183254718780518,
+      "logps/chosen": -2.1882822513580322,
+      "logps/rejected": -94.47828674316406,
+      "loss": 0.47271794080734253,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3363499641418457,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.23398056626319885,
+      "rewards/margins": 7.600938320159912,
+      "rewards/rejected": -7.366957187652588,
+      "step": 459,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 0.9290583186064125,
+      "grad_norm": 3.8238890171051025,
+      "learning_rate": 0.0001199185985103836,
+      "logits/chosen": -1.079871416091919,
+      "logits/rejected": -1.1050368547439575,
+      "logps/chosen": -0.6882074475288391,
+      "logps/rejected": -112.36473846435547,
+      "loss": 0.2024538367986679,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1239171102643013,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.41684651374816895,
+      "rewards/margins": 8.971007347106934,
+      "rewards/rejected": -8.554160118103027,
+      "step": 460,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 0.931078010603383,
+      "grad_norm": 7.065299987792969,
+      "learning_rate": 0.00011959097293106465,
+      "logits/chosen": -1.1408017873764038,
+      "logits/rejected": -1.1611324548721313,
+      "logps/chosen": -0.8949813842773438,
+      "logps/rejected": -44.59485626220703,
+      "loss": 0.26141998171806335,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1390543431043625,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3610275685787201,
+      "rewards/margins": 3.8335914611816406,
+      "rewards/rejected": -3.4725637435913086,
+      "step": 461,
+      "train_speed(iter/s)": 0.02117
+    },
+    {
+      "epoch": 0.9330977026003534,
+      "grad_norm": 4.476932048797607,
+      "learning_rate": 0.00011926312852538455,
+      "logits/chosen": -1.128575086593628,
+      "logits/rejected": -1.157038688659668,
+      "logps/chosen": -3.6390225887298584,
+      "logps/rejected": -63.7474365234375,
+      "loss": 0.5282421708106995,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2585342824459076,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.11798002570867538,
+      "rewards/margins": 5.297179222106934,
+      "rewards/rejected": -5.179199695587158,
+      "step": 462,
+      "train_speed(iter/s)": 0.02117
+    },
+    {
+      "epoch": 0.9351173945973239,
+      "grad_norm": 11.82602310180664,
+      "learning_rate": 0.0001189350689552849,
+      "logits/chosen": -1.1367065906524658,
+      "logits/rejected": -1.156144618988037,
+      "logps/chosen": -1.9807525873184204,
+      "logps/rejected": -49.89271926879883,
+      "loss": 0.6038030385971069,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.34540605545043945,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.37543025612831116,
+      "rewards/margins": 4.289130687713623,
+      "rewards/rejected": -3.9137003421783447,
+      "step": 463,
+      "train_speed(iter/s)": 0.021171
+    },
+    {
+      "epoch": 0.9371370865942944,
+      "grad_norm": 12.497689247131348,
+      "learning_rate": 0.00011860679788511064,
+      "logits/chosen": -1.1170810461044312,
+      "logits/rejected": -1.1292904615402222,
+      "logps/chosen": -4.893659591674805,
+      "logps/rejected": -60.843013763427734,
+      "loss": 0.6962206363677979,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4090201258659363,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.10479210317134857,
+      "rewards/margins": 4.898015022277832,
+      "rewards/rejected": -4.793222427368164,
+      "step": 464,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 0.9391567785912648,
+      "grad_norm": 2.3905272483825684,
+      "learning_rate": 0.00011827831898156905,
+      "logits/chosen": -1.119379997253418,
+      "logits/rejected": -1.0900958776474,
+      "logps/chosen": -1.0332272052764893,
+      "logps/rejected": -82.55847930908203,
+      "loss": 0.21928882598876953,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14558973908424377,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.21076346933841705,
+      "rewards/margins": 6.53924560546875,
+      "rewards/rejected": -6.328482151031494,
+      "step": 465,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 0.9411764705882353,
+      "grad_norm": 2.804384708404541,
+      "learning_rate": 0.00011794963591368893,
+      "logits/chosen": -1.1550871133804321,
+      "logits/rejected": -1.1876434087753296,
+      "logps/chosen": -1.962457299232483,
+      "logps/rejected": -91.43489837646484,
+      "loss": 0.26453521847724915,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18894466757774353,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2565460503101349,
+      "rewards/margins": 7.109828472137451,
+      "rewards/rejected": -6.853282451629639,
+      "step": 466,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 0.9431961625852058,
+      "grad_norm": 6.29252290725708,
+      "learning_rate": 0.00011762075235277943,
+      "logits/chosen": -1.1255277395248413,
+      "logits/rejected": -1.148550271987915,
+      "logps/chosen": -7.0823774337768555,
+      "logps/rejected": -84.76576232910156,
+      "loss": 0.7452417016029358,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6746703386306763,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.21994975209236145,
+      "rewards/margins": 6.5102715492248535,
+      "rewards/rejected": -6.730221271514893,
+      "step": 467,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 0.9452158545821763,
+      "grad_norm": 3.2159340381622314,
+      "learning_rate": 0.00011729167197238935,
+      "logits/chosen": -1.0966346263885498,
+      "logits/rejected": -1.103726863861084,
+      "logps/chosen": -4.399909496307373,
+      "logps/rejected": -42.98252487182617,
+      "loss": 0.7106723785400391,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3980051279067993,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.04321293905377388,
+      "rewards/margins": 3.2986812591552734,
+      "rewards/rejected": -3.2554681301116943,
+      "step": 468,
+      "train_speed(iter/s)": 0.021173
+    },
+    {
+      "epoch": 0.9472355465791467,
+      "grad_norm": 3.3617236614227295,
+      "learning_rate": 0.00011696239844826571,
+      "logits/chosen": -1.1088882684707642,
+      "logits/rejected": -1.1284962892532349,
+      "logps/chosen": -1.639062762260437,
+      "logps/rejected": -72.10785675048828,
+      "loss": 0.24586933851242065,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.12198629230260849,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3906645178794861,
+      "rewards/margins": 6.221713066101074,
+      "rewards/rejected": -5.831048011779785,
+      "step": 469,
+      "train_speed(iter/s)": 0.021174
+    },
+    {
+      "epoch": 0.9492552385761172,
+      "grad_norm": 2.8374667167663574,
+      "learning_rate": 0.00011663293545831302,
+      "logits/chosen": -1.1557269096374512,
+      "logits/rejected": -1.175348162651062,
+      "logps/chosen": -2.1615452766418457,
+      "logps/rejected": -36.723304748535156,
+      "loss": 0.38634181022644043,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22775641083717346,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3754770755767822,
+      "rewards/margins": 3.0530035495758057,
+      "rewards/rejected": -2.6775267124176025,
+      "step": 470,
+      "train_speed(iter/s)": 0.021175
+    },
+    {
+      "epoch": 0.9512749305730877,
+      "grad_norm": 11.482617378234863,
+      "learning_rate": 0.00011630328668255206,
+      "logits/chosen": -1.1401901245117188,
+      "logits/rejected": -1.1286134719848633,
+      "logps/chosen": -6.495123386383057,
+      "logps/rejected": -35.29318618774414,
+      "loss": 0.8619951009750366,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5049220323562622,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.13930773735046387,
+      "rewards/margins": 2.6000585556030273,
+      "rewards/rejected": -2.4607508182525635,
+      "step": 471,
+      "train_speed(iter/s)": 0.021175
+    },
+    {
+      "epoch": 0.953294622570058,
+      "grad_norm": 3.4752252101898193,
+      "learning_rate": 0.00011597345580307875,
+      "logits/chosen": -1.162688970565796,
+      "logits/rejected": -1.192884922027588,
+      "logps/chosen": -2.404099225997925,
+      "logps/rejected": -56.021663665771484,
+      "loss": 0.3634865880012512,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23837605118751526,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.12834902107715607,
+      "rewards/margins": 4.692193508148193,
+      "rewards/rejected": -4.563844203948975,
+      "step": 472,
+      "train_speed(iter/s)": 0.021176
+    },
+    {
+      "epoch": 0.9553143145670285,
+      "grad_norm": 2.5039899349212646,
+      "learning_rate": 0.0001156434465040231,
+      "logits/chosen": -1.1668778657913208,
+      "logits/rejected": -1.1930913925170898,
+      "logps/chosen": -3.766876697540283,
+      "logps/rejected": -63.9132194519043,
+      "loss": 0.3028702735900879,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20875141024589539,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3659513592720032,
+      "rewards/margins": 5.140607833862305,
+      "rewards/rejected": -4.774656295776367,
+      "step": 473,
+      "train_speed(iter/s)": 0.021177
+    },
+    {
+      "epoch": 0.957334006563999,
+      "grad_norm": 2.068408250808716,
+      "learning_rate": 0.00011531326247150803,
+      "logits/chosen": -1.1649010181427002,
+      "logits/rejected": -1.1960549354553223,
+      "logps/chosen": -1.7281254529953003,
+      "logps/rejected": -72.08183288574219,
+      "loss": 0.3556201756000519,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1918855905532837,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.12103867530822754,
+      "rewards/margins": 5.5760884284973145,
+      "rewards/rejected": -5.45504903793335,
+      "step": 474,
+      "train_speed(iter/s)": 0.021177
+    },
+    {
+      "epoch": 0.9593536985609694,
+      "grad_norm": 2.6573944091796875,
+      "learning_rate": 0.00011498290739360815,
+      "logits/chosen": -1.1912912130355835,
+      "logits/rejected": -1.1985927820205688,
+      "logps/chosen": -1.0083485841751099,
+      "logps/rejected": -56.71221923828125,
+      "loss": 0.2199336588382721,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09299471974372864,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2724168002605438,
+      "rewards/margins": 4.650609493255615,
+      "rewards/rejected": -4.37819242477417,
+      "step": 475,
+      "train_speed(iter/s)": 0.021178
+    },
+    {
+      "epoch": 0.9613733905579399,
+      "grad_norm": 5.599519729614258,
+      "learning_rate": 0.00011465238496030868,
+      "logits/chosen": -1.186398983001709,
+      "logits/rejected": -1.2222926616668701,
+      "logps/chosen": -2.25126576423645,
+      "logps/rejected": -65.31526184082031,
+      "loss": 0.6140868067741394,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.350625216960907,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.11485815793275833,
+      "rewards/margins": 5.405522346496582,
+      "rewards/rejected": -5.2906646728515625,
+      "step": 476,
+      "train_speed(iter/s)": 0.021178
+    },
+    {
+      "epoch": 0.9633930825549104,
+      "grad_norm": 1.3937158584594727,
+      "learning_rate": 0.00011432169886346404,
+      "logits/chosen": -1.1549639701843262,
+      "logits/rejected": -1.1829733848571777,
+      "logps/chosen": -2.5441954135894775,
+      "logps/rejected": -70.29151916503906,
+      "loss": 0.24065297842025757,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1281718909740448,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2520023286342621,
+      "rewards/margins": 5.15536642074585,
+      "rewards/rejected": -4.903364181518555,
+      "step": 477,
+      "train_speed(iter/s)": 0.021179
+    },
+    {
+      "epoch": 0.9654127745518808,
+      "grad_norm": 3.0468287467956543,
+      "learning_rate": 0.00011399085279675687,
+      "logits/chosen": -1.1789178848266602,
+      "logits/rejected": -1.2163946628570557,
+      "logps/chosen": -4.268495082855225,
+      "logps/rejected": -53.77469253540039,
+      "loss": 0.40022939443588257,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2632007300853729,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3904189169406891,
+      "rewards/margins": 4.054966926574707,
+      "rewards/rejected": -3.66454815864563,
+      "step": 478,
+      "train_speed(iter/s)": 0.021179
+    },
+    {
+      "epoch": 0.9674324665488513,
+      "grad_norm": 6.615331172943115,
+      "learning_rate": 0.00011365985045565648,
+      "logits/chosen": -1.1960713863372803,
+      "logits/rejected": -1.2022978067398071,
+      "logps/chosen": -4.764765739440918,
+      "logps/rejected": -46.24211120605469,
+      "loss": 0.8375652432441711,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.47868138551712036,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.2317066639661789,
+      "rewards/margins": 3.9557816982269287,
+      "rewards/rejected": -3.7240753173828125,
+      "step": 479,
+      "train_speed(iter/s)": 0.02118
+    },
+    {
+      "epoch": 0.9694521585458218,
+      "grad_norm": 3.0518853664398193,
+      "learning_rate": 0.0001133286955373779,
+      "logits/chosen": -1.1568349599838257,
+      "logits/rejected": -1.1736950874328613,
+      "logps/chosen": -1.8058488368988037,
+      "logps/rejected": -63.62800979614258,
+      "loss": 0.3420032560825348,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1940608024597168,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4025638699531555,
+      "rewards/margins": 5.4827446937561035,
+      "rewards/rejected": -5.080180644989014,
+      "step": 480,
+      "train_speed(iter/s)": 0.02118
+    },
+    {
+      "epoch": 0.9714718505427922,
+      "grad_norm": 6.214394569396973,
+      "learning_rate": 0.00011299739174084025,
+      "logits/chosen": -1.1839512586593628,
+      "logits/rejected": -1.2068456411361694,
+      "logps/chosen": -2.6526894569396973,
+      "logps/rejected": -55.63904571533203,
+      "loss": 0.44367533922195435,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2684517502784729,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1775396466255188,
+      "rewards/margins": 4.404716968536377,
+      "rewards/rejected": -4.227177619934082,
+      "step": 481,
+      "train_speed(iter/s)": 0.021181
+    },
+    {
+      "epoch": 0.9734915425397627,
+      "grad_norm": 5.678071975708008,
+      "learning_rate": 0.0001126659427666257,
+      "logits/chosen": -1.1944388151168823,
+      "logits/rejected": -1.2148536443710327,
+      "logps/chosen": -2.0405924320220947,
+      "logps/rejected": -61.48400115966797,
+      "loss": 0.40944910049438477,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23419791460037231,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34613850712776184,
+      "rewards/margins": 5.159732818603516,
+      "rewards/rejected": -4.813594341278076,
+      "step": 482,
+      "train_speed(iter/s)": 0.021182
+    },
+    {
+      "epoch": 0.9755112345367332,
+      "grad_norm": 26.189855575561523,
+      "learning_rate": 0.00011233435231693794,
+      "logits/chosen": -1.1763508319854736,
+      "logits/rejected": -1.195464015007019,
+      "logps/chosen": -9.439873695373535,
+      "logps/rejected": -40.02541732788086,
+      "loss": 1.6160178184509277,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.8637927174568176,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.2953401505947113,
+      "rewards/margins": 2.8487725257873535,
+      "rewards/rejected": -3.1441128253936768,
+      "step": 483,
+      "train_speed(iter/s)": 0.021182
+    },
+    {
+      "epoch": 0.9775309265337037,
+      "grad_norm": 3.0877511501312256,
+      "learning_rate": 0.00011200262409556097,
+      "logits/chosen": -1.1713593006134033,
+      "logits/rejected": -1.2405827045440674,
+      "logps/chosen": -1.0278502702713013,
+      "logps/rejected": -76.41952514648438,
+      "loss": 0.28457099199295044,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20625659823417664,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.32308924198150635,
+      "rewards/margins": 6.389144420623779,
+      "rewards/rejected": -6.066054821014404,
+      "step": 484,
+      "train_speed(iter/s)": 0.021182
+    },
+    {
+      "epoch": 0.979550618530674,
+      "grad_norm": 9.65100383758545,
+      "learning_rate": 0.00011167076180781764,
+      "logits/chosen": -1.146728277206421,
+      "logits/rejected": -1.2022031545639038,
+      "logps/chosen": -5.121095657348633,
+      "logps/rejected": -74.87187957763672,
+      "loss": 0.5169023871421814,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4096004366874695,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2158682644367218,
+      "rewards/margins": 5.7417778968811035,
+      "rewards/rejected": -5.525909900665283,
+      "step": 485,
+      "train_speed(iter/s)": 0.021182
+    },
+    {
+      "epoch": 0.9815703105276445,
+      "grad_norm": 3.232163190841675,
+      "learning_rate": 0.00011133876916052821,
+      "logits/chosen": -1.1618849039077759,
+      "logits/rejected": -1.19745934009552,
+      "logps/chosen": -4.945662498474121,
+      "logps/rejected": -65.05085754394531,
+      "loss": 0.3440502882003784,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2677401304244995,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.08972585201263428,
+      "rewards/margins": 5.094820976257324,
+      "rewards/rejected": -5.0050950050354,
+      "step": 486,
+      "train_speed(iter/s)": 0.021183
+    },
+    {
+      "epoch": 0.983590002524615,
+      "grad_norm": 1.1338063478469849,
+      "learning_rate": 0.0001110066498619692,
+      "logits/chosen": -1.1489181518554688,
+      "logits/rejected": -1.1883840560913086,
+      "logps/chosen": -0.9724054336547852,
+      "logps/rejected": -69.8716812133789,
+      "loss": 0.16405296325683594,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1183013990521431,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.37503695487976074,
+      "rewards/margins": 6.176860809326172,
+      "rewards/rejected": -5.801823616027832,
+      "step": 487,
+      "train_speed(iter/s)": 0.021183
+    },
+    {
+      "epoch": 0.9856096945215854,
+      "grad_norm": 4.550635814666748,
+      "learning_rate": 0.00011067440762183164,
+      "logits/chosen": -1.127516508102417,
+      "logits/rejected": -1.1615864038467407,
+      "logps/chosen": -1.5935579538345337,
+      "logps/rejected": -70.34913635253906,
+      "loss": 0.32236579060554504,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.21638807654380798,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.30907154083251953,
+      "rewards/margins": 4.958349704742432,
+      "rewards/rejected": -4.649278163909912,
+      "step": 488,
+      "train_speed(iter/s)": 0.021183
+    },
+    {
+      "epoch": 0.9876293865185559,
+      "grad_norm": 4.5942206382751465,
+      "learning_rate": 0.00011034204615117982,
+      "logits/chosen": -1.1207928657531738,
+      "logits/rejected": -1.1431444883346558,
+      "logps/chosen": -2.02649188041687,
+      "logps/rejected": -69.4359359741211,
+      "loss": 0.45521846413612366,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2501518428325653,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.21566975116729736,
+      "rewards/margins": 4.797036647796631,
+      "rewards/rejected": -4.581367492675781,
+      "step": 489,
+      "train_speed(iter/s)": 0.021183
+    },
+    {
+      "epoch": 0.9896490785155264,
+      "grad_norm": 3.483578681945801,
+      "learning_rate": 0.00011000956916240985,
+      "logits/chosen": -1.1226303577423096,
+      "logits/rejected": -1.1501247882843018,
+      "logps/chosen": -5.958000659942627,
+      "logps/rejected": -32.57322311401367,
+      "loss": 0.6992753148078918,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5034775137901306,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.47746312618255615,
+      "rewards/margins": 2.7217843532562256,
+      "rewards/rejected": -2.24432110786438,
+      "step": 490,
+      "train_speed(iter/s)": 0.021184
+    },
+    {
+      "epoch": 0.9916687705124968,
+      "grad_norm": 4.195751667022705,
+      "learning_rate": 0.00010967698036920812,
+      "logits/chosen": -1.10448157787323,
+      "logits/rejected": -1.1570674180984497,
+      "logps/chosen": -4.190779685974121,
+      "logps/rejected": -60.6494255065918,
+      "loss": 0.5290452837944031,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3574896454811096,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.35892462730407715,
+      "rewards/margins": 4.584457874298096,
+      "rewards/rejected": -4.225533485412598,
+      "step": 491,
+      "train_speed(iter/s)": 0.021184
+    },
+    {
+      "epoch": 0.9936884625094673,
+      "grad_norm": 1.556215524673462,
+      "learning_rate": 0.00010934428348650986,
+      "logits/chosen": -1.1513900756835938,
+      "logits/rejected": -1.1773016452789307,
+      "logps/chosen": -0.4516194462776184,
+      "logps/rejected": -54.02649688720703,
+      "loss": 0.15559712052345276,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.06051451712846756,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3142217993736267,
+      "rewards/margins": 4.602743148803711,
+      "rewards/rejected": -4.2885212898254395,
+      "step": 492,
+      "train_speed(iter/s)": 0.021185
+    },
+    {
+      "epoch": 0.9957081545064378,
+      "grad_norm": 1.837632656097412,
+      "learning_rate": 0.00010901148223045761,
+      "logits/chosen": -1.122188925743103,
+      "logits/rejected": -1.1630114316940308,
+      "logps/chosen": -1.2533413171768188,
+      "logps/rejected": -65.71454620361328,
+      "loss": 0.23370426893234253,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1294473558664322,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34939849376678467,
+      "rewards/margins": 4.393534183502197,
+      "rewards/rejected": -4.044135093688965,
+      "step": 493,
+      "train_speed(iter/s)": 0.021185
+    },
+    {
+      "epoch": 0.9977278465034082,
+      "grad_norm": 5.143455982208252,
+      "learning_rate": 0.00010867858031835975,
+      "logits/chosen": -1.160941481590271,
+      "logits/rejected": -1.1693061590194702,
+      "logps/chosen": -2.2333078384399414,
+      "logps/rejected": -62.20633316040039,
+      "loss": 0.33971500396728516,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23756271600723267,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.17624695599079132,
+      "rewards/margins": 5.301555633544922,
+      "rewards/rejected": -5.125308513641357,
+      "step": 494,
+      "train_speed(iter/s)": 0.021186
+    },
+    {
+      "epoch": 0.9997475385003787,
+      "grad_norm": 1.6146841049194336,
+      "learning_rate": 0.000108345581468649,
+      "logits/chosen": -1.156958818435669,
+      "logits/rejected": -1.2084702253341675,
+      "logps/chosen": -1.2683087587356567,
+      "logps/rejected": -73.02035522460938,
+      "loss": 0.2508525550365448,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.12798351049423218,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.23698541522026062,
+      "rewards/margins": 5.670597553253174,
+      "rewards/rejected": -5.433611869812012,
+      "step": 495,
+      "train_speed(iter/s)": 0.021186
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 1.6146841049194336,
+      "learning_rate": 0.00010801248940084074,
+      "logits/chosen": -1.271807074546814,
+      "logits/rejected": -1.3348701000213623,
+      "logps/chosen": -4.720621291198768e-05,
+      "logps/rejected": -91.26646423339844,
+      "loss": 1.2297836292418651e-05,
+      "memory(GiB)": 45.64,
+      "nll_loss": 7.867701242503244e-06,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5421666502952576,
+      "rewards/margins": 9.30994987487793,
+      "rewards/rejected": -8.767783164978027,
+      "step": 496,
+      "train_speed(iter/s)": 0.021226
+    },
+    {
+      "epoch": 1.0020196919969704,
+      "grad_norm": 2.052020311355591,
+      "learning_rate": 0.00010767930783549171,
+      "logits/chosen": -1.1468470096588135,
+      "logits/rejected": -1.1875897645950317,
+      "logps/chosen": -1.5805768966674805,
+      "logps/rejected": -60.07487869262695,
+      "loss": 0.35193732380867004,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22589635848999023,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.43169844150543213,
+      "rewards/margins": 4.954350471496582,
+      "rewards/rejected": -4.522652626037598,
+      "step": 497,
+      "train_speed(iter/s)": 0.021223
+    },
+    {
+      "epoch": 1.004039383993941,
+      "grad_norm": 3.080597400665283,
+      "learning_rate": 0.00010734604049415822,
+      "logits/chosen": -1.1931610107421875,
+      "logits/rejected": -1.2040530443191528,
+      "logps/chosen": -0.3329848051071167,
+      "logps/rejected": -89.8782730102539,
+      "loss": 0.08142131567001343,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.028553346171975136,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3950589895248413,
+      "rewards/margins": 7.771610260009766,
+      "rewards/rejected": -7.376551628112793,
+      "step": 498,
+      "train_speed(iter/s)": 0.021224
+    },
+    {
+      "epoch": 1.0060590759909114,
+      "grad_norm": 1.200395941734314,
+      "learning_rate": 0.00010701269109935474,
+      "logits/chosen": -1.1844416856765747,
+      "logits/rejected": -1.216762900352478,
+      "logps/chosen": -2.2689549922943115,
+      "logps/rejected": -74.18507385253906,
+      "loss": 0.18864388763904572,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13327570259571075,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5378884673118591,
+      "rewards/margins": 6.781070709228516,
+      "rewards/rejected": -6.2431817054748535,
+      "step": 499,
+      "train_speed(iter/s)": 0.021224
+    },
+    {
+      "epoch": 1.0080787679878818,
+      "grad_norm": 1.3004249334335327,
+      "learning_rate": 0.00010667926337451217,
+      "logits/chosen": -1.2056668996810913,
+      "logits/rejected": -1.2564384937286377,
+      "logps/chosen": -2.1434969902038574,
+      "logps/rejected": -107.59019470214844,
+      "loss": 0.14688310027122498,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1056690588593483,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.18269282579421997,
+      "rewards/margins": 9.098722457885742,
+      "rewards/rejected": -8.916029930114746,
+      "step": 500,
+      "train_speed(iter/s)": 0.021224
+    },
+    {
+      "epoch": 1.0100984599848524,
+      "grad_norm": 10.15276050567627,
+      "learning_rate": 0.00010634576104393643,
+      "logits/chosen": -1.2333784103393555,
+      "logits/rejected": -1.2786855697631836,
+      "logps/chosen": -1.4780651330947876,
+      "logps/rejected": -113.30062866210938,
+      "loss": 0.33663061261177063,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18906863033771515,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.19989997148513794,
+      "rewards/margins": 9.928545951843262,
+      "rewards/rejected": -9.728645324707031,
+      "step": 501,
+      "train_speed(iter/s)": 0.021209
+    },
+    {
+      "epoch": 1.0121181519818228,
+      "grad_norm": 1.6605217456817627,
+      "learning_rate": 0.00010601218783276672,
+      "logits/chosen": -1.224414348602295,
+      "logits/rejected": -1.286801815032959,
+      "logps/chosen": -0.5362107753753662,
+      "logps/rejected": -101.3456802368164,
+      "loss": 0.1514112651348114,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.07159439474344254,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.21423012018203735,
+      "rewards/margins": 8.556539535522461,
+      "rewards/rejected": -8.342309951782227,
+      "step": 502,
+      "train_speed(iter/s)": 0.021209
+    },
+    {
+      "epoch": 1.0141378439787931,
+      "grad_norm": 12.038124084472656,
+      "learning_rate": 0.00010567854746693396,
+      "logits/chosen": -1.2550753355026245,
+      "logits/rejected": -1.3067824840545654,
+      "logps/chosen": -4.968113899230957,
+      "logps/rejected": -76.51419067382812,
+      "loss": 0.6970563530921936,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5403850674629211,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.015964236110448837,
+      "rewards/margins": 6.770257472991943,
+      "rewards/rejected": -6.7542924880981445,
+      "step": 503,
+      "train_speed(iter/s)": 0.02121
+    },
+    {
+      "epoch": 1.0161575359757637,
+      "grad_norm": 5.976532936096191,
+      "learning_rate": 0.00010534484367311923,
+      "logits/chosen": -1.2590959072113037,
+      "logits/rejected": -1.3068499565124512,
+      "logps/chosen": -2.158994436264038,
+      "logps/rejected": -74.8353271484375,
+      "loss": 0.34383249282836914,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22420844435691833,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4135916233062744,
+      "rewards/margins": 6.9369611740112305,
+      "rewards/rejected": -6.523369789123535,
+      "step": 504,
+      "train_speed(iter/s)": 0.02121
+    },
+    {
+      "epoch": 1.0181772279727341,
+      "grad_norm": 8.177599906921387,
+      "learning_rate": 0.00010501108017871192,
+      "logits/chosen": -1.2204450368881226,
+      "logits/rejected": -1.2789673805236816,
+      "logps/chosen": -1.5879818201065063,
+      "logps/rejected": -66.489990234375,
+      "loss": 0.35112184286117554,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18649819493293762,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.41614973545074463,
+      "rewards/margins": 5.6356306076049805,
+      "rewards/rejected": -5.219481468200684,
+      "step": 505,
+      "train_speed(iter/s)": 0.021211
+    },
+    {
+      "epoch": 1.0201969199697045,
+      "grad_norm": 1.072225570678711,
+      "learning_rate": 0.00010467726071176853,
+      "logits/chosen": -1.2156474590301514,
+      "logits/rejected": -1.2541887760162354,
+      "logps/chosen": -7.711733341217041,
+      "logps/rejected": -65.3000717163086,
+      "loss": 0.1724158525466919,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.12215537577867508,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6666063070297241,
+      "rewards/margins": 6.159120559692383,
+      "rewards/rejected": -5.492514133453369,
+      "step": 506,
+      "train_speed(iter/s)": 0.021211
+    },
+    {
+      "epoch": 1.0222166119666751,
+      "grad_norm": 2.4010398387908936,
+      "learning_rate": 0.00010434338900097049,
+      "logits/chosen": -1.2528554201126099,
+      "logits/rejected": -1.3154045343399048,
+      "logps/chosen": -0.8416248559951782,
+      "logps/rejected": -69.87122344970703,
+      "loss": 0.12864626944065094,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.07988111674785614,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.31406697630882263,
+      "rewards/margins": 6.217660903930664,
+      "rewards/rejected": -5.903594493865967,
+      "step": 507,
+      "train_speed(iter/s)": 0.021212
+    },
+    {
+      "epoch": 1.0242363039636455,
+      "grad_norm": 1.6103118658065796,
+      "learning_rate": 0.00010400946877558293,
+      "logits/chosen": -1.171745777130127,
+      "logits/rejected": -1.205277919769287,
+      "logps/chosen": -1.2725749015808105,
+      "logps/rejected": -75.42990112304688,
+      "loss": 0.17115575075149536,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11690245568752289,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.42929667234420776,
+      "rewards/margins": 5.8164963722229,
+      "rewards/rejected": -5.387199878692627,
+      "step": 508,
+      "train_speed(iter/s)": 0.021212
+    },
+    {
+      "epoch": 1.0262559959606161,
+      "grad_norm": 7.029423713684082,
+      "learning_rate": 0.00010367550376541283,
+      "logits/chosen": -1.205200433731079,
+      "logits/rejected": -1.2511245012283325,
+      "logps/chosen": -2.5946669578552246,
+      "logps/rejected": -84.32099914550781,
+      "loss": 0.30733728408813477,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2792370319366455,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.25035470724105835,
+      "rewards/margins": 6.80730676651001,
+      "rewards/rejected": -6.556951999664307,
+      "step": 509,
+      "train_speed(iter/s)": 0.021213
+    },
+    {
+      "epoch": 1.0282756879575865,
+      "grad_norm": 0.7719494700431824,
+      "learning_rate": 0.00010334149770076747,
+      "logits/chosen": -1.2033907175064087,
+      "logits/rejected": -1.2671985626220703,
+      "logps/chosen": -0.1477760523557663,
+      "logps/rejected": -67.04902648925781,
+      "loss": 0.06301882117986679,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.029819779098033905,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3393843173980713,
+      "rewards/margins": 5.922677516937256,
+      "rewards/rejected": -5.5832929611206055,
+      "step": 510,
+      "train_speed(iter/s)": 0.021213
+    },
+    {
+      "epoch": 1.030295379954557,
+      "grad_norm": 6.209945201873779,
+      "learning_rate": 0.00010300745431241259,
+      "logits/chosen": -1.1784838438034058,
+      "logits/rejected": -1.2575358152389526,
+      "logps/chosen": -2.9259207248687744,
+      "logps/rejected": -66.25347137451172,
+      "loss": 0.493066668510437,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.34985142946243286,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2988048195838928,
+      "rewards/margins": 5.669159412384033,
+      "rewards/rejected": -5.370354652404785,
+      "step": 511,
+      "train_speed(iter/s)": 0.021214
+    },
+    {
+      "epoch": 1.0323150719515275,
+      "grad_norm": 14.864357948303223,
+      "learning_rate": 0.00010267337733153089,
+      "logits/chosen": -1.211972951889038,
+      "logits/rejected": -1.2457599639892578,
+      "logps/chosen": -4.108205795288086,
+      "logps/rejected": -49.29187774658203,
+      "loss": 0.7474625110626221,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5005552768707275,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1060146689414978,
+      "rewards/margins": 3.993147134780884,
+      "rewards/rejected": -3.8871326446533203,
+      "step": 512,
+      "train_speed(iter/s)": 0.021214
+    },
+    {
+      "epoch": 1.0343347639484979,
+      "grad_norm": 4.5455522537231445,
+      "learning_rate": 0.0001023392704896803,
+      "logits/chosen": -1.1689393520355225,
+      "logits/rejected": -1.1586272716522217,
+      "logps/chosen": -2.678110361099243,
+      "logps/rejected": -66.11392974853516,
+      "loss": 0.3088136315345764,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2110619693994522,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2811347544193268,
+      "rewards/margins": 5.262904167175293,
+      "rewards/rejected": -4.981769561767578,
+      "step": 513,
+      "train_speed(iter/s)": 0.021214
+    },
+    {
+      "epoch": 1.0363544559454683,
+      "grad_norm": 6.412376880645752,
+      "learning_rate": 0.00010200513751875227,
+      "logits/chosen": -1.1625449657440186,
+      "logits/rejected": -1.1948907375335693,
+      "logps/chosen": -2.3596391677856445,
+      "logps/rejected": -68.15946197509766,
+      "loss": 0.39961010217666626,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.26144012808799744,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3064810037612915,
+      "rewards/margins": 5.5451202392578125,
+      "rewards/rejected": -5.2386393547058105,
+      "step": 514,
+      "train_speed(iter/s)": 0.021215
+    },
+    {
+      "epoch": 1.0383741479424389,
+      "grad_norm": 2.85520601272583,
+      "learning_rate": 0.00010167098215093009,
+      "logits/chosen": -1.1633456945419312,
+      "logits/rejected": -1.2228730916976929,
+      "logps/chosen": -3.7188360691070557,
+      "logps/rejected": -95.9930419921875,
+      "loss": 0.2916540801525116,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23056216537952423,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3937748074531555,
+      "rewards/margins": 7.421454429626465,
+      "rewards/rejected": -7.027679443359375,
+      "step": 515,
+      "train_speed(iter/s)": 0.021214
+    },
+    {
+      "epoch": 1.0403938399394093,
+      "grad_norm": 5.375782012939453,
+      "learning_rate": 0.00010133680811864727,
+      "logits/chosen": -1.1782926321029663,
+      "logits/rejected": -1.2417353391647339,
+      "logps/chosen": -1.3107043504714966,
+      "logps/rejected": -53.36506652832031,
+      "loss": 0.19477064907550812,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14968222379684448,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.25208836793899536,
+      "rewards/margins": 4.3324995040893555,
+      "rewards/rejected": -4.080410957336426,
+      "step": 516,
+      "train_speed(iter/s)": 0.021215
+    },
+    {
+      "epoch": 1.0424135319363796,
+      "grad_norm": 1.9422154426574707,
+      "learning_rate": 0.00010100261915454571,
+      "logits/chosen": -1.1514760255813599,
+      "logits/rejected": -1.2427828311920166,
+      "logps/chosen": -0.6266792416572571,
+      "logps/rejected": -79.95481872558594,
+      "loss": 0.09264641255140305,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.06077795848250389,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.45081964135169983,
+      "rewards/margins": 6.714786529541016,
+      "rewards/rejected": -6.263967037200928,
+      "step": 517,
+      "train_speed(iter/s)": 0.021215
+    },
+    {
+      "epoch": 1.0444332239333503,
+      "grad_norm": 20.16422462463379,
+      "learning_rate": 0.00010066841899143425,
+      "logits/chosen": -1.1963735818862915,
+      "logits/rejected": -1.2633908987045288,
+      "logps/chosen": -6.527058124542236,
+      "logps/rejected": -39.10000991821289,
+      "loss": 0.7584791779518127,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5177947878837585,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3015112280845642,
+      "rewards/margins": 3.383577585220337,
+      "rewards/rejected": -3.082066535949707,
+      "step": 518,
+      "train_speed(iter/s)": 0.021215
+    },
+    {
+      "epoch": 1.0464529159303206,
+      "grad_norm": 7.269643783569336,
+      "learning_rate": 0.00010033421136224659,
+      "logits/chosen": -1.1538660526275635,
+      "logits/rejected": -1.1920374631881714,
+      "logps/chosen": -1.6170885562896729,
+      "logps/rejected": -57.34727096557617,
+      "loss": 0.1741914451122284,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11453510820865631,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2959902584552765,
+      "rewards/margins": 4.944981575012207,
+      "rewards/rejected": -4.648991584777832,
+      "step": 519,
+      "train_speed(iter/s)": 0.021216
+    },
+    {
+      "epoch": 1.048472607927291,
+      "grad_norm": 2.5758349895477295,
+      "learning_rate": 0.0001,
+      "logits/chosen": -1.1641168594360352,
+      "logits/rejected": -1.2306013107299805,
+      "logps/chosen": -2.8853821754455566,
+      "logps/rejected": -45.58841323852539,
+      "loss": 0.35295766592025757,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2536754012107849,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34334903955459595,
+      "rewards/margins": 3.8398725986480713,
+      "rewards/rejected": -3.49652361869812,
+      "step": 520,
+      "train_speed(iter/s)": 0.021217
+    },
+    {
+      "epoch": 1.0504922999242616,
+      "grad_norm": 3.192835569381714,
+      "learning_rate": 9.966578863775344e-05,
+      "logits/chosen": -1.1690419912338257,
+      "logits/rejected": -1.2847118377685547,
+      "logps/chosen": -1.6086413860321045,
+      "logps/rejected": -75.8061294555664,
+      "loss": 0.2794322371482849,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20665356516838074,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3336547017097473,
+      "rewards/margins": 6.422711372375488,
+      "rewards/rejected": -6.089056491851807,
+      "step": 521,
+      "train_speed(iter/s)": 0.021217
+    },
+    {
+      "epoch": 1.052511991921232,
+      "grad_norm": 5.373813629150391,
+      "learning_rate": 9.93315810085658e-05,
+      "logits/chosen": -1.099102258682251,
+      "logits/rejected": -1.2499687671661377,
+      "logps/chosen": -0.8937387466430664,
+      "logps/rejected": -83.5426254272461,
+      "loss": 0.22286824882030487,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1412465125322342,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34925389289855957,
+      "rewards/margins": 6.224581718444824,
+      "rewards/rejected": -5.8753275871276855,
+      "step": 522,
+      "train_speed(iter/s)": 0.021217
+    },
+    {
+      "epoch": 1.0545316839182024,
+      "grad_norm": 1.588864803314209,
+      "learning_rate": 9.89973808454543e-05,
+      "logits/chosen": -1.2088499069213867,
+      "logits/rejected": -1.2908425331115723,
+      "logps/chosen": -1.425804615020752,
+      "logps/rejected": -84.73382568359375,
+      "loss": 0.24856942892074585,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17529940605163574,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.33582112193107605,
+      "rewards/margins": 7.493390083312988,
+      "rewards/rejected": -7.157568454742432,
+      "step": 523,
+      "train_speed(iter/s)": 0.021218
+    },
+    {
+      "epoch": 1.056551375915173,
+      "grad_norm": 11.534995079040527,
+      "learning_rate": 9.866319188135274e-05,
+      "logits/chosen": -1.155828833580017,
+      "logits/rejected": -1.226155400276184,
+      "logps/chosen": -2.9224207401275635,
+      "logps/rejected": -76.7179946899414,
+      "loss": 0.39948102831840515,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.28366178274154663,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.40342777967453003,
+      "rewards/margins": 6.173599720001221,
+      "rewards/rejected": -5.770172119140625,
+      "step": 524,
+      "train_speed(iter/s)": 0.021218
+    },
+    {
+      "epoch": 1.0585710679121434,
+      "grad_norm": 3.017298936843872,
+      "learning_rate": 9.83290178490699e-05,
+      "logits/chosen": -1.2128490209579468,
+      "logits/rejected": -1.3072235584259033,
+      "logps/chosen": -2.281188488006592,
+      "logps/rejected": -75.67351531982422,
+      "loss": 0.28667759895324707,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2029131054878235,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.27607351541519165,
+      "rewards/margins": 6.455198287963867,
+      "rewards/rejected": -6.17912483215332,
+      "step": 525,
+      "train_speed(iter/s)": 0.021218
+    },
+    {
+      "epoch": 1.0605907599091138,
+      "grad_norm": 2.768603563308716,
+      "learning_rate": 9.799486248124775e-05,
+      "logits/chosen": -1.1740716695785522,
+      "logits/rejected": -1.267440915107727,
+      "logps/chosen": -3.626737594604492,
+      "logps/rejected": -79.38348388671875,
+      "loss": 0.2453736811876297,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15417639911174774,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.7520807385444641,
+      "rewards/margins": 7.059826850891113,
+      "rewards/rejected": -6.307745933532715,
+      "step": 526,
+      "train_speed(iter/s)": 0.021218
+    },
+    {
+      "epoch": 1.0626104519060844,
+      "grad_norm": 2.207165002822876,
+      "learning_rate": 9.766072951031972e-05,
+      "logits/chosen": -1.2672584056854248,
+      "logits/rejected": -1.32329261302948,
+      "logps/chosen": -1.5716861486434937,
+      "logps/rejected": -68.19523620605469,
+      "loss": 0.20522795617580414,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.16552022099494934,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3385736048221588,
+      "rewards/margins": 5.919947624206543,
+      "rewards/rejected": -5.581373691558838,
+      "step": 527,
+      "train_speed(iter/s)": 0.021219
+    },
+    {
+      "epoch": 1.0646301439030548,
+      "grad_norm": 1.6318920850753784,
+      "learning_rate": 9.732662266846912e-05,
+      "logits/chosen": -1.157897710800171,
+      "logits/rejected": -1.2902889251708984,
+      "logps/chosen": -1.1771454811096191,
+      "logps/rejected": -94.12054443359375,
+      "loss": 0.21492774784564972,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17460428178310394,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3003194034099579,
+      "rewards/margins": 8.205141067504883,
+      "rewards/rejected": -7.904821395874023,
+      "step": 528,
+      "train_speed(iter/s)": 0.021219
+    },
+    {
+      "epoch": 1.0666498359000252,
+      "grad_norm": 5.924353122711182,
+      "learning_rate": 9.699254568758741e-05,
+      "logits/chosen": -1.2643760442733765,
+      "logits/rejected": -1.368143081665039,
+      "logps/chosen": -1.368375301361084,
+      "logps/rejected": -86.12940979003906,
+      "loss": 0.14123007655143738,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11123973876237869,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3200608789920807,
+      "rewards/margins": 7.876832008361816,
+      "rewards/rejected": -7.556771278381348,
+      "step": 529,
+      "train_speed(iter/s)": 0.021219
+    },
+    {
+      "epoch": 1.0686695278969958,
+      "grad_norm": 1.9186508655548096,
+      "learning_rate": 9.665850229923258e-05,
+      "logits/chosen": -1.2655577659606934,
+      "logits/rejected": -1.344153642654419,
+      "logps/chosen": -0.23585116863250732,
+      "logps/rejected": -80.97904968261719,
+      "loss": 0.11053043603897095,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.04247582331299782,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3903675675392151,
+      "rewards/margins": 7.256943702697754,
+      "rewards/rejected": -6.866576194763184,
+      "step": 530,
+      "train_speed(iter/s)": 0.02122
+    },
+    {
+      "epoch": 1.0706892198939661,
+      "grad_norm": 1.6089398860931396,
+      "learning_rate": 9.632449623458718e-05,
+      "logits/chosen": -1.2310450077056885,
+      "logits/rejected": -1.306509256362915,
+      "logps/chosen": -1.039363145828247,
+      "logps/rejected": -56.28333282470703,
+      "loss": 0.18445441126823425,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09440238028764725,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4418891668319702,
+      "rewards/margins": 4.846897602081299,
+      "rewards/rejected": -4.405008792877197,
+      "step": 531,
+      "train_speed(iter/s)": 0.021221
+    },
+    {
+      "epoch": 1.0727089118909365,
+      "grad_norm": 4.852963447570801,
+      "learning_rate": 9.59905312244171e-05,
+      "logits/chosen": -1.2377697229385376,
+      "logits/rejected": -1.3623260259628296,
+      "logps/chosen": -0.7856588363647461,
+      "logps/rejected": -99.22467041015625,
+      "loss": 0.1534590870141983,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11053737998008728,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.27947190403938293,
+      "rewards/margins": 8.624140739440918,
+      "rewards/rejected": -8.3446683883667,
+      "step": 532,
+      "train_speed(iter/s)": 0.021221
+    },
+    {
+      "epoch": 1.0747286038879071,
+      "grad_norm": 10.742178916931152,
+      "learning_rate": 9.565661099902952e-05,
+      "logits/chosen": -1.1463772058486938,
+      "logits/rejected": -1.2955669164657593,
+      "logps/chosen": -3.547930955886841,
+      "logps/rejected": -82.50328063964844,
+      "loss": 0.657089352607727,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4685670733451843,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.0838821679353714,
+      "rewards/margins": 6.732100009918213,
+      "rewards/rejected": -6.64821720123291,
+      "step": 533,
+      "train_speed(iter/s)": 0.021221
+    },
+    {
+      "epoch": 1.0767482958848775,
+      "grad_norm": 1.633371114730835,
+      "learning_rate": 9.532273928823151e-05,
+      "logits/chosen": -1.2212746143341064,
+      "logits/rejected": -1.269982099533081,
+      "logps/chosen": -3.351022243499756,
+      "logps/rejected": -92.98002624511719,
+      "loss": 0.20061077177524567,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09750235080718994,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.6985870599746704,
+      "rewards/margins": 7.949156284332275,
+      "rewards/rejected": -7.250568389892578,
+      "step": 534,
+      "train_speed(iter/s)": 0.021221
+    },
+    {
+      "epoch": 1.078767987881848,
+      "grad_norm": 1.1351590156555176,
+      "learning_rate": 9.498891982128809e-05,
+      "logits/chosen": -1.2288845777511597,
+      "logits/rejected": -1.3357539176940918,
+      "logps/chosen": -0.435973584651947,
+      "logps/rejected": -82.69353485107422,
+      "loss": 0.06550046801567078,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.051096200942993164,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3056626617908478,
+      "rewards/margins": 7.295601844787598,
+      "rewards/rejected": -6.989939212799072,
+      "step": 535,
+      "train_speed(iter/s)": 0.021222
+    },
+    {
+      "epoch": 1.0807876798788185,
+      "grad_norm": 7.175545692443848,
+      "learning_rate": 9.46551563268808e-05,
+      "logits/chosen": -1.1364165544509888,
+      "logits/rejected": -1.2312272787094116,
+      "logps/chosen": -3.2798967361450195,
+      "logps/rejected": -68.39805603027344,
+      "loss": 0.40154242515563965,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.27947506308555603,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.20716847479343414,
+      "rewards/margins": 5.617040634155273,
+      "rewards/rejected": -5.409872531890869,
+      "step": 536,
+      "train_speed(iter/s)": 0.021222
+    },
+    {
+      "epoch": 1.082807371875789,
+      "grad_norm": 16.1717472076416,
+      "learning_rate": 9.432145253306604e-05,
+      "logits/chosen": -1.1581474542617798,
+      "logits/rejected": -1.275297999382019,
+      "logps/chosen": -6.416726112365723,
+      "logps/rejected": -84.11360931396484,
+      "loss": 0.6197578310966492,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6048212647438049,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3198436498641968,
+      "rewards/margins": 7.643919944763184,
+      "rewards/rejected": -7.324074745178223,
+      "step": 537,
+      "train_speed(iter/s)": 0.021223
+    },
+    {
+      "epoch": 1.0848270638727593,
+      "grad_norm": 16.336959838867188,
+      "learning_rate": 9.398781216723331e-05,
+      "logits/chosen": -1.035546898841858,
+      "logits/rejected": -1.2581911087036133,
+      "logps/chosen": -1.4019535779953003,
+      "logps/rejected": -93.62826538085938,
+      "loss": 0.3454136550426483,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23441605269908905,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2636178433895111,
+      "rewards/margins": 8.16170597076416,
+      "rewards/rejected": -7.898087501525879,
+      "step": 538,
+      "train_speed(iter/s)": 0.021223
+    },
+    {
+      "epoch": 1.08684675586973,
+      "grad_norm": 4.587718963623047,
+      "learning_rate": 9.36542389560636e-05,
+      "logits/chosen": -1.0745295286178589,
+      "logits/rejected": -1.2483410835266113,
+      "logps/chosen": -5.092793941497803,
+      "logps/rejected": -87.74403381347656,
+      "loss": 0.5541177988052368,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.42148298025131226,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3391857147216797,
+      "rewards/margins": 7.316706657409668,
+      "rewards/rejected": -6.977520942687988,
+      "step": 539,
+      "train_speed(iter/s)": 0.021223
+    },
+    {
+      "epoch": 1.0888664478667003,
+      "grad_norm": 2.1240127086639404,
+      "learning_rate": 9.332073662548784e-05,
+      "logits/chosen": -1.1243212223052979,
+      "logits/rejected": -1.193894863128662,
+      "logps/chosen": -1.587296724319458,
+      "logps/rejected": -59.78162384033203,
+      "loss": 0.28339654207229614,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18274566531181335,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2662951350212097,
+      "rewards/margins": 5.166958808898926,
+      "rewards/rejected": -4.900664329528809,
+      "step": 540,
+      "train_speed(iter/s)": 0.021224
+    },
+    {
+      "epoch": 1.090886139863671,
+      "grad_norm": 2.6355695724487305,
+      "learning_rate": 9.29873089006453e-05,
+      "logits/chosen": -1.0941351652145386,
+      "logits/rejected": -1.2066431045532227,
+      "logps/chosen": -1.4627405405044556,
+      "logps/rejected": -77.10699462890625,
+      "loss": 0.2207845151424408,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13547611236572266,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3999878466129303,
+      "rewards/margins": 6.711189270019531,
+      "rewards/rejected": -6.311202049255371,
+      "step": 541,
+      "train_speed(iter/s)": 0.021225
+    },
+    {
+      "epoch": 1.0929058318606413,
+      "grad_norm": 2.05720853805542,
+      "learning_rate": 9.26539595058418e-05,
+      "logits/chosen": -1.0910502672195435,
+      "logits/rejected": -1.2283904552459717,
+      "logps/chosen": -1.5382826328277588,
+      "logps/rejected": -76.15080261230469,
+      "loss": 0.25323227047920227,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1447850912809372,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3722470700740814,
+      "rewards/margins": 6.785793304443359,
+      "rewards/rejected": -6.413545608520508,
+      "step": 542,
+      "train_speed(iter/s)": 0.021225
+    },
+    {
+      "epoch": 1.0949255238576117,
+      "grad_norm": 2.266726493835449,
+      "learning_rate": 9.23206921645083e-05,
+      "logits/chosen": -1.0470198392868042,
+      "logits/rejected": -1.150384545326233,
+      "logps/chosen": -3.195725679397583,
+      "logps/rejected": -61.4305534362793,
+      "loss": 0.293194055557251,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2274458259344101,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5585137605667114,
+      "rewards/margins": 5.642398834228516,
+      "rewards/rejected": -5.083885192871094,
+      "step": 543,
+      "train_speed(iter/s)": 0.021226
+    },
+    {
+      "epoch": 1.0969452158545823,
+      "grad_norm": 2.564605474472046,
+      "learning_rate": 9.198751059915925e-05,
+      "logits/chosen": -1.0186107158660889,
+      "logits/rejected": -1.2122492790222168,
+      "logps/chosen": -2.0936737060546875,
+      "logps/rejected": -82.15401458740234,
+      "loss": 0.34008562564849854,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2496323138475418,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.31812718510627747,
+      "rewards/margins": 6.999300479888916,
+      "rewards/rejected": -6.6811723709106445,
+      "step": 544,
+      "train_speed(iter/s)": 0.021226
+    },
+    {
+      "epoch": 1.0989649078515527,
+      "grad_norm": 9.990952491760254,
+      "learning_rate": 9.165441853135104e-05,
+      "logits/chosen": -1.1474440097808838,
+      "logits/rejected": -1.2639394998550415,
+      "logps/chosen": -1.8011548519134521,
+      "logps/rejected": -73.41018676757812,
+      "loss": 0.36625638604164124,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.29132476449012756,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.21145863831043243,
+      "rewards/margins": 6.8196306228637695,
+      "rewards/rejected": -6.608172416687012,
+      "step": 545,
+      "train_speed(iter/s)": 0.021227
+    },
+    {
+      "epoch": 1.100984599848523,
+      "grad_norm": 2.7831358909606934,
+      "learning_rate": 9.132141968164026e-05,
+      "logits/chosen": -1.0405611991882324,
+      "logits/rejected": -1.2216545343399048,
+      "logps/chosen": -1.1892681121826172,
+      "logps/rejected": -94.48490905761719,
+      "loss": 0.25090837478637695,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14087605476379395,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3783609867095947,
+      "rewards/margins": 8.133208274841309,
+      "rewards/rejected": -7.754847526550293,
+      "step": 546,
+      "train_speed(iter/s)": 0.021227
+    },
+    {
+      "epoch": 1.1030042918454936,
+      "grad_norm": 8.550939559936523,
+      "learning_rate": 9.098851776954241e-05,
+      "logits/chosen": -1.0600169897079468,
+      "logits/rejected": -1.1879816055297852,
+      "logps/chosen": -2.9978091716766357,
+      "logps/rejected": -69.18419647216797,
+      "loss": 0.491380512714386,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.32408130168914795,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.21779419481754303,
+      "rewards/margins": 5.817697048187256,
+      "rewards/rejected": -5.599903106689453,
+      "step": 547,
+      "train_speed(iter/s)": 0.021227
+    },
+    {
+      "epoch": 1.105023983842464,
+      "grad_norm": 14.694620132446289,
+      "learning_rate": 9.065571651349015e-05,
+      "logits/chosen": -1.0587307214736938,
+      "logits/rejected": -1.1352782249450684,
+      "logps/chosen": -2.723759174346924,
+      "logps/rejected": -73.38800048828125,
+      "loss": 0.3820952773094177,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.28878253698349,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.25954926013946533,
+      "rewards/margins": 6.028653621673584,
+      "rewards/rejected": -5.769104480743408,
+      "step": 548,
+      "train_speed(iter/s)": 0.021228
+    },
+    {
+      "epoch": 1.1070436758394344,
+      "grad_norm": 3.4368412494659424,
+      "learning_rate": 9.032301963079191e-05,
+      "logits/chosen": -1.0471107959747314,
+      "logits/rejected": -1.216453194618225,
+      "logps/chosen": -2.2245757579803467,
+      "logps/rejected": -64.19511413574219,
+      "loss": 0.2849371135234833,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2175203114748001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3077858090400696,
+      "rewards/margins": 5.670999526977539,
+      "rewards/rejected": -5.363213539123535,
+      "step": 549,
+      "train_speed(iter/s)": 0.021228
+    },
+    {
+      "epoch": 1.109063367836405,
+      "grad_norm": 2.0185515880584717,
+      "learning_rate": 8.999043083759017e-05,
+      "logits/chosen": -1.0703552961349487,
+      "logits/rejected": -1.2551573514938354,
+      "logps/chosen": -1.0120265483856201,
+      "logps/rejected": -85.53215026855469,
+      "loss": 0.15907928347587585,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11441606283187866,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2885012924671173,
+      "rewards/margins": 7.635853290557861,
+      "rewards/rejected": -7.347352504730225,
+      "step": 550,
+      "train_speed(iter/s)": 0.021228
+    },
+    {
+      "epoch": 1.1110830598333754,
+      "grad_norm": 2.8778460025787354,
+      "learning_rate": 8.965795384882021e-05,
+      "logits/chosen": -0.9114001393318176,
+      "logits/rejected": -1.166597604751587,
+      "logps/chosen": -1.139864444732666,
+      "logps/rejected": -100.16082000732422,
+      "loss": 0.30031049251556396,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2430577278137207,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.24268564581871033,
+      "rewards/margins": 8.84631633758545,
+      "rewards/rejected": -8.603631019592285,
+      "step": 551,
+      "train_speed(iter/s)": 0.021214
+    },
+    {
+      "epoch": 1.1131027518303458,
+      "grad_norm": 2.5478785037994385,
+      "learning_rate": 8.932559237816839e-05,
+      "logits/chosen": -0.9818710088729858,
+      "logits/rejected": -1.225954532623291,
+      "logps/chosen": -3.4108481407165527,
+      "logps/rejected": -79.83183288574219,
+      "loss": 0.4420861601829529,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3446241617202759,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.22779923677444458,
+      "rewards/margins": 6.333407402038574,
+      "rewards/rejected": -6.105607509613037,
+      "step": 552,
+      "train_speed(iter/s)": 0.021214
+    },
+    {
+      "epoch": 1.1151224438273164,
+      "grad_norm": 6.415960788726807,
+      "learning_rate": 8.899335013803084e-05,
+      "logits/chosen": -1.091843605041504,
+      "logits/rejected": -1.242721438407898,
+      "logps/chosen": -1.220659613609314,
+      "logps/rejected": -61.345306396484375,
+      "loss": 0.22371630370616913,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1757785677909851,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.40316012501716614,
+      "rewards/margins": 5.553282737731934,
+      "rewards/rejected": -5.150122165679932,
+      "step": 553,
+      "train_speed(iter/s)": 0.021215
+    },
+    {
+      "epoch": 1.1171421358242868,
+      "grad_norm": 1.3796110153198242,
+      "learning_rate": 8.866123083947182e-05,
+      "logits/chosen": -1.075179100036621,
+      "logits/rejected": -1.1752409934997559,
+      "logps/chosen": -1.5424776077270508,
+      "logps/rejected": -65.63243103027344,
+      "loss": 0.23596073687076569,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18632307648658752,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34452810883522034,
+      "rewards/margins": 5.910964012145996,
+      "rewards/rejected": -5.566435813903809,
+      "step": 554,
+      "train_speed(iter/s)": 0.021215
+    },
+    {
+      "epoch": 1.1191618278212572,
+      "grad_norm": 3.3275868892669678,
+      "learning_rate": 8.832923819218238e-05,
+      "logits/chosen": -1.001117467880249,
+      "logits/rejected": -1.1721633672714233,
+      "logps/chosen": -1.894881010055542,
+      "logps/rejected": -61.61512756347656,
+      "loss": 0.35188451409339905,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3188917636871338,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.12245701998472214,
+      "rewards/margins": 5.278988838195801,
+      "rewards/rejected": -5.15653133392334,
+      "step": 555,
+      "train_speed(iter/s)": 0.021216
+    },
+    {
+      "epoch": 1.1211815198182278,
+      "grad_norm": 1.58943510055542,
+      "learning_rate": 8.799737590443902e-05,
+      "logits/chosen": -0.9907103776931763,
+      "logits/rejected": -1.2039836645126343,
+      "logps/chosen": -3.9374332427978516,
+      "logps/rejected": -85.5752944946289,
+      "loss": 0.3835102617740631,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3536885380744934,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1798345446586609,
+      "rewards/margins": 7.102011203765869,
+      "rewards/rejected": -6.922177314758301,
+      "step": 556,
+      "train_speed(iter/s)": 0.021216
+    },
+    {
+      "epoch": 1.1232012118151982,
+      "grad_norm": 4.0305495262146,
+      "learning_rate": 8.766564768306207e-05,
+      "logits/chosen": -1.1515066623687744,
+      "logits/rejected": -1.214231252670288,
+      "logps/chosen": -0.7779844999313354,
+      "logps/rejected": -56.81134796142578,
+      "loss": 0.1507856547832489,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.0650799572467804,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.30766594409942627,
+      "rewards/margins": 5.128314971923828,
+      "rewards/rejected": -4.820648670196533,
+      "step": 557,
+      "train_speed(iter/s)": 0.021217
+    },
+    {
+      "epoch": 1.1252209038121685,
+      "grad_norm": 4.040610313415527,
+      "learning_rate": 8.733405723337432e-05,
+      "logits/chosen": -1.0960428714752197,
+      "logits/rejected": -1.2518525123596191,
+      "logps/chosen": -0.9766073226928711,
+      "logps/rejected": -73.12732696533203,
+      "loss": 0.26937615871429443,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1882752627134323,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.26189538836479187,
+      "rewards/margins": 6.15379524230957,
+      "rewards/rejected": -5.891900062561035,
+      "step": 558,
+      "train_speed(iter/s)": 0.021217
+    },
+    {
+      "epoch": 1.1272405958091392,
+      "grad_norm": 2.961078643798828,
+      "learning_rate": 8.700260825915976e-05,
+      "logits/chosen": -0.997047483921051,
+      "logits/rejected": -1.174019455909729,
+      "logps/chosen": -1.8312853574752808,
+      "logps/rejected": -93.381591796875,
+      "loss": 0.2475411295890808,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.21483805775642395,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3925812244415283,
+      "rewards/margins": 7.863485336303711,
+      "rewards/rejected": -7.470904350280762,
+      "step": 559,
+      "train_speed(iter/s)": 0.021217
+    },
+    {
+      "epoch": 1.1292602878061095,
+      "grad_norm": 13.051665306091309,
+      "learning_rate": 8.667130446262214e-05,
+      "logits/chosen": -0.8055945634841919,
+      "logits/rejected": -1.1168835163116455,
+      "logps/chosen": -1.7664347887039185,
+      "logps/rejected": -100.53841400146484,
+      "loss": 0.30384474992752075,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2042587250471115,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1575130969285965,
+      "rewards/margins": 8.052501678466797,
+      "rewards/rejected": -7.894989490509033,
+      "step": 560,
+      "train_speed(iter/s)": 0.021217
+    },
+    {
+      "epoch": 1.1312799798030801,
+      "grad_norm": 2.204517364501953,
+      "learning_rate": 8.634014954434355e-05,
+      "logits/chosen": -0.9541032314300537,
+      "logits/rejected": -1.1428359746932983,
+      "logps/chosen": -0.8130196928977966,
+      "logps/rejected": -77.63139343261719,
+      "loss": 0.11135084927082062,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.06836092472076416,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.31184935569763184,
+      "rewards/margins": 5.629817485809326,
+      "rewards/rejected": -5.317968368530273,
+      "step": 561,
+      "train_speed(iter/s)": 0.021217
+    },
+    {
+      "epoch": 1.1332996718000505,
+      "grad_norm": 0.7750061750411987,
+      "learning_rate": 8.600914720324316e-05,
+      "logits/chosen": -1.127307415008545,
+      "logits/rejected": -1.2096644639968872,
+      "logps/chosen": -0.43007510900497437,
+      "logps/rejected": -60.772300720214844,
+      "loss": 0.0971565991640091,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.06539256125688553,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.367216557264328,
+      "rewards/margins": 5.615878105163574,
+      "rewards/rejected": -5.248661041259766,
+      "step": 562,
+      "train_speed(iter/s)": 0.021218
+    },
+    {
+      "epoch": 1.135319363797021,
+      "grad_norm": 2.891634464263916,
+      "learning_rate": 8.567830113653597e-05,
+      "logits/chosen": -1.1409940719604492,
+      "logits/rejected": -1.1916230916976929,
+      "logps/chosen": -1.7586002349853516,
+      "logps/rejected": -63.676971435546875,
+      "loss": 0.2738097012042999,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18838034570217133,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.5218551158905029,
+      "rewards/margins": 5.294318675994873,
+      "rewards/rejected": -4.772463798522949,
+      "step": 563,
+      "train_speed(iter/s)": 0.021219
+    },
+    {
+      "epoch": 1.1373390557939915,
+      "grad_norm": 1.045423150062561,
+      "learning_rate": 8.534761503969136e-05,
+      "logits/chosen": -1.0010972023010254,
+      "logits/rejected": -1.1904897689819336,
+      "logps/chosen": -1.6170446872711182,
+      "logps/rejected": -74.0367660522461,
+      "loss": 0.19317412376403809,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13384878635406494,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2380320429801941,
+      "rewards/margins": 5.972710132598877,
+      "rewards/rejected": -5.734678745269775,
+      "step": 564,
+      "train_speed(iter/s)": 0.021219
+    },
+    {
+      "epoch": 1.139358747790962,
+      "grad_norm": 2.1970374584198,
+      "learning_rate": 8.501709260639186e-05,
+      "logits/chosen": -1.0450353622436523,
+      "logits/rejected": -1.163125991821289,
+      "logps/chosen": -1.5424336194992065,
+      "logps/rejected": -73.43540954589844,
+      "loss": 0.21883387863636017,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1583956778049469,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.27178311347961426,
+      "rewards/margins": 5.603139877319336,
+      "rewards/rejected": -5.331357002258301,
+      "step": 565,
+      "train_speed(iter/s)": 0.021219
+    },
+    {
+      "epoch": 1.1413784397879323,
+      "grad_norm": 2.8700995445251465,
+      "learning_rate": 8.4686737528492e-05,
+      "logits/chosen": -1.1470935344696045,
+      "logits/rejected": -1.2003036737442017,
+      "logps/chosen": -2.7194712162017822,
+      "logps/rejected": -73.11854553222656,
+      "loss": 0.24578091502189636,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14729218184947968,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.32028070092201233,
+      "rewards/margins": 6.802761077880859,
+      "rewards/rejected": -6.482481002807617,
+      "step": 566,
+      "train_speed(iter/s)": 0.021219
+    },
+    {
+      "epoch": 1.143398131784903,
+      "grad_norm": 1.381791114807129,
+      "learning_rate": 8.435655349597689e-05,
+      "logits/chosen": -1.0863614082336426,
+      "logits/rejected": -1.2500827312469482,
+      "logps/chosen": -0.5668889880180359,
+      "logps/rejected": -100.23611450195312,
+      "loss": 0.11013483256101608,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.0759727880358696,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.25675472617149353,
+      "rewards/margins": 8.975456237792969,
+      "rewards/rejected": -8.71870231628418,
+      "step": 567,
+      "train_speed(iter/s)": 0.02122
+    },
+    {
+      "epoch": 1.1454178237818733,
+      "grad_norm": 1.6486469507217407,
+      "learning_rate": 8.40265441969213e-05,
+      "logits/chosen": -1.108482837677002,
+      "logits/rejected": -1.2181017398834229,
+      "logps/chosen": -0.9796234965324402,
+      "logps/rejected": -72.5177993774414,
+      "loss": 0.17747226357460022,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.0940045714378357,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4885254204273224,
+      "rewards/margins": 5.819605350494385,
+      "rewards/rejected": -5.331079959869385,
+      "step": 568,
+      "train_speed(iter/s)": 0.02122
+    },
+    {
+      "epoch": 1.1474375157788437,
+      "grad_norm": 5.995792865753174,
+      "learning_rate": 8.369671331744798e-05,
+      "logits/chosen": -1.119094729423523,
+      "logits/rejected": -1.2429081201553345,
+      "logps/chosen": -1.075197696685791,
+      "logps/rejected": -81.46918487548828,
+      "loss": 0.21839378774166107,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15385104715824127,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.457529217004776,
+      "rewards/margins": 7.371258735656738,
+      "rewards/rejected": -6.913729667663574,
+      "step": 569,
+      "train_speed(iter/s)": 0.021221
+    },
+    {
+      "epoch": 1.1494572077758143,
+      "grad_norm": 5.149206638336182,
+      "learning_rate": 8.336706454168701e-05,
+      "logits/chosen": -1.1530694961547852,
+      "logits/rejected": -1.29622220993042,
+      "logps/chosen": -1.0280964374542236,
+      "logps/rejected": -119.51797485351562,
+      "loss": 0.20643891394138336,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.08810500800609589,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.43384313583374023,
+      "rewards/margins": 10.894699096679688,
+      "rewards/rejected": -10.460856437683105,
+      "step": 570,
+      "train_speed(iter/s)": 0.021221
+    },
+    {
+      "epoch": 1.1514768997727847,
+      "grad_norm": 1.2678836584091187,
+      "learning_rate": 8.303760155173431e-05,
+      "logits/chosen": -1.1424627304077148,
+      "logits/rejected": -1.3004976511001587,
+      "logps/chosen": -0.5466084480285645,
+      "logps/rejected": -118.68138122558594,
+      "loss": 0.12502805888652802,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1129414513707161,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3263624608516693,
+      "rewards/margins": 10.448878288269043,
+      "rewards/rejected": -10.122516632080078,
+      "step": 571,
+      "train_speed(iter/s)": 0.021222
+    },
+    {
+      "epoch": 1.153496591769755,
+      "grad_norm": 1.4974409341812134,
+      "learning_rate": 8.27083280276107e-05,
+      "logits/chosen": -1.1443991661071777,
+      "logits/rejected": -1.3150042295455933,
+      "logps/chosen": -0.3272748589515686,
+      "logps/rejected": -139.84246826171875,
+      "loss": 0.08136013895273209,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.050167154520750046,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4843906760215759,
+      "rewards/margins": 12.638895034790039,
+      "rewards/rejected": -12.15450382232666,
+      "step": 572,
+      "train_speed(iter/s)": 0.021222
+    },
+    {
+      "epoch": 1.1555162837667257,
+      "grad_norm": 1.1129848957061768,
+      "learning_rate": 8.237924764722058e-05,
+      "logits/chosen": -1.1491975784301758,
+      "logits/rejected": -1.307010531425476,
+      "logps/chosen": -1.0129919052124023,
+      "logps/rejected": -127.45437622070312,
+      "loss": 0.11947691440582275,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10945737361907959,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2766776978969574,
+      "rewards/margins": 11.592185974121094,
+      "rewards/rejected": -11.315506935119629,
+      "step": 573,
+      "train_speed(iter/s)": 0.021222
+    },
+    {
+      "epoch": 1.157535975763696,
+      "grad_norm": 12.299660682678223,
+      "learning_rate": 8.20503640863111e-05,
+      "logits/chosen": -1.2407050132751465,
+      "logits/rejected": -1.3422547578811646,
+      "logps/chosen": -0.5153883099555969,
+      "logps/rejected": -133.4427032470703,
+      "loss": 0.14274229109287262,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09612210839986801,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.25790077447891235,
+      "rewards/margins": 12.522222518920898,
+      "rewards/rejected": -12.264321327209473,
+      "step": 574,
+      "train_speed(iter/s)": 0.021222
+    },
+    {
+      "epoch": 1.1595556677606664,
+      "grad_norm": 4.597177505493164,
+      "learning_rate": 8.172168101843099e-05,
+      "logits/chosen": -1.2469947338104248,
+      "logits/rejected": -1.3221312761306763,
+      "logps/chosen": -2.5218143463134766,
+      "logps/rejected": -64.33858489990234,
+      "loss": 0.33642834424972534,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2502836585044861,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3977520167827606,
+      "rewards/margins": 6.087981700897217,
+      "rewards/rejected": -5.690229415893555,
+      "step": 575,
+      "train_speed(iter/s)": 0.021223
+    },
+    {
+      "epoch": 1.161575359757637,
+      "grad_norm": 1.4237756729125977,
+      "learning_rate": 8.139320211488938e-05,
+      "logits/chosen": -1.20731520652771,
+      "logits/rejected": -1.3155488967895508,
+      "logps/chosen": -1.9141697883605957,
+      "logps/rejected": -84.08015441894531,
+      "loss": 0.20295384526252747,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1318996697664261,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3964309096336365,
+      "rewards/margins": 8.057353973388672,
+      "rewards/rejected": -7.660923480987549,
+      "step": 576,
+      "train_speed(iter/s)": 0.021224
+    },
+    {
+      "epoch": 1.1635950517546074,
+      "grad_norm": 2.7795519828796387,
+      "learning_rate": 8.106493104471512e-05,
+      "logits/chosen": -1.1746073961257935,
+      "logits/rejected": -1.2740343809127808,
+      "logps/chosen": -2.3649284839630127,
+      "logps/rejected": -110.2456283569336,
+      "loss": 0.2869383990764618,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.21996352076530457,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.37983080744743347,
+      "rewards/margins": 9.394953727722168,
+      "rewards/rejected": -9.015122413635254,
+      "step": 577,
+      "train_speed(iter/s)": 0.021224
+    },
+    {
+      "epoch": 1.1656147437515778,
+      "grad_norm": 2.63078236579895,
+      "learning_rate": 8.073687147461547e-05,
+      "logits/chosen": -1.1673715114593506,
+      "logits/rejected": -1.330165147781372,
+      "logps/chosen": -2.938627004623413,
+      "logps/rejected": -126.129638671875,
+      "loss": 0.36669260263442993,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3255380392074585,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19922614097595215,
+      "rewards/margins": 10.992043495178223,
+      "rewards/rejected": -10.792817115783691,
+      "step": 578,
+      "train_speed(iter/s)": 0.021224
+    },
+    {
+      "epoch": 1.1676344357485484,
+      "grad_norm": 3.1014156341552734,
+      "learning_rate": 8.04090270689354e-05,
+      "logits/chosen": -1.2527494430541992,
+      "logits/rejected": -1.3397026062011719,
+      "logps/chosen": -2.1539173126220703,
+      "logps/rejected": -119.85777282714844,
+      "loss": 0.35629451274871826,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.26880595088005066,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.17089465260505676,
+      "rewards/margins": 10.79810619354248,
+      "rewards/rejected": -10.62721061706543,
+      "step": 579,
+      "train_speed(iter/s)": 0.021225
+    },
+    {
+      "epoch": 1.1696541277455188,
+      "grad_norm": 2.183499813079834,
+      "learning_rate": 8.008140148961641e-05,
+      "logits/chosen": -1.2131707668304443,
+      "logits/rejected": -1.3254077434539795,
+      "logps/chosen": -1.9817386865615845,
+      "logps/rejected": -97.93504333496094,
+      "loss": 0.2147722840309143,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13111397624015808,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3702988922595978,
+      "rewards/margins": 8.80711555480957,
+      "rewards/rejected": -8.436817169189453,
+      "step": 580,
+      "train_speed(iter/s)": 0.021225
+    },
+    {
+      "epoch": 1.1716738197424892,
+      "grad_norm": 1.777718186378479,
+      "learning_rate": 7.975399839615588e-05,
+      "logits/chosen": -1.2131725549697876,
+      "logits/rejected": -1.3256409168243408,
+      "logps/chosen": -1.6898517608642578,
+      "logps/rejected": -143.40451049804688,
+      "loss": 0.10389101505279541,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.07294470816850662,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7127897143363953,
+      "rewards/margins": 13.449665069580078,
+      "rewards/rejected": -12.736875534057617,
+      "step": 581,
+      "train_speed(iter/s)": 0.021225
+    },
+    {
+      "epoch": 1.1736935117394598,
+      "grad_norm": 3.9212543964385986,
+      "learning_rate": 7.942682144556604e-05,
+      "logits/chosen": -1.1815121173858643,
+      "logits/rejected": -1.2933522462844849,
+      "logps/chosen": -1.2875924110412598,
+      "logps/rejected": -102.97988891601562,
+      "loss": 0.25881722569465637,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17525863647460938,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.23393197357654572,
+      "rewards/margins": 9.580009460449219,
+      "rewards/rejected": -9.346076011657715,
+      "step": 582,
+      "train_speed(iter/s)": 0.021226
+    },
+    {
+      "epoch": 1.1757132037364302,
+      "grad_norm": 10.820318222045898,
+      "learning_rate": 7.909987429233317e-05,
+      "logits/chosen": -1.1671289205551147,
+      "logits/rejected": -1.2844188213348389,
+      "logps/chosen": -3.267451047897339,
+      "logps/rejected": -142.51571655273438,
+      "loss": 0.3873688876628876,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2906343936920166,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.45434269309043884,
+      "rewards/margins": 12.700260162353516,
+      "rewards/rejected": -12.245917320251465,
+      "step": 583,
+      "train_speed(iter/s)": 0.021226
+    },
+    {
+      "epoch": 1.1777328957334006,
+      "grad_norm": 8.162153244018555,
+      "learning_rate": 7.877316058837674e-05,
+      "logits/chosen": -1.0726277828216553,
+      "logits/rejected": -1.2778667211532593,
+      "logps/chosen": -2.711777687072754,
+      "logps/rejected": -147.33900451660156,
+      "loss": 0.43292930722236633,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.37318992614746094,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.31741955876350403,
+      "rewards/margins": 13.22387981414795,
+      "rewards/rejected": -12.906461715698242,
+      "step": 584,
+      "train_speed(iter/s)": 0.021226
+    },
+    {
+      "epoch": 1.1797525877303712,
+      "grad_norm": 8.373458862304688,
+      "learning_rate": 7.844668398300865e-05,
+      "logits/chosen": -1.1396088600158691,
+      "logits/rejected": -1.260819435119629,
+      "logps/chosen": -2.466878890991211,
+      "logps/rejected": -117.13096618652344,
+      "loss": 0.3733997344970703,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3299589157104492,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.13099858164787292,
+      "rewards/margins": 10.681720733642578,
+      "rewards/rejected": -10.550722122192383,
+      "step": 585,
+      "train_speed(iter/s)": 0.021226
+    },
+    {
+      "epoch": 1.1817722797273416,
+      "grad_norm": 20.348384857177734,
+      "learning_rate": 7.812044812289249e-05,
+      "logits/chosen": -1.2010091543197632,
+      "logits/rejected": -1.3245834112167358,
+      "logps/chosen": -3.3453171253204346,
+      "logps/rejected": -117.00688171386719,
+      "loss": 0.6515809893608093,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3382578492164612,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.15548963844776154,
+      "rewards/margins": 10.246532440185547,
+      "rewards/rejected": -10.091042518615723,
+      "step": 586,
+      "train_speed(iter/s)": 0.021226
+    },
+    {
+      "epoch": 1.183791971724312,
+      "grad_norm": 14.155014991760254,
+      "learning_rate": 7.779445665200284e-05,
+      "logits/chosen": -1.1876106262207031,
+      "logits/rejected": -1.2901134490966797,
+      "logps/chosen": -3.768019437789917,
+      "logps/rejected": -83.8199691772461,
+      "loss": 0.7886047959327698,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5176796913146973,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16025924682617188,
+      "rewards/margins": 7.629866600036621,
+      "rewards/rejected": -7.469607353210449,
+      "step": 587,
+      "train_speed(iter/s)": 0.021227
+    },
+    {
+      "epoch": 1.1858116637212825,
+      "grad_norm": 2.6726722717285156,
+      "learning_rate": 7.746871321158434e-05,
+      "logits/chosen": -1.2011233568191528,
+      "logits/rejected": -1.2838491201400757,
+      "logps/chosen": -1.551110863685608,
+      "logps/rejected": -74.54078674316406,
+      "loss": 0.3482269048690796,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20897427201271057,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.13612668216228485,
+      "rewards/margins": 6.496390342712402,
+      "rewards/rejected": -6.360263824462891,
+      "step": 588,
+      "train_speed(iter/s)": 0.021227
+    },
+    {
+      "epoch": 1.187831355718253,
+      "grad_norm": 11.104055404663086,
+      "learning_rate": 7.71432214401114e-05,
+      "logits/chosen": -1.184781551361084,
+      "logits/rejected": -1.367568850517273,
+      "logps/chosen": -2.4659273624420166,
+      "logps/rejected": -158.41024780273438,
+      "loss": 0.46998411417007446,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.37064406275749207,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.14929750561714172,
+      "rewards/margins": 14.278914451599121,
+      "rewards/rejected": -14.129616737365723,
+      "step": 589,
+      "train_speed(iter/s)": 0.021228
+    },
+    {
+      "epoch": 1.1898510477152233,
+      "grad_norm": 1.3642657995224,
+      "learning_rate": 7.681798497324716e-05,
+      "logits/chosen": -1.1494981050491333,
+      "logits/rejected": -1.2755074501037598,
+      "logps/chosen": -3.920079231262207,
+      "logps/rejected": -99.15157318115234,
+      "loss": 0.1694917380809784,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1302870661020279,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6689797043800354,
+      "rewards/margins": 8.986592292785645,
+      "rewards/rejected": -8.317612648010254,
+      "step": 590,
+      "train_speed(iter/s)": 0.021228
+    },
+    {
+      "epoch": 1.191870739712194,
+      "grad_norm": 2.9878756999969482,
+      "learning_rate": 7.649300744380328e-05,
+      "logits/chosen": -1.2234166860580444,
+      "logits/rejected": -1.3413805961608887,
+      "logps/chosen": -3.245425224304199,
+      "logps/rejected": -97.82754516601562,
+      "loss": 0.6991743445396423,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5142159461975098,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.04035697877407074,
+      "rewards/margins": 8.96734619140625,
+      "rewards/rejected": -8.926989555358887,
+      "step": 591,
+      "train_speed(iter/s)": 0.021228
+    },
+    {
+      "epoch": 1.1938904317091643,
+      "grad_norm": 3.8199944496154785,
+      "learning_rate": 7.616829248169891e-05,
+      "logits/chosen": -1.168949842453003,
+      "logits/rejected": -1.2326381206512451,
+      "logps/chosen": -2.8712589740753174,
+      "logps/rejected": -63.083431243896484,
+      "loss": 0.4011121094226837,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2862622141838074,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.291637659072876,
+      "rewards/margins": 5.633680820465088,
+      "rewards/rejected": -5.342043399810791,
+      "step": 592,
+      "train_speed(iter/s)": 0.021228
+    },
+    {
+      "epoch": 1.1959101237061347,
+      "grad_norm": 2.4625227451324463,
+      "learning_rate": 7.584384371392055e-05,
+      "logits/chosen": -1.2142324447631836,
+      "logits/rejected": -1.2614374160766602,
+      "logps/chosen": -1.5779057741165161,
+      "logps/rejected": -86.51065826416016,
+      "loss": 0.1223965585231781,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09837526828050613,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.9265052676200867,
+      "rewards/margins": 8.553476333618164,
+      "rewards/rejected": -7.62697172164917,
+      "step": 593,
+      "train_speed(iter/s)": 0.021229
+    },
+    {
+      "epoch": 1.1979298157031053,
+      "grad_norm": 3.092069149017334,
+      "learning_rate": 7.55196647644814e-05,
+      "logits/chosen": -1.1460374593734741,
+      "logits/rejected": -1.325160264968872,
+      "logps/chosen": -0.844829261302948,
+      "logps/rejected": -130.2720184326172,
+      "loss": 0.1258152425289154,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09725081920623779,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.22555547952651978,
+      "rewards/margins": 11.706364631652832,
+      "rewards/rejected": -11.480809211730957,
+      "step": 594,
+      "train_speed(iter/s)": 0.021229
+    },
+    {
+      "epoch": 1.1999495077000757,
+      "grad_norm": 4.890739440917969,
+      "learning_rate": 7.519575925438067e-05,
+      "logits/chosen": -1.1363235712051392,
+      "logits/rejected": -1.3047442436218262,
+      "logps/chosen": -3.450829267501831,
+      "logps/rejected": -122.64608001708984,
+      "loss": 0.40785038471221924,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2815420627593994,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.30323055386543274,
+      "rewards/margins": 10.626569747924805,
+      "rewards/rejected": -10.323339462280273,
+      "step": 595,
+      "train_speed(iter/s)": 0.021229
+    },
+    {
+      "epoch": 1.201969199697046,
+      "grad_norm": 9.498205184936523,
+      "learning_rate": 7.487213080156355e-05,
+      "logits/chosen": -1.2033909559249878,
+      "logits/rejected": -1.2762019634246826,
+      "logps/chosen": -1.286055326461792,
+      "logps/rejected": -79.14070129394531,
+      "loss": 0.24982167780399323,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14821632206439972,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2594597637653351,
+      "rewards/margins": 7.015608787536621,
+      "rewards/rejected": -6.756147861480713,
+      "step": 596,
+      "train_speed(iter/s)": 0.02123
+    },
+    {
+      "epoch": 1.2039888916940167,
+      "grad_norm": 5.124003887176514,
+      "learning_rate": 7.454878302088039e-05,
+      "logits/chosen": -1.1855096817016602,
+      "logits/rejected": -1.2832536697387695,
+      "logps/chosen": -2.1323423385620117,
+      "logps/rejected": -77.67525482177734,
+      "loss": 0.3236369490623474,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22611530125141144,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2844330668449402,
+      "rewards/margins": 6.800313472747803,
+      "rewards/rejected": -6.515880584716797,
+      "step": 597,
+      "train_speed(iter/s)": 0.02123
+    },
+    {
+      "epoch": 1.206008583690987,
+      "grad_norm": 7.811437606811523,
+      "learning_rate": 7.422571952404663e-05,
+      "logits/chosen": -1.1619443893432617,
+      "logits/rejected": -1.2858407497406006,
+      "logps/chosen": -3.6687519550323486,
+      "logps/rejected": -89.48348236083984,
+      "loss": 0.6337468028068542,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.430719256401062,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.09281295537948608,
+      "rewards/margins": 7.673408508300781,
+      "rewards/rejected": -7.58059549331665,
+      "step": 598,
+      "train_speed(iter/s)": 0.021231
+    },
+    {
+      "epoch": 1.2080282756879577,
+      "grad_norm": 6.1173577308654785,
+      "learning_rate": 7.390294391960217e-05,
+      "logits/chosen": -1.2435667514801025,
+      "logits/rejected": -1.3407572507858276,
+      "logps/chosen": -0.851167619228363,
+      "logps/rejected": -90.95581817626953,
+      "loss": 0.1322077065706253,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.07706882804632187,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3039519786834717,
+      "rewards/margins": 8.613190650939941,
+      "rewards/rejected": -8.309237480163574,
+      "step": 599,
+      "train_speed(iter/s)": 0.021231
+    },
+    {
+      "epoch": 1.210047967684928,
+      "grad_norm": 1.1325865983963013,
+      "learning_rate": 7.358045981287141e-05,
+      "logits/chosen": -1.1153496503829956,
+      "logits/rejected": -1.2983583211898804,
+      "logps/chosen": -0.8740055561065674,
+      "logps/rejected": -100.239990234375,
+      "loss": 0.1091877669095993,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.0765417069196701,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.24975469708442688,
+      "rewards/margins": 8.55569076538086,
+      "rewards/rejected": -8.305935859680176,
+      "step": 600,
+      "train_speed(iter/s)": 0.021231
+    },
+    {
+      "epoch": 1.210047967684928,
+      "eval_logits/chosen": -1.1828652620315552,
+      "eval_logits/rejected": -1.3658164739608765,
+      "eval_logps/chosen": -1.8309376239776611,
+      "eval_logps/rejected": -125.0011215209961,
+      "eval_loss": 0.24665305018424988,
+      "eval_nll_loss": 0.17694206535816193,
+      "eval_rewards/accuracies": 0.9750000238418579,
+      "eval_rewards/chosen": 0.26265937089920044,
+      "eval_rewards/margins": 11.061813354492188,
+      "eval_rewards/rejected": -10.799153327941895,
+      "eval_runtime": 92.2944,
+      "eval_samples_per_second": 0.867,
+      "eval_steps_per_second": 0.433,
+      "step": 600
+    },
+    {
+      "epoch": 1.2120676596818984,
+      "grad_norm": 9.913939476013184,
+      "learning_rate": 7.325827080592266e-05,
+      "logits/chosen": -1.2156214714050293,
+      "logits/rejected": -1.35942542552948,
+      "logps/chosen": -6.396952152252197,
+      "logps/rejected": -114.74188232421875,
+      "loss": 0.4289969205856323,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.40381112694740295,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.08136852085590363,
+      "rewards/margins": 9.671481132507324,
+      "rewards/rejected": -9.752849578857422,
+      "step": 601,
+      "train_speed(iter/s)": 0.021149
+    },
+    {
+      "epoch": 1.214087351678869,
+      "grad_norm": 2.413109064102173,
+      "learning_rate": 7.293638049752812e-05,
+      "logits/chosen": -1.1656737327575684,
+      "logits/rejected": -1.414901614189148,
+      "logps/chosen": -2.074061155319214,
+      "logps/rejected": -168.7305908203125,
+      "loss": 0.19876618683338165,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11121080070734024,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3010483682155609,
+      "rewards/margins": 15.196366310119629,
+      "rewards/rejected": -14.895317077636719,
+      "step": 602,
+      "train_speed(iter/s)": 0.02115
+    },
+    {
+      "epoch": 1.2161070436758394,
+      "grad_norm": 14.77046012878418,
+      "learning_rate": 7.261479248312352e-05,
+      "logits/chosen": -1.3167017698287964,
+      "logits/rejected": -1.370416283607483,
+      "logps/chosen": -4.046845436096191,
+      "logps/rejected": -92.28170776367188,
+      "loss": 0.5522013306617737,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.44899022579193115,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.05194016173481941,
+      "rewards/margins": 8.308582305908203,
+      "rewards/rejected": -8.360522270202637,
+      "step": 603,
+      "train_speed(iter/s)": 0.02115
+    },
+    {
+      "epoch": 1.2181267356728098,
+      "grad_norm": 4.001037120819092,
+      "learning_rate": 7.22935103547681e-05,
+      "logits/chosen": -1.2455226182937622,
+      "logits/rejected": -1.4458909034729004,
+      "logps/chosen": -1.4079478979110718,
+      "logps/rejected": -145.70974731445312,
+      "loss": 0.11345479637384415,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09860379248857498,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3669095039367676,
+      "rewards/margins": 13.468746185302734,
+      "rewards/rejected": -13.101836204528809,
+      "step": 604,
+      "train_speed(iter/s)": 0.021151
+    },
+    {
+      "epoch": 1.2201464276697804,
+      "grad_norm": 3.84513521194458,
+      "learning_rate": 7.197253770110438e-05,
+      "logits/chosen": -1.1828511953353882,
+      "logits/rejected": -1.3776271343231201,
+      "logps/chosen": -1.2730638980865479,
+      "logps/rejected": -138.26931762695312,
+      "loss": 0.15900050103664398,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14723443984985352,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.39036068320274353,
+      "rewards/margins": 12.166678428649902,
+      "rewards/rejected": -11.776317596435547,
+      "step": 605,
+      "train_speed(iter/s)": 0.021151
+    },
+    {
+      "epoch": 1.2221661196667508,
+      "grad_norm": 17.166400909423828,
+      "learning_rate": 7.165187810731823e-05,
+      "logits/chosen": -1.2092819213867188,
+      "logits/rejected": -1.3941118717193604,
+      "logps/chosen": -1.860000491142273,
+      "logps/rejected": -152.60494995117188,
+      "loss": 0.31487131118774414,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22380079329013824,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.492226779460907,
+      "rewards/margins": 14.125248908996582,
+      "rewards/rejected": -13.633021354675293,
+      "step": 606,
+      "train_speed(iter/s)": 0.021151
+    },
+    {
+      "epoch": 1.2241858116637212,
+      "grad_norm": 1.6913998126983643,
+      "learning_rate": 7.133153515509855e-05,
+      "logits/chosen": -1.2237271070480347,
+      "logits/rejected": -1.433366060256958,
+      "logps/chosen": -0.5559945106506348,
+      "logps/rejected": -155.11065673828125,
+      "loss": 0.16471454501152039,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11912833154201508,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.32575783133506775,
+      "rewards/margins": 14.4934720993042,
+      "rewards/rejected": -14.167715072631836,
+      "step": 607,
+      "train_speed(iter/s)": 0.021152
+    },
+    {
+      "epoch": 1.2262055036606918,
+      "grad_norm": 1.9620745182037354,
+      "learning_rate": 7.101151242259756e-05,
+      "logits/chosen": -1.1885676383972168,
+      "logits/rejected": -1.378993272781372,
+      "logps/chosen": -2.593254566192627,
+      "logps/rejected": -117.13745880126953,
+      "loss": 0.23732948303222656,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1710463911294937,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3385048508644104,
+      "rewards/margins": 10.88075065612793,
+      "rewards/rejected": -10.54224681854248,
+      "step": 608,
+      "train_speed(iter/s)": 0.021153
+    },
+    {
+      "epoch": 1.2282251956576622,
+      "grad_norm": 2.9237098693847656,
+      "learning_rate": 7.06918134843907e-05,
+      "logits/chosen": -1.18954336643219,
+      "logits/rejected": -1.3267593383789062,
+      "logps/chosen": -3.154736280441284,
+      "logps/rejected": -91.14146423339844,
+      "loss": 0.29007232189178467,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17210328578948975,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.397045373916626,
+      "rewards/margins": 8.095890045166016,
+      "rewards/rejected": -7.698844909667969,
+      "step": 609,
+      "train_speed(iter/s)": 0.021153
+    },
+    {
+      "epoch": 1.2302448876546326,
+      "grad_norm": 8.460159301757812,
+      "learning_rate": 7.037244191143661e-05,
+      "logits/chosen": -1.214463233947754,
+      "logits/rejected": -1.3371509313583374,
+      "logps/chosen": -3.574279308319092,
+      "logps/rejected": -89.81301879882812,
+      "loss": 0.42950019240379333,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2784547805786133,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.067209392786026,
+      "rewards/margins": 7.711863040924072,
+      "rewards/rejected": -7.644654273986816,
+      "step": 610,
+      "train_speed(iter/s)": 0.021153
+    },
+    {
+      "epoch": 1.2322645796516032,
+      "grad_norm": 13.689557075500488,
+      "learning_rate": 7.005340127103745e-05,
+      "logits/chosen": -1.1511098146438599,
+      "logits/rejected": -1.3526651859283447,
+      "logps/chosen": -2.9878780841827393,
+      "logps/rejected": -127.11341857910156,
+      "loss": 0.40525269508361816,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3287810981273651,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.019121740013360977,
+      "rewards/margins": 10.934074401855469,
+      "rewards/rejected": -10.953195571899414,
+      "step": 611,
+      "train_speed(iter/s)": 0.021154
+    },
+    {
+      "epoch": 1.2342842716485736,
+      "grad_norm": 2.4316670894622803,
+      "learning_rate": 6.973469512679886e-05,
+      "logits/chosen": -1.1571348905563354,
+      "logits/rejected": -1.3065540790557861,
+      "logps/chosen": -1.8132346868515015,
+      "logps/rejected": -108.02491760253906,
+      "loss": 0.07915420830249786,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.0534609816968441,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.805472731590271,
+      "rewards/margins": 10.520647048950195,
+      "rewards/rejected": -9.71517562866211,
+      "step": 612,
+      "train_speed(iter/s)": 0.021154
+    },
+    {
+      "epoch": 1.2363039636455442,
+      "grad_norm": 1.8934969902038574,
+      "learning_rate": 6.941632703859036e-05,
+      "logits/chosen": -1.1191542148590088,
+      "logits/rejected": -1.3276458978652954,
+      "logps/chosen": -0.9014697074890137,
+      "logps/rejected": -94.8488540649414,
+      "loss": 0.1784897893667221,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1215212345123291,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3536786437034607,
+      "rewards/margins": 8.533373832702637,
+      "rewards/rejected": -8.179695129394531,
+      "step": 613,
+      "train_speed(iter/s)": 0.021155
+    },
+    {
+      "epoch": 1.2383236556425146,
+      "grad_norm": 7.0670166015625,
+      "learning_rate": 6.909830056250527e-05,
+      "logits/chosen": -1.1881051063537598,
+      "logits/rejected": -1.2666465044021606,
+      "logps/chosen": -4.071624279022217,
+      "logps/rejected": -76.82312774658203,
+      "loss": 0.29166367650032043,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.19953405857086182,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3121744990348816,
+      "rewards/margins": 6.753731727600098,
+      "rewards/rejected": -6.441556930541992,
+      "step": 614,
+      "train_speed(iter/s)": 0.021155
+    },
+    {
+      "epoch": 1.240343347639485,
+      "grad_norm": 2.0287578105926514,
+      "learning_rate": 6.878061925082137e-05,
+      "logits/chosen": -1.1983009576797485,
+      "logits/rejected": -1.3032424449920654,
+      "logps/chosen": -2.237928867340088,
+      "logps/rejected": -78.43489074707031,
+      "loss": 0.24645480513572693,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17541807889938354,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4067380428314209,
+      "rewards/margins": 7.369697570800781,
+      "rewards/rejected": -6.962960243225098,
+      "step": 615,
+      "train_speed(iter/s)": 0.021156
+    },
+    {
+      "epoch": 1.2423630396364556,
+      "grad_norm": 8.199972152709961,
+      "learning_rate": 6.846328665196098e-05,
+      "logits/chosen": -1.052995204925537,
+      "logits/rejected": -1.3107107877731323,
+      "logps/chosen": -2.1211647987365723,
+      "logps/rejected": -109.34521484375,
+      "loss": 0.31173238158226013,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22831910848617554,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.201967254281044,
+      "rewards/margins": 9.036110877990723,
+      "rewards/rejected": -8.83414363861084,
+      "step": 616,
+      "train_speed(iter/s)": 0.021156
+    },
+    {
+      "epoch": 1.244382731633426,
+      "grad_norm": 3.1504924297332764,
+      "learning_rate": 6.814630631045136e-05,
+      "logits/chosen": -1.116700530052185,
+      "logits/rejected": -1.274092197418213,
+      "logps/chosen": -0.9862836599349976,
+      "logps/rejected": -78.76496887207031,
+      "loss": 0.2599913775920868,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.12422175705432892,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3174135982990265,
+      "rewards/margins": 6.622673034667969,
+      "rewards/rejected": -6.305260181427002,
+      "step": 617,
+      "train_speed(iter/s)": 0.021157
+    },
+    {
+      "epoch": 1.2464024236303963,
+      "grad_norm": 2.1881184577941895,
+      "learning_rate": 6.782968176688514e-05,
+      "logits/chosen": -1.148500919342041,
+      "logits/rejected": -1.3377251625061035,
+      "logps/chosen": -0.6570072174072266,
+      "logps/rejected": -96.93216705322266,
+      "loss": 0.10727754235267639,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.08484382927417755,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34505102038383484,
+      "rewards/margins": 8.49383544921875,
+      "rewards/rejected": -8.148785591125488,
+      "step": 618,
+      "train_speed(iter/s)": 0.021157
+    },
+    {
+      "epoch": 1.248422115627367,
+      "grad_norm": 1.6966434717178345,
+      "learning_rate": 6.751341655788077e-05,
+      "logits/chosen": -1.2084660530090332,
+      "logits/rejected": -1.2793188095092773,
+      "logps/chosen": -1.3728337287902832,
+      "logps/rejected": -60.3212890625,
+      "loss": 0.2079794555902481,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13015279173851013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.28127437829971313,
+      "rewards/margins": 5.467742919921875,
+      "rewards/rejected": -5.186468601226807,
+      "step": 619,
+      "train_speed(iter/s)": 0.021158
+    },
+    {
+      "epoch": 1.2504418076243373,
+      "grad_norm": 1.073513150215149,
+      "learning_rate": 6.719751421604309e-05,
+      "logits/chosen": -1.1013176441192627,
+      "logits/rejected": -1.2899197340011597,
+      "logps/chosen": -0.6654816269874573,
+      "logps/rejected": -87.93569946289062,
+      "loss": 0.10443835705518723,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.06041412800550461,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3901611566543579,
+      "rewards/margins": 7.748969078063965,
+      "rewards/rejected": -7.358807563781738,
+      "step": 620,
+      "train_speed(iter/s)": 0.021158
+    },
+    {
+      "epoch": 1.2524614996213077,
+      "grad_norm": 2.8936827182769775,
+      "learning_rate": 6.688197826992375e-05,
+      "logits/chosen": -1.1164414882659912,
+      "logits/rejected": -1.2044141292572021,
+      "logps/chosen": -2.608881950378418,
+      "logps/rejected": -42.977203369140625,
+      "loss": 0.2782241702079773,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1327691674232483,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5928317904472351,
+      "rewards/margins": 3.942835807800293,
+      "rewards/rejected": -3.350004196166992,
+      "step": 621,
+      "train_speed(iter/s)": 0.021159
+    },
+    {
+      "epoch": 1.2544811916182783,
+      "grad_norm": 1.7200703620910645,
+      "learning_rate": 6.656681224398183e-05,
+      "logits/chosen": -1.0104758739471436,
+      "logits/rejected": -1.0320334434509277,
+      "logps/chosen": -6.659427165985107,
+      "logps/rejected": -66.0435562133789,
+      "loss": 0.2288062870502472,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18073612451553345,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5307789444923401,
+      "rewards/margins": 5.470790863037109,
+      "rewards/rejected": -4.940011978149414,
+      "step": 622,
+      "train_speed(iter/s)": 0.021159
+    },
+    {
+      "epoch": 1.2565008836152487,
+      "grad_norm": 1.7575489282608032,
+      "learning_rate": 6.625201965854453e-05,
+      "logits/chosen": -1.082411527633667,
+      "logits/rejected": -1.252202033996582,
+      "logps/chosen": -0.7623193860054016,
+      "logps/rejected": -68.75422668457031,
+      "loss": 0.12465101480484009,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.08836035430431366,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2589690089225769,
+      "rewards/margins": 5.73964262008667,
+      "rewards/rejected": -5.480673789978027,
+      "step": 623,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 1.258520575612219,
+      "grad_norm": 6.982329368591309,
+      "learning_rate": 6.593760402976775e-05,
+      "logits/chosen": -0.9956216812133789,
+      "logits/rejected": -1.23116934299469,
+      "logps/chosen": -1.50418221950531,
+      "logps/rejected": -69.6702651977539,
+      "loss": 0.2932169437408447,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18249881267547607,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.37432289123535156,
+      "rewards/margins": 5.484264373779297,
+      "rewards/rejected": -5.1099419593811035,
+      "step": 624,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 1.2605402676091897,
+      "grad_norm": 0.5276002287864685,
+      "learning_rate": 6.562356886959704e-05,
+      "logits/chosen": -1.0346295833587646,
+      "logits/rejected": -1.2541873455047607,
+      "logps/chosen": -0.6328487396240234,
+      "logps/rejected": -76.94085693359375,
+      "loss": 0.06354828923940659,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.03634391352534294,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5320701599121094,
+      "rewards/margins": 6.873848915100098,
+      "rewards/rejected": -6.341780185699463,
+      "step": 625,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 1.26255995960616,
+      "grad_norm": 11.666182518005371,
+      "learning_rate": 6.530991768572794e-05,
+      "logits/chosen": -0.7891740202903748,
+      "logits/rejected": -1.2037701606750488,
+      "logps/chosen": -2.5640218257904053,
+      "logps/rejected": -141.39793395996094,
+      "loss": 0.4891372323036194,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3772202134132385,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.26330137252807617,
+      "rewards/margins": 11.030693054199219,
+      "rewards/rejected": -10.7673921585083,
+      "step": 626,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 1.2645796516031305,
+      "grad_norm": 2.8686726093292236,
+      "learning_rate": 6.499665398156733e-05,
+      "logits/chosen": -1.138472080230713,
+      "logits/rejected": -1.2411797046661377,
+      "logps/chosen": -2.9183404445648193,
+      "logps/rejected": -67.09615325927734,
+      "loss": 0.24029400944709778,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18298767507076263,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.05490890145301819,
+      "rewards/margins": 5.573385238647461,
+      "rewards/rejected": -5.518476486206055,
+      "step": 627,
+      "train_speed(iter/s)": 0.021161
+    },
+    {
+      "epoch": 1.266599343600101,
+      "grad_norm": 1.658783197402954,
+      "learning_rate": 6.46837812561939e-05,
+      "logits/chosen": -0.9565812945365906,
+      "logits/rejected": -1.2289061546325684,
+      "logps/chosen": -2.3054585456848145,
+      "logps/rejected": -100.20985412597656,
+      "loss": 0.14678572118282318,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09979239851236343,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.35647761821746826,
+      "rewards/margins": 8.205042839050293,
+      "rewards/rejected": -7.848564624786377,
+      "step": 628,
+      "train_speed(iter/s)": 0.021161
+    },
+    {
+      "epoch": 1.2686190355970715,
+      "grad_norm": 1.3928836584091187,
+      "learning_rate": 6.437130300431924e-05,
+      "logits/chosen": -0.9374477863311768,
+      "logits/rejected": -1.2098945379257202,
+      "logps/chosen": -3.1727993488311768,
+      "logps/rejected": -111.17371368408203,
+      "loss": 0.19910651445388794,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13950186967849731,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4299960136413574,
+      "rewards/margins": 8.817339897155762,
+      "rewards/rejected": -8.387343406677246,
+      "step": 629,
+      "train_speed(iter/s)": 0.021161
+    },
+    {
+      "epoch": 1.2706387275940418,
+      "grad_norm": 1.6878154277801514,
+      "learning_rate": 6.405922271624874e-05,
+      "logits/chosen": -1.1196115016937256,
+      "logits/rejected": -1.298462152481079,
+      "logps/chosen": -1.360905647277832,
+      "logps/rejected": -88.43049621582031,
+      "loss": 0.23307307064533234,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14912521839141846,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3634467124938965,
+      "rewards/margins": 7.924455642700195,
+      "rewards/rejected": -7.561008930206299,
+      "step": 630,
+      "train_speed(iter/s)": 0.021161
+    },
+    {
+      "epoch": 1.2726584195910124,
+      "grad_norm": 2.8349177837371826,
+      "learning_rate": 6.374754387784262e-05,
+      "logits/chosen": -1.094043493270874,
+      "logits/rejected": -1.310784101486206,
+      "logps/chosen": -0.5979782938957214,
+      "logps/rejected": -76.4305648803711,
+      "loss": 0.09544818848371506,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.0627792477607727,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34104666113853455,
+      "rewards/margins": 6.593111991882324,
+      "rewards/rejected": -6.252065658569336,
+      "step": 631,
+      "train_speed(iter/s)": 0.021162
+    },
+    {
+      "epoch": 1.2746781115879828,
+      "grad_norm": 2.06506609916687,
+      "learning_rate": 6.343626997047713e-05,
+      "logits/chosen": -1.0237411260604858,
+      "logits/rejected": -1.3066352605819702,
+      "logps/chosen": -1.3768390417099,
+      "logps/rejected": -100.54714965820312,
+      "loss": 0.23874782025814056,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.19472825527191162,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3415237367153168,
+      "rewards/margins": 8.822443962097168,
+      "rewards/rejected": -8.480920791625977,
+      "step": 632,
+      "train_speed(iter/s)": 0.021162
+    },
+    {
+      "epoch": 1.2766978035849532,
+      "grad_norm": 5.690185070037842,
+      "learning_rate": 6.312540447100534e-05,
+      "logits/chosen": -1.2119157314300537,
+      "logits/rejected": -1.338576316833496,
+      "logps/chosen": -0.42053210735321045,
+      "logps/rejected": -72.25818634033203,
+      "loss": 0.17603927850723267,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.05525693669915199,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.32522186636924744,
+      "rewards/margins": 6.52457332611084,
+      "rewards/rejected": -6.199350833892822,
+      "step": 633,
+      "train_speed(iter/s)": 0.021163
+    },
+    {
+      "epoch": 1.2787174955819238,
+      "grad_norm": 3.5418388843536377,
+      "learning_rate": 6.281495085171869e-05,
+      "logits/chosen": -1.2703211307525635,
+      "logits/rejected": -1.366028070449829,
+      "logps/chosen": -2.1424360275268555,
+      "logps/rejected": -72.02389526367188,
+      "loss": 0.3543678820133209,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.25578010082244873,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.22544154524803162,
+      "rewards/margins": 6.657296657562256,
+      "rewards/rejected": -6.431855201721191,
+      "step": 634,
+      "train_speed(iter/s)": 0.021163
+    },
+    {
+      "epoch": 1.2807371875788942,
+      "grad_norm": 2.0642120838165283,
+      "learning_rate": 6.250491258030791e-05,
+      "logits/chosen": -1.0552889108657837,
+      "logits/rejected": -1.3846908807754517,
+      "logps/chosen": -1.9661707878112793,
+      "logps/rejected": -124.82286834716797,
+      "loss": 0.1997641623020172,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17810964584350586,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.11012859642505646,
+      "rewards/margins": 10.5875244140625,
+      "rewards/rejected": -10.477395057678223,
+      "step": 635,
+      "train_speed(iter/s)": 0.021163
+    },
+    {
+      "epoch": 1.2827568795758646,
+      "grad_norm": 1.7234506607055664,
+      "learning_rate": 6.219529311982442e-05,
+      "logits/chosen": -0.9691514372825623,
+      "logits/rejected": -1.3024271726608276,
+      "logps/chosen": -1.6869831085205078,
+      "logps/rejected": -89.70926666259766,
+      "loss": 0.16458135843276978,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11242479830980301,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19229495525360107,
+      "rewards/margins": 7.334975719451904,
+      "rewards/rejected": -7.142680644989014,
+      "step": 636,
+      "train_speed(iter/s)": 0.021163
+    },
+    {
+      "epoch": 1.2847765715728352,
+      "grad_norm": 1.581983208656311,
+      "learning_rate": 6.188609592864163e-05,
+      "logits/chosen": -1.0093014240264893,
+      "logits/rejected": -1.3331385850906372,
+      "logps/chosen": -1.1868222951889038,
+      "logps/rejected": -147.7542724609375,
+      "loss": 0.11227454990148544,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.07105088979005814,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.324317991733551,
+      "rewards/margins": 12.817794799804688,
+      "rewards/rejected": -12.493476867675781,
+      "step": 637,
+      "train_speed(iter/s)": 0.021164
+    },
+    {
+      "epoch": 1.2867962635698056,
+      "grad_norm": 1.4069782495498657,
+      "learning_rate": 6.15773244604163e-05,
+      "logits/chosen": -1.2011477947235107,
+      "logits/rejected": -1.4000192880630493,
+      "logps/chosen": -1.3291258811950684,
+      "logps/rejected": -114.03934478759766,
+      "loss": 0.1708032190799713,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15742947161197662,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.27321162819862366,
+      "rewards/margins": 10.484274864196777,
+      "rewards/rejected": -10.21106243133545,
+      "step": 638,
+      "train_speed(iter/s)": 0.021164
+    },
+    {
+      "epoch": 1.288815955566776,
+      "grad_norm": 1.703295111656189,
+      "learning_rate": 6.126898216405e-05,
+      "logits/chosen": -1.1628754138946533,
+      "logits/rejected": -1.3372544050216675,
+      "logps/chosen": -1.2863965034484863,
+      "logps/rejected": -109.2572250366211,
+      "loss": 0.22218197584152222,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15462332963943481,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3368228077888489,
+      "rewards/margins": 9.793524742126465,
+      "rewards/rejected": -9.45670223236084,
+      "step": 639,
+      "train_speed(iter/s)": 0.021164
+    },
+    {
+      "epoch": 1.2908356475637466,
+      "grad_norm": 1.5797827243804932,
+      "learning_rate": 6.0961072483650526e-05,
+      "logits/chosen": -1.2602944374084473,
+      "logits/rejected": -1.4602419137954712,
+      "logps/chosen": -1.9707204103469849,
+      "logps/rejected": -125.09149169921875,
+      "loss": 0.14470840990543365,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.12814298272132874,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.28131091594696045,
+      "rewards/margins": 11.473433494567871,
+      "rewards/rejected": -11.192121505737305,
+      "step": 640,
+      "train_speed(iter/s)": 0.021165
+    },
+    {
+      "epoch": 1.292855339560717,
+      "grad_norm": 18.529470443725586,
+      "learning_rate": 6.065359885849345e-05,
+      "logits/chosen": -1.0401921272277832,
+      "logits/rejected": -1.3627612590789795,
+      "logps/chosen": -2.5566060543060303,
+      "logps/rejected": -133.431640625,
+      "loss": 0.2834732234477997,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2489144504070282,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34254688024520874,
+      "rewards/margins": 11.843310356140137,
+      "rewards/rejected": -11.500763893127441,
+      "step": 641,
+      "train_speed(iter/s)": 0.021165
+    },
+    {
+      "epoch": 1.2948750315576874,
+      "grad_norm": 2.119724750518799,
+      "learning_rate": 6.0346564722983736e-05,
+      "logits/chosen": -1.0882471799850464,
+      "logits/rejected": -1.3279931545257568,
+      "logps/chosen": -2.8448848724365234,
+      "logps/rejected": -114.02204132080078,
+      "loss": 0.3005206882953644,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23894429206848145,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.21939946711063385,
+      "rewards/margins": 9.814103126525879,
+      "rewards/rejected": -9.594703674316406,
+      "step": 642,
+      "train_speed(iter/s)": 0.021165
+    },
+    {
+      "epoch": 1.296894723554658,
+      "grad_norm": 2.0415537357330322,
+      "learning_rate": 6.003997350661732e-05,
+      "logits/chosen": -0.9607599973678589,
+      "logits/rejected": -1.3674266338348389,
+      "logps/chosen": -0.11069926619529724,
+      "logps/rejected": -164.1722412109375,
+      "loss": 0.06574507802724838,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.023912711068987846,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3115651607513428,
+      "rewards/margins": 14.507678985595703,
+      "rewards/rejected": -14.196113586425781,
+      "step": 643,
+      "train_speed(iter/s)": 0.021165
+    },
+    {
+      "epoch": 1.2989144155516283,
+      "grad_norm": 9.165788650512695,
+      "learning_rate": 5.9733828633942955e-05,
+      "logits/chosen": -1.2315356731414795,
+      "logits/rejected": -1.3989313840866089,
+      "logps/chosen": -2.512181520462036,
+      "logps/rejected": -121.66224670410156,
+      "loss": 0.5127250552177429,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3775697648525238,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.5036662817001343,
+      "rewards/margins": 11.449174880981445,
+      "rewards/rejected": -10.94550895690918,
+      "step": 644,
+      "train_speed(iter/s)": 0.021166
+    },
+    {
+      "epoch": 1.3009341075485987,
+      "grad_norm": 1.520035982131958,
+      "learning_rate": 5.9428133524523646e-05,
+      "logits/chosen": -1.0076013803482056,
+      "logits/rejected": -1.3784972429275513,
+      "logps/chosen": -0.653781533241272,
+      "logps/rejected": -153.15548706054688,
+      "loss": 0.0643075481057167,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.060876213014125824,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.48861101269721985,
+      "rewards/margins": 13.001919746398926,
+      "rewards/rejected": -12.513309478759766,
+      "step": 645,
+      "train_speed(iter/s)": 0.021166
+    },
+    {
+      "epoch": 1.3029537995455693,
+      "grad_norm": 2.1273088455200195,
+      "learning_rate": 5.912289159289883e-05,
+      "logits/chosen": -1.238785743713379,
+      "logits/rejected": -1.3599903583526611,
+      "logps/chosen": -5.028998851776123,
+      "logps/rejected": -101.78237915039062,
+      "loss": 0.2955174446105957,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.234098419547081,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3761712908744812,
+      "rewards/margins": 9.599595069885254,
+      "rewards/rejected": -9.223423957824707,
+      "step": 646,
+      "train_speed(iter/s)": 0.021166
+    },
+    {
+      "epoch": 1.3049734915425397,
+      "grad_norm": 1.553829550743103,
+      "learning_rate": 5.8818106248546004e-05,
+      "logits/chosen": -1.2518084049224854,
+      "logits/rejected": -1.3711079359054565,
+      "logps/chosen": -0.7378253936767578,
+      "logps/rejected": -73.98809814453125,
+      "loss": 0.14814817905426025,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10143674910068512,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2884601950645447,
+      "rewards/margins": 6.529684066772461,
+      "rewards/rejected": -6.2412238121032715,
+      "step": 647,
+      "train_speed(iter/s)": 0.021167
+    },
+    {
+      "epoch": 1.30699318353951,
+      "grad_norm": 4.927335262298584,
+      "learning_rate": 5.851378089584264e-05,
+      "logits/chosen": -1.2225008010864258,
+      "logits/rejected": -1.4799065589904785,
+      "logps/chosen": -2.824509620666504,
+      "logps/rejected": -155.8907470703125,
+      "loss": 0.4188971221446991,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.37693387269973755,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.197892427444458,
+      "rewards/margins": 14.133217811584473,
+      "rewards/rejected": -13.935325622558594,
+      "step": 648,
+      "train_speed(iter/s)": 0.021167
+    },
+    {
+      "epoch": 1.3090128755364807,
+      "grad_norm": 3.685182571411133,
+      "learning_rate": 5.8209918934028275e-05,
+      "logits/chosen": -1.2832986116409302,
+      "logits/rejected": -1.498741626739502,
+      "logps/chosen": -0.19548237323760986,
+      "logps/rejected": -183.37681579589844,
+      "loss": 0.034012358635663986,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.026802418753504753,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3500330150127411,
+      "rewards/margins": 17.256229400634766,
+      "rewards/rejected": -16.90619659423828,
+      "step": 649,
+      "train_speed(iter/s)": 0.021167
+    },
+    {
+      "epoch": 1.311032567533451,
+      "grad_norm": 11.082085609436035,
+      "learning_rate": 5.790652375716652e-05,
+      "logits/chosen": -1.2703983783721924,
+      "logits/rejected": -1.3829171657562256,
+      "logps/chosen": -3.457026720046997,
+      "logps/rejected": -93.01042938232422,
+      "loss": 0.5071526765823364,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4225212335586548,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16669617593288422,
+      "rewards/margins": 8.420509338378906,
+      "rewards/rejected": -8.253813743591309,
+      "step": 650,
+      "train_speed(iter/s)": 0.021168
+    },
+    {
+      "epoch": 1.3130522595304215,
+      "grad_norm": 5.953857898712158,
+      "learning_rate": 5.760359875410702e-05,
+      "logits/chosen": -1.1639339923858643,
+      "logits/rejected": -1.382941722869873,
+      "logps/chosen": -2.26224946975708,
+      "logps/rejected": -131.36766052246094,
+      "loss": 0.3274594843387604,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.28436633944511414,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2177920788526535,
+      "rewards/margins": 11.934349060058594,
+      "rewards/rejected": -11.716556549072266,
+      "step": 651,
+      "train_speed(iter/s)": 0.021156
+    },
+    {
+      "epoch": 1.315071951527392,
+      "grad_norm": 5.2896318435668945,
+      "learning_rate": 5.730114730844772e-05,
+      "logits/chosen": -1.3035253286361694,
+      "logits/rejected": -1.446520209312439,
+      "logps/chosen": -0.8383687734603882,
+      "logps/rejected": -158.22140502929688,
+      "loss": 0.1792777180671692,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.12140916287899017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.22891457378864288,
+      "rewards/margins": 14.84164810180664,
+      "rewards/rejected": -14.612733840942383,
+      "step": 652,
+      "train_speed(iter/s)": 0.021157
+    },
+    {
+      "epoch": 1.3170916435243625,
+      "grad_norm": 0.33124715089797974,
+      "learning_rate": 5.699917279849714e-05,
+      "logits/chosen": -1.0527029037475586,
+      "logits/rejected": -1.5235481262207031,
+      "logps/chosen": -1.6428444723715074e-05,
+      "logps/rejected": -246.20037841796875,
+      "loss": 6.130961992312223e-05,
+      "memory(GiB)": 45.64,
+      "nll_loss": 2.7380742722016294e-06,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.29877859354019165,
+      "rewards/margins": 22.724132537841797,
+      "rewards/rejected": -22.425352096557617,
+      "step": 653,
+      "train_speed(iter/s)": 0.021157
+    },
+    {
+      "epoch": 1.3191113355213329,
+      "grad_norm": 0.9578924775123596,
+      "learning_rate": 5.6697678597236356e-05,
+      "logits/chosen": -1.2435551881790161,
+      "logits/rejected": -1.3243225812911987,
+      "logps/chosen": -1.4107168912887573,
+      "logps/rejected": -97.84590148925781,
+      "loss": 0.1316772699356079,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.07347413152456284,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.49246057868003845,
+      "rewards/margins": 9.10840892791748,
+      "rewards/rejected": -8.615947723388672,
+      "step": 654,
+      "train_speed(iter/s)": 0.021157
+    },
+    {
+      "epoch": 1.3211310275183035,
+      "grad_norm": 0.9858250617980957,
+      "learning_rate": 5.639666807228175e-05,
+      "logits/chosen": -1.2371559143066406,
+      "logits/rejected": -1.4588568210601807,
+      "logps/chosen": -0.6821967959403992,
+      "logps/rejected": -150.42994689941406,
+      "loss": 0.11689744889736176,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.08747828751802444,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.27471041679382324,
+      "rewards/margins": 13.550159454345703,
+      "rewards/rejected": -13.275447845458984,
+      "step": 655,
+      "train_speed(iter/s)": 0.021158
+    },
+    {
+      "epoch": 1.3231507195152739,
+      "grad_norm": 7.005616188049316,
+      "learning_rate": 5.609614458584693e-05,
+      "logits/chosen": -1.3301862478256226,
+      "logits/rejected": -1.4930702447891235,
+      "logps/chosen": -1.5043425559997559,
+      "logps/rejected": -138.81788635253906,
+      "loss": 0.3016442358493805,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22508452832698822,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.26623108983039856,
+      "rewards/margins": 13.041426658630371,
+      "rewards/rejected": -12.775196075439453,
+      "step": 656,
+      "train_speed(iter/s)": 0.021158
+    },
+    {
+      "epoch": 1.3251704115122445,
+      "grad_norm": 1.5643805265426636,
+      "learning_rate": 5.5796111494705584e-05,
+      "logits/chosen": -1.2056901454925537,
+      "logits/rejected": -1.4967139959335327,
+      "logps/chosen": -0.9908508658409119,
+      "logps/rejected": -175.29811096191406,
+      "loss": 0.13119570910930634,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.08715493232011795,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3701123893260956,
+      "rewards/margins": 15.8663330078125,
+      "rewards/rejected": -15.496219635009766,
+      "step": 657,
+      "train_speed(iter/s)": 0.021159
+    },
+    {
+      "epoch": 1.3271901035092148,
+      "grad_norm": 1.4744075536727905,
+      "learning_rate": 5.549657215015367e-05,
+      "logits/chosen": -1.2551229000091553,
+      "logits/rejected": -1.445619821548462,
+      "logps/chosen": -5.067073822021484,
+      "logps/rejected": -158.08531188964844,
+      "loss": 0.16623428463935852,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15509991347789764,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5507426261901855,
+      "rewards/margins": 14.81292724609375,
+      "rewards/rejected": -14.262184143066406,
+      "step": 658,
+      "train_speed(iter/s)": 0.021159
+    },
+    {
+      "epoch": 1.3292097955061852,
+      "grad_norm": 1.9796504974365234,
+      "learning_rate": 5.519752989797224e-05,
+      "logits/chosen": -1.225923776626587,
+      "logits/rejected": -1.3867952823638916,
+      "logps/chosen": -9.267533302307129,
+      "logps/rejected": -127.44146728515625,
+      "loss": 0.306078165769577,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2250090092420578,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.48099297285079956,
+      "rewards/margins": 11.609613418579102,
+      "rewards/rejected": -11.128621101379395,
+      "step": 659,
+      "train_speed(iter/s)": 0.021159
+    },
+    {
+      "epoch": 1.3312294875031558,
+      "grad_norm": 7.617762088775635,
+      "learning_rate": 5.48989880783898e-05,
+      "logits/chosen": -1.332612156867981,
+      "logits/rejected": -1.4683401584625244,
+      "logps/chosen": -1.6133828163146973,
+      "logps/rejected": -134.66595458984375,
+      "loss": 0.314396470785141,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.174350768327713,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.28070563077926636,
+      "rewards/margins": 12.366264343261719,
+      "rewards/rejected": -12.085558891296387,
+      "step": 660,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 1.3332491795001262,
+      "grad_norm": 1.6940075159072876,
+      "learning_rate": 5.4600950026045326e-05,
+      "logits/chosen": -1.2701005935668945,
+      "logits/rejected": -1.4827098846435547,
+      "logps/chosen": -0.814773440361023,
+      "logps/rejected": -159.13258361816406,
+      "loss": 0.08944887667894363,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.06748656183481216,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.42033714056015015,
+      "rewards/margins": 15.079629898071289,
+      "rewards/rejected": -14.659292221069336,
+      "step": 661,
+      "train_speed(iter/s)": 0.02116
+    },
+    {
+      "epoch": 1.3352688714970966,
+      "grad_norm": 4.607296466827393,
+      "learning_rate": 5.430341906995065e-05,
+      "logits/chosen": -1.3689677715301514,
+      "logits/rejected": -1.4671341180801392,
+      "logps/chosen": -1.0908613204956055,
+      "logps/rejected": -107.647705078125,
+      "loss": 0.1359582096338272,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09177005290985107,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3954192101955414,
+      "rewards/margins": 10.061091423034668,
+      "rewards/rejected": -9.665672302246094,
+      "step": 662,
+      "train_speed(iter/s)": 0.021161
+    },
+    {
+      "epoch": 1.3372885634940672,
+      "grad_norm": 3.405909538269043,
+      "learning_rate": 5.400639853345364e-05,
+      "logits/chosen": -1.337336778640747,
+      "logits/rejected": -1.4998462200164795,
+      "logps/chosen": -1.6724298000335693,
+      "logps/rejected": -140.56777954101562,
+      "loss": 0.1607973426580429,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1261008381843567,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3773120045661926,
+      "rewards/margins": 13.164057731628418,
+      "rewards/rejected": -12.78674602508545,
+      "step": 663,
+      "train_speed(iter/s)": 0.021161
+    },
+    {
+      "epoch": 1.3393082554910376,
+      "grad_norm": 3.410588026046753,
+      "learning_rate": 5.370989173420076e-05,
+      "logits/chosen": -1.151210069656372,
+      "logits/rejected": -1.506962537765503,
+      "logps/chosen": -1.3926831483840942,
+      "logps/rejected": -190.99147033691406,
+      "loss": 0.21182231605052948,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18281033635139465,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.33187681436538696,
+      "rewards/margins": 17.34160614013672,
+      "rewards/rejected": -17.009729385375977,
+      "step": 664,
+      "train_speed(iter/s)": 0.021161
+    },
+    {
+      "epoch": 1.3413279474880082,
+      "grad_norm": 3.943718910217285,
+      "learning_rate": 5.341390198410019e-05,
+      "logits/chosen": -1.2449796199798584,
+      "logits/rejected": -1.4735636711120605,
+      "logps/chosen": -2.5048630237579346,
+      "logps/rejected": -115.57294464111328,
+      "loss": 0.20854948461055756,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18741244077682495,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.17739172279834747,
+      "rewards/margins": 10.402885437011719,
+      "rewards/rejected": -10.225493431091309,
+      "step": 665,
+      "train_speed(iter/s)": 0.021161
+    },
+    {
+      "epoch": 1.3433476394849786,
+      "grad_norm": 1.966831922531128,
+      "learning_rate": 5.31184325892849e-05,
+      "logits/chosen": -1.3403204679489136,
+      "logits/rejected": -1.4594730138778687,
+      "logps/chosen": -3.696135997772217,
+      "logps/rejected": -96.65306091308594,
+      "loss": 0.29288676381111145,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.27750274538993835,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1646392047405243,
+      "rewards/margins": 8.705083847045898,
+      "rewards/rejected": -8.540444374084473,
+      "step": 666,
+      "train_speed(iter/s)": 0.021162
+    },
+    {
+      "epoch": 1.345367331481949,
+      "grad_norm": 6.142731666564941,
+      "learning_rate": 5.282348685007543e-05,
+      "logits/chosen": -1.250907301902771,
+      "logits/rejected": -1.4630358219146729,
+      "logps/chosen": -9.278131484985352,
+      "logps/rejected": -111.81947326660156,
+      "loss": 0.6875588893890381,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6116375923156738,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3264651894569397,
+      "rewards/margins": 9.587200164794922,
+      "rewards/rejected": -9.913666725158691,
+      "step": 667,
+      "train_speed(iter/s)": 0.021162
+    },
+    {
+      "epoch": 1.3473870234789196,
+      "grad_norm": 3.65250301361084,
+      "learning_rate": 5.252906806094344e-05,
+      "logits/chosen": -1.2841596603393555,
+      "logits/rejected": -1.481567621231079,
+      "logps/chosen": -6.503849506378174,
+      "logps/rejected": -145.4832763671875,
+      "loss": 0.2576192319393158,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22354330122470856,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5191047191619873,
+      "rewards/margins": 13.803091049194336,
+      "rewards/rejected": -13.28398609161377,
+      "step": 668,
+      "train_speed(iter/s)": 0.021163
+    },
+    {
+      "epoch": 1.34940671547589,
+      "grad_norm": 38.141231536865234,
+      "learning_rate": 5.223517951047449e-05,
+      "logits/chosen": -1.3074294328689575,
+      "logits/rejected": -1.4011731147766113,
+      "logps/chosen": -2.9717133045196533,
+      "logps/rejected": -87.94474029541016,
+      "loss": 0.8502455353736877,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.5621344447135925,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.08282925933599472,
+      "rewards/margins": 7.743957042694092,
+      "rewards/rejected": -7.661128044128418,
+      "step": 669,
+      "train_speed(iter/s)": 0.021163
+    },
+    {
+      "epoch": 1.3514264074728604,
+      "grad_norm": 12.709877014160156,
+      "learning_rate": 5.1941824481331626e-05,
+      "logits/chosen": -1.2951457500457764,
+      "logits/rejected": -1.451993703842163,
+      "logps/chosen": -3.700683116912842,
+      "logps/rejected": -118.45479583740234,
+      "loss": 0.44092750549316406,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.334817498922348,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4615364074707031,
+      "rewards/margins": 11.104840278625488,
+      "rewards/rejected": -10.643303871154785,
+      "step": 670,
+      "train_speed(iter/s)": 0.021164
+    },
+    {
+      "epoch": 1.353446099469831,
+      "grad_norm": 1.8442193269729614,
+      "learning_rate": 5.164900625021856e-05,
+      "logits/chosen": -1.3582431077957153,
+      "logits/rejected": -1.4244040250778198,
+      "logps/chosen": -2.293272018432617,
+      "logps/rejected": -76.9024429321289,
+      "loss": 0.2617197334766388,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22223389148712158,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.37559252977371216,
+      "rewards/margins": 7.079739093780518,
+      "rewards/rejected": -6.704146385192871,
+      "step": 671,
+      "train_speed(iter/s)": 0.021164
+    },
+    {
+      "epoch": 1.3554657914668014,
+      "grad_norm": 11.00622272491455,
+      "learning_rate": 5.135672808784301e-05,
+      "logits/chosen": -1.1831167936325073,
+      "logits/rejected": -1.3369331359863281,
+      "logps/chosen": -10.682836532592773,
+      "logps/rejected": -78.8557357788086,
+      "loss": 0.47703832387924194,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3703742027282715,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.301474928855896,
+      "rewards/margins": 7.111950874328613,
+      "rewards/rejected": -6.810475826263428,
+      "step": 672,
+      "train_speed(iter/s)": 0.021164
+    },
+    {
+      "epoch": 1.3574854834637717,
+      "grad_norm": 1.9490995407104492,
+      "learning_rate": 5.106499325888041e-05,
+      "logits/chosen": -1.2989749908447266,
+      "logits/rejected": -1.4706417322158813,
+      "logps/chosen": -2.24629807472229,
+      "logps/rejected": -148.9947967529297,
+      "loss": 0.20434732735157013,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.19584883749485016,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.31430450081825256,
+      "rewards/margins": 13.695998191833496,
+      "rewards/rejected": -13.381694793701172,
+      "step": 673,
+      "train_speed(iter/s)": 0.021165
+    },
+    {
+      "epoch": 1.3595051754607423,
+      "grad_norm": 2.571312427520752,
+      "learning_rate": 5.077380502193725e-05,
+      "logits/chosen": -1.229909896850586,
+      "logits/rejected": -1.378341794013977,
+      "logps/chosen": -2.009305953979492,
+      "logps/rejected": -87.10649108886719,
+      "loss": 0.28070059418678284,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.200593501329422,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.22044259309768677,
+      "rewards/margins": 7.651946067810059,
+      "rewards/rejected": -7.431502819061279,
+      "step": 674,
+      "train_speed(iter/s)": 0.021165
+    },
+    {
+      "epoch": 1.3615248674577127,
+      "grad_norm": 1.1776663064956665,
+      "learning_rate": 5.0483166629514654e-05,
+      "logits/chosen": -1.1630738973617554,
+      "logits/rejected": -1.434651255607605,
+      "logps/chosen": -0.332209050655365,
+      "logps/rejected": -144.20486450195312,
+      "loss": 0.08161605149507523,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.06819590926170349,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3886221945285797,
+      "rewards/margins": 12.766104698181152,
+      "rewards/rejected": -12.377482414245605,
+      "step": 675,
+      "train_speed(iter/s)": 0.021165
+    },
+    {
+      "epoch": 1.3635445594546831,
+      "grad_norm": 2.123082399368286,
+      "learning_rate": 5.019308132797213e-05,
+      "logits/chosen": -1.2997647523880005,
+      "logits/rejected": -1.4661848545074463,
+      "logps/chosen": -0.9574002027511597,
+      "logps/rejected": -100.84864044189453,
+      "loss": 0.1859961897134781,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17034102976322174,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.31249648332595825,
+      "rewards/margins": 9.160298347473145,
+      "rewards/rejected": -8.847803115844727,
+      "step": 676,
+      "train_speed(iter/s)": 0.021166
+    },
+    {
+      "epoch": 1.3655642514516537,
+      "grad_norm": 6.4801788330078125,
+      "learning_rate": 4.9903552357491404e-05,
+      "logits/chosen": -1.2616076469421387,
+      "logits/rejected": -1.4036049842834473,
+      "logps/chosen": -0.8396381735801697,
+      "logps/rejected": -79.37264251708984,
+      "loss": 0.12827752530574799,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10109595954418182,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3446037173271179,
+      "rewards/margins": 7.113584041595459,
+      "rewards/rejected": -6.768980026245117,
+      "step": 677,
+      "train_speed(iter/s)": 0.021166
+    },
+    {
+      "epoch": 1.367583943448624,
+      "grad_norm": 1.0385584831237793,
+      "learning_rate": 4.961458295203999e-05,
+      "logits/chosen": -1.2414830923080444,
+      "logits/rejected": -1.3784379959106445,
+      "logps/chosen": -0.7634080052375793,
+      "logps/rejected": -101.95245361328125,
+      "loss": 0.10871831327676773,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.060792580246925354,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.38261836767196655,
+      "rewards/margins": 9.370853424072266,
+      "rewards/rejected": -8.988234519958496,
+      "step": 678,
+      "train_speed(iter/s)": 0.021167
+    },
+    {
+      "epoch": 1.3696036354455945,
+      "grad_norm": 1.0497952699661255,
+      "learning_rate": 4.9326176339335225e-05,
+      "logits/chosen": -1.2115139961242676,
+      "logits/rejected": -1.3533506393432617,
+      "logps/chosen": -2.428312301635742,
+      "logps/rejected": -129.19033813476562,
+      "loss": 0.1941956877708435,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1578986644744873,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5029597282409668,
+      "rewards/margins": 11.33251953125,
+      "rewards/rejected": -10.829560279846191,
+      "step": 679,
+      "train_speed(iter/s)": 0.021167
+    },
+    {
+      "epoch": 1.371623327442565,
+      "grad_norm": 1.8122177124023438,
+      "learning_rate": 4.903833574080825e-05,
+      "logits/chosen": -1.1344064474105835,
+      "logits/rejected": -1.374717116355896,
+      "logps/chosen": -1.7576682567596436,
+      "logps/rejected": -111.03709411621094,
+      "loss": 0.2686014473438263,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15585042536258698,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.30624550580978394,
+      "rewards/margins": 9.679397583007812,
+      "rewards/rejected": -9.373152732849121,
+      "step": 680,
+      "train_speed(iter/s)": 0.021167
+    },
+    {
+      "epoch": 1.3736430194395355,
+      "grad_norm": 2.180088996887207,
+      "learning_rate": 4.875106437156795e-05,
+      "logits/chosen": -1.273840069770813,
+      "logits/rejected": -1.4243693351745605,
+      "logps/chosen": -0.3663364052772522,
+      "logps/rejected": -97.48326110839844,
+      "loss": 0.0809452086687088,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.03888875991106033,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.42584162950515747,
+      "rewards/margins": 8.95561408996582,
+      "rewards/rejected": -8.529773712158203,
+      "step": 681,
+      "train_speed(iter/s)": 0.021168
+    },
+    {
+      "epoch": 1.3756627114365059,
+      "grad_norm": 9.817505836486816,
+      "learning_rate": 4.8464365440365044e-05,
+      "logits/chosen": -1.1696536540985107,
+      "logits/rejected": -1.3185362815856934,
+      "logps/chosen": -2.9259347915649414,
+      "logps/rejected": -73.75517272949219,
+      "loss": 0.5601015090942383,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3666420876979828,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.03908505663275719,
+      "rewards/margins": 5.313644886016846,
+      "rewards/rejected": -5.274559020996094,
+      "step": 682,
+      "train_speed(iter/s)": 0.021168
+    },
+    {
+      "epoch": 1.3776824034334765,
+      "grad_norm": 1.6399908065795898,
+      "learning_rate": 4.8178242149556176e-05,
+      "logits/chosen": -1.162152647972107,
+      "logits/rejected": -1.3488810062408447,
+      "logps/chosen": -0.7934271097183228,
+      "logps/rejected": -120.67758178710938,
+      "loss": 0.233425110578537,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14084455370903015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3075992166996002,
+      "rewards/margins": 10.230250358581543,
+      "rewards/rejected": -9.922650337219238,
+      "step": 683,
+      "train_speed(iter/s)": 0.021168
+    },
+    {
+      "epoch": 1.3797020954304469,
+      "grad_norm": 3.952440023422241,
+      "learning_rate": 4.789269769506839e-05,
+      "logits/chosen": -1.1867936849594116,
+      "logits/rejected": -1.448725938796997,
+      "logps/chosen": -0.45201483368873596,
+      "logps/rejected": -132.04908752441406,
+      "loss": 0.05989331752061844,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.05165192112326622,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.46122118830680847,
+      "rewards/margins": 11.5491304397583,
+      "rewards/rejected": -11.087908744812012,
+      "step": 684,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 1.3817217874274172,
+      "grad_norm": 1.8802335262298584,
+      "learning_rate": 4.760773526636315e-05,
+      "logits/chosen": -1.1008230447769165,
+      "logits/rejected": -1.442441701889038,
+      "logps/chosen": -1.509798526763916,
+      "logps/rejected": -166.89305114746094,
+      "loss": 0.1596652716398239,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14110448956489563,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3266116678714752,
+      "rewards/margins": 15.062488555908203,
+      "rewards/rejected": -14.735878944396973,
+      "step": 685,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 1.3837414794243879,
+      "grad_norm": 1.448838710784912,
+      "learning_rate": 4.7323358046400844e-05,
+      "logits/chosen": -1.2354135513305664,
+      "logits/rejected": -1.4121716022491455,
+      "logps/chosen": -0.17699043452739716,
+      "logps/rejected": -110.69224548339844,
+      "loss": 0.10782967507839203,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.035099636763334274,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34154537320137024,
+      "rewards/margins": 10.394100189208984,
+      "rewards/rejected": -10.052555084228516,
+      "step": 686,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 1.3857611714213582,
+      "grad_norm": 3.616986036300659,
+      "learning_rate": 4.703956921160528e-05,
+      "logits/chosen": -1.3047581911087036,
+      "logits/rejected": -1.3844478130340576,
+      "logps/chosen": -1.7377244234085083,
+      "logps/rejected": -86.59591674804688,
+      "loss": 0.21381613612174988,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17451715469360352,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2591663897037506,
+      "rewards/margins": 7.664936065673828,
+      "rewards/rejected": -7.405769348144531,
+      "step": 687,
+      "train_speed(iter/s)": 0.02117
+    },
+    {
+      "epoch": 1.3877808634183286,
+      "grad_norm": 1.3300586938858032,
+      "learning_rate": 4.675637193182816e-05,
+      "logits/chosen": -1.3291690349578857,
+      "logits/rejected": -1.4771991968154907,
+      "logps/chosen": -0.6744148135185242,
+      "logps/rejected": -122.29110717773438,
+      "loss": 0.08174961805343628,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.07034464925527573,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4019295871257782,
+      "rewards/margins": 11.68201732635498,
+      "rewards/rejected": -11.2800874710083,
+      "step": 688,
+      "train_speed(iter/s)": 0.02117
+    },
+    {
+      "epoch": 1.3898005554152992,
+      "grad_norm": 19.863229751586914,
+      "learning_rate": 4.647376937031356e-05,
+      "logits/chosen": -1.3024847507476807,
+      "logits/rejected": -1.3900057077407837,
+      "logps/chosen": -5.704400539398193,
+      "logps/rejected": -89.47821044921875,
+      "loss": 0.5266599655151367,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4258812963962555,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.12338393926620483,
+      "rewards/margins": 7.841525554656982,
+      "rewards/rejected": -7.964909076690674,
+      "step": 689,
+      "train_speed(iter/s)": 0.021171
+    },
+    {
+      "epoch": 1.3918202474122696,
+      "grad_norm": 21.22760581970215,
+      "learning_rate": 4.6191764683662744e-05,
+      "logits/chosen": -1.2463624477386475,
+      "logits/rejected": -1.479274034500122,
+      "logps/chosen": -3.5922951698303223,
+      "logps/rejected": -145.2352294921875,
+      "loss": 0.41582149267196655,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3625841736793518,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.009728386998176575,
+      "rewards/margins": 12.781889915466309,
+      "rewards/rejected": -12.772160530090332,
+      "step": 690,
+      "train_speed(iter/s)": 0.021171
+    },
+    {
+      "epoch": 1.39383993940924,
+      "grad_norm": 5.069929599761963,
+      "learning_rate": 4.591036102179893e-05,
+      "logits/chosen": -1.2319940328598022,
+      "logits/rejected": -1.4201147556304932,
+      "logps/chosen": -1.8396357297897339,
+      "logps/rejected": -105.60858154296875,
+      "loss": 0.24667870998382568,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.21403905749320984,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4658468961715698,
+      "rewards/margins": 9.344257354736328,
+      "rewards/rejected": -8.878410339355469,
+      "step": 691,
+      "train_speed(iter/s)": 0.021171
+    },
+    {
+      "epoch": 1.3958596314062106,
+      "grad_norm": 1.9479291439056396,
+      "learning_rate": 4.562956152793192e-05,
+      "logits/chosen": -1.1664063930511475,
+      "logits/rejected": -1.3733720779418945,
+      "logps/chosen": -1.7960572242736816,
+      "logps/rejected": -87.40716552734375,
+      "loss": 0.24663159251213074,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.21220353245735168,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.37847164273262024,
+      "rewards/margins": 7.924840927124023,
+      "rewards/rejected": -7.546369552612305,
+      "step": 692,
+      "train_speed(iter/s)": 0.021171
+    },
+    {
+      "epoch": 1.397879323403181,
+      "grad_norm": 7.135278701782227,
+      "learning_rate": 4.534936933852324e-05,
+      "logits/chosen": -1.175508737564087,
+      "logits/rejected": -1.3563309907913208,
+      "logps/chosen": -2.5826776027679443,
+      "logps/rejected": -92.12893676757812,
+      "loss": 0.3086884021759033,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2583051919937134,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2889597415924072,
+      "rewards/margins": 8.378761291503906,
+      "rewards/rejected": -8.089801788330078,
+      "step": 693,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 1.3998990154001514,
+      "grad_norm": 0.927460789680481,
+      "learning_rate": 4.506978758325081e-05,
+      "logits/chosen": -0.9685267210006714,
+      "logits/rejected": -1.3303056955337524,
+      "logps/chosen": -9.243148803710938,
+      "logps/rejected": -114.05810546875,
+      "loss": 0.2084275633096695,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17222465574741364,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5202786922454834,
+      "rewards/margins": 8.965426445007324,
+      "rewards/rejected": -8.445147514343262,
+      "step": 694,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 1.401918707397122,
+      "grad_norm": 0.6815046668052673,
+      "learning_rate": 4.479081938497435e-05,
+      "logits/chosen": -1.2229392528533936,
+      "logits/rejected": -1.4068939685821533,
+      "logps/chosen": -1.0758274793624878,
+      "logps/rejected": -93.91361236572266,
+      "loss": 0.0781669169664383,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.0379214808344841,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5448591709136963,
+      "rewards/margins": 8.357100486755371,
+      "rewards/rejected": -7.812242031097412,
+      "step": 695,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 1.4039383993940924,
+      "grad_norm": 0.6699221134185791,
+      "learning_rate": 4.451246785970009e-05,
+      "logits/chosen": -1.2351534366607666,
+      "logits/rejected": -1.4253066778182983,
+      "logps/chosen": -2.743697166442871,
+      "logps/rejected": -105.41436767578125,
+      "loss": 0.43233761191368103,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.40566331148147583,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1658860445022583,
+      "rewards/margins": 9.486268997192383,
+      "rewards/rejected": -9.320383071899414,
+      "step": 696,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 1.4059580913910628,
+      "grad_norm": 7.279857635498047,
+      "learning_rate": 4.4234736116546364e-05,
+      "logits/chosen": -1.1237012147903442,
+      "logits/rejected": -1.3446521759033203,
+      "logps/chosen": -2.647087335586548,
+      "logps/rejected": -93.25931549072266,
+      "loss": 0.3315366804599762,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.29119834303855896,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2183208465576172,
+      "rewards/margins": 7.42087459564209,
+      "rewards/rejected": -7.202553749084473,
+      "step": 697,
+      "train_speed(iter/s)": 0.021173
+    },
+    {
+      "epoch": 1.4079777833880334,
+      "grad_norm": 0.8701071739196777,
+      "learning_rate": 4.395762725770852e-05,
+      "logits/chosen": -1.21685791015625,
+      "logits/rejected": -1.4127399921417236,
+      "logps/chosen": -1.876451849937439,
+      "logps/rejected": -99.66693115234375,
+      "loss": 0.14667458832263947,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13231584429740906,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.42442944645881653,
+      "rewards/margins": 9.00214958190918,
+      "rewards/rejected": -8.577720642089844,
+      "step": 698,
+      "train_speed(iter/s)": 0.021173
+    },
+    {
+      "epoch": 1.4099974753850038,
+      "grad_norm": 8.796870231628418,
+      "learning_rate": 4.368114437842461e-05,
+      "logits/chosen": -1.150916576385498,
+      "logits/rejected": -1.3707295656204224,
+      "logps/chosen": -4.650566101074219,
+      "logps/rejected": -81.81768798828125,
+      "loss": 0.3437148928642273,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.27169081568717957,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.6446067094802856,
+      "rewards/margins": 7.310153007507324,
+      "rewards/rejected": -6.665546894073486,
+      "step": 699,
+      "train_speed(iter/s)": 0.021173
+    },
+    {
+      "epoch": 1.4120171673819741,
+      "grad_norm": 1.7056961059570312,
+      "learning_rate": 4.340529056694047e-05,
+      "logits/chosen": -1.163533091545105,
+      "logits/rejected": -1.381466031074524,
+      "logps/chosen": -2.1018874645233154,
+      "logps/rejected": -105.22046661376953,
+      "loss": 0.24114225804805756,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.21665070950984955,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.23153725266456604,
+      "rewards/margins": 9.193727493286133,
+      "rewards/rejected": -8.962190628051758,
+      "step": 700,
+      "train_speed(iter/s)": 0.021173
+    },
+    {
+      "epoch": 1.4140368593789447,
+      "grad_norm": 2.044799566268921,
+      "learning_rate": 4.3130068904475586e-05,
+      "logits/chosen": -1.2229055166244507,
+      "logits/rejected": -1.3745191097259521,
+      "logps/chosen": -3.618516445159912,
+      "logps/rejected": -105.83740997314453,
+      "loss": 0.43210887908935547,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.39272040128707886,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.18585002422332764,
+      "rewards/margins": 9.373245239257812,
+      "rewards/rejected": -9.187395095825195,
+      "step": 701,
+      "train_speed(iter/s)": 0.021163
+    },
+    {
+      "epoch": 1.4160565513759151,
+      "grad_norm": 13.033117294311523,
+      "learning_rate": 4.285548246518837e-05,
+      "logits/chosen": -1.1940739154815674,
+      "logits/rejected": -1.3720226287841797,
+      "logps/chosen": -1.3294414281845093,
+      "logps/rejected": -86.94921875,
+      "loss": 0.20416998863220215,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1334892064332962,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.413254976272583,
+      "rewards/margins": 7.377900123596191,
+      "rewards/rejected": -6.9646453857421875,
+      "step": 702,
+      "train_speed(iter/s)": 0.021163
+    },
+    {
+      "epoch": 1.4180762433728855,
+      "grad_norm": 46.98183822631836,
+      "learning_rate": 4.258153431614193e-05,
+      "logits/chosen": -1.0818350315093994,
+      "logits/rejected": -1.396226167678833,
+      "logps/chosen": -1.1283810138702393,
+      "logps/rejected": -125.59014892578125,
+      "loss": 0.1630658358335495,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14043158292770386,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3703669309616089,
+      "rewards/margins": 10.535307884216309,
+      "rewards/rejected": -10.16494083404541,
+      "step": 703,
+      "train_speed(iter/s)": 0.021163
+    },
+    {
+      "epoch": 1.4200959353698561,
+      "grad_norm": 1.6561558246612549,
+      "learning_rate": 4.230822751726994e-05,
+      "logits/chosen": -1.1182897090911865,
+      "logits/rejected": -1.4314367771148682,
+      "logps/chosen": -1.1943761110305786,
+      "logps/rejected": -109.91300201416016,
+      "loss": 0.19304288923740387,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18118199706077576,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.263763427734375,
+      "rewards/margins": 10.072196960449219,
+      "rewards/rejected": -9.808433532714844,
+      "step": 704,
+      "train_speed(iter/s)": 0.021163
+    },
+    {
+      "epoch": 1.4221156273668265,
+      "grad_norm": 0.6058415770530701,
+      "learning_rate": 4.2035565121342246e-05,
+      "logits/chosen": -1.325484275817871,
+      "logits/rejected": -1.4248348474502563,
+      "logps/chosen": -3.050715446472168,
+      "logps/rejected": -94.10456848144531,
+      "loss": 0.16054768860340118,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13168105483055115,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4427938461303711,
+      "rewards/margins": 9.043136596679688,
+      "rewards/rejected": -8.600342750549316,
+      "step": 705,
+      "train_speed(iter/s)": 0.021164
+    },
+    {
+      "epoch": 1.424135319363797,
+      "grad_norm": 2.4991536140441895,
+      "learning_rate": 4.176355017393099e-05,
+      "logits/chosen": -1.2388986349105835,
+      "logits/rejected": -1.4082176685333252,
+      "logps/chosen": -2.5669615268707275,
+      "logps/rejected": -79.71957397460938,
+      "loss": 0.26363304257392883,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20085805654525757,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2871832847595215,
+      "rewards/margins": 6.903397560119629,
+      "rewards/rejected": -6.616213798522949,
+      "step": 706,
+      "train_speed(iter/s)": 0.021164
+    },
+    {
+      "epoch": 1.4261550113607675,
+      "grad_norm": 1.4340723752975464,
+      "learning_rate": 4.14921857133763e-05,
+      "logits/chosen": -1.3228248357772827,
+      "logits/rejected": -1.5218312740325928,
+      "logps/chosen": -1.2171344757080078,
+      "logps/rejected": -146.4058837890625,
+      "loss": 0.12822310626506805,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11732035130262375,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.30606240034103394,
+      "rewards/margins": 13.73283863067627,
+      "rewards/rejected": -13.426776885986328,
+      "step": 707,
+      "train_speed(iter/s)": 0.021165
+    },
+    {
+      "epoch": 1.4281747033577379,
+      "grad_norm": 11.203118324279785,
+      "learning_rate": 4.12214747707527e-05,
+      "logits/chosen": -1.1985714435577393,
+      "logits/rejected": -1.4072692394256592,
+      "logps/chosen": -6.1920013427734375,
+      "logps/rejected": -98.05803680419922,
+      "loss": 0.9179109334945679,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6197484731674194,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.07338708639144897,
+      "rewards/margins": 8.524528503417969,
+      "rewards/rejected": -8.451142311096191,
+      "step": 708,
+      "train_speed(iter/s)": 0.021165
+    },
+    {
+      "epoch": 1.4301943953547083,
+      "grad_norm": 3.682178258895874,
+      "learning_rate": 4.095142036983497e-05,
+      "logits/chosen": -1.2648918628692627,
+      "logits/rejected": -1.4843016862869263,
+      "logps/chosen": -2.780545949935913,
+      "logps/rejected": -132.77816772460938,
+      "loss": 0.21526435017585754,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20382842421531677,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2568470537662506,
+      "rewards/margins": 12.083222389221191,
+      "rewards/rejected": -11.826375961303711,
+      "step": 709,
+      "train_speed(iter/s)": 0.021165
+    },
+    {
+      "epoch": 1.4322140873516789,
+      "grad_norm": 1.2915520668029785,
+      "learning_rate": 4.0682025527064486e-05,
+      "logits/chosen": -1.2681580781936646,
+      "logits/rejected": -1.3704899549484253,
+      "logps/chosen": -1.0966300964355469,
+      "logps/rejected": -93.09785461425781,
+      "loss": 0.1752842217683792,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13495945930480957,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3077487349510193,
+      "rewards/margins": 8.806397438049316,
+      "rewards/rejected": -8.498648643493652,
+      "step": 710,
+      "train_speed(iter/s)": 0.021166
+    },
+    {
+      "epoch": 1.4342337793486493,
+      "grad_norm": 1.2456876039505005,
+      "learning_rate": 4.0413293251515574e-05,
+      "logits/chosen": -1.329200029373169,
+      "logits/rejected": -1.4942477941513062,
+      "logps/chosen": -1.2735542058944702,
+      "logps/rejected": -104.48162078857422,
+      "loss": 0.10679938644170761,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10188594460487366,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.42877647280693054,
+      "rewards/margins": 9.919543266296387,
+      "rewards/rejected": -9.490765571594238,
+      "step": 711,
+      "train_speed(iter/s)": 0.021166
+    },
+    {
+      "epoch": 1.4362534713456199,
+      "grad_norm": 14.501618385314941,
+      "learning_rate": 4.0145226544861845e-05,
+      "logits/chosen": -1.3506288528442383,
+      "logits/rejected": -1.527575969696045,
+      "logps/chosen": -1.667263150215149,
+      "logps/rejected": -138.45535278320312,
+      "loss": 0.38350167870521545,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2976124882698059,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.20130714774131775,
+      "rewards/margins": 13.026888847351074,
+      "rewards/rejected": -12.825581550598145,
+      "step": 712,
+      "train_speed(iter/s)": 0.021167
+    },
+    {
+      "epoch": 1.4382731633425903,
+      "grad_norm": 2.30352783203125,
+      "learning_rate": 3.987782840134263e-05,
+      "logits/chosen": -1.1699286699295044,
+      "logits/rejected": -1.3781887292861938,
+      "logps/chosen": -2.1137773990631104,
+      "logps/rejected": -102.24137115478516,
+      "loss": 0.3341729938983917,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.29472023248672485,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.24791941046714783,
+      "rewards/margins": 8.627683639526367,
+      "rewards/rejected": -8.379764556884766,
+      "step": 713,
+      "train_speed(iter/s)": 0.021167
+    },
+    {
+      "epoch": 1.4402928553395606,
+      "grad_norm": 1.966960072517395,
+      "learning_rate": 3.961110180772955e-05,
+      "logits/chosen": -1.4148898124694824,
+      "logits/rejected": -1.4885958433151245,
+      "logps/chosen": -0.3124823272228241,
+      "logps/rejected": -79.99415588378906,
+      "loss": 0.11379528045654297,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.04322899505496025,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3067057430744171,
+      "rewards/margins": 7.7095441818237305,
+      "rewards/rejected": -7.402838706970215,
+      "step": 714,
+      "train_speed(iter/s)": 0.021168
+    },
+    {
+      "epoch": 1.4423125473365312,
+      "grad_norm": 14.858046531677246,
+      "learning_rate": 3.934504974329326e-05,
+      "logits/chosen": -1.247122049331665,
+      "logits/rejected": -1.4954911470413208,
+      "logps/chosen": -1.0672775506973267,
+      "logps/rejected": -131.54306030273438,
+      "loss": 0.25801706314086914,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1741979718208313,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2108839750289917,
+      "rewards/margins": 11.704009056091309,
+      "rewards/rejected": -11.493123054504395,
+      "step": 715,
+      "train_speed(iter/s)": 0.021168
+    },
+    {
+      "epoch": 1.4443322393335016,
+      "grad_norm": 13.160477638244629,
+      "learning_rate": 3.907967517977005e-05,
+      "logits/chosen": -1.1280797719955444,
+      "logits/rejected": -1.4235178232192993,
+      "logps/chosen": -3.1211905479431152,
+      "logps/rejected": -145.0430145263672,
+      "loss": 0.6609184145927429,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.4466950297355652,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4032019078731537,
+      "rewards/margins": 13.173227310180664,
+      "rewards/rejected": -12.770024299621582,
+      "step": 716,
+      "train_speed(iter/s)": 0.021168
+    },
+    {
+      "epoch": 1.4463519313304722,
+      "grad_norm": 19.675289154052734,
+      "learning_rate": 3.8814981081328615e-05,
+      "logits/chosen": -1.168630838394165,
+      "logits/rejected": -1.3688334226608276,
+      "logps/chosen": -4.860542297363281,
+      "logps/rejected": -90.74357604980469,
+      "loss": 0.36712172627449036,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3287407159805298,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3620649576187134,
+      "rewards/margins": 7.89828634262085,
+      "rewards/rejected": -7.536221504211426,
+      "step": 717,
+      "train_speed(iter/s)": 0.021168
+    },
+    {
+      "epoch": 1.4483716233274426,
+      "grad_norm": 0.5230522155761719,
+      "learning_rate": 3.8550970404537144e-05,
+      "logits/chosen": -1.3261909484863281,
+      "logits/rejected": -1.5047823190689087,
+      "logps/chosen": -0.7877066135406494,
+      "logps/rejected": -118.50739288330078,
+      "loss": 0.07164949178695679,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.06458094716072083,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3282886743545532,
+      "rewards/margins": 11.048782348632812,
+      "rewards/rejected": -10.72049331665039,
+      "step": 718,
+      "train_speed(iter/s)": 0.021168
+    },
+    {
+      "epoch": 1.450391315324413,
+      "grad_norm": 1.3397313356399536,
+      "learning_rate": 3.8287646098330166e-05,
+      "logits/chosen": -1.3261269330978394,
+      "logits/rejected": -1.5037808418273926,
+      "logps/chosen": -1.6821765899658203,
+      "logps/rejected": -135.7525177001953,
+      "loss": 0.15260693430900574,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09921140968799591,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3897606134414673,
+      "rewards/margins": 12.938263893127441,
+      "rewards/rejected": -12.548502922058105,
+      "step": 719,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 1.4524110073213836,
+      "grad_norm": 1.1280800104141235,
+      "learning_rate": 3.802501110397553e-05,
+      "logits/chosen": -1.3106818199157715,
+      "logits/rejected": -1.465780258178711,
+      "logps/chosen": -0.23740831017494202,
+      "logps/rejected": -103.62336730957031,
+      "loss": 0.07739432156085968,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.04916374757885933,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4229488968849182,
+      "rewards/margins": 9.546566009521484,
+      "rewards/rejected": -9.123618125915527,
+      "step": 720,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 1.454430699318354,
+      "grad_norm": 2.9534316062927246,
+      "learning_rate": 3.776306835504166e-05,
+      "logits/chosen": -1.338354468345642,
+      "logits/rejected": -1.4439857006072998,
+      "logps/chosen": -1.9907044172286987,
+      "logps/rejected": -91.03913879394531,
+      "loss": 0.231755793094635,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1922505795955658,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2891916334629059,
+      "rewards/margins": 8.375207901000977,
+      "rewards/rejected": -8.086015701293945,
+      "step": 721,
+      "train_speed(iter/s)": 0.02117
+    },
+    {
+      "epoch": 1.4564503913153244,
+      "grad_norm": 2.9279937744140625,
+      "learning_rate": 3.750182077736486e-05,
+      "logits/chosen": -1.2446619272232056,
+      "logits/rejected": -1.3896995782852173,
+      "logps/chosen": -2.385638952255249,
+      "logps/rejected": -141.6939239501953,
+      "loss": 0.2721683382987976,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.21835589408874512,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2517452836036682,
+      "rewards/margins": 12.755172729492188,
+      "rewards/rejected": -12.503426551818848,
+      "step": 722,
+      "train_speed(iter/s)": 0.02117
+    },
+    {
+      "epoch": 1.458470083312295,
+      "grad_norm": 2.318556308746338,
+      "learning_rate": 3.724127128901644e-05,
+      "logits/chosen": -1.2406973838806152,
+      "logits/rejected": -1.3330278396606445,
+      "logps/chosen": -1.9295787811279297,
+      "logps/rejected": -79.50251770019531,
+      "loss": 0.30775219202041626,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2230408787727356,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.32601049542427063,
+      "rewards/margins": 7.316903114318848,
+      "rewards/rejected": -6.9908928871154785,
+      "step": 723,
+      "train_speed(iter/s)": 0.02117
+    },
+    {
+      "epoch": 1.4604897753092654,
+      "grad_norm": 6.0814900398254395,
+      "learning_rate": 3.6981422800270235e-05,
+      "logits/chosen": -1.2400330305099487,
+      "logits/rejected": -1.5108119249343872,
+      "logps/chosen": -2.0213303565979004,
+      "logps/rejected": -122.81996154785156,
+      "loss": 0.3561140298843384,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2918931841850281,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.18052934110164642,
+      "rewards/margins": 10.716202735900879,
+      "rewards/rejected": -10.535673141479492,
+      "step": 724,
+      "train_speed(iter/s)": 0.021171
+    },
+    {
+      "epoch": 1.4625094673062358,
+      "grad_norm": 1.218264102935791,
+      "learning_rate": 3.672227821357014e-05,
+      "logits/chosen": -1.3618268966674805,
+      "logits/rejected": -1.5020407438278198,
+      "logps/chosen": -0.8248708844184875,
+      "logps/rejected": -119.432861328125,
+      "loss": 0.08305380493402481,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.061092063784599304,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.41972440481185913,
+      "rewards/margins": 11.186538696289062,
+      "rewards/rejected": -10.766813278198242,
+      "step": 725,
+      "train_speed(iter/s)": 0.021171
+    },
+    {
+      "epoch": 1.4645291593032064,
+      "grad_norm": 1.82322096824646,
+      "learning_rate": 3.646384042349764e-05,
+      "logits/chosen": -1.3026429414749146,
+      "logits/rejected": -1.3172602653503418,
+      "logps/chosen": -4.971906661987305,
+      "logps/rejected": -72.15706634521484,
+      "loss": 0.29972898960113525,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24978867173194885,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.8750786781311035,
+      "rewards/margins": 6.622191905975342,
+      "rewards/rejected": -5.747113227844238,
+      "step": 726,
+      "train_speed(iter/s)": 0.021171
+    },
+    {
+      "epoch": 1.4665488513001768,
+      "grad_norm": 1.8178704977035522,
+      "learning_rate": 3.6206112316739424e-05,
+      "logits/chosen": -1.3585789203643799,
+      "logits/rejected": -1.4767429828643799,
+      "logps/chosen": -1.1017827987670898,
+      "logps/rejected": -115.45960235595703,
+      "loss": 0.18051394820213318,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1251862794160843,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2346717268228531,
+      "rewards/margins": 10.644471168518066,
+      "rewards/rejected": -10.409799575805664,
+      "step": 727,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 1.4685685432971471,
+      "grad_norm": 2.5391275882720947,
+      "learning_rate": 3.594909677205517e-05,
+      "logits/chosen": -1.347773790359497,
+      "logits/rejected": -1.5261428356170654,
+      "logps/chosen": -1.0800597667694092,
+      "logps/rejected": -139.0677032470703,
+      "loss": 0.18077687919139862,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.161635160446167,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3401648998260498,
+      "rewards/margins": 12.93256950378418,
+      "rewards/rejected": -12.59240436553955,
+      "step": 728,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 1.4705882352941178,
+      "grad_norm": 1.0226455926895142,
+      "learning_rate": 3.5692796660245507e-05,
+      "logits/chosen": -0.9919037222862244,
+      "logits/rejected": -1.387916088104248,
+      "logps/chosen": -2.1228020191192627,
+      "logps/rejected": -182.4185791015625,
+      "loss": 0.4151538014411926,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.34759047627449036,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.14504505693912506,
+      "rewards/margins": 16.098854064941406,
+      "rewards/rejected": -15.95380973815918,
+      "step": 729,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 1.4726079272910881,
+      "grad_norm": 15.697002410888672,
+      "learning_rate": 3.543721484411976e-05,
+      "logits/chosen": -1.3747676610946655,
+      "logits/rejected": -1.4820277690887451,
+      "logps/chosen": -0.6890533566474915,
+      "logps/rejected": -95.2104721069336,
+      "loss": 0.2115727961063385,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.08876544237136841,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.38683223724365234,
+      "rewards/margins": 9.076919555664062,
+      "rewards/rejected": -8.69008731842041,
+      "step": 730,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 1.4746276192880585,
+      "grad_norm": 1.1154985427856445,
+      "learning_rate": 3.5182354178464175e-05,
+      "logits/chosen": -1.3362171649932861,
+      "logits/rejected": -1.5471057891845703,
+      "logps/chosen": -0.4180063009262085,
+      "logps/rejected": -162.81857299804688,
+      "loss": 0.06482063978910446,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.034465301781892776,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5066577196121216,
+      "rewards/margins": 15.355396270751953,
+      "rewards/rejected": -14.848737716674805,
+      "step": 731,
+      "train_speed(iter/s)": 0.021173
+    },
+    {
+      "epoch": 1.4766473112850291,
+      "grad_norm": 0.8502868413925171,
+      "learning_rate": 3.492821751000976e-05,
+      "logits/chosen": -1.1731257438659668,
+      "logits/rejected": -1.4166922569274902,
+      "logps/chosen": -0.5787484645843506,
+      "logps/rejected": -129.0984344482422,
+      "loss": 0.14126057922840118,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.07160625606775284,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.26274749636650085,
+      "rewards/margins": 10.521753311157227,
+      "rewards/rejected": -10.25900650024414,
+      "step": 732,
+      "train_speed(iter/s)": 0.021173
+    },
+    {
+      "epoch": 1.4786670032819995,
+      "grad_norm": 1.1046440601348877,
+      "learning_rate": 3.4674807677400865e-05,
+      "logits/chosen": -1.375756025314331,
+      "logits/rejected": -1.5594062805175781,
+      "logps/chosen": -0.4635104537010193,
+      "logps/rejected": -156.51016235351562,
+      "loss": 0.05521240830421448,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.05156245455145836,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3216698169708252,
+      "rewards/margins": 14.742155075073242,
+      "rewards/rejected": -14.420485496520996,
+      "step": 733,
+      "train_speed(iter/s)": 0.021173
+    },
+    {
+      "epoch": 1.48068669527897,
+      "grad_norm": 12.377058982849121,
+      "learning_rate": 3.442212751116305e-05,
+      "logits/chosen": -1.0884863138198853,
+      "logits/rejected": -1.4107279777526855,
+      "logps/chosen": -2.824897289276123,
+      "logps/rejected": -166.00961303710938,
+      "loss": 0.39974313974380493,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2855280339717865,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1118754893541336,
+      "rewards/margins": 14.611434936523438,
+      "rewards/rejected": -14.499560356140137,
+      "step": 734,
+      "train_speed(iter/s)": 0.021173
+    },
+    {
+      "epoch": 1.4827063872759405,
+      "grad_norm": 1.2210673093795776,
+      "learning_rate": 3.4170179833671846e-05,
+      "logits/chosen": -1.1905813217163086,
+      "logits/rejected": -1.4067723751068115,
+      "logps/chosen": -1.2237229347229004,
+      "logps/rejected": -131.0060577392578,
+      "loss": 0.14345374703407288,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.07419726252555847,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4797264337539673,
+      "rewards/margins": 11.75821590423584,
+      "rewards/rejected": -11.278491020202637,
+      "step": 735,
+      "train_speed(iter/s)": 0.021173
+    },
+    {
+      "epoch": 1.484726079272911,
+      "grad_norm": 1.677512288093567,
+      "learning_rate": 3.391896745912094e-05,
+      "logits/chosen": -1.2383729219436646,
+      "logits/rejected": -1.486020803451538,
+      "logps/chosen": -1.5792930126190186,
+      "logps/rejected": -105.1240463256836,
+      "loss": 0.1709045171737671,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.13735303282737732,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.45899084210395813,
+      "rewards/margins": 9.465791702270508,
+      "rewards/rejected": -9.006800651550293,
+      "step": 736,
+      "train_speed(iter/s)": 0.021174
+    },
+    {
+      "epoch": 1.4867457712698813,
+      "grad_norm": 1.3144184350967407,
+      "learning_rate": 3.366849319349099e-05,
+      "logits/chosen": -1.360811471939087,
+      "logits/rejected": -1.4882185459136963,
+      "logps/chosen": -2.445948362350464,
+      "logps/rejected": -171.1454315185547,
+      "loss": 0.09988545626401901,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.06584716588258743,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4912615716457367,
+      "rewards/margins": 16.239545822143555,
+      "rewards/rejected": -15.748284339904785,
+      "step": 737,
+      "train_speed(iter/s)": 0.021174
+    },
+    {
+      "epoch": 1.4887654632668519,
+      "grad_norm": 8.858131408691406,
+      "learning_rate": 3.3418759834518056e-05,
+      "logits/chosen": -1.3772072792053223,
+      "logits/rejected": -1.4881007671356201,
+      "logps/chosen": -2.1875855922698975,
+      "logps/rejected": -127.1390151977539,
+      "loss": 0.465101420879364,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3819086253643036,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3905561864376068,
+      "rewards/margins": 12.209409713745117,
+      "rewards/rejected": -11.818853378295898,
+      "step": 738,
+      "train_speed(iter/s)": 0.021175
+    },
+    {
+      "epoch": 1.4907851552638223,
+      "grad_norm": 1.4290043115615845,
+      "learning_rate": 3.316977017166244e-05,
+      "logits/chosen": -1.3368918895721436,
+      "logits/rejected": -1.4668309688568115,
+      "logps/chosen": -3.5408458709716797,
+      "logps/rejected": -113.14617156982422,
+      "loss": 0.18968448042869568,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1604820191860199,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5777198672294617,
+      "rewards/margins": 10.534330368041992,
+      "rewards/rejected": -9.956611633300781,
+      "step": 739,
+      "train_speed(iter/s)": 0.021175
+    },
+    {
+      "epoch": 1.4928048472607927,
+      "grad_norm": 2.0293209552764893,
+      "learning_rate": 3.292152698607768e-05,
+      "logits/chosen": -1.2035475969314575,
+      "logits/rejected": -1.4047435522079468,
+      "logps/chosen": -3.9670791625976562,
+      "logps/rejected": -127.44470977783203,
+      "loss": 0.2798480987548828,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18252626061439514,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6036968231201172,
+      "rewards/margins": 11.327651023864746,
+      "rewards/rejected": -10.723954200744629,
+      "step": 740,
+      "train_speed(iter/s)": 0.021175
+    },
+    {
+      "epoch": 1.4948245392577633,
+      "grad_norm": 6.465835094451904,
+      "learning_rate": 3.267403305057916e-05,
+      "logits/chosen": -1.3113141059875488,
+      "logits/rejected": -1.4887977838516235,
+      "logps/chosen": -1.2720156908035278,
+      "logps/rejected": -143.11294555664062,
+      "loss": 0.15806855261325836,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1515921652317047,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.32133498787879944,
+      "rewards/margins": 13.046092987060547,
+      "rewards/rejected": -12.724757194519043,
+      "step": 741,
+      "train_speed(iter/s)": 0.021175
+    },
+    {
+      "epoch": 1.4968442312547336,
+      "grad_norm": 9.529487609863281,
+      "learning_rate": 3.24272911296135e-05,
+      "logits/chosen": -1.3097560405731201,
+      "logits/rejected": -1.5280407667160034,
+      "logps/chosen": -1.1988142728805542,
+      "logps/rejected": -129.46568298339844,
+      "loss": 0.200734943151474,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.18056869506835938,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.29430529475212097,
+      "rewards/margins": 12.073684692382812,
+      "rewards/rejected": -11.779377937316895,
+      "step": 742,
+      "train_speed(iter/s)": 0.021176
+    },
+    {
+      "epoch": 1.498863923251704,
+      "grad_norm": 2.6711034774780273,
+      "learning_rate": 3.218130397922735e-05,
+      "logits/chosen": -1.2700159549713135,
+      "logits/rejected": -1.4580274820327759,
+      "logps/chosen": -6.290490627288818,
+      "logps/rejected": -114.78617858886719,
+      "loss": 0.3417443335056305,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.3109719455242157,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5350229144096375,
+      "rewards/margins": 10.513776779174805,
+      "rewards/rejected": -9.978754997253418,
+      "step": 743,
+      "train_speed(iter/s)": 0.021176
+    },
+    {
+      "epoch": 1.5008836152486746,
+      "grad_norm": 1.3751882314682007,
+      "learning_rate": 3.19360743470369e-05,
+      "logits/chosen": -1.1223201751708984,
+      "logits/rejected": -1.4623578786849976,
+      "logps/chosen": -0.8023467063903809,
+      "logps/rejected": -232.16357421875,
+      "loss": 0.14550745487213135,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.11616021394729614,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.32895487546920776,
+      "rewards/margins": 20.819808959960938,
+      "rewards/rejected": -20.49085235595703,
+      "step": 744,
+      "train_speed(iter/s)": 0.021176
+    },
+    {
+      "epoch": 1.502903307245645,
+      "grad_norm": 6.538708209991455,
+      "learning_rate": 3.169160497219692e-05,
+      "logits/chosen": -1.1104850769042969,
+      "logits/rejected": -1.3502544164657593,
+      "logps/chosen": -5.819465160369873,
+      "logps/rejected": -154.62669372558594,
+      "loss": 0.22419646382331848,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17467880249023438,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4808841347694397,
+      "rewards/margins": 14.223871231079102,
+      "rewards/rejected": -13.742988586425781,
+      "step": 745,
+      "train_speed(iter/s)": 0.021176
+    },
+    {
+      "epoch": 1.5049229992426154,
+      "grad_norm": 3.28774356842041,
+      "learning_rate": 3.1447898585370384e-05,
+      "logits/chosen": -1.327174425125122,
+      "logits/rejected": -1.4644252061843872,
+      "logps/chosen": -2.708258867263794,
+      "logps/rejected": -107.84658813476562,
+      "loss": 0.317179799079895,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2389330416917801,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3801038861274719,
+      "rewards/margins": 10.307621955871582,
+      "rewards/rejected": -9.927518844604492,
+      "step": 746,
+      "train_speed(iter/s)": 0.021176
+    },
+    {
+      "epoch": 1.506942691239586,
+      "grad_norm": 1.626086950302124,
+      "learning_rate": 3.120495790869783e-05,
+      "logits/chosen": -1.0340219736099243,
+      "logits/rejected": -1.4414012432098389,
+      "logps/chosen": -1.7292864322662354,
+      "logps/rejected": -148.7239990234375,
+      "loss": 0.2304137498140335,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.17849569022655487,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34534165263175964,
+      "rewards/margins": 12.805582046508789,
+      "rewards/rejected": -12.460240364074707,
+      "step": 747,
+      "train_speed(iter/s)": 0.021176
+    },
+    {
+      "epoch": 1.5089623832365564,
+      "grad_norm": 5.187267780303955,
+      "learning_rate": 3.096278565576698e-05,
+      "logits/chosen": -1.3892465829849243,
+      "logits/rejected": -1.4590100049972534,
+      "logps/chosen": -3.5674116611480713,
+      "logps/rejected": -88.74075317382812,
+      "loss": 0.5602635145187378,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.49122655391693115,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1743486523628235,
+      "rewards/margins": 8.105524063110352,
+      "rewards/rejected": -7.931175708770752,
+      "step": 748,
+      "train_speed(iter/s)": 0.021177
+    },
+    {
+      "epoch": 1.5109820752335268,
+      "grad_norm": 3.1810035705566406,
+      "learning_rate": 3.07213845315825e-05,
+      "logits/chosen": -1.1909027099609375,
+      "logits/rejected": -1.331541895866394,
+      "logps/chosen": -4.226578235626221,
+      "logps/rejected": -87.8639144897461,
+      "loss": 0.3404882550239563,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2267073690891266,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.489352822303772,
+      "rewards/margins": 8.109345436096191,
+      "rewards/rejected": -7.619992256164551,
+      "step": 749,
+      "train_speed(iter/s)": 0.021177
+    },
+    {
+      "epoch": 1.5130017672304974,
+      "grad_norm": 4.147840976715088,
+      "learning_rate": 3.0480757232535772e-05,
+      "logits/chosen": -1.2313125133514404,
+      "logits/rejected": -1.4576904773712158,
+      "logps/chosen": -1.2580523490905762,
+      "logps/rejected": -148.97238159179688,
+      "loss": 0.2519887685775757,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.189640611410141,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.21735075116157532,
+      "rewards/margins": 13.471217155456543,
+      "rewards/rejected": -13.253866195678711,
+      "step": 750,
+      "train_speed(iter/s)": 0.021177
+    },
+    {
+      "epoch": 1.5150214592274678,
+      "grad_norm": 2.2532622814178467,
+      "learning_rate": 3.024090644637465e-05,
+      "logits/chosen": -1.2693455219268799,
+      "logits/rejected": -1.4010496139526367,
+      "logps/chosen": -4.09386682510376,
+      "logps/rejected": -120.09562683105469,
+      "loss": 0.23183885216712952,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1668306440114975,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3415793478488922,
+      "rewards/margins": 11.19216537475586,
+      "rewards/rejected": -10.8505859375,
+      "step": 751,
+      "train_speed(iter/s)": 0.021168
+    },
+    {
+      "epoch": 1.5170411512244382,
+      "grad_norm": 3.6028339862823486,
+      "learning_rate": 3.0001834852173537e-05,
+      "logits/chosen": -1.2393250465393066,
+      "logits/rejected": -1.4767005443572998,
+      "logps/chosen": -1.057918906211853,
+      "logps/rejected": -108.03105163574219,
+      "loss": 0.20541587471961975,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.12951552867889404,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.34247100353240967,
+      "rewards/margins": 9.704581260681152,
+      "rewards/rejected": -9.36211109161377,
+      "step": 752,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 1.5190608432214088,
+      "grad_norm": 2.4015023708343506,
+      "learning_rate": 2.9763545120303548e-05,
+      "logits/chosen": -1.122794270515442,
+      "logits/rejected": -1.3994086980819702,
+      "logps/chosen": -1.4307366609573364,
+      "logps/rejected": -148.82347106933594,
+      "loss": 0.2855421006679535,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.26716095209121704,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2603963613510132,
+      "rewards/margins": 12.873067855834961,
+      "rewards/rejected": -12.6126708984375,
+      "step": 753,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 1.5210805352183792,
+      "grad_norm": 2.1218819618225098,
+      "learning_rate": 2.9526039912402503e-05,
+      "logits/chosen": -1.1363654136657715,
+      "logits/rejected": -1.43178391456604,
+      "logps/chosen": -1.4727904796600342,
+      "logps/rejected": -135.5030517578125,
+      "loss": 0.13307449221611023,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10365013778209686,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.39402905106544495,
+      "rewards/margins": 11.632341384887695,
+      "rewards/rejected": -11.238310813903809,
+      "step": 754,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 1.5231002272153495,
+      "grad_norm": 1.7406599521636963,
+      "learning_rate": 2.9289321881345254e-05,
+      "logits/chosen": -1.2162468433380127,
+      "logits/rejected": -1.4458454847335815,
+      "logps/chosen": -5.626956939697266,
+      "logps/rejected": -102.26946258544922,
+      "loss": 0.261200487613678,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.21312278509140015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.12907829880714417,
+      "rewards/margins": 8.760376930236816,
+      "rewards/rejected": -8.63129997253418,
+      "step": 755,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 1.5251199192123202,
+      "grad_norm": 3.6318564414978027,
+      "learning_rate": 2.905339367121418e-05,
+      "logits/chosen": -1.112025499343872,
+      "logits/rejected": -1.412227749824524,
+      "logps/chosen": -0.9260848164558411,
+      "logps/rejected": -115.6212387084961,
+      "loss": 0.16580034792423248,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.12949852645397186,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2384950816631317,
+      "rewards/margins": 9.828255653381348,
+      "rewards/rejected": -9.589760780334473,
+      "step": 756,
+      "train_speed(iter/s)": 0.021169
+    },
+    {
+      "epoch": 1.5271396112092905,
+      "grad_norm": 1.2222281694412231,
+      "learning_rate": 2.881825791726952e-05,
+      "logits/chosen": -1.3425182104110718,
+      "logits/rejected": -1.4633957147598267,
+      "logps/chosen": -1.9715039730072021,
+      "logps/rejected": -113.19708251953125,
+      "loss": 0.219611257314682,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.19716471433639526,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2599337100982666,
+      "rewards/margins": 10.554387092590332,
+      "rewards/rejected": -10.294453620910645,
+      "step": 757,
+      "train_speed(iter/s)": 0.02117
+    },
+    {
+      "epoch": 1.529159303206261,
+      "grad_norm": 1.8520249128341675,
+      "learning_rate": 2.8583917245919945e-05,
+      "logits/chosen": -1.2967931032180786,
+      "logits/rejected": -1.466963768005371,
+      "logps/chosen": -1.880911111831665,
+      "logps/rejected": -135.49337768554688,
+      "loss": 0.2678569555282593,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2080049067735672,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3048132658004761,
+      "rewards/margins": 12.353228569030762,
+      "rewards/rejected": -12.048416137695312,
+      "step": 758,
+      "train_speed(iter/s)": 0.02117
+    },
+    {
+      "epoch": 1.5311789952032315,
+      "grad_norm": 2.2175347805023193,
+      "learning_rate": 2.8350374274693213e-05,
+      "logits/chosen": -1.2903900146484375,
+      "logits/rejected": -1.430172085762024,
+      "logps/chosen": -2.0055158138275146,
+      "logps/rejected": -92.69169616699219,
+      "loss": 0.28425323963165283,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.20914305746555328,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5304386615753174,
+      "rewards/margins": 8.536468505859375,
+      "rewards/rejected": -8.00602912902832,
+      "step": 759,
+      "train_speed(iter/s)": 0.021171
+    },
+    {
+      "epoch": 1.533198687200202,
+      "grad_norm": 1.383278727531433,
+      "learning_rate": 2.8117631612207084e-05,
+      "logits/chosen": -1.302045226097107,
+      "logits/rejected": -1.4697225093841553,
+      "logps/chosen": -1.8129011392593384,
+      "logps/rejected": -124.70085144042969,
+      "loss": 0.10829529911279678,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.08068909496068954,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2857769727706909,
+      "rewards/margins": 11.438787460327148,
+      "rewards/rejected": -11.153010368347168,
+      "step": 760,
+      "train_speed(iter/s)": 0.021171
+    },
+    {
+      "epoch": 1.5352183791971723,
+      "grad_norm": 1.0347193479537964,
+      "learning_rate": 2.7885691858139963e-05,
+      "logits/chosen": -1.2177413702011108,
+      "logits/rejected": -1.498382568359375,
+      "logps/chosen": -1.4776089191436768,
+      "logps/rejected": -136.19607543945312,
+      "loss": 0.1053784191608429,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09817221015691757,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4567747116088867,
+      "rewards/margins": 12.35906982421875,
+      "rewards/rejected": -11.902295112609863,
+      "step": 761,
+      "train_speed(iter/s)": 0.021171
+    },
+    {
+      "epoch": 1.537238071194143,
+      "grad_norm": 4.66424036026001,
+      "learning_rate": 2.7654557603201957e-05,
+      "logits/chosen": -1.1047477722167969,
+      "logits/rejected": -1.3985992670059204,
+      "logps/chosen": -3.208644151687622,
+      "logps/rejected": -153.35877990722656,
+      "loss": 0.8918479681015015,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.6579436659812927,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.041088301688432693,
+      "rewards/margins": 13.52843952178955,
+      "rewards/rejected": -13.487350463867188,
+      "step": 762,
+      "train_speed(iter/s)": 0.021171
+    },
+    {
+      "epoch": 1.5392577631911135,
+      "grad_norm": 1.2521512508392334,
+      "learning_rate": 2.742423142910603e-05,
+      "logits/chosen": -1.33512282371521,
+      "logits/rejected": -1.4375691413879395,
+      "logps/chosen": -1.5388550758361816,
+      "logps/rejected": -101.17770385742188,
+      "loss": 0.1395106464624405,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.120069220662117,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4132011830806732,
+      "rewards/margins": 9.373456001281738,
+      "rewards/rejected": -8.960253715515137,
+      "step": 763,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 1.5412774551880837,
+      "grad_norm": 1.202734112739563,
+      "learning_rate": 2.719471590853907e-05,
+      "logits/chosen": -1.3484524488449097,
+      "logits/rejected": -1.47200608253479,
+      "logps/chosen": -0.4913441240787506,
+      "logps/rejected": -87.54550170898438,
+      "loss": 0.07030540704727173,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.06171432510018349,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.354159951210022,
+      "rewards/margins": 8.304422378540039,
+      "rewards/rejected": -7.950262069702148,
+      "step": 764,
+      "train_speed(iter/s)": 0.021172
+    },
+    {
+      "epoch": 1.5432971471850543,
+      "grad_norm": 0.9708402156829834,
+      "learning_rate": 2.6966013605133088e-05,
+      "logits/chosen": -1.2900387048721313,
+      "logits/rejected": -1.4735229015350342,
+      "logps/chosen": -2.4513182640075684,
+      "logps/rejected": -145.91192626953125,
+      "loss": 0.10491526871919632,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09214503318071365,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6740211248397827,
+      "rewards/margins": 14.10338306427002,
+      "rewards/rejected": -13.429362297058105,
+      "step": 765,
+      "train_speed(iter/s)": 0.021173
+    },
+    {
+      "epoch": 1.545316839182025,
+      "grad_norm": 3.5615055561065674,
+      "learning_rate": 2.673812707343669e-05,
+      "logits/chosen": -1.337423324584961,
+      "logits/rejected": -1.411500334739685,
+      "logps/chosen": -1.9122189283370972,
+      "logps/rejected": -80.02887725830078,
+      "loss": 0.26139193773269653,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.21918295323848724,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.22392795979976654,
+      "rewards/margins": 7.230677604675293,
+      "rewards/rejected": -7.006749629974365,
+      "step": 766,
+      "train_speed(iter/s)": 0.021173
+    },
+    {
+      "epoch": 1.547336531178995,
+      "grad_norm": 1.3090108633041382,
+      "learning_rate": 2.6511058858886573e-05,
+      "logits/chosen": -1.249571442604065,
+      "logits/rejected": -1.4248452186584473,
+      "logps/chosen": -1.653437852859497,
+      "logps/rejected": -128.087646484375,
+      "loss": 0.11293576657772064,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10526332259178162,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5344476699829102,
+      "rewards/margins": 11.581936836242676,
+      "rewards/rejected": -11.04748821258545,
+      "step": 767,
+      "train_speed(iter/s)": 0.021174
+    },
+    {
+      "epoch": 1.5493562231759657,
+      "grad_norm": 1.412434697151184,
+      "learning_rate": 2.6284811497778915e-05,
+      "logits/chosen": -1.2566429376602173,
+      "logits/rejected": -1.516014814376831,
+      "logps/chosen": -0.6881668567657471,
+      "logps/rejected": -159.653564453125,
+      "loss": 0.14191856980323792,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.070882149040699,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2940995395183563,
+      "rewards/margins": 14.506892204284668,
+      "rewards/rejected": -14.212791442871094,
+      "step": 768,
+      "train_speed(iter/s)": 0.021174
+    },
+    {
+      "epoch": 1.5513759151729363,
+      "grad_norm": 1.4652314186096191,
+      "learning_rate": 2.6059387517241307e-05,
+      "logits/chosen": -1.3113449811935425,
+      "logits/rejected": -1.55575430393219,
+      "logps/chosen": -1.2395075559616089,
+      "logps/rejected": -196.9792022705078,
+      "loss": 0.06537880003452301,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.06293562799692154,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6358461380004883,
+      "rewards/margins": 18.485309600830078,
+      "rewards/rejected": -17.849464416503906,
+      "step": 769,
+      "train_speed(iter/s)": 0.021174
+    },
+    {
+      "epoch": 1.5533956071699064,
+      "grad_norm": 1.7013682126998901,
+      "learning_rate": 2.5834789435204243e-05,
+      "logits/chosen": -1.2813678979873657,
+      "logits/rejected": -1.4381935596466064,
+      "logps/chosen": -1.3362274169921875,
+      "logps/rejected": -96.55331420898438,
+      "loss": 0.24336016178131104,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.15523691475391388,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2544943690299988,
+      "rewards/margins": 8.931314468383789,
+      "rewards/rejected": -8.676820755004883,
+      "step": 770,
+      "train_speed(iter/s)": 0.021175
+    },
+    {
+      "epoch": 1.555415299166877,
+      "grad_norm": 1.327748417854309,
+      "learning_rate": 2.5611019760373255e-05,
+      "logits/chosen": -1.2529298067092896,
+      "logits/rejected": -1.4934749603271484,
+      "logps/chosen": -0.6677508354187012,
+      "logps/rejected": -145.9220428466797,
+      "loss": 0.2080720067024231,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10923931002616882,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3528885841369629,
+      "rewards/margins": 13.13940143585205,
+      "rewards/rejected": -12.78651237487793,
+      "step": 771,
+      "train_speed(iter/s)": 0.021175
+    },
+    {
+      "epoch": 1.5574349911638476,
+      "grad_norm": 1.456613302230835,
+      "learning_rate": 2.5388080992200636e-05,
+      "logits/chosen": -1.3448463678359985,
+      "logits/rejected": -1.451443076133728,
+      "logps/chosen": -0.420827180147171,
+      "logps/rejected": -105.18292236328125,
+      "loss": 0.07830143719911575,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.06242983788251877,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5701310038566589,
+      "rewards/margins": 10.055427551269531,
+      "rewards/rejected": -9.485297203063965,
+      "step": 772,
+      "train_speed(iter/s)": 0.021175
+    },
+    {
+      "epoch": 1.5594546831608178,
+      "grad_norm": 1.3975962400436401,
+      "learning_rate": 2.5165975620857818e-05,
+      "logits/chosen": -1.301852822303772,
+      "logits/rejected": -1.4368289709091187,
+      "logps/chosen": -1.5585435628890991,
+      "logps/rejected": -100.274658203125,
+      "loss": 0.19546322524547577,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.14822641015052795,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4045298993587494,
+      "rewards/margins": 9.32353401184082,
+      "rewards/rejected": -8.919004440307617,
+      "step": 773,
+      "train_speed(iter/s)": 0.021176
+    },
+    {
+      "epoch": 1.5614743751577884,
+      "grad_norm": 1.8541526794433594,
+      "learning_rate": 2.494470612720725e-05,
+      "logits/chosen": -1.3901867866516113,
+      "logits/rejected": -1.5561095476150513,
+      "logps/chosen": -0.15552577376365662,
+      "logps/rejected": -116.71949768066406,
+      "loss": 0.06297751516103745,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.03428273648023605,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.42397540807724,
+      "rewards/margins": 11.099949836730957,
+      "rewards/rejected": -10.675973892211914,
+      "step": 774,
+      "train_speed(iter/s)": 0.021176
+    },
+    {
+      "epoch": 1.563494067154759,
+      "grad_norm": 3.5473546981811523,
+      "learning_rate": 2.4724274982774865e-05,
+      "logits/chosen": -1.316893219947815,
+      "logits/rejected": -1.471555471420288,
+      "logps/chosen": -2.295685052871704,
+      "logps/rejected": -118.34683227539062,
+      "loss": 0.27752485871315,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2106786072254181,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.42384564876556396,
+      "rewards/margins": 11.249652862548828,
+      "rewards/rejected": -10.825806617736816,
+      "step": 775,
+      "train_speed(iter/s)": 0.021177
+    },
+    {
+      "epoch": 1.5655137591517294,
+      "grad_norm": 2.9827053546905518,
+      "learning_rate": 2.4504684649722544e-05,
+      "logits/chosen": -1.3508703708648682,
+      "logits/rejected": -1.454907774925232,
+      "logps/chosen": -2.8311688899993896,
+      "logps/rejected": -92.16709899902344,
+      "loss": 0.3435622751712799,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.2923312187194824,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.25256916880607605,
+      "rewards/margins": 8.624588966369629,
+      "rewards/rejected": -8.372020721435547,
+      "step": 776,
+      "train_speed(iter/s)": 0.021177
+    },
+    {
+      "epoch": 1.5675334511486998,
+      "grad_norm": 0.21882030367851257,
+      "learning_rate": 2.428593758082035e-05,
+      "logits/chosen": -1.1836020946502686,
+      "logits/rejected": -1.5535478591918945,
+      "logps/chosen": -0.60710608959198,
+      "logps/rejected": -203.61517333984375,
+      "loss": 0.054839178919792175,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.047202836722135544,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2823777496814728,
+      "rewards/margins": 18.952245712280273,
+      "rewards/rejected": -18.66986846923828,
+      "step": 777,
+      "train_speed(iter/s)": 0.021177
+    },
+    {
+      "epoch": 1.5695531431456704,
+      "grad_norm": 2.3607430458068848,
+      "learning_rate": 2.4068036219419432e-05,
+      "logits/chosen": -1.1538668870925903,
+      "logits/rejected": -1.5722734928131104,
+      "logps/chosen": -2.0943477153778076,
+      "logps/rejected": -208.68576049804688,
+      "loss": 0.10892216861248016,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10658928006887436,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3742532730102539,
+      "rewards/margins": 18.359811782836914,
+      "rewards/rejected": -17.985557556152344,
+      "step": 778,
+      "train_speed(iter/s)": 0.021177
+    },
+    {
+      "epoch": 1.5715728351426408,
+      "grad_norm": 1.0467064380645752,
+      "learning_rate": 2.3850982999424498e-05,
+      "logits/chosen": -1.3144843578338623,
+      "logits/rejected": -1.5946495532989502,
+      "logps/chosen": -0.9175739884376526,
+      "logps/rejected": -183.0205078125,
+      "loss": 0.06769800931215286,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.06343834847211838,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.40800103545188904,
+      "rewards/margins": 17.426477432250977,
+      "rewards/rejected": -17.018476486206055,
+      "step": 779,
+      "train_speed(iter/s)": 0.021177
+    },
+    {
+      "epoch": 1.5735925271396112,
+      "grad_norm": 7.271415710449219,
+      "learning_rate": 2.3634780345266806e-05,
+      "logits/chosen": -1.354949712753296,
+      "logits/rejected": -1.4896622896194458,
+      "logps/chosen": -1.5982024669647217,
+      "logps/rejected": -118.48434448242188,
+      "loss": 0.32608869671821594,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.24822936952114105,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.25157132744789124,
+      "rewards/margins": 10.491283416748047,
+      "rewards/rejected": -10.239712715148926,
+      "step": 780,
+      "train_speed(iter/s)": 0.021178
+    },
+    {
+      "epoch": 1.5756122191365818,
+      "grad_norm": 1.095346212387085,
+      "learning_rate": 2.341943067187691e-05,
+      "logits/chosen": -1.0725579261779785,
+      "logits/rejected": -1.4255238771438599,
+      "logps/chosen": -1.4045692682266235,
+      "logps/rejected": -196.7482147216797,
+      "loss": 0.135026752948761,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.12142231315374374,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.44816428422927856,
+      "rewards/margins": 17.575708389282227,
+      "rewards/rejected": -17.127544403076172,
+      "step": 781,
+      "train_speed(iter/s)": 0.021177
+    },
+    {
+      "epoch": 1.5776319111335522,
+      "grad_norm": 1.5681113004684448,
+      "learning_rate": 2.3204936384657872e-05,
+      "logits/chosen": -1.1062440872192383,
+      "logits/rejected": -1.4810614585876465,
+      "logps/chosen": -0.8656953573226929,
+      "logps/rejected": -185.3040313720703,
+      "loss": 0.10629081726074219,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09828759729862213,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2712135910987854,
+      "rewards/margins": 16.67391586303711,
+      "rewards/rejected": -16.40270233154297,
+      "step": 782,
+      "train_speed(iter/s)": 0.021177
+    },
+    {
+      "epoch": 1.5796516031305226,
+      "grad_norm": 1.000409722328186,
+      "learning_rate": 2.2991299879458217e-05,
+      "logits/chosen": -1.3829787969589233,
+      "logits/rejected": -1.528799057006836,
+      "logps/chosen": -1.4075665473937988,
+      "logps/rejected": -108.40474700927734,
+      "loss": 0.11096090823411942,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09907718747854233,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5557278990745544,
+      "rewards/margins": 10.103904724121094,
+      "rewards/rejected": -9.548175811767578,
+      "step": 783,
+      "train_speed(iter/s)": 0.021178
+    },
+    {
+      "epoch": 1.5816712951274932,
+      "grad_norm": 2.282076120376587,
+      "learning_rate": 2.2778523542545326e-05,
+      "logits/chosen": -1.3425467014312744,
+      "logits/rejected": -1.5713337659835815,
+      "logps/chosen": -1.9913424253463745,
+      "logps/rejected": -154.333740234375,
+      "loss": 0.24184367060661316,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22620907425880432,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6124590635299683,
+      "rewards/margins": 14.911059379577637,
+      "rewards/rejected": -14.298600196838379,
+      "step": 784,
+      "train_speed(iter/s)": 0.021178
+    },
+    {
+      "epoch": 1.5836909871244635,
+      "grad_norm": 5.127676963806152,
+      "learning_rate": 2.2566609750578673e-05,
+      "logits/chosen": -1.3819358348846436,
+      "logits/rejected": -1.5853811502456665,
+      "logps/chosen": -0.5152206420898438,
+      "logps/rejected": -172.47085571289062,
+      "loss": 0.0796223133802414,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.04822954535484314,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.46276915073394775,
+      "rewards/margins": 16.34103775024414,
+      "rewards/rejected": -15.878271102905273,
+      "step": 785,
+      "train_speed(iter/s)": 0.021179
+    },
+    {
+      "epoch": 1.585710679121434,
+      "grad_norm": 1.2354482412338257,
+      "learning_rate": 2.235556087058328e-05,
+      "logits/chosen": -1.3431332111358643,
+      "logits/rejected": -1.4766733646392822,
+      "logps/chosen": -2.4101006984710693,
+      "logps/rejected": -125.0008773803711,
+      "loss": 0.15823666751384735,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.1488935798406601,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5909098386764526,
+      "rewards/margins": 11.000923156738281,
+      "rewards/rejected": -10.410013198852539,
+      "step": 786,
+      "train_speed(iter/s)": 0.021179
+    },
+    {
+      "epoch": 1.5877303711184045,
+      "grad_norm": 1.0751861333847046,
+      "learning_rate": 2.2145379259923392e-05,
+      "logits/chosen": -1.3757960796356201,
+      "logits/rejected": -1.4742145538330078,
+      "logps/chosen": -2.099942445755005,
+      "logps/rejected": -110.02182006835938,
+      "loss": 0.16085366904735565,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09588892012834549,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.731009304523468,
+      "rewards/margins": 10.84728717803955,
+      "rewards/rejected": -10.116278648376465,
+      "step": 787,
+      "train_speed(iter/s)": 0.021179
+    },
+    {
+      "epoch": 1.589750063115375,
+      "grad_norm": 0.45493581891059875,
+      "learning_rate": 2.193606726627606e-05,
+      "logits/chosen": -1.1831483840942383,
+      "logits/rejected": -1.475436806678772,
+      "logps/chosen": -0.21247398853302002,
+      "logps/rejected": -148.15493774414062,
+      "loss": 0.06328378617763519,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.03598284348845482,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.43760454654693604,
+      "rewards/margins": 12.907350540161133,
+      "rewards/rejected": -12.469746589660645,
+      "step": 788,
+      "train_speed(iter/s)": 0.021179
+    },
+    {
+      "epoch": 1.5917697551123453,
+      "grad_norm": 2.2279629707336426,
+      "learning_rate": 2.1727627227604874e-05,
+      "logits/chosen": -1.2100962400436401,
+      "logits/rejected": -1.4708439111709595,
+      "logps/chosen": -1.3257055282592773,
+      "logps/rejected": -111.88347625732422,
+      "loss": 0.10991434752941132,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.09015247970819473,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4133949279785156,
+      "rewards/margins": 9.966840744018555,
+      "rewards/rejected": -9.553445816040039,
+      "step": 789,
+      "train_speed(iter/s)": 0.021179
+    },
+    {
+      "epoch": 1.593789447109316,
+      "grad_norm": 1.2698335647583008,
+      "learning_rate": 2.1520061472133902e-05,
+      "logits/chosen": -1.228274941444397,
+      "logits/rejected": -1.5459834337234497,
+      "logps/chosen": -1.5582325458526611,
+      "logps/rejected": -205.62095642089844,
+      "loss": 0.16958492994308472,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.16615107655525208,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3384014368057251,
+      "rewards/margins": 19.099943161010742,
+      "rewards/rejected": -18.76154136657715,
+      "step": 790,
+      "train_speed(iter/s)": 0.02118
+    },
+    {
+      "epoch": 1.5958091391062863,
+      "grad_norm": 1.6259820461273193,
+      "learning_rate": 2.1313372318321746e-05,
+      "logits/chosen": -1.3327020406723022,
+      "logits/rejected": -1.5858879089355469,
+      "logps/chosen": -0.6086030602455139,
+      "logps/rejected": -229.32562255859375,
+      "loss": 0.09911161661148071,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.08497641980648041,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.35119205713272095,
+      "rewards/margins": 21.821672439575195,
+      "rewards/rejected": -21.470483779907227,
+      "step": 791,
+      "train_speed(iter/s)": 0.02118
+    },
+    {
+      "epoch": 1.5978288311032567,
+      "grad_norm": 11.140735626220703,
+      "learning_rate": 2.110756207483554e-05,
+      "logits/chosen": -1.279889464378357,
+      "logits/rejected": -1.467369794845581,
+      "logps/chosen": -5.798372268676758,
+      "logps/rejected": -144.6669158935547,
+      "loss": 0.31990110874176025,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.26768559217453003,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6251093745231628,
+      "rewards/margins": 13.777156829833984,
+      "rewards/rejected": -13.152047157287598,
+      "step": 792,
+      "train_speed(iter/s)": 0.02118
+    },
+    {
+      "epoch": 1.5998485231002273,
+      "grad_norm": 2.8799500465393066,
+      "learning_rate": 2.0902633040525155e-05,
+      "logits/chosen": -1.2789034843444824,
+      "logits/rejected": -1.4255249500274658,
+      "logps/chosen": -1.9766592979431152,
+      "logps/rejected": -114.88394165039062,
+      "loss": 0.26157423853874207,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.23831398785114288,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.143344447016716,
+      "rewards/margins": 9.978683471679688,
+      "rewards/rejected": -9.835338592529297,
+      "step": 793,
+      "train_speed(iter/s)": 0.021181
+    },
+    {
+      "epoch": 1.6018682150971977,
+      "grad_norm": 1.2032852172851562,
+      "learning_rate": 2.069858750439768e-05,
+      "logits/chosen": -1.330883502960205,
+      "logits/rejected": -1.5647773742675781,
+      "logps/chosen": -0.7675846815109253,
+      "logps/rejected": -169.77903747558594,
+      "loss": 0.18148253858089447,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10654448717832565,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.29082542657852173,
+      "rewards/margins": 15.883310317993164,
+      "rewards/rejected": -15.592485427856445,
+      "step": 794,
+      "train_speed(iter/s)": 0.021181
+    },
+    {
+      "epoch": 1.603887907094168,
+      "grad_norm": 19.920034408569336,
+      "learning_rate": 2.04954277455917e-05,
+      "logits/chosen": -1.385201334953308,
+      "logits/rejected": -1.5076872110366821,
+      "logps/chosen": -1.5061265230178833,
+      "logps/rejected": -112.52576446533203,
+      "loss": 0.3391573131084442,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.22222107648849487,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.26670733094215393,
+      "rewards/margins": 10.322530746459961,
+      "rewards/rejected": -10.055824279785156,
+      "step": 795,
+      "train_speed(iter/s)": 0.021182
+    },
+    {
+      "epoch": 1.6059075990911387,
+      "grad_norm": 1.2854934930801392,
+      "learning_rate": 2.029315603335187e-05,
+      "logits/chosen": -1.291205644607544,
+      "logits/rejected": -1.5413569211959839,
+      "logps/chosen": -0.7821458578109741,
+      "logps/rejected": -151.5994415283203,
+      "loss": 0.11608272045850754,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.10654986649751663,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.28300514817237854,
+      "rewards/margins": 13.741439819335938,
+      "rewards/rejected": -13.458434104919434,
+      "step": 796,
+      "train_speed(iter/s)": 0.021182
+    },
+    {
+      "epoch": 1.607927291088109,
+      "grad_norm": 2.0723013877868652,
+      "learning_rate": 2.0091774627003534e-05,
+      "logits/chosen": -1.2769379615783691,
+      "logits/rejected": -1.5271036624908447,
+      "logps/chosen": -1.09879469871521,
+      "logps/rejected": -147.4831085205078,
+      "loss": 0.14599668979644775,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.07294835895299911,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.32040083408355713,
+      "rewards/margins": 12.92518424987793,
+      "rewards/rejected": -12.604782104492188,
+      "step": 797,
+      "train_speed(iter/s)": 0.021182
+    },
+    {
+      "epoch": 1.6099469830850794,
+      "grad_norm": 6.424983024597168,
+      "learning_rate": 1.9891285775927682e-05,
+      "logits/chosen": -1.226479411125183,
+      "logits/rejected": -1.4615333080291748,
+      "logps/chosen": -7.138606071472168,
+      "logps/rejected": -123.69717407226562,
+      "loss": 0.5906609296798706,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.46497058868408203,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.19697651267051697,
+      "rewards/margins": 10.924787521362305,
+      "rewards/rejected": -10.727810859680176,
+      "step": 798,
+      "train_speed(iter/s)": 0.021182
+    },
+    {
+      "epoch": 1.61196667508205,
+      "grad_norm": 1.8099446296691895,
+      "learning_rate": 1.969169171953553e-05,
+      "logits/chosen": -1.2584270238876343,
+      "logits/rejected": -1.537156105041504,
+      "logps/chosen": -1.6354960203170776,
+      "logps/rejected": -146.7123565673828,
+      "loss": 0.17544996738433838,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.12089676409959793,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3379935026168823,
+      "rewards/margins": 13.37423324584961,
+      "rewards/rejected": -13.036240577697754,
+      "step": 799,
+      "train_speed(iter/s)": 0.021182
+    },
+    {
+      "epoch": 1.6139863670790204,
+      "grad_norm": 1.2541955709457397,
+      "learning_rate": 1.9492994687243714e-05,
+      "logits/chosen": -1.4045718908309937,
+      "logits/rejected": -1.5673680305480957,
+      "logps/chosen": -1.2418478727340698,
+      "logps/rejected": -128.81858825683594,
+      "loss": 0.17549119889736176,
+      "memory(GiB)": 45.64,
+      "nll_loss": 0.16011641919612885,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3217834532260895,
+      "rewards/margins": 12.120386123657227,
+      "rewards/rejected": -11.798602104187012,
+      "step": 800,
+      "train_speed(iter/s)": 0.021183
     }
   ],
   "logging_steps": 1,
-  "max_steps": 596,
+  "max_steps": 990,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 4,
-  "save_steps": 5,
+  "num_train_epochs": 2,
+  "save_steps": 50,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
@@ -5355,8 +14461,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 0.0,
-  "train_batch_size": 8,
+  "total_flos": 2.316997131763712e+17,
+  "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null
 }