diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,6351 @@
+{
+  "best_global_step": 300,
+  "best_metric": 0.79432702,
+  "best_model_checkpoint": "/data2/kdd/crag/cjz/output/task1+3_dpo_absgt10_noidk__pari0.8_0608/v0-20250611-010914/checkpoint-300",
+  "epoch": 1.563758389261745,
+  "eval_steps": 300,
+  "global_step": 350,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0044742729306487695,
+      "grad_norm": 4.521431922912598,
+      "learning_rate": 8.695652173913044e-06,
+      "logits/chosen": -0.37893733382225037,
+      "logits/rejected": -0.4597730040550232,
+      "logps/chosen": -11.001541137695312,
+      "logps/rejected": -17.05784034729004,
+      "loss": 1.3586950302124023,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6655479669570923,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1,
+      "train_speed(iter/s)": 0.005374
+    },
+    {
+      "epoch": 0.008948545861297539,
+      "grad_norm": 3.470855951309204,
+      "learning_rate": 1.739130434782609e-05,
+      "logits/chosen": -0.3633067309856415,
+      "logits/rejected": -0.4392029941082001,
+      "logps/chosen": -7.973292350769043,
+      "logps/rejected": -13.90955924987793,
+      "loss": 1.2437297105789185,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5505824685096741,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 2,
+      "train_speed(iter/s)": 0.005501
+    },
+    {
+      "epoch": 0.013422818791946308,
+      "grad_norm": 5.51271390914917,
+      "learning_rate": 2.608695652173913e-05,
+      "logits/chosen": -0.26578277349472046,
+      "logits/rejected": -0.45589232444763184,
+      "logps/chosen": -9.600592613220215,
+      "logps/rejected": -22.355058670043945,
+      "loss": 1.392338514328003,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6997992396354675,
+      "rewards/accuracies": 0.453125,
+      "rewards/chosen": 4.354381235316396e-05,
+      "rewards/margins": 0.0013819998130202293,
+      "rewards/rejected": -0.001338455593213439,
+      "step": 3,
+      "train_speed(iter/s)": 0.005521
+    },
+    {
+      "epoch": 0.017897091722595078,
+      "grad_norm": 6.5741682052612305,
+      "learning_rate": 3.478260869565218e-05,
+      "logits/chosen": -0.40403854846954346,
+      "logits/rejected": -0.5402908325195312,
+      "logps/chosen": -6.373080253601074,
+      "logps/rejected": -16.791393280029297,
+      "loss": 1.222740888595581,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5450983047485352,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.010452846065163612,
+      "rewards/margins": 0.031955912709236145,
+      "rewards/rejected": -0.021503068506717682,
+      "step": 4,
+      "train_speed(iter/s)": 0.005551
+    },
+    {
+      "epoch": 0.02237136465324385,
+      "grad_norm": 3.046576738357544,
+      "learning_rate": 4.347826086956522e-05,
+      "logits/chosen": -0.3187038004398346,
+      "logits/rejected": -0.4441537857055664,
+      "logps/chosen": -9.070188522338867,
+      "logps/rejected": -17.255901336669922,
+      "loss": 1.2276911735534668,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5655408501625061,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.007649715524166822,
+      "rewards/margins": 0.07260076701641083,
+      "rewards/rejected": -0.08025049418210983,
+      "step": 5,
+      "train_speed(iter/s)": 0.005557
+    },
+    {
+      "epoch": 0.026845637583892617,
+      "grad_norm": 4.846445560455322,
+      "learning_rate": 5.217391304347826e-05,
+      "logits/chosen": -0.30176016688346863,
+      "logits/rejected": -0.45638489723205566,
+      "logps/chosen": -8.63841438293457,
+      "logps/rejected": -18.97266960144043,
+      "loss": 1.1484646797180176,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5371410250663757,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.024880727753043175,
+      "rewards/margins": 0.23170879483222961,
+      "rewards/rejected": -0.25658950209617615,
+      "step": 6,
+      "train_speed(iter/s)": 0.005568
+    },
+    {
+      "epoch": 0.03131991051454139,
+      "grad_norm": 7.562114715576172,
+      "learning_rate": 6.086956521739131e-05,
+      "logits/chosen": -0.3937205672264099,
+      "logits/rejected": -0.48791196942329407,
+      "logps/chosen": -10.315498352050781,
+      "logps/rejected": -20.046728134155273,
+      "loss": 1.3093780279159546,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6873191595077515,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.14371810853481293,
+      "rewards/margins": 0.2797008454799652,
+      "rewards/rejected": -0.42341893911361694,
+      "step": 7,
+      "train_speed(iter/s)": 0.005577
+    },
+    {
+      "epoch": 0.035794183445190156,
+      "grad_norm": 7.459152698516846,
+      "learning_rate": 6.956521739130436e-05,
+      "logits/chosen": -0.3540106415748596,
+      "logits/rejected": -0.43836480379104614,
+      "logps/chosen": -8.758136749267578,
+      "logps/rejected": -18.294340133666992,
+      "loss": 1.3215807676315308,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6817622184753418,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.19053705036640167,
+      "rewards/margins": 0.2214164435863495,
+      "rewards/rejected": -0.41195347905158997,
+      "step": 8,
+      "train_speed(iter/s)": 0.005585
+    },
+    {
+      "epoch": 0.040268456375838924,
+      "grad_norm": 3.0581202507019043,
+      "learning_rate": 7.82608695652174e-05,
+      "logits/chosen": -0.3466319143772125,
+      "logits/rejected": -0.4212740361690521,
+      "logps/chosen": -10.387115478515625,
+      "logps/rejected": -20.363929748535156,
+      "loss": 1.1448516845703125,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5626436471939087,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.06380077451467514,
+      "rewards/margins": 0.3404861092567444,
+      "rewards/rejected": -0.40428686141967773,
+      "step": 9,
+      "train_speed(iter/s)": 0.005583
+    },
+    {
+      "epoch": 0.0447427293064877,
+      "grad_norm": 3.9353108406066895,
+      "learning_rate": 8.695652173913044e-05,
+      "logits/chosen": -0.4188399314880371,
+      "logits/rejected": -0.5007613301277161,
+      "logps/chosen": -9.096799850463867,
+      "logps/rejected": -18.05609130859375,
+      "loss": 1.2927519083023071,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6729815006256104,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.060015156865119934,
+      "rewards/margins": 0.20498159527778625,
+      "rewards/rejected": -0.264996737241745,
+      "step": 10,
+      "train_speed(iter/s)": 0.00559
+    },
+    {
+      "epoch": 0.049217002237136466,
+      "grad_norm": 2.2350685596466064,
+      "learning_rate": 9.565217391304348e-05,
+      "logits/chosen": -0.41371461749076843,
+      "logits/rejected": -0.4829203188419342,
+      "logps/chosen": -7.282920837402344,
+      "logps/rejected": -15.015157699584961,
+      "loss": 1.167080044746399,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5672391653060913,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.07586651295423508,
+      "rewards/margins": 0.2256288230419159,
+      "rewards/rejected": -0.14976228773593903,
+      "step": 11,
+      "train_speed(iter/s)": 0.005596
+    },
+    {
+      "epoch": 0.053691275167785234,
+      "grad_norm": 2.6733052730560303,
+      "learning_rate": 0.00010434782608695653,
+      "logits/chosen": -0.38637417554855347,
+      "logits/rejected": -0.45029905438423157,
+      "logps/chosen": -11.042367935180664,
+      "logps/rejected": -16.67047882080078,
+      "loss": 1.2763197422027588,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6531897783279419,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": 0.06360332667827606,
+      "rewards/margins": 0.17586028575897217,
+      "rewards/rejected": -0.1122569665312767,
+      "step": 12,
+      "train_speed(iter/s)": 0.005598
+    },
+    {
+      "epoch": 0.058165548098434,
+      "grad_norm": 1.9472479820251465,
+      "learning_rate": 0.00011304347826086956,
+      "logits/chosen": -0.4195230007171631,
+      "logits/rejected": -0.4734322428703308,
+      "logps/chosen": -7.205465316772461,
+      "logps/rejected": -13.222125053405762,
+      "loss": 1.1376935243606567,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4900382459163666,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.018632210791110992,
+      "rewards/margins": 0.11014734208583832,
+      "rewards/rejected": -0.09151513874530792,
+      "step": 13,
+      "train_speed(iter/s)": 0.005601
+    },
+    {
+      "epoch": 0.06263982102908278,
+      "grad_norm": 1.828997015953064,
+      "learning_rate": 0.00012173913043478263,
+      "logits/chosen": -0.399485319852829,
+      "logits/rejected": -0.4682197570800781,
+      "logps/chosen": -8.005407333374023,
+      "logps/rejected": -16.06272315979004,
+      "loss": 1.1697134971618652,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5445835590362549,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.0030609495006501675,
+      "rewards/margins": 0.16438230872154236,
+      "rewards/rejected": -0.16132137179374695,
+      "step": 14,
+      "train_speed(iter/s)": 0.005604
+    },
+    {
+      "epoch": 0.06711409395973154,
+      "grad_norm": 2.1229512691497803,
+      "learning_rate": 0.00013043478260869567,
+      "logits/chosen": -0.3835911750793457,
+      "logits/rejected": -0.4592927396297455,
+      "logps/chosen": -6.499156475067139,
+      "logps/rejected": -15.707530975341797,
+      "loss": 1.0722893476486206,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.503006637096405,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.06762155145406723,
+      "rewards/margins": 0.31041574478149414,
+      "rewards/rejected": -0.24279415607452393,
+      "step": 15,
+      "train_speed(iter/s)": 0.005608
+    },
+    {
+      "epoch": 0.07158836689038031,
+      "grad_norm": 4.802647113800049,
+      "learning_rate": 0.0001391304347826087,
+      "logits/chosen": -0.3306896388530731,
+      "logits/rejected": -0.41153809428215027,
+      "logps/chosen": -10.432010650634766,
+      "logps/rejected": -21.58324432373047,
+      "loss": 1.3572790622711182,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.7762681245803833,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.06817664206027985,
+      "rewards/margins": 0.363118052482605,
+      "rewards/rejected": -0.4312947690486908,
+      "step": 16,
+      "train_speed(iter/s)": 0.005608
+    },
+    {
+      "epoch": 0.07606263982102908,
+      "grad_norm": 4.122814178466797,
+      "learning_rate": 0.00014782608695652173,
+      "logits/chosen": -0.32593876123428345,
+      "logits/rejected": -0.4176383316516876,
+      "logps/chosen": -9.068134307861328,
+      "logps/rejected": -21.460546493530273,
+      "loss": 1.1545155048370361,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5955549478530884,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.0028565023094415665,
+      "rewards/margins": 0.3992985486984253,
+      "rewards/rejected": -0.3964419960975647,
+      "step": 17,
+      "train_speed(iter/s)": 0.00561
+    },
+    {
+      "epoch": 0.08053691275167785,
+      "grad_norm": 1.9508388042449951,
+      "learning_rate": 0.0001565217391304348,
+      "logits/chosen": -0.3363662660121918,
+      "logits/rejected": -0.40299132466316223,
+      "logps/chosen": -7.306541919708252,
+      "logps/rejected": -15.145895004272461,
+      "loss": 1.1673789024353027,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5968848466873169,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.13542409241199493,
+      "rewards/margins": 0.3165813684463501,
+      "rewards/rejected": -0.18115724623203278,
+      "step": 18,
+      "train_speed(iter/s)": 0.005611
+    },
+    {
+      "epoch": 0.08501118568232663,
+      "grad_norm": 4.076289176940918,
+      "learning_rate": 0.00016521739130434784,
+      "logits/chosen": -0.33677053451538086,
+      "logits/rejected": -0.42119455337524414,
+      "logps/chosen": -6.263703346252441,
+      "logps/rejected": -17.470884323120117,
+      "loss": 0.9588873982429504,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.42532750964164734,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.17247594892978668,
+      "rewards/margins": 0.41344738006591797,
+      "rewards/rejected": -0.24097144603729248,
+      "step": 19,
+      "train_speed(iter/s)": 0.005613
+    },
+    {
+      "epoch": 0.0894854586129754,
+      "grad_norm": 2.8480069637298584,
+      "learning_rate": 0.00017391304347826088,
+      "logits/chosen": -0.27281832695007324,
+      "logits/rejected": -0.3382987976074219,
+      "logps/chosen": -8.123124122619629,
+      "logps/rejected": -17.66653060913086,
+      "loss": 1.1668881177902222,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5924651622772217,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.00576989259570837,
+      "rewards/margins": 0.35684970021247864,
+      "rewards/rejected": -0.36261963844299316,
+      "step": 20,
+      "train_speed(iter/s)": 0.005615
+    },
+    {
+      "epoch": 0.09395973154362416,
+      "grad_norm": 2.664872646331787,
+      "learning_rate": 0.00018260869565217392,
+      "logits/chosen": -0.2556913197040558,
+      "logits/rejected": -0.3474789261817932,
+      "logps/chosen": -5.220917701721191,
+      "logps/rejected": -23.547605514526367,
+      "loss": 0.9417474865913391,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4541093111038208,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": 0.043988656252622604,
+      "rewards/margins": 0.6349199414253235,
+      "rewards/rejected": -0.5909312963485718,
+      "step": 21,
+      "train_speed(iter/s)": 0.005615
+    },
+    {
+      "epoch": 0.09843400447427293,
+      "grad_norm": 2.325854539871216,
+      "learning_rate": 0.00019130434782608697,
+      "logits/chosen": -0.2636699676513672,
+      "logits/rejected": -0.33711570501327515,
+      "logps/chosen": -8.976820945739746,
+      "logps/rejected": -19.829130172729492,
+      "loss": 1.0729048252105713,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.595395028591156,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.09026271104812622,
+      "rewards/margins": 0.623466432094574,
+      "rewards/rejected": -0.5332037210464478,
+      "step": 22,
+      "train_speed(iter/s)": 0.005617
+    },
+    {
+      "epoch": 0.1029082774049217,
+      "grad_norm": 2.684507369995117,
+      "learning_rate": 0.0002,
+      "logits/chosen": -0.24679847061634064,
+      "logits/rejected": -0.3378849923610687,
+      "logps/chosen": -7.14666223526001,
+      "logps/rejected": -23.644922256469727,
+      "loss": 0.9377666115760803,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.427379846572876,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.0773518830537796,
+      "rewards/margins": 0.6581345796585083,
+      "rewards/rejected": -0.7354865074157715,
+      "step": 23,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.10738255033557047,
+      "grad_norm": 2.6381781101226807,
+      "learning_rate": 0.00019999724204599747,
+      "logits/chosen": -0.246232807636261,
+      "logits/rejected": -0.3716464936733246,
+      "logps/chosen": -6.111550331115723,
+      "logps/rejected": -24.672321319580078,
+      "loss": 0.9850844740867615,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5655074119567871,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.027247514575719833,
+      "rewards/margins": 0.8617139458656311,
+      "rewards/rejected": -0.8344664573669434,
+      "step": 24,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.11185682326621924,
+      "grad_norm": 2.5178840160369873,
+      "learning_rate": 0.00019998896833611603,
+      "logits/chosen": -0.23005428910255432,
+      "logits/rejected": -0.3447636365890503,
+      "logps/chosen": -8.92835807800293,
+      "logps/rejected": -25.173357009887695,
+      "loss": 1.1135735511779785,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5982357263565063,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.07063186913728714,
+      "rewards/margins": 0.6180722117424011,
+      "rewards/rejected": -0.6887040138244629,
+      "step": 25,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.116331096196868,
+      "grad_norm": 1.9486284255981445,
+      "learning_rate": 0.0001999751793267259,
+      "logits/chosen": -0.17476646602153778,
+      "logits/rejected": -0.28910771012306213,
+      "logps/chosen": -6.982065200805664,
+      "logps/rejected": -25.235271453857422,
+      "loss": 0.9616149067878723,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4814175069332123,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.017612511292099953,
+      "rewards/margins": 0.7644082307815552,
+      "rewards/rejected": -0.7467955350875854,
+      "step": 26,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.12080536912751678,
+      "grad_norm": 2.024102210998535,
+      "learning_rate": 0.0001999558757784162,
+      "logits/chosen": -0.22182828187942505,
+      "logits/rejected": -0.33474183082580566,
+      "logps/chosen": -8.283013343811035,
+      "logps/rejected": -23.36388397216797,
+      "loss": 1.071007251739502,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5651244521141052,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.005644451826810837,
+      "rewards/margins": 0.549856424331665,
+      "rewards/rejected": -0.5555009245872498,
+      "step": 27,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.12527964205816555,
+      "grad_norm": 2.334916830062866,
+      "learning_rate": 0.0001999310587559529,
+      "logits/chosen": -0.1745743602514267,
+      "logits/rejected": -0.28861111402511597,
+      "logps/chosen": -7.148108959197998,
+      "logps/rejected": -22.50945472717285,
+      "loss": 1.0254650115966797,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5092177391052246,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.024923313409090042,
+      "rewards/margins": 0.6763253211975098,
+      "rewards/rejected": -0.6514019966125488,
+      "step": 28,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.1297539149888143,
+      "grad_norm": 4.392460823059082,
+      "learning_rate": 0.00019990072962822007,
+      "logits/chosen": -0.1607450246810913,
+      "logits/rejected": -0.30258849263191223,
+      "logps/chosen": -8.330341339111328,
+      "logps/rejected": -23.302955627441406,
+      "loss": 1.1018993854522705,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6234543919563293,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1296255737543106,
+      "rewards/margins": 0.7057831883430481,
+      "rewards/rejected": -0.5761575102806091,
+      "step": 29,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.1342281879194631,
+      "grad_norm": 1.9557713270187378,
+      "learning_rate": 0.00019986489006814452,
+      "logits/chosen": -0.15406475961208344,
+      "logits/rejected": -0.32140052318573,
+      "logps/chosen": -6.338605880737305,
+      "logps/rejected": -27.934860229492188,
+      "loss": 0.8741156458854675,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.43781277537345886,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.005505967885255814,
+      "rewards/margins": 0.9152470827102661,
+      "rewards/rejected": -0.9097411632537842,
+      "step": 30,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.13870246085011187,
+      "grad_norm": 3.570540189743042,
+      "learning_rate": 0.00019982354205260347,
+      "logits/chosen": -0.13595342636108398,
+      "logits/rejected": -0.24140222370624542,
+      "logps/chosen": -9.938725471496582,
+      "logps/rejected": -26.17535972595215,
+      "loss": 1.1157177686691284,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6451507806777954,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.06947211921215057,
+      "rewards/margins": 0.9398424029350281,
+      "rewards/rejected": -1.0093145370483398,
+      "step": 31,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.14317673378076062,
+      "grad_norm": 2.263465404510498,
+      "learning_rate": 0.00019977668786231534,
+      "logits/chosen": -0.13489112257957458,
+      "logits/rejected": -0.2546093165874481,
+      "logps/chosen": -9.485723495483398,
+      "logps/rejected": -30.659465789794922,
+      "loss": 1.1024084091186523,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.7035719752311707,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.094739630818367,
+      "rewards/margins": 1.1513433456420898,
+      "rewards/rejected": -1.0566036701202393,
+      "step": 32,
+      "train_speed(iter/s)": 0.005617
+    },
+    {
+      "epoch": 0.1476510067114094,
+      "grad_norm": 4.003927230834961,
+      "learning_rate": 0.00019972433008171416,
+      "logits/chosen": -0.2770022451877594,
+      "logits/rejected": -0.3445180356502533,
+      "logps/chosen": -7.854065895080566,
+      "logps/rejected": -17.47032356262207,
+      "loss": 1.1709215641021729,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.635529637336731,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.03749306499958038,
+      "rewards/margins": 0.6043045520782471,
+      "rewards/rejected": -0.5668113827705383,
+      "step": 33,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.15212527964205816,
+      "grad_norm": 2.930697202682495,
+      "learning_rate": 0.00019966647159880703,
+      "logits/chosen": -0.20737791061401367,
+      "logits/rejected": -0.3068307638168335,
+      "logps/chosen": -5.924644470214844,
+      "logps/rejected": -20.47597885131836,
+      "loss": 0.9971151947975159,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4416566789150238,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.04966810718178749,
+      "rewards/margins": 0.48412013053894043,
+      "rewards/rejected": -0.4344520568847656,
+      "step": 34,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.15659955257270694,
+      "grad_norm": 2.292167901992798,
+      "learning_rate": 0.00019960311560501454,
+      "logits/chosen": -0.20404092967510223,
+      "logits/rejected": -0.2819429636001587,
+      "logps/chosen": -9.679859161376953,
+      "logps/rejected": -22.75543785095215,
+      "loss": 1.1032215356826782,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6184647679328918,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.07092233002185822,
+      "rewards/margins": 0.8317320942878723,
+      "rewards/rejected": -0.7608097791671753,
+      "step": 35,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.1610738255033557,
+      "grad_norm": 1.5171147584915161,
+      "learning_rate": 0.0001995342655949951,
+      "logits/chosen": -0.18677955865859985,
+      "logits/rejected": -0.30433592200279236,
+      "logps/chosen": -7.317014217376709,
+      "logps/rejected": -23.58348274230957,
+      "loss": 1.0045303106307983,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5375936031341553,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.13633473217487335,
+      "rewards/margins": 0.7853459119796753,
+      "rewards/rejected": -0.6490110754966736,
+      "step": 36,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.16554809843400448,
+      "grad_norm": 1.8760979175567627,
+      "learning_rate": 0.00019945992536645187,
+      "logits/chosen": -0.17943330109119415,
+      "logits/rejected": -0.27492690086364746,
+      "logps/chosen": -9.596693992614746,
+      "logps/rejected": -25.057254791259766,
+      "loss": 1.0518603324890137,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5895128846168518,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 4.468485713005066e-06,
+      "rewards/margins": 0.8271102905273438,
+      "rewards/rejected": -0.827105700969696,
+      "step": 37,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.17002237136465326,
+      "grad_norm": 2.1765875816345215,
+      "learning_rate": 0.0001993800990199235,
+      "logits/chosen": -0.17796888947486877,
+      "logits/rejected": -0.27494707703590393,
+      "logps/chosen": -8.854700088500977,
+      "logps/rejected": -22.589916229248047,
+      "loss": 1.1736024618148804,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6680553555488586,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": 0.027395736426115036,
+      "rewards/margins": 0.702995777130127,
+      "rewards/rejected": -0.6756000518798828,
+      "step": 38,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.174496644295302,
+      "grad_norm": 2.0398788452148438,
+      "learning_rate": 0.0001992947909585578,
+      "logits/chosen": -0.15950845181941986,
+      "logits/rejected": -0.285598486661911,
+      "logps/chosen": -7.358231544494629,
+      "logps/rejected": -23.962072372436523,
+      "loss": 0.9235795736312866,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5008021593093872,
+      "rewards/accuracies": 0.890625,
+      "rewards/chosen": 0.1876223385334015,
+      "rewards/margins": 0.9989414215087891,
+      "rewards/rejected": -0.8113190531730652,
+      "step": 39,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.1789709172259508,
+      "grad_norm": 2.137016773223877,
+      "learning_rate": 0.000199204005887869,
+      "logits/chosen": -0.18114963173866272,
+      "logits/rejected": -0.23074296116828918,
+      "logps/chosen": -9.713254928588867,
+      "logps/rejected": -15.0142822265625,
+      "loss": 1.1612474918365479,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5743556618690491,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.08143825083971024,
+      "rewards/margins": 0.3889071047306061,
+      "rewards/rejected": -0.30746886134147644,
+      "step": 40,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.18344519015659955,
+      "grad_norm": 2.5089111328125,
+      "learning_rate": 0.000199107748815478,
+      "logits/chosen": -0.17036814987659454,
+      "logits/rejected": -0.25284841656684875,
+      "logps/chosen": -7.5790181159973145,
+      "logps/rejected": -22.757585525512695,
+      "loss": 0.9598883986473083,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5350608229637146,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.10258667171001434,
+      "rewards/margins": 0.982707679271698,
+      "rewards/rejected": -0.8801209926605225,
+      "step": 41,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.18791946308724833,
+      "grad_norm": 2.425978660583496,
+      "learning_rate": 0.00019900602505083648,
+      "logits/chosen": -0.17056672275066376,
+      "logits/rejected": -0.24402382969856262,
+      "logps/chosen": -9.11720085144043,
+      "logps/rejected": -26.321962356567383,
+      "loss": 0.9588575959205627,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5199980735778809,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.02633209154009819,
+      "rewards/margins": 1.148821234703064,
+      "rewards/rejected": -1.1751532554626465,
+      "step": 42,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.19239373601789708,
+      "grad_norm": 4.055136203765869,
+      "learning_rate": 0.0001988988402049336,
+      "logits/chosen": -0.06654427945613861,
+      "logits/rejected": -0.19404850900173187,
+      "logps/chosen": -10.976348876953125,
+      "logps/rejected": -35.80363464355469,
+      "loss": 1.205069661140442,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.7504922151565552,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.11243688315153122,
+      "rewards/margins": 1.5014084577560425,
+      "rewards/rejected": -1.6138453483581543,
+      "step": 43,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.19686800894854586,
+      "grad_norm": 2.9351325035095215,
+      "learning_rate": 0.00019878620018998696,
+      "logits/chosen": -0.21978600323200226,
+      "logits/rejected": -0.32882797718048096,
+      "logps/chosen": -6.976373195648193,
+      "logps/rejected": -30.215131759643555,
+      "loss": 0.9490640759468079,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5300356149673462,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": -0.11029693484306335,
+      "rewards/margins": 1.1996657848358154,
+      "rewards/rejected": -1.3099626302719116,
+      "step": 44,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.20134228187919462,
+      "grad_norm": 2.6874403953552246,
+      "learning_rate": 0.00019866811121911607,
+      "logits/chosen": -0.1936551034450531,
+      "logits/rejected": -0.284149706363678,
+      "logps/chosen": -6.9035515785217285,
+      "logps/rejected": -22.284351348876953,
+      "loss": 0.8805959820747375,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4068413972854614,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.07986758649349213,
+      "rewards/margins": 0.8386067152023315,
+      "rewards/rejected": -0.7587391138076782,
+      "step": 45,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.2058165548098434,
+      "grad_norm": 1.7893315553665161,
+      "learning_rate": 0.000198544579806,
+      "logits/chosen": -0.20961534976959229,
+      "logits/rejected": -0.33521464467048645,
+      "logps/chosen": -7.002056121826172,
+      "logps/rejected": -25.93248748779297,
+      "loss": 0.9666993618011475,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4805544912815094,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": 0.07972076535224915,
+      "rewards/margins": 0.8979825377464294,
+      "rewards/rejected": -0.8182618021965027,
+      "step": 46,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.21029082774049218,
+      "grad_norm": 2.2033731937408447,
+      "learning_rate": 0.0001984156127645178,
+      "logits/chosen": -0.21569889783859253,
+      "logits/rejected": -0.38949355483055115,
+      "logps/chosen": -5.476630687713623,
+      "logps/rejected": -27.572505950927734,
+      "loss": 0.8785698413848877,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.42979833483695984,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.17229261994361877,
+      "rewards/margins": 1.044838547706604,
+      "rewards/rejected": -0.8725459575653076,
+      "step": 47,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.21476510067114093,
+      "grad_norm": 2.1163032054901123,
+      "learning_rate": 0.00019828121720837286,
+      "logits/chosen": -0.34833359718322754,
+      "logits/rejected": -0.41568437218666077,
+      "logps/chosen": -7.711272239685059,
+      "logps/rejected": -25.60678482055664,
+      "loss": 0.9890228509902954,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.548301100730896,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.07654333114624023,
+      "rewards/margins": 1.2576009035110474,
+      "rewards/rejected": -1.1810575723648071,
+      "step": 48,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.21923937360178972,
+      "grad_norm": 1.987210988998413,
+      "learning_rate": 0.00019814140055070042,
+      "logits/chosen": -0.2835751473903656,
+      "logits/rejected": -0.38263481855392456,
+      "logps/chosen": -8.624626159667969,
+      "logps/rejected": -25.244403839111328,
+      "loss": 0.9971789121627808,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5687417387962341,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 0.09697108715772629,
+      "rewards/margins": 1.0362679958343506,
+      "rewards/rejected": -0.9392969608306885,
+      "step": 49,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.22371364653243847,
+      "grad_norm": 1.9501866102218628,
+      "learning_rate": 0.0001979961705036587,
+      "logits/chosen": -0.38833191990852356,
+      "logits/rejected": -0.4256289303302765,
+      "logps/chosen": -8.22580623626709,
+      "logps/rejected": -23.093332290649414,
+      "loss": 0.9298551082611084,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4849992096424103,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.04563959687948227,
+      "rewards/margins": 1.0999000072479248,
+      "rewards/rejected": -1.054260492324829,
+      "step": 50,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.22818791946308725,
+      "grad_norm": 2.496234893798828,
+      "learning_rate": 0.00019784553507800349,
+      "logits/chosen": -0.2710403501987457,
+      "logits/rejected": -0.41735631227493286,
+      "logps/chosen": -8.894371032714844,
+      "logps/rejected": -28.586936950683594,
+      "loss": 1.1125377416610718,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6875916719436646,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": 0.10749680548906326,
+      "rewards/margins": 1.2817683219909668,
+      "rewards/rejected": -1.174271583557129,
+      "step": 51,
+      "train_speed(iter/s)": 0.005611
+    },
+    {
+      "epoch": 0.232662192393736,
+      "grad_norm": 3.274833917617798,
+      "learning_rate": 0.00019768950258264623,
+      "logits/chosen": -0.39167118072509766,
+      "logits/rejected": -0.4531523883342743,
+      "logps/chosen": -9.231929779052734,
+      "logps/rejected": -22.90334701538086,
+      "loss": 1.1200237274169922,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6103911399841309,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.16369852423667908,
+      "rewards/margins": 0.9332659244537354,
+      "rewards/rejected": -1.0969644784927368,
+      "step": 52,
+      "train_speed(iter/s)": 0.005612
+    },
+    {
+      "epoch": 0.2371364653243848,
+      "grad_norm": 2.9111413955688477,
+      "learning_rate": 0.0001975280816241959,
+      "logits/chosen": -0.3340080976486206,
+      "logits/rejected": -0.3975221514701843,
+      "logps/chosen": -11.631303787231445,
+      "logps/rejected": -26.26183319091797,
+      "loss": 1.1326872110366821,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6245000958442688,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.032076627016067505,
+      "rewards/margins": 0.9003015160560608,
+      "rewards/rejected": -0.9323781132698059,
+      "step": 53,
+      "train_speed(iter/s)": 0.005612
+    },
+    {
+      "epoch": 0.24161073825503357,
+      "grad_norm": 2.7828478813171387,
+      "learning_rate": 0.00019736128110648407,
+      "logits/chosen": -0.345708429813385,
+      "logits/rejected": -0.447677880525589,
+      "logps/chosen": -8.686080932617188,
+      "logps/rejected": -23.45962142944336,
+      "loss": 0.9978072047233582,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5644747018814087,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.18989036977291107,
+      "rewards/margins": 1.1108869314193726,
+      "rewards/rejected": -0.9209966659545898,
+      "step": 54,
+      "train_speed(iter/s)": 0.005613
+    },
+    {
+      "epoch": 0.24608501118568232,
+      "grad_norm": 2.1220943927764893,
+      "learning_rate": 0.0001971891102300738,
+      "logits/chosen": -0.3758760690689087,
+      "logits/rejected": -0.4768344461917877,
+      "logps/chosen": -7.041341304779053,
+      "logps/rejected": -22.334056854248047,
+      "loss": 1.0144665241241455,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5401796698570251,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1189827248454094,
+      "rewards/margins": 0.939737856388092,
+      "rewards/rejected": -0.820755124092102,
+      "step": 55,
+      "train_speed(iter/s)": 0.005613
+    },
+    {
+      "epoch": 0.2505592841163311,
+      "grad_norm": 1.843038558959961,
+      "learning_rate": 0.00019701157849175228,
+      "logits/chosen": -0.37703099846839905,
+      "logits/rejected": -0.46533748507499695,
+      "logps/chosen": -8.012426376342773,
+      "logps/rejected": -22.621742248535156,
+      "loss": 1.0123478174209595,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5707098245620728,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.15585632622241974,
+      "rewards/margins": 0.9487887620925903,
+      "rewards/rejected": -0.792932391166687,
+      "step": 56,
+      "train_speed(iter/s)": 0.005614
+    },
+    {
+      "epoch": 0.2550335570469799,
+      "grad_norm": 2.230550527572632,
+      "learning_rate": 0.00019682869568400684,
+      "logits/chosen": -0.27077603340148926,
+      "logits/rejected": -0.43189266324043274,
+      "logps/chosen": -6.5583930015563965,
+      "logps/rejected": -29.555484771728516,
+      "loss": 0.9619537591934204,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5382556915283203,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.012836463749408722,
+      "rewards/margins": 1.0998661518096924,
+      "rewards/rejected": -1.0870296955108643,
+      "step": 57,
+      "train_speed(iter/s)": 0.005614
+    },
+    {
+      "epoch": 0.2595078299776286,
+      "grad_norm": 2.252734661102295,
+      "learning_rate": 0.00019664047189448493,
+      "logits/chosen": -0.30061304569244385,
+      "logits/rejected": -0.4390646517276764,
+      "logps/chosen": -5.5507659912109375,
+      "logps/rejected": -26.51093292236328,
+      "loss": 0.9082356095314026,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.41620010137557983,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.027434132993221283,
+      "rewards/margins": 0.9383218288421631,
+      "rewards/rejected": -0.9657559394836426,
+      "step": 58,
+      "train_speed(iter/s)": 0.005615
+    },
+    {
+      "epoch": 0.2639821029082774,
+      "grad_norm": 2.943512201309204,
+      "learning_rate": 0.00019644691750543767,
+      "logits/chosen": -0.2617231011390686,
+      "logits/rejected": -0.38011983036994934,
+      "logps/chosen": -10.003762245178223,
+      "logps/rejected": -27.785303115844727,
+      "loss": 1.1763017177581787,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6572035551071167,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.07653994858264923,
+      "rewards/margins": 0.9367327690124512,
+      "rewards/rejected": -1.0132726430892944,
+      "step": 59,
+      "train_speed(iter/s)": 0.005615
+    },
+    {
+      "epoch": 0.2684563758389262,
+      "grad_norm": 1.736883521080017,
+      "learning_rate": 0.00019624804319314705,
+      "logits/chosen": -0.3555205464363098,
+      "logits/rejected": -0.4549139738082886,
+      "logps/chosen": -7.022237300872803,
+      "logps/rejected": -28.04250717163086,
+      "loss": 0.9159735441207886,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.47989311814308167,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.060450125485658646,
+      "rewards/margins": 1.1664522886276245,
+      "rewards/rejected": -1.1060020923614502,
+      "step": 60,
+      "train_speed(iter/s)": 0.005616
+    },
+    {
+      "epoch": 0.27293064876957496,
+      "grad_norm": 1.8152859210968018,
+      "learning_rate": 0.00019604385992733715,
+      "logits/chosen": -0.2845993638038635,
+      "logits/rejected": -0.3513396680355072,
+      "logps/chosen": -9.077548027038574,
+      "logps/rejected": -22.780668258666992,
+      "loss": 0.9680572748184204,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4960174560546875,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": 0.06926427036523819,
+      "rewards/margins": 0.9312706589698792,
+      "rewards/rejected": -0.862006425857544,
+      "step": 61,
+      "train_speed(iter/s)": 0.005616
+    },
+    {
+      "epoch": 0.27740492170022374,
+      "grad_norm": 2.46378755569458,
+      "learning_rate": 0.00019583437897056915,
+      "logits/chosen": -0.3284386098384857,
+      "logits/rejected": -0.4009109139442444,
+      "logps/chosen": -8.893238067626953,
+      "logps/rejected": -21.27986717224121,
+      "loss": 1.128482699394226,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5518131256103516,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": 0.010286218486726284,
+      "rewards/margins": 0.5274713635444641,
+      "rewards/rejected": -0.5171850919723511,
+      "step": 62,
+      "train_speed(iter/s)": 0.005617
+    },
+    {
+      "epoch": 0.28187919463087246,
+      "grad_norm": 2.1732046604156494,
+      "learning_rate": 0.00019561961187761985,
+      "logits/chosen": -0.24422433972358704,
+      "logits/rejected": -0.375420480966568,
+      "logps/chosen": -6.340362071990967,
+      "logps/rejected": -20.184795379638672,
+      "loss": 0.987078070640564,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4728015065193176,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.168831467628479,
+      "rewards/margins": 0.6444362998008728,
+      "rewards/rejected": -0.4756048321723938,
+      "step": 63,
+      "train_speed(iter/s)": 0.005617
+    },
+    {
+      "epoch": 0.28635346756152125,
+      "grad_norm": 2.231933355331421,
+      "learning_rate": 0.00019539957049484458,
+      "logits/chosen": -0.22632239758968353,
+      "logits/rejected": -0.3298073410987854,
+      "logps/chosen": -5.790369987487793,
+      "logps/rejected": -21.647865295410156,
+      "loss": 0.9348481893539429,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4110260307788849,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.11635202169418335,
+      "rewards/margins": 0.6466613411903381,
+      "rewards/rejected": -0.5303093194961548,
+      "step": 64,
+      "train_speed(iter/s)": 0.005617
+    },
+    {
+      "epoch": 0.29082774049217003,
+      "grad_norm": 2.592883825302124,
+      "learning_rate": 0.00019517426695952358,
+      "logits/chosen": -0.2631714344024658,
+      "logits/rejected": -0.3835735321044922,
+      "logps/chosen": -8.772464752197266,
+      "logps/rejected": -23.13977813720703,
+      "loss": 1.0820575952529907,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6241056323051453,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.1670496016740799,
+      "rewards/margins": 0.9779586791992188,
+      "rewards/rejected": -0.8109091520309448,
+      "step": 65,
+      "train_speed(iter/s)": 0.005617
+    },
+    {
+      "epoch": 0.2953020134228188,
+      "grad_norm": 2.1508402824401855,
+      "learning_rate": 0.0001949437136991925,
+      "logits/chosen": -0.28555068373680115,
+      "logits/rejected": -0.42386704683303833,
+      "logps/chosen": -4.788064002990723,
+      "logps/rejected": -26.989431381225586,
+      "loss": 0.7586299777030945,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3784462809562683,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.09788702428340912,
+      "rewards/margins": 1.3373408317565918,
+      "rewards/rejected": -1.239453673362732,
+      "step": 66,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.29977628635346754,
+      "grad_norm": 4.065814971923828,
+      "learning_rate": 0.00019470792343095718,
+      "logits/chosen": -0.3336545526981354,
+      "logits/rejected": -0.3990672826766968,
+      "logps/chosen": -8.33244514465332,
+      "logps/rejected": -29.013874053955078,
+      "loss": 1.0058256387710571,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6062498092651367,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.020362887531518936,
+      "rewards/margins": 1.5735464096069336,
+      "rewards/rejected": -1.5939093828201294,
+      "step": 67,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.3042505592841163,
+      "grad_norm": 2.998826265335083,
+      "learning_rate": 0.0001944669091607919,
+      "logits/chosen": -0.17095105350017548,
+      "logits/rejected": -0.33402499556541443,
+      "logps/chosen": -7.172041893005371,
+      "logps/rejected": -36.20872497558594,
+      "loss": 0.8435176610946655,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5263906121253967,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.01939973793923855,
+      "rewards/margins": 2.0239157676696777,
+      "rewards/rejected": -2.004516124725342,
+      "step": 68,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.3087248322147651,
+      "grad_norm": 2.8316242694854736,
+      "learning_rate": 0.00019422068418282202,
+      "logits/chosen": -0.23822994530200958,
+      "logits/rejected": -0.3510916531085968,
+      "logps/chosen": -8.204334259033203,
+      "logps/rejected": -26.101930618286133,
+      "loss": 0.9929779171943665,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.592566967010498,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.07919630408287048,
+      "rewards/margins": 1.4216609001159668,
+      "rewards/rejected": -1.3424646854400635,
+      "step": 69,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.3131991051454139,
+      "grad_norm": 2.1727654933929443,
+      "learning_rate": 0.00019396926207859084,
+      "logits/chosen": -0.19429409503936768,
+      "logits/rejected": -0.33817699551582336,
+      "logps/chosen": -7.17151403427124,
+      "logps/rejected": -29.4168701171875,
+      "loss": 0.8446124196052551,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4697123169898987,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 0.09260577708482742,
+      "rewards/margins": 1.373153567314148,
+      "rewards/rejected": -1.280547857284546,
+      "step": 70,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.31767337807606266,
+      "grad_norm": 2.131971597671509,
+      "learning_rate": 0.00019371265671631037,
+      "logits/chosen": -0.2500344514846802,
+      "logits/rejected": -0.391252338886261,
+      "logps/chosen": -6.454396724700928,
+      "logps/rejected": -31.325042724609375,
+      "loss": 0.857275128364563,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.45489931106567383,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.12299442291259766,
+      "rewards/margins": 1.4080641269683838,
+      "rewards/rejected": -1.2850697040557861,
+      "step": 71,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.3221476510067114,
+      "grad_norm": 2.6743502616882324,
+      "learning_rate": 0.00019345088225009626,
+      "logits/chosen": -0.20935188233852386,
+      "logits/rejected": -0.3504689633846283,
+      "logps/chosen": -8.028803825378418,
+      "logps/rejected": -33.473976135253906,
+      "loss": 0.951294481754303,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5516812205314636,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.002318259794265032,
+      "rewards/margins": 1.4767383337020874,
+      "rewards/rejected": -1.4744200706481934,
+      "step": 72,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.32662192393736017,
+      "grad_norm": 2.1057374477386475,
+      "learning_rate": 0.0001931839531191873,
+      "logits/chosen": -0.21616998314857483,
+      "logits/rejected": -0.3215841054916382,
+      "logps/chosen": -6.386109352111816,
+      "logps/rejected": -26.058643341064453,
+      "loss": 0.9168461561203003,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5115207433700562,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 0.16049177944660187,
+      "rewards/margins": 1.1623506546020508,
+      "rewards/rejected": -1.0018587112426758,
+      "step": 73,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.33109619686800895,
+      "grad_norm": 1.8313034772872925,
+      "learning_rate": 0.00019291188404714878,
+      "logits/chosen": -0.2762296795845032,
+      "logits/rejected": -0.3759133815765381,
+      "logps/chosen": -7.998234748840332,
+      "logps/rejected": -29.753068923950195,
+      "loss": 0.9865212440490723,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5828378796577454,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.04827752709388733,
+      "rewards/margins": 1.679243564605713,
+      "rewards/rejected": -1.6309659481048584,
+      "step": 74,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.33557046979865773,
+      "grad_norm": 2.7458698749542236,
+      "learning_rate": 0.0001926346900410604,
+      "logits/chosen": -0.26945143938064575,
+      "logits/rejected": -0.319683313369751,
+      "logps/chosen": -9.152531623840332,
+      "logps/rejected": -22.447328567504883,
+      "loss": 1.118042230606079,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6387289762496948,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.012468192726373672,
+      "rewards/margins": 0.904109537601471,
+      "rewards/rejected": -0.8916412591934204,
+      "step": 75,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.3400447427293065,
+      "grad_norm": 2.5607635974884033,
+      "learning_rate": 0.00019235238639068856,
+      "logits/chosen": -0.19932252168655396,
+      "logits/rejected": -0.37197503447532654,
+      "logps/chosen": -7.51588773727417,
+      "logps/rejected": -33.714637756347656,
+      "loss": 0.9608560800552368,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5613290667533875,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.0078031644225120544,
+      "rewards/margins": 1.3550933599472046,
+      "rewards/rejected": -1.347290277481079,
+      "step": 76,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.34451901565995524,
+      "grad_norm": 2.152766227722168,
+      "learning_rate": 0.00019206498866764288,
+      "logits/chosen": -0.18406139314174652,
+      "logits/rejected": -0.36746495962142944,
+      "logps/chosen": -4.768266677856445,
+      "logps/rejected": -35.82173156738281,
+      "loss": 0.8752678632736206,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5213672518730164,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": -0.024980902671813965,
+      "rewards/margins": 1.6603260040283203,
+      "rewards/rejected": -1.6853069067001343,
+      "step": 77,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.348993288590604,
+      "grad_norm": 1.957088589668274,
+      "learning_rate": 0.0001917725127245174,
+      "logits/chosen": -0.2368219494819641,
+      "logits/rejected": -0.31324779987335205,
+      "logps/chosen": -7.644411087036133,
+      "logps/rejected": -25.68874740600586,
+      "loss": 0.9143100380897522,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.47288480401039124,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": 0.04239855334162712,
+      "rewards/margins": 0.925534188747406,
+      "rewards/rejected": -0.8831356167793274,
+      "step": 78,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.3534675615212528,
+      "grad_norm": 2.0980629920959473,
+      "learning_rate": 0.0001914749746940161,
+      "logits/chosen": -0.3016536831855774,
+      "logits/rejected": -0.3982566297054291,
+      "logps/chosen": -7.566556930541992,
+      "logps/rejected": -26.53971290588379,
+      "loss": 0.9413108825683594,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5161536931991577,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.1749323159456253,
+      "rewards/margins": 1.1360664367675781,
+      "rewards/rejected": -0.9611341953277588,
+      "step": 79,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.3579418344519016,
+      "grad_norm": 2.1753523349761963,
+      "learning_rate": 0.00019117239098806295,
+      "logits/chosen": -0.25202423334121704,
+      "logits/rejected": -0.3296201527118683,
+      "logps/chosen": -8.623554229736328,
+      "logps/rejected": -24.96216583251953,
+      "loss": 0.9156360626220703,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.521922767162323,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.17782656848430634,
+      "rewards/margins": 1.1749255657196045,
+      "rewards/rejected": -0.997098982334137,
+      "step": 80,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.3624161073825503,
+      "grad_norm": 3.2816970348358154,
+      "learning_rate": 0.00019086477829689685,
+      "logits/chosen": -0.22371181845664978,
+      "logits/rejected": -0.3009035885334015,
+      "logps/chosen": -10.19958782196045,
+      "logps/rejected": -26.129743576049805,
+      "loss": 1.161747932434082,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.7456332445144653,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.06418374180793762,
+      "rewards/margins": 1.1640924215316772,
+      "rewards/rejected": -1.099908709526062,
+      "step": 81,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.3668903803131991,
+      "grad_norm": 4.27427339553833,
+      "learning_rate": 0.0001905521535881509,
+      "logits/chosen": -0.15121029317378998,
+      "logits/rejected": -0.3036370873451233,
+      "logps/chosen": -7.66187858581543,
+      "logps/rejected": -30.927831649780273,
+      "loss": 1.1210309267044067,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6641408205032349,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.00867693591862917,
+      "rewards/margins": 1.272807002067566,
+      "rewards/rejected": -1.2814840078353882,
+      "step": 82,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.3713646532438479,
+      "grad_norm": 2.29978084564209,
+      "learning_rate": 0.00019023453410591635,
+      "logits/chosen": -0.21486631035804749,
+      "logits/rejected": -0.3746005892753601,
+      "logps/chosen": -9.569436073303223,
+      "logps/rejected": -32.21381378173828,
+      "loss": 1.1229475736618042,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6794871687889099,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.0131741464138031,
+      "rewards/margins": 1.4518442153930664,
+      "rewards/rejected": -1.4386701583862305,
+      "step": 83,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.37583892617449666,
+      "grad_norm": 2.6593234539031982,
+      "learning_rate": 0.00018991193736979175,
+      "logits/chosen": -0.12227392941713333,
+      "logits/rejected": -0.2726154327392578,
+      "logps/chosen": -7.078165054321289,
+      "logps/rejected": -30.666397094726562,
+      "loss": 0.7911984920501709,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4572717845439911,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.06052548438310623,
+      "rewards/margins": 1.428847074508667,
+      "rewards/rejected": -1.3683216571807861,
+      "step": 84,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.38031319910514544,
+      "grad_norm": 2.8958120346069336,
+      "learning_rate": 0.00018958438117391618,
+      "logits/chosen": -0.0972181037068367,
+      "logits/rejected": -0.2590752840042114,
+      "logps/chosen": -8.193891525268555,
+      "logps/rejected": -35.56235885620117,
+      "loss": 1.0837842226028442,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6906030774116516,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.06835082173347473,
+      "rewards/margins": 1.79606294631958,
+      "rewards/rejected": -1.8644137382507324,
+      "step": 85,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.38478747203579416,
+      "grad_norm": 3.5083513259887695,
+      "learning_rate": 0.00018925188358598813,
+      "logits/chosen": -0.17531399428844452,
+      "logits/rejected": -0.25995591282844543,
+      "logps/chosen": -8.639050483703613,
+      "logps/rejected": -29.226892471313477,
+      "loss": 1.0494576692581177,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6720495820045471,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.03724604845046997,
+      "rewards/margins": 1.4595208168029785,
+      "rewards/rejected": -1.4967668056488037,
+      "step": 86,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.38926174496644295,
+      "grad_norm": 2.0132222175598145,
+      "learning_rate": 0.00018891446294626866,
+      "logits/chosen": -0.05051232874393463,
+      "logits/rejected": -0.25516456365585327,
+      "logps/chosen": -4.587294578552246,
+      "logps/rejected": -36.072322845458984,
+      "loss": 0.602871298789978,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.32869917154312134,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.17169487476348877,
+      "rewards/margins": 2.032069444656372,
+      "rewards/rejected": -1.8603744506835938,
+      "step": 87,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.39373601789709173,
+      "grad_norm": 3.1251611709594727,
+      "learning_rate": 0.00018857213786656985,
+      "logits/chosen": -0.16258063912391663,
+      "logits/rejected": -0.2599830627441406,
+      "logps/chosen": -9.219761848449707,
+      "logps/rejected": -27.451026916503906,
+      "loss": 1.0880504846572876,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6885117292404175,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 0.04242437332868576,
+      "rewards/margins": 1.207942247390747,
+      "rewards/rejected": -1.1655179262161255,
+      "step": 88,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.3982102908277405,
+      "grad_norm": 2.0855207443237305,
+      "learning_rate": 0.0001882249272292282,
+      "logits/chosen": -0.1683093011379242,
+      "logits/rejected": -0.3001120686531067,
+      "logps/chosen": -7.336337089538574,
+      "logps/rejected": -24.294296264648438,
+      "loss": 0.9775981903076172,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.515565812587738,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.09023457765579224,
+      "rewards/margins": 0.9091135859489441,
+      "rewards/rejected": -0.8188790082931519,
+      "step": 89,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.40268456375838924,
+      "grad_norm": 1.8228232860565186,
+      "learning_rate": 0.00018787285018606297,
+      "logits/chosen": -0.1059221625328064,
+      "logits/rejected": -0.30855828523635864,
+      "logps/chosen": -3.3506364822387695,
+      "logps/rejected": -32.792762756347656,
+      "loss": 0.6121629476547241,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.2924441993236542,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.11779851466417313,
+      "rewards/margins": 1.558281660079956,
+      "rewards/rejected": -1.4404832124710083,
+      "step": 90,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.407158836689038,
+      "grad_norm": 2.038564443588257,
+      "learning_rate": 0.00018751592615732005,
+      "logits/chosen": -0.23549917340278625,
+      "logits/rejected": -0.28893160820007324,
+      "logps/chosen": -7.049911975860596,
+      "logps/rejected": -23.307397842407227,
+      "loss": 0.8128443956375122,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.42805472016334534,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.235755056142807,
+      "rewards/margins": 1.322766900062561,
+      "rewards/rejected": -1.0870119333267212,
+      "step": 91,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.4116331096196868,
+      "grad_norm": 3.0157337188720703,
+      "learning_rate": 0.0001871541748306005,
+      "logits/chosen": -0.16576483845710754,
+      "logits/rejected": -0.3216527998447418,
+      "logps/chosen": -3.5259528160095215,
+      "logps/rejected": -34.49477005004883,
+      "loss": 0.6575559377670288,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.324476957321167,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": 0.0956890657544136,
+      "rewards/margins": 2.1255316734313965,
+      "rewards/rejected": -2.0298423767089844,
+      "step": 92,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.4161073825503356,
+      "grad_norm": 4.184682846069336,
+      "learning_rate": 0.00018678761615977468,
+      "logits/chosen": -0.08753462135791779,
+      "logits/rejected": -0.2798512876033783,
+      "logps/chosen": -9.157657623291016,
+      "logps/rejected": -36.74687576293945,
+      "loss": 1.0873180627822876,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.7294520139694214,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.07273296266794205,
+      "rewards/margins": 1.809075117111206,
+      "rewards/rejected": -1.736342430114746,
+      "step": 93,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.42058165548098436,
+      "grad_norm": 3.6453614234924316,
+      "learning_rate": 0.00018641627036388169,
+      "logits/chosen": -0.26587241888046265,
+      "logits/rejected": -0.35532236099243164,
+      "logps/chosen": -6.705877780914307,
+      "logps/rejected": -34.05841827392578,
+      "loss": 0.8682178258895874,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5556175112724304,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.0699356198310852,
+      "rewards/margins": 2.082659959793091,
+      "rewards/rejected": -2.0127243995666504,
+      "step": 94,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.4250559284116331,
+      "grad_norm": 3.926342725753784,
+      "learning_rate": 0.00018604015792601396,
+      "logits/chosen": -0.18198826909065247,
+      "logits/rejected": -0.28424450755119324,
+      "logps/chosen": -9.228765487670898,
+      "logps/rejected": -28.494712829589844,
+      "loss": 1.100342869758606,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6449709534645081,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.0022647292353212833,
+      "rewards/margins": 1.3801387548446655,
+      "rewards/rejected": -1.3778741359710693,
+      "step": 95,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.42953020134228187,
+      "grad_norm": 6.2491960525512695,
+      "learning_rate": 0.00018565929959218758,
+      "logits/chosen": -0.162868469953537,
+      "logits/rejected": -0.35664084553718567,
+      "logps/chosen": -5.505702495574951,
+      "logps/rejected": -40.82157516479492,
+      "loss": 0.8373050689697266,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.48504531383514404,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.03489302098751068,
+      "rewards/margins": 1.9277093410491943,
+      "rewards/rejected": -1.892816424369812,
+      "step": 96,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.43400447427293065,
+      "grad_norm": 3.155294895172119,
+      "learning_rate": 0.0001852737163701979,
+      "logits/chosen": -0.14200377464294434,
+      "logits/rejected": -0.3238810896873474,
+      "logps/chosen": -6.1366353034973145,
+      "logps/rejected": -36.68225860595703,
+      "loss": 0.7804915904998779,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.47483596205711365,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.05882677063345909,
+      "rewards/margins": 1.9737701416015625,
+      "rewards/rejected": -1.9149433374404907,
+      "step": 97,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.43847874720357943,
+      "grad_norm": 2.6832268238067627,
+      "learning_rate": 0.00018488342952846073,
+      "logits/chosen": -0.12941551208496094,
+      "logits/rejected": -0.2959509789943695,
+      "logps/chosen": -12.032288551330566,
+      "logps/rejected": -35.381710052490234,
+      "loss": 0.838473916053772,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.49494707584381104,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.032307837158441544,
+      "rewards/margins": 1.5512968301773071,
+      "rewards/rejected": -1.5189889669418335,
+      "step": 98,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.4429530201342282,
+      "grad_norm": 2.0462474822998047,
+      "learning_rate": 0.0001844884605948392,
+      "logits/chosen": -0.23558180034160614,
+      "logits/rejected": -0.3692886531352997,
+      "logps/chosen": -9.6895112991333,
+      "logps/rejected": -26.33759117126465,
+      "loss": 1.0117727518081665,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6607507467269897,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.148264542222023,
+      "rewards/margins": 1.4499516487121582,
+      "rewards/rejected": -1.3016871213912964,
+      "step": 99,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.44742729306487694,
+      "grad_norm": 2.056586503982544,
+      "learning_rate": 0.00018408883135545632,
+      "logits/chosen": -0.20895381271839142,
+      "logits/rejected": -0.350538432598114,
+      "logps/chosen": -7.372556209564209,
+      "logps/rejected": -30.752912521362305,
+      "loss": 0.8857133388519287,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.49547722935676575,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.0997147411108017,
+      "rewards/margins": 1.3622983694076538,
+      "rewards/rejected": -1.262583613395691,
+      "step": 100,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.4519015659955257,
+      "grad_norm": 2.2281575202941895,
+      "learning_rate": 0.00018368456385349334,
+      "logits/chosen": -0.22281751036643982,
+      "logits/rejected": -0.34848061203956604,
+      "logps/chosen": -6.225516319274902,
+      "logps/rejected": -27.612606048583984,
+      "loss": 0.9511269330978394,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5547913312911987,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.09937570989131927,
+      "rewards/margins": 1.3068878650665283,
+      "rewards/rejected": -1.2075122594833374,
+      "step": 101,
+      "train_speed(iter/s)": 0.005617
+    },
+    {
+      "epoch": 0.4563758389261745,
+      "grad_norm": 2.278520345687866,
+      "learning_rate": 0.0001832756803879737,
+      "logits/chosen": -0.20784702897071838,
+      "logits/rejected": -0.3433951139450073,
+      "logps/chosen": -9.43735122680664,
+      "logps/rejected": -25.318925857543945,
+      "loss": 0.8974735140800476,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5149669647216797,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.10113431513309479,
+      "rewards/margins": 1.3135309219360352,
+      "rewards/rejected": -1.2123966217041016,
+      "step": 102,
+      "train_speed(iter/s)": 0.005617
+    },
+    {
+      "epoch": 0.4608501118568233,
+      "grad_norm": 2.4370529651641846,
+      "learning_rate": 0.0001828622035125332,
+      "logits/chosen": -0.21997469663619995,
+      "logits/rejected": -0.3634868562221527,
+      "logps/chosen": -6.314621925354004,
+      "logps/rejected": -29.342872619628906,
+      "loss": 0.8530542254447937,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5132930874824524,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.12123695015907288,
+      "rewards/margins": 1.667711615562439,
+      "rewards/rejected": -1.546474575996399,
+      "step": 103,
+      "train_speed(iter/s)": 0.005617
+    },
+    {
+      "epoch": 0.465324384787472,
+      "grad_norm": 2.014124870300293,
+      "learning_rate": 0.00018244415603417603,
+      "logits/chosen": -0.24184371531009674,
+      "logits/rejected": -0.36744868755340576,
+      "logps/chosen": -9.540416717529297,
+      "logps/rejected": -29.473955154418945,
+      "loss": 0.9970014095306396,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6495711803436279,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.11495855450630188,
+      "rewards/margins": 1.480215311050415,
+      "rewards/rejected": -1.365256667137146,
+      "step": 104,
+      "train_speed(iter/s)": 0.005617
+    },
+    {
+      "epoch": 0.4697986577181208,
+      "grad_norm": 3.0877394676208496,
+      "learning_rate": 0.00018202156101201645,
+      "logits/chosen": -0.3040401041507721,
+      "logits/rejected": -0.3883563280105591,
+      "logps/chosen": -5.42517614364624,
+      "logps/rejected": -24.144115447998047,
+      "loss": 0.8570873737335205,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.447566956281662,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.056889522820711136,
+      "rewards/margins": 1.262117624282837,
+      "rewards/rejected": -1.2052280902862549,
+      "step": 105,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.4742729306487696,
+      "grad_norm": 2.858959674835205,
+      "learning_rate": 0.00018159444175600703,
+      "logits/chosen": -0.2462853342294693,
+      "logits/rejected": -0.3662327527999878,
+      "logps/chosen": -9.698307991027832,
+      "logps/rejected": -28.979686737060547,
+      "loss": 0.9490512609481812,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5636053681373596,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.21260645985603333,
+      "rewards/margins": 1.5004974603652954,
+      "rewards/rejected": -1.2878910303115845,
+      "step": 106,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.47874720357941836,
+      "grad_norm": 2.4962282180786133,
+      "learning_rate": 0.00018116282182565311,
+      "logits/chosen": -0.33677104115486145,
+      "logits/rejected": -0.3884970545768738,
+      "logps/chosen": -6.480227947235107,
+      "logps/rejected": -24.565458297729492,
+      "loss": 0.779257595539093,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4241567850112915,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.19994103908538818,
+      "rewards/margins": 1.3964340686798096,
+      "rewards/rejected": -1.196493148803711,
+      "step": 107,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.48322147651006714,
+      "grad_norm": 5.664783000946045,
+      "learning_rate": 0.00018072672502871296,
+      "logits/chosen": -0.25596198439598083,
+      "logits/rejected": -0.39605844020843506,
+      "logps/chosen": -5.608950138092041,
+      "logps/rejected": -35.774356842041016,
+      "loss": 0.7417677640914917,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4080236554145813,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.06679990887641907,
+      "rewards/margins": 1.873131275177002,
+      "rewards/rejected": -1.8063313961029053,
+      "step": 108,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.48769574944071586,
+      "grad_norm": 2.3131730556488037,
+      "learning_rate": 0.00018028617541988472,
+      "logits/chosen": -0.24975106120109558,
+      "logits/rejected": -0.3782438635826111,
+      "logps/chosen": -7.196217060089111,
+      "logps/rejected": -30.633535385131836,
+      "loss": 0.8902066946029663,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5435032844543457,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": -0.005009647458791733,
+      "rewards/margins": 1.4205687046051025,
+      "rewards/rejected": -1.4255784749984741,
+      "step": 109,
+      "train_speed(iter/s)": 0.005618
+    },
+    {
+      "epoch": 0.49217002237136465,
+      "grad_norm": 1.9771348237991333,
+      "learning_rate": 0.00017984119729947944,
+      "logits/chosen": -0.24366146326065063,
+      "logits/rejected": -0.3742184340953827,
+      "logps/chosen": -7.5738325119018555,
+      "logps/rejected": -36.54035186767578,
+      "loss": 0.8050943613052368,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5340873003005981,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.21246597170829773,
+      "rewards/margins": 2.065727710723877,
+      "rewards/rejected": -1.853261947631836,
+      "step": 110,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.4966442953020134,
+      "grad_norm": 2.3881213665008545,
+      "learning_rate": 0.000179391815212081,
+      "logits/chosen": -0.2847563922405243,
+      "logits/rejected": -0.41209399700164795,
+      "logps/chosen": -7.6792683601379395,
+      "logps/rejected": -35.70549011230469,
+      "loss": 0.8006418347358704,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.482678085565567,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.05708705633878708,
+      "rewards/margins": 1.9757572412490845,
+      "rewards/rejected": -1.9186700582504272,
+      "step": 111,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.5011185682326622,
+      "grad_norm": 2.492396116256714,
+      "learning_rate": 0.0001789380539451919,
+      "logits/chosen": -0.28943008184432983,
+      "logits/rejected": -0.36423763632774353,
+      "logps/chosen": -11.887781143188477,
+      "logps/rejected": -28.29849624633789,
+      "loss": 1.1252532005310059,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.730536162853241,
+      "rewards/accuracies": 0.890625,
+      "rewards/chosen": 0.11893969774246216,
+      "rewards/margins": 1.626894474029541,
+      "rewards/rejected": -1.5079548358917236,
+      "step": 112,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.5055928411633109,
+      "grad_norm": 2.185915946960449,
+      "learning_rate": 0.0001784799385278661,
+      "logits/chosen": -0.27568331360816956,
+      "logits/rejected": -0.40015357732772827,
+      "logps/chosen": -6.916875839233398,
+      "logps/rejected": -33.39084243774414,
+      "loss": 0.7926216125488281,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4541390538215637,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.04535483196377754,
+      "rewards/margins": 1.5393131971359253,
+      "rewards/rejected": -1.4939584732055664,
+      "step": 113,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.5100671140939598,
+      "grad_norm": 2.4158413410186768,
+      "learning_rate": 0.0001780174942293287,
+      "logits/chosen": -0.3091110587120056,
+      "logits/rejected": -0.40796011686325073,
+      "logps/chosen": -6.973065376281738,
+      "logps/rejected": -28.748104095458984,
+      "loss": 0.9577688574790955,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6081997156143188,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.1668429970741272,
+      "rewards/margins": 1.543600082397461,
+      "rewards/rejected": -1.376757025718689,
+      "step": 114,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.5145413870246085,
+      "grad_norm": 1.9534834623336792,
+      "learning_rate": 0.00017755074655758174,
+      "logits/chosen": -0.2860637605190277,
+      "logits/rejected": -0.407674103975296,
+      "logps/chosen": -7.297882556915283,
+      "logps/rejected": -31.946149826049805,
+      "loss": 0.8219610452651978,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5011268854141235,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.18072012066841125,
+      "rewards/margins": 1.9877647161483765,
+      "rewards/rejected": -1.807044267654419,
+      "step": 115,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.5190156599552572,
+      "grad_norm": 4.979251384735107,
+      "learning_rate": 0.00017707972125799735,
+      "logits/chosen": -0.3695046305656433,
+      "logits/rejected": -0.4696848392486572,
+      "logps/chosen": -6.748042583465576,
+      "logps/rejected": -28.866714477539062,
+      "loss": 0.8665784597396851,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5186231732368469,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.20859460532665253,
+      "rewards/margins": 1.632576584815979,
+      "rewards/rejected": -1.4239819049835205,
+      "step": 116,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.5234899328859061,
+      "grad_norm": 2.385962963104248,
+      "learning_rate": 0.0001766044443118978,
+      "logits/chosen": -0.3143601715564728,
+      "logits/rejected": -0.3878296911716461,
+      "logps/chosen": -8.803234100341797,
+      "logps/rejected": -24.038074493408203,
+      "loss": 1.0012811422348022,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5274536609649658,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.05610959604382515,
+      "rewards/margins": 1.1446653604507446,
+      "rewards/rejected": -1.0885558128356934,
+      "step": 117,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.5279642058165548,
+      "grad_norm": 4.134023189544678,
+      "learning_rate": 0.0001761249419351222,
+      "logits/chosen": -0.3163571357727051,
+      "logits/rejected": -0.4659003019332886,
+      "logps/chosen": -5.800329208374023,
+      "logps/rejected": -33.57259750366211,
+      "loss": 0.824342668056488,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4846615791320801,
+      "rewards/accuracies": 0.890625,
+      "rewards/chosen": 0.10410095751285553,
+      "rewards/margins": 1.8639620542526245,
+      "rewards/rejected": -1.7598611116409302,
+      "step": 118,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.5324384787472036,
+      "grad_norm": 2.1616837978363037,
+      "learning_rate": 0.00017564124057658056,
+      "logits/chosen": -0.32389989495277405,
+      "logits/rejected": -0.4355754852294922,
+      "logps/chosen": -9.749198913574219,
+      "logps/rejected": -28.6905517578125,
+      "loss": 0.9396259784698486,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5555200576782227,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.15960067510604858,
+      "rewards/margins": 1.569477915763855,
+      "rewards/rejected": -1.4098771810531616,
+      "step": 119,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.5369127516778524,
+      "grad_norm": 6.066991806030273,
+      "learning_rate": 0.00017515336691679477,
+      "logits/chosen": -0.3255332112312317,
+      "logits/rejected": -0.4361691474914551,
+      "logps/chosen": -5.765564918518066,
+      "logps/rejected": -29.787982940673828,
+      "loss": 0.8768988847732544,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4620920717716217,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": 0.06438101083040237,
+      "rewards/margins": 1.6063313484191895,
+      "rewards/rejected": -1.5419504642486572,
+      "step": 120,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.5413870246085011,
+      "grad_norm": 3.371220827102661,
+      "learning_rate": 0.0001746613478664271,
+      "logits/chosen": -0.3099900186061859,
+      "logits/rejected": -0.3865930438041687,
+      "logps/chosen": -9.693756103515625,
+      "logps/rejected": -29.932273864746094,
+      "loss": 0.9306422472000122,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5374558568000793,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": 0.13387000560760498,
+      "rewards/margins": 1.695232629776001,
+      "rewards/rejected": -1.5613627433776855,
+      "step": 121,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.5458612975391499,
+      "grad_norm": 2.195613384246826,
+      "learning_rate": 0.00017416521056479577,
+      "logits/chosen": -0.26501592993736267,
+      "logits/rejected": -0.42681097984313965,
+      "logps/chosen": -6.962400436401367,
+      "logps/rejected": -33.85113525390625,
+      "loss": 0.81517094373703,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4922808110713959,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1830664575099945,
+      "rewards/margins": 2.055619478225708,
+      "rewards/rejected": -1.8725531101226807,
+      "step": 122,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.5503355704697986,
+      "grad_norm": 2.068668842315674,
+      "learning_rate": 0.0001736649823783779,
+      "logits/chosen": -0.30316585302352905,
+      "logits/rejected": -0.45119816064834595,
+      "logps/chosen": -7.222802639007568,
+      "logps/rejected": -36.19852066040039,
+      "loss": 0.8215571045875549,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5099644660949707,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.15236249566078186,
+      "rewards/margins": 2.028899669647217,
+      "rewards/rejected": -1.8765370845794678,
+      "step": 123,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.5548098434004475,
+      "grad_norm": 3.336085557937622,
+      "learning_rate": 0.00017316069089930007,
+      "logits/chosen": -0.30727842450141907,
+      "logits/rejected": -0.440543532371521,
+      "logps/chosen": -7.512927532196045,
+      "logps/rejected": -35.85497283935547,
+      "loss": 0.913392186164856,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5231627821922302,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.062082529067993164,
+      "rewards/margins": 1.9021214246749878,
+      "rewards/rejected": -1.840038776397705,
+      "step": 124,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.5592841163310962,
+      "grad_norm": 2.1725635528564453,
+      "learning_rate": 0.00017265236394381633,
+      "logits/chosen": -0.32654696702957153,
+      "logits/rejected": -0.4225502014160156,
+      "logps/chosen": -7.848655700683594,
+      "logps/rejected": -28.869049072265625,
+      "loss": 0.9009624123573303,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4998895227909088,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": 0.15675391256809235,
+      "rewards/margins": 1.4675108194351196,
+      "rewards/rejected": -1.3107569217681885,
+      "step": 125,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.5637583892617449,
+      "grad_norm": 3.518026113510132,
+      "learning_rate": 0.00017214002955077393,
+      "logits/chosen": -0.28199663758277893,
+      "logits/rejected": -0.3950486481189728,
+      "logps/chosen": -6.079558849334717,
+      "logps/rejected": -29.816913604736328,
+      "loss": 0.8342326283454895,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.49542737007141113,
+      "rewards/accuracies": 0.890625,
+      "rewards/chosen": 0.1437724232673645,
+      "rewards/margins": 1.676330804824829,
+      "rewards/rejected": -1.5325582027435303,
+      "step": 126,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.5682326621923938,
+      "grad_norm": 2.908057928085327,
+      "learning_rate": 0.00017162371598006666,
+      "logits/chosen": -0.2865302562713623,
+      "logits/rejected": -0.39790090918540955,
+      "logps/chosen": -8.84012508392334,
+      "logps/rejected": -25.36516761779785,
+      "loss": 0.9509469866752625,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5248587131500244,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": 0.13465049862861633,
+      "rewards/margins": 1.1559473276138306,
+      "rewards/rejected": -1.0212968587875366,
+      "step": 127,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.5727069351230425,
+      "grad_norm": 3.0232553482055664,
+      "learning_rate": 0.0001711034517110761,
+      "logits/chosen": -0.29277676343917847,
+      "logits/rejected": -0.42875176668167114,
+      "logps/chosen": -5.9850897789001465,
+      "logps/rejected": -23.161203384399414,
+      "loss": 0.9273078441619873,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.49909183382987976,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.23866456747055054,
+      "rewards/margins": 1.1174073219299316,
+      "rewards/rejected": -0.8787426352500916,
+      "step": 128,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.5771812080536913,
+      "grad_norm": 2.585862636566162,
+      "learning_rate": 0.0001705792654411007,
+      "logits/chosen": -0.28234124183654785,
+      "logits/rejected": -0.43345722556114197,
+      "logps/chosen": -6.964176177978516,
+      "logps/rejected": -29.49521827697754,
+      "loss": 0.8882912397384644,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5149023532867432,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.13531282544136047,
+      "rewards/margins": 1.5991878509521484,
+      "rewards/rejected": -1.4638750553131104,
+      "step": 129,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.5816554809843401,
+      "grad_norm": 1.718246579170227,
+      "learning_rate": 0.00017005118608377288,
+      "logits/chosen": -0.25894033908843994,
+      "logits/rejected": -0.35234570503234863,
+      "logps/chosen": -7.487059116363525,
+      "logps/rejected": -31.42106056213379,
+      "loss": 0.8475052714347839,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4482796788215637,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.10318747907876968,
+      "rewards/margins": 1.6407992839813232,
+      "rewards/rejected": -1.537611722946167,
+      "step": 130,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.5861297539149888,
+      "grad_norm": 1.766929268836975,
+      "learning_rate": 0.00016951924276746425,
+      "logits/chosen": -0.2688104212284088,
+      "logits/rejected": -0.3982466757297516,
+      "logps/chosen": -4.793079853057861,
+      "logps/rejected": -32.5130615234375,
+      "loss": 0.6818744540214539,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3626863360404968,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.21576498448848724,
+      "rewards/margins": 2.161926746368408,
+      "rewards/rejected": -1.9461617469787598,
+      "step": 131,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.5906040268456376,
+      "grad_norm": 2.9491937160491943,
+      "learning_rate": 0.00016898346483367867,
+      "logits/chosen": -0.21997107565402985,
+      "logits/rejected": -0.4120599627494812,
+      "logps/chosen": -5.662740230560303,
+      "logps/rejected": -35.41563034057617,
+      "loss": 0.8450470566749573,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4608619809150696,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.16963335871696472,
+      "rewards/margins": 2.1595871448516846,
+      "rewards/rejected": -1.989953637123108,
+      "step": 132,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.5950782997762863,
+      "grad_norm": 3.147080659866333,
+      "learning_rate": 0.00016844388183543418,
+      "logits/chosen": -0.24002769589424133,
+      "logits/rejected": -0.40396854281425476,
+      "logps/chosen": -4.948556423187256,
+      "logps/rejected": -46.72270965576172,
+      "loss": 0.6112457513809204,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.35614243149757385,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.04689760506153107,
+      "rewards/margins": 3.0640969276428223,
+      "rewards/rejected": -3.0171995162963867,
+      "step": 133,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.5995525727069351,
+      "grad_norm": 3.0668933391571045,
+      "learning_rate": 0.00016790052353563253,
+      "logits/chosen": -0.23955634236335754,
+      "logits/rejected": -0.4039704203605652,
+      "logps/chosen": -7.274659156799316,
+      "logps/rejected": -42.970394134521484,
+      "loss": 0.8683923482894897,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5945435762405396,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.08974725753068924,
+      "rewards/margins": 2.8205695152282715,
+      "rewards/rejected": -2.7308225631713867,
+      "step": 134,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.6040268456375839,
+      "grad_norm": 5.152528762817383,
+      "learning_rate": 0.00016735341990541764,
+      "logits/chosen": -0.2829990088939667,
+      "logits/rejected": -0.3635609447956085,
+      "logps/chosen": -13.53715705871582,
+      "logps/rejected": -37.386192321777344,
+      "loss": 1.2928409576416016,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.8582869172096252,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.11900941282510757,
+      "rewards/margins": 1.9026870727539062,
+      "rewards/rejected": -2.0216965675354004,
+      "step": 135,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.6085011185682326,
+      "grad_norm": 3.047576665878296,
+      "learning_rate": 0.0001668026011225225,
+      "logits/chosen": -0.1925339251756668,
+      "logits/rejected": -0.3699888586997986,
+      "logps/chosen": -7.0672407150268555,
+      "logps/rejected": -38.66764831542969,
+      "loss": 0.8304687738418579,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5318548083305359,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 0.06892228871583939,
+      "rewards/margins": 2.3139286041259766,
+      "rewards/rejected": -2.245006561279297,
+      "step": 136,
+      "train_speed(iter/s)": 0.005623
+    },
+    {
+      "epoch": 0.6129753914988815,
+      "grad_norm": 2.2745347023010254,
+      "learning_rate": 0.00016624809756960444,
+      "logits/chosen": -0.11182728409767151,
+      "logits/rejected": -0.2984737455844879,
+      "logps/chosen": -5.8051066398620605,
+      "logps/rejected": -31.151941299438477,
+      "loss": 0.7885230779647827,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.38472679257392883,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.11600922793149948,
+      "rewards/margins": 1.5462919473648071,
+      "rewards/rejected": -1.430282711982727,
+      "step": 137,
+      "train_speed(iter/s)": 0.005623
+    },
+    {
+      "epoch": 0.6174496644295302,
+      "grad_norm": 2.21187162399292,
+      "learning_rate": 0.0001656899398325693,
+      "logits/chosen": -0.17814204096794128,
+      "logits/rejected": -0.28958842158317566,
+      "logps/chosen": -11.657821655273438,
+      "logps/rejected": -28.357452392578125,
+      "loss": 1.0776360034942627,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6752557754516602,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.15066924691200256,
+      "rewards/margins": 1.488224744796753,
+      "rewards/rejected": -1.3375554084777832,
+      "step": 138,
+      "train_speed(iter/s)": 0.005623
+    },
+    {
+      "epoch": 0.6219239373601789,
+      "grad_norm": 2.826819896697998,
+      "learning_rate": 0.0001651281586988844,
+      "logits/chosen": -0.2112751007080078,
+      "logits/rejected": -0.3410148024559021,
+      "logps/chosen": -6.036767482757568,
+      "logps/rejected": -22.628429412841797,
+      "loss": 0.9106030464172363,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4903098940849304,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": 0.15719373524188995,
+      "rewards/margins": 1.1676839590072632,
+      "rewards/rejected": -1.0104901790618896,
+      "step": 139,
+      "train_speed(iter/s)": 0.005623
+    },
+    {
+      "epoch": 0.6263982102908278,
+      "grad_norm": 1.9086319208145142,
+      "learning_rate": 0.00016456278515588024,
+      "logits/chosen": -0.17578895390033722,
+      "logits/rejected": -0.2394012212753296,
+      "logps/chosen": -7.572815895080566,
+      "logps/rejected": -27.15605926513672,
+      "loss": 0.9325375556945801,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5454995036125183,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.1564534306526184,
+      "rewards/margins": 1.4798097610473633,
+      "rewards/rejected": -1.3233562707901,
+      "step": 140,
+      "train_speed(iter/s)": 0.005623
+    },
+    {
+      "epoch": 0.6308724832214765,
+      "grad_norm": 2.0129153728485107,
+      "learning_rate": 0.00016399385038904138,
+      "logits/chosen": -0.15921340882778168,
+      "logits/rejected": -0.3579505681991577,
+      "logps/chosen": -4.274261951446533,
+      "logps/rejected": -33.25766372680664,
+      "loss": 0.767472505569458,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.40454328060150146,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 0.07659953087568283,
+      "rewards/margins": 1.6252936124801636,
+      "rewards/rejected": -1.5486938953399658,
+      "step": 141,
+      "train_speed(iter/s)": 0.005624
+    },
+    {
+      "epoch": 0.6353467561521253,
+      "grad_norm": 2.377077579498291,
+      "learning_rate": 0.00016342138578028613,
+      "logits/chosen": -0.0892738476395607,
+      "logits/rejected": -0.2357676923274994,
+      "logps/chosen": -7.721086502075195,
+      "logps/rejected": -30.33781623840332,
+      "loss": 0.948746919631958,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5438387393951416,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": 0.15941840410232544,
+      "rewards/margins": 1.4621765613555908,
+      "rewards/rejected": -1.3027582168579102,
+      "step": 142,
+      "train_speed(iter/s)": 0.005623
+    },
+    {
+      "epoch": 0.639821029082774,
+      "grad_norm": 2.5466043949127197,
+      "learning_rate": 0.00016284542290623567,
+      "logits/chosen": -0.2347170114517212,
+      "logits/rejected": -0.37242478132247925,
+      "logps/chosen": -6.517569065093994,
+      "logps/rejected": -34.21914291381836,
+      "loss": 0.7467401027679443,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4813250005245209,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.12270703911781311,
+      "rewards/margins": 2.2491090297698975,
+      "rewards/rejected": -2.1264021396636963,
+      "step": 143,
+      "train_speed(iter/s)": 0.005624
+    },
+    {
+      "epoch": 0.6442953020134228,
+      "grad_norm": 2.4996299743652344,
+      "learning_rate": 0.00016226599353647228,
+      "logits/chosen": -0.21299152076244354,
+      "logits/rejected": -0.3434564769268036,
+      "logps/chosen": -8.105118751525879,
+      "logps/rejected": -33.501346588134766,
+      "loss": 0.8418401479721069,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4816139042377472,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.0775795578956604,
+      "rewards/margins": 1.7935457229614258,
+      "rewards/rejected": -1.7159662246704102,
+      "step": 144,
+      "train_speed(iter/s)": 0.005624
+    },
+    {
+      "epoch": 0.6487695749440716,
+      "grad_norm": 2.852815866470337,
+      "learning_rate": 0.00016168312963178697,
+      "logits/chosen": -0.14896008372306824,
+      "logits/rejected": -0.29324355721473694,
+      "logps/chosen": -8.402843475341797,
+      "logps/rejected": -30.785614013671875,
+      "loss": 0.966815173625946,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6201918721199036,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.06825412809848785,
+      "rewards/margins": 1.6479313373565674,
+      "rewards/rejected": -1.5796774625778198,
+      "step": 145,
+      "train_speed(iter/s)": 0.005624
+    },
+    {
+      "epoch": 0.6532438478747203,
+      "grad_norm": 3.7648990154266357,
+      "learning_rate": 0.00016109686334241655,
+      "logits/chosen": -0.19154362380504608,
+      "logits/rejected": -0.3218660056591034,
+      "logps/chosen": -10.407678604125977,
+      "logps/rejected": -36.2245979309082,
+      "loss": 0.9397022128105164,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6243451833724976,
+      "rewards/accuracies": 0.890625,
+      "rewards/chosen": 0.07954104989767075,
+      "rewards/margins": 2.1638622283935547,
+      "rewards/rejected": -2.0843210220336914,
+      "step": 146,
+      "train_speed(iter/s)": 0.005624
+    },
+    {
+      "epoch": 0.6577181208053692,
+      "grad_norm": 5.735119819641113,
+      "learning_rate": 0.00016050722700627012,
+      "logits/chosen": -0.18544235825538635,
+      "logits/rejected": -0.38329729437828064,
+      "logps/chosen": -5.036187171936035,
+      "logps/rejected": -37.17473602294922,
+      "loss": 0.7683266401290894,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4119165539741516,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.026729077100753784,
+      "rewards/margins": 1.973728895187378,
+      "rewards/rejected": -1.9470000267028809,
+      "step": 147,
+      "train_speed(iter/s)": 0.005624
+    },
+    {
+      "epoch": 0.6621923937360179,
+      "grad_norm": 2.958143472671509,
+      "learning_rate": 0.0001599142531471456,
+      "logits/chosen": -0.22263091802597046,
+      "logits/rejected": -0.3673292100429535,
+      "logps/chosen": -8.627145767211914,
+      "logps/rejected": -34.2421875,
+      "loss": 0.8043665885925293,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4987104833126068,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": 0.11155197024345398,
+      "rewards/margins": 1.846522569656372,
+      "rewards/rejected": -1.7349706888198853,
+      "step": 148,
+      "train_speed(iter/s)": 0.005624
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 1.810313105583191,
+      "learning_rate": 0.00015931797447293552,
+      "logits/chosen": -0.25553733110427856,
+      "logits/rejected": -0.38629549741744995,
+      "logps/chosen": -5.752464294433594,
+      "logps/rejected": -25.610708236694336,
+      "loss": 0.7433949708938599,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3995458483695984,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.1192181408405304,
+      "rewards/margins": 1.3859959840774536,
+      "rewards/rejected": -1.266777753829956,
+      "step": 149,
+      "train_speed(iter/s)": 0.005625
+    },
+    {
+      "epoch": 0.6711409395973155,
+      "grad_norm": 4.102981090545654,
+      "learning_rate": 0.00015871842387382305,
+      "logits/chosen": -0.2806379497051239,
+      "logits/rejected": -0.390668660402298,
+      "logps/chosen": -15.568105697631836,
+      "logps/rejected": -25.584888458251953,
+      "loss": 1.1545464992523193,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6800742745399475,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.030628615990281105,
+      "rewards/margins": 1.1961755752563477,
+      "rewards/rejected": -1.1655468940734863,
+      "step": 150,
+      "train_speed(iter/s)": 0.005625
+    },
+    {
+      "epoch": 0.6756152125279642,
+      "grad_norm": 2.4007022380828857,
+      "learning_rate": 0.00015811563442046767,
+      "logits/chosen": -0.21862031519412994,
+      "logits/rejected": -0.4358934164047241,
+      "logps/chosen": -7.3923821449279785,
+      "logps/rejected": -38.984134674072266,
+      "loss": 0.8094646334648132,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.510733425617218,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 0.1613074541091919,
+      "rewards/margins": 2.380244016647339,
+      "rewards/rejected": -2.2189366817474365,
+      "step": 151,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.680089485458613,
+      "grad_norm": 1.7411408424377441,
+      "learning_rate": 0.00015750963936218105,
+      "logits/chosen": -0.31933140754699707,
+      "logits/rejected": -0.47491735219955444,
+      "logps/chosen": -6.779457092285156,
+      "logps/rejected": -43.27799987792969,
+      "loss": 0.8203473091125488,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4936409592628479,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.09017263352870941,
+      "rewards/margins": 2.683919906616211,
+      "rewards/rejected": -2.593747138977051,
+      "step": 152,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.6845637583892618,
+      "grad_norm": 4.146765232086182,
+      "learning_rate": 0.00015690047212509316,
+      "logits/chosen": -0.37495219707489014,
+      "logits/rejected": -0.4795547127723694,
+      "logps/chosen": -7.767773628234863,
+      "logps/rejected": -37.96726608276367,
+      "loss": 0.8648288249969482,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5447298884391785,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.08953765779733658,
+      "rewards/margins": 2.2388553619384766,
+      "rewards/rejected": -2.149317741394043,
+      "step": 153,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.6890380313199105,
+      "grad_norm": 2.1461992263793945,
+      "learning_rate": 0.00015628816631030836,
+      "logits/chosen": -0.32161566615104675,
+      "logits/rejected": -0.43094250559806824,
+      "logps/chosen": -8.232239723205566,
+      "logps/rejected": -36.47972106933594,
+      "loss": 0.7230132222175598,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.391425758600235,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.12491930276155472,
+      "rewards/margins": 2.1603236198425293,
+      "rewards/rejected": -2.035404682159424,
+      "step": 154,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.6935123042505593,
+      "grad_norm": 4.374569416046143,
+      "learning_rate": 0.00015567275569205218,
+      "logits/chosen": -0.25436070561408997,
+      "logits/rejected": -0.45995038747787476,
+      "logps/chosen": -8.461811065673828,
+      "logps/rejected": -44.51617431640625,
+      "loss": 0.9842564463615417,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6817278861999512,
+      "rewards/accuracies": 0.890625,
+      "rewards/chosen": 0.040151700377464294,
+      "rewards/margins": 2.510289430618286,
+      "rewards/rejected": -2.47013783454895,
+      "step": 155,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.697986577181208,
+      "grad_norm": 4.502596378326416,
+      "learning_rate": 0.00015505427421580808,
+      "logits/chosen": -0.4160749316215515,
+      "logits/rejected": -0.506524384021759,
+      "logps/chosen": -8.063937187194824,
+      "logps/rejected": -32.82113265991211,
+      "loss": 0.8426942825317383,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5383368730545044,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 0.11457733064889908,
+      "rewards/margins": 2.155301094055176,
+      "rewards/rejected": -2.0407238006591797,
+      "step": 156,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 0.7024608501118568,
+      "grad_norm": 3.176142692565918,
+      "learning_rate": 0.00015443275599644538,
+      "logits/chosen": -0.42277562618255615,
+      "logits/rejected": -0.5972986817359924,
+      "logps/chosen": -5.387449264526367,
+      "logps/rejected": -45.159637451171875,
+      "loss": 0.6948708295822144,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4455086886882782,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1367429941892624,
+      "rewards/margins": 3.0624241828918457,
+      "rewards/rejected": -2.9256811141967773,
+      "step": 157,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.7069351230425056,
+      "grad_norm": 3.3241329193115234,
+      "learning_rate": 0.00015380823531633729,
+      "logits/chosen": -0.3996984660625458,
+      "logits/rejected": -0.5501067042350769,
+      "logps/chosen": -6.42928409576416,
+      "logps/rejected": -34.65045166015625,
+      "loss": 0.7898067235946655,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.474100798368454,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 0.12420983612537384,
+      "rewards/margins": 2.1587226390838623,
+      "rewards/rejected": -2.034512758255005,
+      "step": 158,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.7114093959731543,
+      "grad_norm": 3.497190237045288,
+      "learning_rate": 0.00015318074662346994,
+      "logits/chosen": -0.41886377334594727,
+      "logits/rejected": -0.5915812849998474,
+      "logps/chosen": -7.062689781188965,
+      "logps/rejected": -42.2303466796875,
+      "loss": 0.7893521785736084,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.46673545241355896,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.0015753507614135742,
+      "rewards/margins": 2.453634738922119,
+      "rewards/rejected": -2.452059745788574,
+      "step": 159,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.7158836689038032,
+      "grad_norm": 2.7327327728271484,
+      "learning_rate": 0.00015255032452954245,
+      "logits/chosen": -0.3371175527572632,
+      "logits/rejected": -0.5264066457748413,
+      "logps/chosen": -7.435405731201172,
+      "logps/rejected": -50.61220932006836,
+      "loss": 0.7223600149154663,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4396481215953827,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.025311682373285294,
+      "rewards/margins": 3.0541183948516846,
+      "rewards/rejected": -3.028806447982788,
+      "step": 160,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.7203579418344519,
+      "grad_norm": 4.352429389953613,
+      "learning_rate": 0.00015191700380805752,
+      "logits/chosen": -0.44862544536590576,
+      "logits/rejected": -0.5382689833641052,
+      "logps/chosen": -10.93178939819336,
+      "logps/rejected": -35.30765151977539,
+      "loss": 0.9672499895095825,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5727404356002808,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.02203945443034172,
+      "rewards/margins": 2.1007864475250244,
+      "rewards/rejected": -2.122826099395752,
+      "step": 161,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.7248322147651006,
+      "grad_norm": 4.718836784362793,
+      "learning_rate": 0.00015128081939240357,
+      "logits/chosen": -0.3789413869380951,
+      "logits/rejected": -0.5269034504890442,
+      "logps/chosen": -6.491231441497803,
+      "logps/rejected": -39.26808547973633,
+      "loss": 0.8075002431869507,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4541192948818207,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 0.02522008866071701,
+      "rewards/margins": 2.450805187225342,
+      "rewards/rejected": -2.4255850315093994,
+      "step": 162,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.7293064876957495,
+      "grad_norm": 3.2440578937530518,
+      "learning_rate": 0.00015064180637392764,
+      "logits/chosen": -0.3883664608001709,
+      "logits/rejected": -0.5545072555541992,
+      "logps/chosen": -4.190845012664795,
+      "logps/rejected": -28.679349899291992,
+      "loss": 0.7410864233970642,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3713621199131012,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.13914553821086884,
+      "rewards/margins": 1.7679030895233154,
+      "rewards/rejected": -1.6287575960159302,
+      "step": 163,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.7337807606263982,
+      "grad_norm": 2.2508671283721924,
+      "learning_rate": 0.00015000000000000001,
+      "logits/chosen": -0.41260772943496704,
+      "logits/rejected": -0.5085133910179138,
+      "logps/chosen": -9.365473747253418,
+      "logps/rejected": -26.65607452392578,
+      "loss": 0.8695790767669678,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5267202258110046,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 0.19111280143260956,
+      "rewards/margins": 1.517545461654663,
+      "rewards/rejected": -1.3264328241348267,
+      "step": 164,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.738255033557047,
+      "grad_norm": 1.987011194229126,
+      "learning_rate": 0.00014935543567206984,
+      "logits/chosen": -0.4246593117713928,
+      "logits/rejected": -0.5221028327941895,
+      "logps/chosen": -7.673718452453613,
+      "logps/rejected": -29.984445571899414,
+      "loss": 0.9060878157615662,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5306745171546936,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 0.12876029312610626,
+      "rewards/margins": 1.761435627937317,
+      "rewards/rejected": -1.6326754093170166,
+      "step": 165,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.7427293064876958,
+      "grad_norm": 2.439631700515747,
+      "learning_rate": 0.00014870814894371245,
+      "logits/chosen": -0.4619531035423279,
+      "logits/rejected": -0.5249654054641724,
+      "logps/chosen": -10.422987937927246,
+      "logps/rejected": -28.858501434326172,
+      "loss": 0.9730715155601501,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6119397282600403,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.0721518024802208,
+      "rewards/margins": 1.6050055027008057,
+      "rewards/rejected": -1.532853603363037,
+      "step": 166,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.7472035794183445,
+      "grad_norm": 2.184906482696533,
+      "learning_rate": 0.00014805817551866838,
+      "logits/chosen": -0.4090336859226227,
+      "logits/rejected": -0.5729665160179138,
+      "logps/chosen": -6.691109657287598,
+      "logps/rejected": -31.740489959716797,
+      "loss": 0.7506605386734009,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4180562496185303,
+      "rewards/accuracies": 0.890625,
+      "rewards/chosen": 0.16018937528133392,
+      "rewards/margins": 1.805705189704895,
+      "rewards/rejected": -1.6455157995224,
+      "step": 167,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.7516778523489933,
+      "grad_norm": 3.0194168090820312,
+      "learning_rate": 0.00014740555124887375,
+      "logits/chosen": -0.4370940029621124,
+      "logits/rejected": -0.5650765895843506,
+      "logps/chosen": -8.651978492736816,
+      "logps/rejected": -27.381715774536133,
+      "loss": 1.0154253244400024,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5850539207458496,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.10986748337745667,
+      "rewards/margins": 1.2420449256896973,
+      "rewards/rejected": -1.1321773529052734,
+      "step": 168,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.756152125279642,
+      "grad_norm": 2.3874940872192383,
+      "learning_rate": 0.00014675031213248296,
+      "logits/chosen": -0.47348707914352417,
+      "logits/rejected": -0.6095517873764038,
+      "logps/chosen": -6.874096870422363,
+      "logps/rejected": -32.03157424926758,
+      "loss": 0.9490684866905212,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5718879699707031,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 0.11709172278642654,
+      "rewards/margins": 1.9521950483322144,
+      "rewards/rejected": -1.8351032733917236,
+      "step": 169,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.7606263982102909,
+      "grad_norm": 2.2490291595458984,
+      "learning_rate": 0.00014609249431188278,
+      "logits/chosen": -0.47727060317993164,
+      "logits/rejected": -0.629499077796936,
+      "logps/chosen": -9.573102951049805,
+      "logps/rejected": -40.56207275390625,
+      "loss": 0.8739838600158691,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5675899982452393,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 0.07386627048254013,
+      "rewards/margins": 2.4693443775177,
+      "rewards/rejected": -2.3954782485961914,
+      "step": 170,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.7651006711409396,
+      "grad_norm": 2.2746880054473877,
+      "learning_rate": 0.0001454321340716992,
+      "logits/chosen": -0.4521310031414032,
+      "logits/rejected": -0.6028440594673157,
+      "logps/chosen": -6.765651702880859,
+      "logps/rejected": -34.42289352416992,
+      "loss": 0.8103550672531128,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4473325312137604,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.11047148704528809,
+      "rewards/margins": 2.000337600708008,
+      "rewards/rejected": -1.8898662328720093,
+      "step": 171,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.7695749440715883,
+      "grad_norm": 3.201479196548462,
+      "learning_rate": 0.00014476926783679538,
+      "logits/chosen": -0.5274945497512817,
+      "logits/rejected": -0.6449275612831116,
+      "logps/chosen": -7.808686256408691,
+      "logps/rejected": -29.541404724121094,
+      "loss": 0.8972499370574951,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5542759895324707,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.13792771100997925,
+      "rewards/margins": 1.9822808504104614,
+      "rewards/rejected": -1.8443530797958374,
+      "step": 172,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.7740492170022372,
+      "grad_norm": 2.665815830230713,
+      "learning_rate": 0.00014410393217026318,
+      "logits/chosen": -0.4524228870868683,
+      "logits/rejected": -0.6132454872131348,
+      "logps/chosen": -10.15006160736084,
+      "logps/rejected": -41.52241516113281,
+      "loss": 0.8796108365058899,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5965855717658997,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.20403775572776794,
+      "rewards/margins": 2.6676454544067383,
+      "rewards/rejected": -2.4636075496673584,
+      "step": 173,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.7785234899328859,
+      "grad_norm": 3.2595131397247314,
+      "learning_rate": 0.00014343616377140582,
+      "logits/chosen": -0.5274872779846191,
+      "logits/rejected": -0.6744863986968994,
+      "logps/chosen": -8.180425643920898,
+      "logps/rejected": -43.29051971435547,
+      "loss": 0.7626029253005981,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4893677234649658,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.07820628583431244,
+      "rewards/margins": 2.6948142051696777,
+      "rewards/rejected": -2.616607904434204,
+      "step": 174,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.7829977628635347,
+      "grad_norm": 3.828737735748291,
+      "learning_rate": 0.00014276599947371388,
+      "logits/chosen": -0.49092575907707214,
+      "logits/rejected": -0.6571353673934937,
+      "logps/chosen": -5.790119647979736,
+      "logps/rejected": -36.81107711791992,
+      "loss": 0.7012597918510437,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.41794249415397644,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.11556562781333923,
+      "rewards/margins": 2.311169385910034,
+      "rewards/rejected": -2.195603847503662,
+      "step": 175,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.7874720357941835,
+      "grad_norm": 8.090876579284668,
+      "learning_rate": 0.0001420934762428335,
+      "logits/chosen": -0.47237420082092285,
+      "logits/rejected": -0.6282271146774292,
+      "logps/chosen": -8.150810241699219,
+      "logps/rejected": -43.93907928466797,
+      "loss": 0.943342924118042,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6251461505889893,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": -0.03655552491545677,
+      "rewards/margins": 2.5700528621673584,
+      "rewards/rejected": -2.6066083908081055,
+      "step": 176,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.7919463087248322,
+      "grad_norm": 2.371922016143799,
+      "learning_rate": 0.00014141863117452745,
+      "logits/chosen": -0.5399549007415771,
+      "logits/rejected": -0.6778814196586609,
+      "logps/chosen": -5.152353763580322,
+      "logps/rejected": -44.89213562011719,
+      "loss": 0.6812291741371155,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.40526461601257324,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.09105440229177475,
+      "rewards/margins": 3.134105682373047,
+      "rewards/rejected": -3.043051242828369,
+      "step": 177,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.796420581655481,
+      "grad_norm": 3.8354110717773438,
+      "learning_rate": 0.0001407415014926288,
+      "logits/chosen": -0.5333717465400696,
+      "logits/rejected": -0.6879783868789673,
+      "logps/chosen": -8.797846794128418,
+      "logps/rejected": -42.53145980834961,
+      "loss": 0.8743501901626587,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6057224273681641,
+      "rewards/accuracies": 0.890625,
+      "rewards/chosen": 0.014064118266105652,
+      "rewards/margins": 2.73359751701355,
+      "rewards/rejected": -2.7195334434509277,
+      "step": 178,
+      "train_speed(iter/s)": 0.005623
+    },
+    {
+      "epoch": 0.8008948545861297,
+      "grad_norm": 2.9632174968719482,
+      "learning_rate": 0.00014006212454698797,
+      "logits/chosen": -0.5371872782707214,
+      "logits/rejected": -0.6701904535293579,
+      "logps/chosen": -8.69965648651123,
+      "logps/rejected": -35.25392150878906,
+      "loss": 0.9904757738113403,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.7375240325927734,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": 0.1597575545310974,
+      "rewards/margins": 2.451249599456787,
+      "rewards/rejected": -2.291491746902466,
+      "step": 179,
+      "train_speed(iter/s)": 0.005623
+    },
+    {
+      "epoch": 0.8053691275167785,
+      "grad_norm": 3.797736406326294,
+      "learning_rate": 0.00013938053781141222,
+      "logits/chosen": -0.4403976798057556,
+      "logits/rejected": -0.6186584234237671,
+      "logps/chosen": -6.943508148193359,
+      "logps/rejected": -43.18886947631836,
+      "loss": 0.7113572955131531,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.47942519187927246,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.0965544655919075,
+      "rewards/margins": 2.6151046752929688,
+      "rewards/rejected": -2.518550157546997,
+      "step": 180,
+      "train_speed(iter/s)": 0.005623
+    },
+    {
+      "epoch": 0.8098434004474273,
+      "grad_norm": 2.0015857219696045,
+      "learning_rate": 0.00013869677888159887,
+      "logits/chosen": -0.4414394497871399,
+      "logits/rejected": -0.5980672240257263,
+      "logps/chosen": -6.416171550750732,
+      "logps/rejected": -34.68084716796875,
+      "loss": 0.7498472929000854,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4237560033798218,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.1907028704881668,
+      "rewards/margins": 1.996950626373291,
+      "rewards/rejected": -1.8062477111816406,
+      "step": 181,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.814317673378076,
+      "grad_norm": 3.0869336128234863,
+      "learning_rate": 0.00013801088547306148,
+      "logits/chosen": -0.38094401359558105,
+      "logits/rejected": -0.5588433742523193,
+      "logps/chosen": -6.33656120300293,
+      "logps/rejected": -34.21479797363281,
+      "loss": 0.7289522886276245,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.42591309547424316,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": 0.25475332140922546,
+      "rewards/margins": 2.0227596759796143,
+      "rewards/rejected": -1.7680062055587769,
+      "step": 182,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.8187919463087249,
+      "grad_norm": 3.356872081756592,
+      "learning_rate": 0.00013732289541904948,
+      "logits/chosen": -0.4178617596626282,
+      "logits/rejected": -0.6289569139480591,
+      "logps/chosen": -7.5165696144104,
+      "logps/rejected": -40.291988372802734,
+      "loss": 0.7296820282936096,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.44231122732162476,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.10255959630012512,
+      "rewards/margins": 2.4512202739715576,
+      "rewards/rejected": -2.348660707473755,
+      "step": 183,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.8232662192393736,
+      "grad_norm": 3.37188720703125,
+      "learning_rate": 0.00013663284666846134,
+      "logits/chosen": -0.4483550786972046,
+      "logits/rejected": -0.6116241812705994,
+      "logps/chosen": -7.468384742736816,
+      "logps/rejected": -36.9821662902832,
+      "loss": 0.8177734017372131,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5343176126480103,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.13721218705177307,
+      "rewards/margins": 2.2863237857818604,
+      "rewards/rejected": -2.14911150932312,
+      "step": 184,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.8277404921700223,
+      "grad_norm": 5.6198577880859375,
+      "learning_rate": 0.00013594077728375128,
+      "logits/chosen": -0.40004104375839233,
+      "logits/rejected": -0.6040188074111938,
+      "logps/chosen": -4.515679359436035,
+      "logps/rejected": -34.73591232299805,
+      "loss": 0.7695606350898743,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.43972742557525635,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.13223202526569366,
+      "rewards/margins": 2.1134281158447266,
+      "rewards/rejected": -1.9811959266662598,
+      "step": 185,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.8322147651006712,
+      "grad_norm": 2.3837716579437256,
+      "learning_rate": 0.00013524672543882996,
+      "logits/chosen": -0.36424440145492554,
+      "logits/rejected": -0.5849202871322632,
+      "logps/chosen": -5.077719688415527,
+      "logps/rejected": -37.2129020690918,
+      "loss": 0.7024149894714355,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.419405460357666,
+      "rewards/accuracies": 0.890625,
+      "rewards/chosen": 0.14517350494861603,
+      "rewards/margins": 2.114969253540039,
+      "rewards/rejected": -1.9697959423065186,
+      "step": 186,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.8366890380313199,
+      "grad_norm": 1.8797435760498047,
+      "learning_rate": 0.00013455072941695863,
+      "logits/chosen": -0.46017298102378845,
+      "logits/rejected": -0.6270426511764526,
+      "logps/chosen": -5.515599250793457,
+      "logps/rejected": -35.95061492919922,
+      "loss": 0.6589202284812927,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.38542789220809937,
+      "rewards/accuracies": 0.890625,
+      "rewards/chosen": 0.22691603004932404,
+      "rewards/margins": 2.2506327629089355,
+      "rewards/rejected": -2.023716688156128,
+      "step": 187,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.8411633109619687,
+      "grad_norm": 3.041163206100464,
+      "learning_rate": 0.00013385282760863758,
+      "logits/chosen": -0.4110674560070038,
+      "logits/rejected": -0.5675216317176819,
+      "logps/chosen": -8.159733772277832,
+      "logps/rejected": -37.882545471191406,
+      "loss": 0.8134464025497437,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5169687867164612,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.21555905044078827,
+      "rewards/margins": 2.2340266704559326,
+      "rewards/rejected": -2.018467664718628,
+      "step": 188,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.8456375838926175,
+      "grad_norm": 4.433757781982422,
+      "learning_rate": 0.00013315305850948846,
+      "logits/chosen": -0.42173799872398376,
+      "logits/rejected": -0.577892005443573,
+      "logps/chosen": -6.5279011726379395,
+      "logps/rejected": -32.7200927734375,
+      "loss": 0.7939549684524536,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.45606493949890137,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.06638915091753006,
+      "rewards/margins": 1.937835931777954,
+      "rewards/rejected": -1.8714466094970703,
+      "step": 189,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.8501118568232662,
+      "grad_norm": 4.739625453948975,
+      "learning_rate": 0.00013245146071813114,
+      "logits/chosen": -0.325115442276001,
+      "logits/rejected": -0.5505204796791077,
+      "logps/chosen": -5.985814571380615,
+      "logps/rejected": -38.72880172729492,
+      "loss": 0.8021903038024902,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.45717212557792664,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.034357890486717224,
+      "rewards/margins": 2.132763147354126,
+      "rewards/rejected": -2.098405122756958,
+      "step": 190,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.854586129753915,
+      "grad_norm": 2.447476625442505,
+      "learning_rate": 0.00013174807293405428,
+      "logits/chosen": -0.4180159270763397,
+      "logits/rejected": -0.5316296219825745,
+      "logps/chosen": -9.652694702148438,
+      "logps/rejected": -40.23739242553711,
+      "loss": 1.0005970001220703,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6700254678726196,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.08515516668558121,
+      "rewards/margins": 2.6961252689361572,
+      "rewards/rejected": -2.6109700202941895,
+      "step": 191,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.8590604026845637,
+      "grad_norm": 1.7246663570404053,
+      "learning_rate": 0.00013104293395548098,
+      "logits/chosen": -0.3204466998577118,
+      "logits/rejected": -0.5718462467193604,
+      "logps/chosen": -4.133203029632568,
+      "logps/rejected": -42.92612075805664,
+      "loss": 0.6551041603088379,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4175110161304474,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2626938819885254,
+      "rewards/margins": 2.859964370727539,
+      "rewards/rejected": -2.5972704887390137,
+      "step": 192,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.8635346756152126,
+      "grad_norm": 2.3944220542907715,
+      "learning_rate": 0.00013033608267722858,
+      "logits/chosen": -0.2919933497905731,
+      "logits/rejected": -0.521720290184021,
+      "logps/chosen": -5.744174003601074,
+      "logps/rejected": -42.762874603271484,
+      "loss": 0.7408826351165771,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4414190948009491,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.16091150045394897,
+      "rewards/margins": 2.5740907192230225,
+      "rewards/rejected": -2.413179397583008,
+      "step": 193,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.8680089485458613,
+      "grad_norm": 2.7282540798187256,
+      "learning_rate": 0.00012962755808856342,
+      "logits/chosen": -0.3888423442840576,
+      "logits/rejected": -0.5426087379455566,
+      "logps/chosen": -9.648605346679688,
+      "logps/rejected": -37.68000793457031,
+      "loss": 0.9185003638267517,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.596512496471405,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 0.0373542495071888,
+      "rewards/margins": 2.42608904838562,
+      "rewards/rejected": -2.388734817504883,
+      "step": 194,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.87248322147651,
+      "grad_norm": 1.9054832458496094,
+      "learning_rate": 0.0001289173992710499,
+      "logits/chosen": -0.3781818151473999,
+      "logits/rejected": -0.4687708914279938,
+      "logps/chosen": -6.976923942565918,
+      "logps/rejected": -33.893470764160156,
+      "loss": 0.7810498476028442,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4621986150741577,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1968202143907547,
+      "rewards/margins": 2.0472464561462402,
+      "rewards/rejected": -1.8504258394241333,
+      "step": 195,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.8769574944071589,
+      "grad_norm": 4.798694610595703,
+      "learning_rate": 0.00012820564539639512,
+      "logits/chosen": -0.2861044704914093,
+      "logits/rejected": -0.47894322872161865,
+      "logps/chosen": -6.239299774169922,
+      "logps/rejected": -34.055992126464844,
+      "loss": 0.9598355889320374,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5919869542121887,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 0.1414777934551239,
+      "rewards/margins": 2.0753138065338135,
+      "rewards/rejected": -1.9338361024856567,
+      "step": 196,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.8814317673378076,
+      "grad_norm": 2.859259843826294,
+      "learning_rate": 0.00012749233572428804,
+      "logits/chosen": -0.3430550694465637,
+      "logits/rejected": -0.5082629323005676,
+      "logps/chosen": -5.32232666015625,
+      "logps/rejected": -38.88724136352539,
+      "loss": 0.6508762240409851,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4472533166408539,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.28352218866348267,
+      "rewards/margins": 2.8338985443115234,
+      "rewards/rejected": -2.5503764152526855,
+      "step": 197,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.8859060402684564,
+      "grad_norm": 3.0246400833129883,
+      "learning_rate": 0.00012677750960023396,
+      "logits/chosen": -0.32787883281707764,
+      "logits/rejected": -0.4495905637741089,
+      "logps/chosen": -10.30122184753418,
+      "logps/rejected": -42.09465026855469,
+      "loss": 0.866395115852356,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.574297308921814,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": 0.13904830813407898,
+      "rewards/margins": 2.8916144371032715,
+      "rewards/rejected": -2.75256609916687,
+      "step": 198,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.8903803131991052,
+      "grad_norm": 4.209658622741699,
+      "learning_rate": 0.0001260612064533843,
+      "logits/chosen": -0.2963576018810272,
+      "logits/rejected": -0.5197300314903259,
+      "logps/chosen": -7.585211753845215,
+      "logps/rejected": -38.58053970336914,
+      "loss": 0.9138990640640259,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6033953428268433,
+      "rewards/accuracies": 0.890625,
+      "rewards/chosen": 0.2075960785150528,
+      "rewards/margins": 2.4362571239471436,
+      "rewards/rejected": -2.228661298751831,
+      "step": 199,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.8948545861297539,
+      "grad_norm": 5.18996524810791,
+      "learning_rate": 0.0001253434657943616,
+      "logits/chosen": -0.2213418185710907,
+      "logits/rejected": -0.45573991537094116,
+      "logps/chosen": -8.23147201538086,
+      "logps/rejected": -50.46209716796875,
+      "loss": 0.974190890789032,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6812121868133545,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.15396177768707275,
+      "rewards/margins": 3.1692397594451904,
+      "rewards/rejected": -3.015277862548828,
+      "step": 200,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 0.8993288590604027,
+      "grad_norm": 6.953693389892578,
+      "learning_rate": 0.0001246243272130804,
+      "logits/chosen": -0.3670811057090759,
+      "logits/rejected": -0.5695959329605103,
+      "logps/chosen": -4.348814964294434,
+      "logps/rejected": -44.83324432373047,
+      "loss": 0.6207172870635986,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4008825123310089,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.020788919180631638,
+      "rewards/margins": 3.0340218544006348,
+      "rewards/rejected": -3.013232469558716,
+      "step": 201,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.9038031319910514,
+      "grad_norm": 2.0709781646728516,
+      "learning_rate": 0.00012390383037656327,
+      "logits/chosen": -0.2781023383140564,
+      "logits/rejected": -0.5001654624938965,
+      "logps/chosen": -5.481650352478027,
+      "logps/rejected": -45.52299499511719,
+      "loss": 0.6335952877998352,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.41790321469306946,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": 0.14746525883674622,
+      "rewards/margins": 3.0953919887542725,
+      "rewards/rejected": -2.9479269981384277,
+      "step": 202,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.9082774049217002,
+      "grad_norm": 3.936102867126465,
+      "learning_rate": 0.00012318201502675285,
+      "logits/chosen": -0.4156622588634491,
+      "logits/rejected": -0.49767816066741943,
+      "logps/chosen": -9.21568775177002,
+      "logps/rejected": -37.2863883972168,
+      "loss": 0.8709239959716797,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.568230390548706,
+      "rewards/accuracies": 0.890625,
+      "rewards/chosen": 0.08218996226787567,
+      "rewards/margins": 2.5877761840820312,
+      "rewards/rejected": -2.5055861473083496,
+      "step": 203,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.912751677852349,
+      "grad_norm": 3.1774184703826904,
+      "learning_rate": 0.00012245892097831982,
+      "logits/chosen": -0.3529960513114929,
+      "logits/rejected": -0.5200711488723755,
+      "logps/chosen": -5.73190450668335,
+      "logps/rejected": -33.8062858581543,
+      "loss": 0.7089971303939819,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4285321831703186,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.19104257225990295,
+      "rewards/margins": 2.2398736476898193,
+      "rewards/rejected": -2.0488312244415283,
+      "step": 204,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.9172259507829977,
+      "grad_norm": 5.392505168914795,
+      "learning_rate": 0.0001217345881164667,
+      "logits/chosen": -0.26341456174850464,
+      "logits/rejected": -0.4720497131347656,
+      "logps/chosen": -5.948171138763428,
+      "logps/rejected": -39.315799713134766,
+      "loss": 0.7282085418701172,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4318191409111023,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.0930575281381607,
+      "rewards/margins": 2.5829875469207764,
+      "rewards/rejected": -2.4899299144744873,
+      "step": 205,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.9217002237136466,
+      "grad_norm": 1.9011708498001099,
+      "learning_rate": 0.00012100905639472779,
+      "logits/chosen": -0.30916672945022583,
+      "logits/rejected": -0.4543130397796631,
+      "logps/chosen": -6.238247394561768,
+      "logps/rejected": -37.510353088378906,
+      "loss": 0.6386366486549377,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4259726405143738,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.22653764486312866,
+      "rewards/margins": 2.5835399627685547,
+      "rewards/rejected": -2.3570022583007812,
+      "step": 206,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.9261744966442953,
+      "grad_norm": 2.14213228225708,
+      "learning_rate": 0.00012028236583276542,
+      "logits/chosen": -0.2883036136627197,
+      "logits/rejected": -0.41744503378868103,
+      "logps/chosen": -8.626989364624023,
+      "logps/rejected": -35.49958038330078,
+      "loss": 0.8952888250350952,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5360022783279419,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.08197470754384995,
+      "rewards/margins": 1.8212658166885376,
+      "rewards/rejected": -1.7392911911010742,
+      "step": 207,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.930648769574944,
+      "grad_norm": 4.2299604415893555,
+      "learning_rate": 0.00011955455651416246,
+      "logits/chosen": -0.27569687366485596,
+      "logits/rejected": -0.46719983220100403,
+      "logps/chosen": -6.777812480926514,
+      "logps/rejected": -37.963653564453125,
+      "loss": 0.6860868334770203,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4499008357524872,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": 0.2759566307067871,
+      "rewards/margins": 2.4637763500213623,
+      "rewards/rejected": -2.187819480895996,
+      "step": 208,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.9351230425055929,
+      "grad_norm": 2.7029130458831787,
+      "learning_rate": 0.00011882566858421135,
+      "logits/chosen": -0.3265005946159363,
+      "logits/rejected": -0.4556988775730133,
+      "logps/chosen": -8.206356048583984,
+      "logps/rejected": -35.63490676879883,
+      "loss": 0.8148421049118042,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5701502561569214,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.17454853653907776,
+      "rewards/margins": 2.2981252670288086,
+      "rewards/rejected": -2.1235766410827637,
+      "step": 209,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.9395973154362416,
+      "grad_norm": 3.4696102142333984,
+      "learning_rate": 0.00011809574224769981,
+      "logits/chosen": -0.30080342292785645,
+      "logits/rejected": -0.5164678692817688,
+      "logps/chosen": -8.895907402038574,
+      "logps/rejected": -46.112876892089844,
+      "loss": 0.830041766166687,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6021772623062134,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.16136810183525085,
+      "rewards/margins": 3.008272171020508,
+      "rewards/rejected": -2.8469040393829346,
+      "step": 210,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.9440715883668904,
+      "grad_norm": 2.8616750240325928,
+      "learning_rate": 0.00011736481776669306,
+      "logits/chosen": -0.31334638595581055,
+      "logits/rejected": -0.45864373445510864,
+      "logps/chosen": -5.779447555541992,
+      "logps/rejected": -32.93850326538086,
+      "loss": 0.6704469919204712,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3824280798435211,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": 0.16728241741657257,
+      "rewards/margins": 2.098851442337036,
+      "rewards/rejected": -1.9315690994262695,
+      "step": 211,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.9485458612975392,
+      "grad_norm": 2.4147164821624756,
+      "learning_rate": 0.00011663293545831302,
+      "logits/chosen": -0.30772557854652405,
+      "logits/rejected": -0.5058435201644897,
+      "logps/chosen": -9.233728408813477,
+      "logps/rejected": -48.16905975341797,
+      "loss": 0.8766963481903076,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6797882318496704,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.21987132728099823,
+      "rewards/margins": 3.1963741779327393,
+      "rewards/rejected": -2.9765028953552246,
+      "step": 212,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.9530201342281879,
+      "grad_norm": 2.5486626625061035,
+      "learning_rate": 0.00011590013569251457,
+      "logits/chosen": -0.29882898926734924,
+      "logits/rejected": -0.43773284554481506,
+      "logps/chosen": -6.394394874572754,
+      "logps/rejected": -34.92805862426758,
+      "loss": 0.6322169303894043,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3928750455379486,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.24000707268714905,
+      "rewards/margins": 2.3915932178497314,
+      "rewards/rejected": -2.1515860557556152,
+      "step": 213,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.9574944071588367,
+      "grad_norm": 2.4576425552368164,
+      "learning_rate": 0.0001151664588898586,
+      "logits/chosen": -0.35491204261779785,
+      "logits/rejected": -0.584037721157074,
+      "logps/chosen": -5.498895645141602,
+      "logps/rejected": -50.16877365112305,
+      "loss": 0.6617028117179871,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.45270809531211853,
+      "rewards/accuracies": 0.890625,
+      "rewards/chosen": 0.15284521877765656,
+      "rewards/margins": 3.232778310775757,
+      "rewards/rejected": -3.0799331665039062,
+      "step": 214,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.9619686800894854,
+      "grad_norm": 2.428880214691162,
+      "learning_rate": 0.00011443194551928266,
+      "logits/chosen": -0.4301077425479889,
+      "logits/rejected": -0.5748926401138306,
+      "logps/chosen": -5.792450904846191,
+      "logps/rejected": -41.20557403564453,
+      "loss": 0.6007997989654541,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.35522550344467163,
+      "rewards/accuracies": 0.890625,
+      "rewards/chosen": 0.08507393300533295,
+      "rewards/margins": 2.962244987487793,
+      "rewards/rejected": -2.8771708011627197,
+      "step": 215,
+      "train_speed(iter/s)": 0.005619
+    },
+    {
+      "epoch": 0.9664429530201343,
+      "grad_norm": 5.455321788787842,
+      "learning_rate": 0.00011369663609586854,
+      "logits/chosen": -0.4297526478767395,
+      "logits/rejected": -0.4847024381160736,
+      "logps/chosen": -8.678275108337402,
+      "logps/rejected": -29.509740829467773,
+      "loss": 0.8770935535430908,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5399242639541626,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.08710916340351105,
+      "rewards/margins": 1.9487600326538086,
+      "rewards/rejected": -1.8616507053375244,
+      "step": 216,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.970917225950783,
+      "grad_norm": 3.3811686038970947,
+      "learning_rate": 0.00011296057117860759,
+      "logits/chosen": -0.31279319524765015,
+      "logits/rejected": -0.5541050434112549,
+      "logps/chosen": -9.380926132202148,
+      "logps/rejected": -47.00410461425781,
+      "loss": 0.8845710158348083,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6761906147003174,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": 0.0901801735162735,
+      "rewards/margins": 3.160248041152954,
+      "rewards/rejected": -3.0700676441192627,
+      "step": 217,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.9753914988814317,
+      "grad_norm": 7.690444469451904,
+      "learning_rate": 0.00011222379136816345,
+      "logits/chosen": -0.3524364233016968,
+      "logits/rejected": -0.5617326498031616,
+      "logps/chosen": -10.020103454589844,
+      "logps/rejected": -47.92871856689453,
+      "loss": 0.9561252593994141,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6848851442337036,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.15353024005889893,
+      "rewards/margins": 3.196129083633423,
+      "rewards/rejected": -3.0425989627838135,
+      "step": 218,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.9798657718120806,
+      "grad_norm": 2.4010486602783203,
+      "learning_rate": 0.00011148633730463273,
+      "logits/chosen": -0.33735328912734985,
+      "logits/rejected": -0.6036848425865173,
+      "logps/chosen": -5.477067470550537,
+      "logps/rejected": -46.78923034667969,
+      "loss": 0.6661429405212402,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.45900487899780273,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": 0.16355790197849274,
+      "rewards/margins": 3.1154041290283203,
+      "rewards/rejected": -2.9518463611602783,
+      "step": 219,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.9843400447427293,
+      "grad_norm": 3.927243232727051,
+      "learning_rate": 0.00011074824966530312,
+      "logits/chosen": -0.37031301856040955,
+      "logits/rejected": -0.5702263712882996,
+      "logps/chosen": -7.515779495239258,
+      "logps/rejected": -40.670631408691406,
+      "loss": 0.9489113688468933,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5751282572746277,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.11724560707807541,
+      "rewards/margins": 2.502910614013672,
+      "rewards/rejected": -2.385664939880371,
+      "step": 220,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.9888143176733781,
+      "grad_norm": 6.59840726852417,
+      "learning_rate": 0.00011000956916240985,
+      "logits/chosen": -0.41073113679885864,
+      "logits/rejected": -0.6303737759590149,
+      "logps/chosen": -6.878203392028809,
+      "logps/rejected": -45.99734878540039,
+      "loss": 0.7122887372970581,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5226024389266968,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": 0.11505140364170074,
+      "rewards/margins": 3.235778570175171,
+      "rewards/rejected": -3.1207275390625,
+      "step": 221,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.9932885906040269,
+      "grad_norm": 2.5927021503448486,
+      "learning_rate": 0.00010927033654088983,
+      "logits/chosen": -0.3365384638309479,
+      "logits/rejected": -0.5594428777694702,
+      "logps/chosen": -8.96605396270752,
+      "logps/rejected": -43.94550323486328,
+      "loss": 1.002770185470581,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.6635525226593018,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.014833219349384308,
+      "rewards/margins": 2.641629219055176,
+      "rewards/rejected": -2.6564626693725586,
+      "step": 222,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 0.9977628635346756,
+      "grad_norm": 2.235867977142334,
+      "learning_rate": 0.00010853059257613448,
+      "logits/chosen": -0.3795309066772461,
+      "logits/rejected": -0.5900553464889526,
+      "logps/chosen": -6.937759876251221,
+      "logps/rejected": -40.11079406738281,
+      "loss": 0.5907022953033447,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.33843478560447693,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.07233436405658722,
+      "rewards/margins": 2.356292724609375,
+      "rewards/rejected": -2.2839584350585938,
+      "step": 223,
+      "train_speed(iter/s)": 0.00562
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 2.235867977142334,
+      "learning_rate": 0.00010779037807174033,
+      "logits/chosen": -0.38307642936706543,
+      "logits/rejected": -0.575429379940033,
+      "logps/chosen": -10.307394981384277,
+      "logps/rejected": -36.11070251464844,
+      "loss": 0.5518274903297424,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.7534576058387756,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.2281292825937271,
+      "rewards/margins": 2.14729905128479,
+      "rewards/rejected": -1.9191696643829346,
+      "step": 224,
+      "train_speed(iter/s)": 0.005633
+    },
+    {
+      "epoch": 1.0044742729306488,
+      "grad_norm": 2.484886407852173,
+      "learning_rate": 0.00010704973385725851,
+      "logits/chosen": -0.4444534182548523,
+      "logits/rejected": -0.5337757468223572,
+      "logps/chosen": -7.323111534118652,
+      "logps/rejected": -35.633766174316406,
+      "loss": 0.6172289848327637,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.39472728967666626,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": 0.29545480012893677,
+      "rewards/margins": 2.4496452808380127,
+      "rewards/rejected": -2.1541905403137207,
+      "step": 225,
+      "train_speed(iter/s)": 0.005632
+    },
+    {
+      "epoch": 1.0089485458612975,
+      "grad_norm": 2.8453030586242676,
+      "learning_rate": 0.00010630870078594249,
+      "logits/chosen": -0.3669445812702179,
+      "logits/rejected": -0.5369352698326111,
+      "logps/chosen": -6.009266376495361,
+      "logps/rejected": -43.218292236328125,
+      "loss": 0.47680002450942993,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.31007498502731323,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4155438244342804,
+      "rewards/margins": 3.0658767223358154,
+      "rewards/rejected": -2.6503331661224365,
+      "step": 226,
+      "train_speed(iter/s)": 0.005632
+    },
+    {
+      "epoch": 1.0134228187919463,
+      "grad_norm": 4.40702486038208,
+      "learning_rate": 0.00010556731973249485,
+      "logits/chosen": -0.3794156312942505,
+      "logits/rejected": -0.5609134435653687,
+      "logps/chosen": -6.1214189529418945,
+      "logps/rejected": -38.3588981628418,
+      "loss": 0.5431226491928101,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3845002055168152,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.422393262386322,
+      "rewards/margins": 2.9117181301116943,
+      "rewards/rejected": -2.4893248081207275,
+      "step": 227,
+      "train_speed(iter/s)": 0.005632
+    },
+    {
+      "epoch": 1.0178970917225951,
+      "grad_norm": 1.7455840110778809,
+      "learning_rate": 0.00010482563159081238,
+      "logits/chosen": -0.3268698453903198,
+      "logits/rejected": -0.6033288836479187,
+      "logps/chosen": -5.496387004852295,
+      "logps/rejected": -50.4907112121582,
+      "loss": 0.5993185639381409,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4379004240036011,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.21979132294654846,
+      "rewards/margins": 3.0987210273742676,
+      "rewards/rejected": -2.878929615020752,
+      "step": 228,
+      "train_speed(iter/s)": 0.005632
+    },
+    {
+      "epoch": 1.0223713646532437,
+      "grad_norm": 3.708115339279175,
+      "learning_rate": 0.00010408367727173067,
+      "logits/chosen": -0.335446834564209,
+      "logits/rejected": -0.6248437166213989,
+      "logps/chosen": -1.8182162046432495,
+      "logps/rejected": -49.31162643432617,
+      "loss": 0.30696067214012146,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.16442172229290009,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.1987520307302475,
+      "rewards/margins": 3.270106792449951,
+      "rewards/rejected": -3.071354866027832,
+      "step": 229,
+      "train_speed(iter/s)": 0.005631
+    },
+    {
+      "epoch": 1.0268456375838926,
+      "grad_norm": 1.4245268106460571,
+      "learning_rate": 0.00010334149770076747,
+      "logits/chosen": -0.4321066439151764,
+      "logits/rejected": -0.5824412107467651,
+      "logps/chosen": -5.075246810913086,
+      "logps/rejected": -35.597496032714844,
+      "loss": 0.527398943901062,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3494449853897095,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3942062258720398,
+      "rewards/margins": 2.7244553565979004,
+      "rewards/rejected": -2.330249071121216,
+      "step": 230,
+      "train_speed(iter/s)": 0.005632
+    },
+    {
+      "epoch": 1.0313199105145414,
+      "grad_norm": 2.983712673187256,
+      "learning_rate": 0.0001025991338158651,
+      "logits/chosen": -0.3966450095176697,
+      "logits/rejected": -0.5776999592781067,
+      "logps/chosen": -3.8128936290740967,
+      "logps/rejected": -38.07972717285156,
+      "loss": 0.46247074007987976,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.28802502155303955,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.2343234121799469,
+      "rewards/margins": 2.575010061264038,
+      "rewards/rejected": -2.340686559677124,
+      "step": 231,
+      "train_speed(iter/s)": 0.005632
+    },
+    {
+      "epoch": 1.0357941834451903,
+      "grad_norm": 1.7103675603866577,
+      "learning_rate": 0.00010185662656513251,
+      "logits/chosen": -0.33908575773239136,
+      "logits/rejected": -0.543910801410675,
+      "logps/chosen": -6.781057834625244,
+      "logps/rejected": -40.72597885131836,
+      "loss": 0.5812743902206421,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.39541077613830566,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.32517990469932556,
+      "rewards/margins": 2.686272144317627,
+      "rewards/rejected": -2.3610920906066895,
+      "step": 232,
+      "train_speed(iter/s)": 0.005631
+    },
+    {
+      "epoch": 1.0402684563758389,
+      "grad_norm": 1.6416723728179932,
+      "learning_rate": 0.00010111401690458654,
+      "logits/chosen": -0.37221670150756836,
+      "logits/rejected": -0.5374215841293335,
+      "logps/chosen": -5.313752174377441,
+      "logps/rejected": -39.40772247314453,
+      "loss": 0.47626128792762756,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3095937967300415,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.38376766443252563,
+      "rewards/margins": 2.872239828109741,
+      "rewards/rejected": -2.4884719848632812,
+      "step": 233,
+      "train_speed(iter/s)": 0.005631
+    },
+    {
+      "epoch": 1.0447427293064877,
+      "grad_norm": 1.6093069314956665,
+      "learning_rate": 0.00010037134579589302,
+      "logits/chosen": -0.4185706377029419,
+      "logits/rejected": -0.6050505638122559,
+      "logps/chosen": -7.153165817260742,
+      "logps/rejected": -43.014652252197266,
+      "loss": 0.6177042126655579,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4057208001613617,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.21249428391456604,
+      "rewards/margins": 3.0852949619293213,
+      "rewards/rejected": -2.872800588607788,
+      "step": 234,
+      "train_speed(iter/s)": 0.005631
+    },
+    {
+      "epoch": 1.0492170022371365,
+      "grad_norm": 2.2540111541748047,
+      "learning_rate": 9.962865420410701e-05,
+      "logits/chosen": -0.3561246991157532,
+      "logits/rejected": -0.5401315093040466,
+      "logps/chosen": -6.133809566497803,
+      "logps/rejected": -40.84746170043945,
+      "loss": 0.4942439794540405,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3529200553894043,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2851286828517914,
+      "rewards/margins": 2.9554595947265625,
+      "rewards/rejected": -2.6703310012817383,
+      "step": 235,
+      "train_speed(iter/s)": 0.005631
+    },
+    {
+      "epoch": 1.0536912751677852,
+      "grad_norm": 2.0389819145202637,
+      "learning_rate": 9.888598309541347e-05,
+      "logits/chosen": -0.3859875798225403,
+      "logits/rejected": -0.6725212335586548,
+      "logps/chosen": -4.989171981811523,
+      "logps/rejected": -44.53968811035156,
+      "loss": 0.5320685505867004,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3883105516433716,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.3034836947917938,
+      "rewards/margins": 3.2500219345092773,
+      "rewards/rejected": -2.946537971496582,
+      "step": 236,
+      "train_speed(iter/s)": 0.005631
+    },
+    {
+      "epoch": 1.058165548098434,
+      "grad_norm": 11.425275802612305,
+      "learning_rate": 9.814337343486754e-05,
+      "logits/chosen": -0.41677045822143555,
+      "logits/rejected": -0.681226909160614,
+      "logps/chosen": -6.433794021606445,
+      "logps/rejected": -47.453330993652344,
+      "loss": 0.6826303601264954,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5648487210273743,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.19153328239917755,
+      "rewards/margins": 3.4548466205596924,
+      "rewards/rejected": -3.2633132934570312,
+      "step": 237,
+      "train_speed(iter/s)": 0.005631
+    },
+    {
+      "epoch": 1.0626398210290828,
+      "grad_norm": 2.6108782291412354,
+      "learning_rate": 9.740086618413495e-05,
+      "logits/chosen": -0.35633864998817444,
+      "logits/rejected": -0.6819493174552917,
+      "logps/chosen": -4.880098819732666,
+      "logps/rejected": -58.097110748291016,
+      "loss": 0.44784390926361084,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.37838631868362427,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.29481351375579834,
+      "rewards/margins": 4.305008411407471,
+      "rewards/rejected": -4.010194778442383,
+      "step": 238,
+      "train_speed(iter/s)": 0.005631
+    },
+    {
+      "epoch": 1.0671140939597314,
+      "grad_norm": 4.4547438621521,
+      "learning_rate": 9.665850229923258e-05,
+      "logits/chosen": -0.43452098965644836,
+      "logits/rejected": -0.6579560041427612,
+      "logps/chosen": -4.716073513031006,
+      "logps/rejected": -48.85848617553711,
+      "loss": 0.49553921818733215,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3607966899871826,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.2058188021183014,
+      "rewards/margins": 3.5923969745635986,
+      "rewards/rejected": -3.38657808303833,
+      "step": 239,
+      "train_speed(iter/s)": 0.005631
+    },
+    {
+      "epoch": 1.0715883668903803,
+      "grad_norm": 3.9305038452148438,
+      "learning_rate": 9.591632272826934e-05,
+      "logits/chosen": -0.3088409900665283,
+      "logits/rejected": -0.6176849603652954,
+      "logps/chosen": -4.434784889221191,
+      "logps/rejected": -54.108062744140625,
+      "loss": 0.4914236068725586,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3651493787765503,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.24546122550964355,
+      "rewards/margins": 3.803105354309082,
+      "rewards/rejected": -3.5576443672180176,
+      "step": 240,
+      "train_speed(iter/s)": 0.005631
+    },
+    {
+      "epoch": 1.0760626398210291,
+      "grad_norm": 2.7532975673675537,
+      "learning_rate": 9.517436840918766e-05,
+      "logits/chosen": -0.41035979986190796,
+      "logits/rejected": -0.6430486440658569,
+      "logps/chosen": -5.0145182609558105,
+      "logps/rejected": -46.19696807861328,
+      "loss": 0.5373749732971191,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.362954318523407,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.17752307653427124,
+      "rewards/margins": 3.215165138244629,
+      "rewards/rejected": -3.037642002105713,
+      "step": 241,
+      "train_speed(iter/s)": 0.005631
+    },
+    {
+      "epoch": 1.0805369127516777,
+      "grad_norm": 2.9417192935943604,
+      "learning_rate": 9.44326802675052e-05,
+      "logits/chosen": -0.3006266951560974,
+      "logits/rejected": -0.5898244976997375,
+      "logps/chosen": -5.02177619934082,
+      "logps/rejected": -47.1878662109375,
+      "loss": 0.44721511006355286,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.32431328296661377,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.39047926664352417,
+      "rewards/margins": 3.3298583030700684,
+      "rewards/rejected": -2.9393789768218994,
+      "step": 242,
+      "train_speed(iter/s)": 0.005631
+    },
+    {
+      "epoch": 1.0850111856823266,
+      "grad_norm": 2.234625816345215,
+      "learning_rate": 9.369129921405754e-05,
+      "logits/chosen": -0.3297862112522125,
+      "logits/rejected": -0.6249620318412781,
+      "logps/chosen": -4.588490009307861,
+      "logps/rejected": -43.63856506347656,
+      "loss": 0.44313275814056396,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.30556169152259827,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.2911630868911743,
+      "rewards/margins": 3.1379594802856445,
+      "rewards/rejected": -2.846796989440918,
+      "step": 243,
+      "train_speed(iter/s)": 0.005631
+    },
+    {
+      "epoch": 1.0894854586129754,
+      "grad_norm": 2.9266371726989746,
+      "learning_rate": 9.295026614274152e-05,
+      "logits/chosen": -0.33700501918792725,
+      "logits/rejected": -0.6508822441101074,
+      "logps/chosen": -7.202463150024414,
+      "logps/rejected": -48.011070251464844,
+      "loss": 0.5501227974891663,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.41064733266830444,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.466447114944458,
+      "rewards/margins": 3.4353857040405273,
+      "rewards/rejected": -2.9689383506774902,
+      "step": 244,
+      "train_speed(iter/s)": 0.00563
+    },
+    {
+      "epoch": 1.0939597315436242,
+      "grad_norm": 1.520435094833374,
+      "learning_rate": 9.220962192825968e-05,
+      "logits/chosen": -0.2550147771835327,
+      "logits/rejected": -0.5666214823722839,
+      "logps/chosen": -4.967221736907959,
+      "logps/rejected": -48.09080123901367,
+      "loss": 0.42909708619117737,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.33424443006515503,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.5164206027984619,
+      "rewards/margins": 3.6842832565307617,
+      "rewards/rejected": -3.167862892150879,
+      "step": 245,
+      "train_speed(iter/s)": 0.00563
+    },
+    {
+      "epoch": 1.0984340044742729,
+      "grad_norm": 1.9133579730987549,
+      "learning_rate": 9.146940742386553e-05,
+      "logits/chosen": -0.3395351767539978,
+      "logits/rejected": -0.6194986701011658,
+      "logps/chosen": -3.245999574661255,
+      "logps/rejected": -48.67194747924805,
+      "loss": 0.40019088983535767,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.2735483944416046,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2056996375322342,
+      "rewards/margins": 3.698310375213623,
+      "rewards/rejected": -3.492610454559326,
+      "step": 246,
+      "train_speed(iter/s)": 0.00563
+    },
+    {
+      "epoch": 1.1029082774049217,
+      "grad_norm": 3.2224693298339844,
+      "learning_rate": 9.072966345911019e-05,
+      "logits/chosen": -0.29714155197143555,
+      "logits/rejected": -0.5788121223449707,
+      "logps/chosen": -5.69664192199707,
+      "logps/rejected": -56.27704620361328,
+      "loss": 0.5205512642860413,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.42394617199897766,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.22148963809013367,
+      "rewards/margins": 4.184146881103516,
+      "rewards/rejected": -3.9626572132110596,
+      "step": 247,
+      "train_speed(iter/s)": 0.00563
+    },
+    {
+      "epoch": 1.1073825503355705,
+      "grad_norm": 3.369208335876465,
+      "learning_rate": 8.999043083759017e-05,
+      "logits/chosen": -0.3414077162742615,
+      "logits/rejected": -0.6023144721984863,
+      "logps/chosen": -5.8809943199157715,
+      "logps/rejected": -46.12065505981445,
+      "loss": 0.5429152250289917,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4344760477542877,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.37368032336235046,
+      "rewards/margins": 3.512446403503418,
+      "rewards/rejected": -3.138766288757324,
+      "step": 248,
+      "train_speed(iter/s)": 0.00563
+    },
+    {
+      "epoch": 1.1118568232662192,
+      "grad_norm": 2.0529885292053223,
+      "learning_rate": 8.925175033469688e-05,
+      "logits/chosen": -0.24800914525985718,
+      "logits/rejected": -0.5255987644195557,
+      "logps/chosen": -6.151613235473633,
+      "logps/rejected": -52.948631286621094,
+      "loss": 0.5082452893257141,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.352613627910614,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3551517128944397,
+      "rewards/margins": 4.031224250793457,
+      "rewards/rejected": -3.676072359085083,
+      "step": 249,
+      "train_speed(iter/s)": 0.00563
+    },
+    {
+      "epoch": 1.116331096196868,
+      "grad_norm": 2.5124740600585938,
+      "learning_rate": 8.851366269536729e-05,
+      "logits/chosen": -0.26961249113082886,
+      "logits/rejected": -0.5675927996635437,
+      "logps/chosen": -5.100187301635742,
+      "logps/rejected": -49.04716110229492,
+      "loss": 0.5566443800926208,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.44095391035079956,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.4250047504901886,
+      "rewards/margins": 4.00430965423584,
+      "rewards/rejected": -3.5793049335479736,
+      "step": 250,
+      "train_speed(iter/s)": 0.00563
+    },
+    {
+      "epoch": 1.1208053691275168,
+      "grad_norm": 3.4243483543395996,
+      "learning_rate": 8.777620863183657e-05,
+      "logits/chosen": -0.34957340359687805,
+      "logits/rejected": -0.6483319997787476,
+      "logps/chosen": -6.03017520904541,
+      "logps/rejected": -50.83137130737305,
+      "loss": 0.6758670210838318,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.47578099370002747,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.1781749427318573,
+      "rewards/margins": 3.6373672485351562,
+      "rewards/rejected": -3.4591920375823975,
+      "step": 251,
+      "train_speed(iter/s)": 0.005627
+    },
+    {
+      "epoch": 1.1252796420581657,
+      "grad_norm": 3.0111682415008545,
+      "learning_rate": 8.703942882139245e-05,
+      "logits/chosen": -0.34100106358528137,
+      "logits/rejected": -0.5722084045410156,
+      "logps/chosen": -5.588647365570068,
+      "logps/rejected": -47.537479400634766,
+      "loss": 0.47489383816719055,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.33142027258872986,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4165104031562805,
+      "rewards/margins": 3.5505828857421875,
+      "rewards/rejected": -3.1340725421905518,
+      "step": 252,
+      "train_speed(iter/s)": 0.005627
+    },
+    {
+      "epoch": 1.1297539149888143,
+      "grad_norm": 1.6837427616119385,
+      "learning_rate": 8.630336390413147e-05,
+      "logits/chosen": -0.318341463804245,
+      "logits/rejected": -0.602148175239563,
+      "logps/chosen": -4.762698650360107,
+      "logps/rejected": -57.818058013916016,
+      "loss": 0.41530388593673706,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3407127857208252,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.3078892230987549,
+      "rewards/margins": 4.42728853225708,
+      "rewards/rejected": -4.119399070739746,
+      "step": 253,
+      "train_speed(iter/s)": 0.005627
+    },
+    {
+      "epoch": 1.1342281879194631,
+      "grad_norm": 1.7377208471298218,
+      "learning_rate": 8.556805448071735e-05,
+      "logits/chosen": -0.2768436372280121,
+      "logits/rejected": -0.5286397933959961,
+      "logps/chosen": -5.270069122314453,
+      "logps/rejected": -44.92286682128906,
+      "loss": 0.5719718933105469,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.40346431732177734,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.35468900203704834,
+      "rewards/margins": 3.2416746616363525,
+      "rewards/rejected": -2.8869855403900146,
+      "step": 254,
+      "train_speed(iter/s)": 0.005627
+    },
+    {
+      "epoch": 1.138702460850112,
+      "grad_norm": 2.1543498039245605,
+      "learning_rate": 8.483354111014141e-05,
+      "logits/chosen": -0.4536677300930023,
+      "logits/rejected": -0.6746389865875244,
+      "logps/chosen": -6.907723903656006,
+      "logps/rejected": -43.56280517578125,
+      "loss": 0.5296320915222168,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.41071781516075134,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4868796467781067,
+      "rewards/margins": 3.4957876205444336,
+      "rewards/rejected": -3.0089080333709717,
+      "step": 255,
+      "train_speed(iter/s)": 0.005627
+    },
+    {
+      "epoch": 1.1431767337807606,
+      "grad_norm": 1.8656744956970215,
+      "learning_rate": 8.409986430748545e-05,
+      "logits/chosen": -0.39294540882110596,
+      "logits/rejected": -0.6587871313095093,
+      "logps/chosen": -5.210620880126953,
+      "logps/rejected": -46.816986083984375,
+      "loss": 0.5252071619033813,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3719002604484558,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3239804208278656,
+      "rewards/margins": 3.495149612426758,
+      "rewards/rejected": -3.1711692810058594,
+      "step": 256,
+      "train_speed(iter/s)": 0.005626
+    },
+    {
+      "epoch": 1.1476510067114094,
+      "grad_norm": 2.010622024536133,
+      "learning_rate": 8.336706454168701e-05,
+      "logits/chosen": -0.32442259788513184,
+      "logits/rejected": -0.6206884384155273,
+      "logps/chosen": -6.981322765350342,
+      "logps/rejected": -54.52667236328125,
+      "loss": 0.4600639343261719,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3734227120876312,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.3917311429977417,
+      "rewards/margins": 4.217864513397217,
+      "rewards/rejected": -3.8261334896087646,
+      "step": 257,
+      "train_speed(iter/s)": 0.005626
+    },
+    {
+      "epoch": 1.1521252796420582,
+      "grad_norm": 1.5634533166885376,
+      "learning_rate": 8.263518223330697e-05,
+      "logits/chosen": -0.44610998034477234,
+      "logits/rejected": -0.6434642672538757,
+      "logps/chosen": -7.3278350830078125,
+      "logps/rejected": -48.34819412231445,
+      "loss": 0.5413830876350403,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4026271402835846,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.5039491057395935,
+      "rewards/margins": 3.8213727474212646,
+      "rewards/rejected": -3.3174238204956055,
+      "step": 258,
+      "train_speed(iter/s)": 0.005626
+    },
+    {
+      "epoch": 1.1565995525727069,
+      "grad_norm": 4.007801532745361,
+      "learning_rate": 8.190425775230021e-05,
+      "logits/chosen": -0.3476124405860901,
+      "logits/rejected": -0.6711318492889404,
+      "logps/chosen": -5.098440170288086,
+      "logps/rejected": -48.90070724487305,
+      "loss": 0.42464467883110046,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3378307819366455,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4362422227859497,
+      "rewards/margins": 3.8473596572875977,
+      "rewards/rejected": -3.4111177921295166,
+      "step": 259,
+      "train_speed(iter/s)": 0.005626
+    },
+    {
+      "epoch": 1.1610738255033557,
+      "grad_norm": 2.5361435413360596,
+      "learning_rate": 8.117433141578866e-05,
+      "logits/chosen": -0.4177584946155548,
+      "logits/rejected": -0.5863391160964966,
+      "logps/chosen": -6.138072967529297,
+      "logps/rejected": -46.63839340209961,
+      "loss": 0.5487118363380432,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.40140300989151,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2436431646347046,
+      "rewards/margins": 3.2071709632873535,
+      "rewards/rejected": -2.9635279178619385,
+      "step": 260,
+      "train_speed(iter/s)": 0.005626
+    },
+    {
+      "epoch": 1.1655480984340045,
+      "grad_norm": 3.6130049228668213,
+      "learning_rate": 8.044544348583755e-05,
+      "logits/chosen": -0.38566017150878906,
+      "logits/rejected": -0.6825127005577087,
+      "logps/chosen": -3.661987781524658,
+      "logps/rejected": -47.40394592285156,
+      "loss": 0.40936416387557983,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.28705930709838867,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.34029075503349304,
+      "rewards/margins": 3.613295316696167,
+      "rewards/rejected": -3.2730045318603516,
+      "step": 261,
+      "train_speed(iter/s)": 0.005626
+    },
+    {
+      "epoch": 1.1700223713646531,
+      "grad_norm": 2.4069151878356934,
+      "learning_rate": 7.971763416723459e-05,
+      "logits/chosen": -0.32102227210998535,
+      "logits/rejected": -0.6520596146583557,
+      "logps/chosen": -7.717752933502197,
+      "logps/rejected": -56.516021728515625,
+      "loss": 0.5944215059280396,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.46641111373901367,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.31582900881767273,
+      "rewards/margins": 4.05399227142334,
+      "rewards/rejected": -3.738163709640503,
+      "step": 262,
+      "train_speed(iter/s)": 0.005625
+    },
+    {
+      "epoch": 1.174496644295302,
+      "grad_norm": 2.279827833175659,
+      "learning_rate": 7.89909436052722e-05,
+      "logits/chosen": -0.4377071261405945,
+      "logits/rejected": -0.6339465975761414,
+      "logps/chosen": -6.656128406524658,
+      "logps/rejected": -49.15144348144531,
+      "loss": 0.6149309873580933,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.45076847076416016,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.3392454981803894,
+      "rewards/margins": 3.6513326168060303,
+      "rewards/rejected": -3.312087059020996,
+      "step": 263,
+      "train_speed(iter/s)": 0.005625
+    },
+    {
+      "epoch": 1.1789709172259508,
+      "grad_norm": 3.1244442462921143,
+      "learning_rate": 7.826541188353329e-05,
+      "logits/chosen": -0.35084378719329834,
+      "logits/rejected": -0.6840541958808899,
+      "logps/chosen": -3.9023451805114746,
+      "logps/rejected": -53.137367248535156,
+      "loss": 0.43266385793685913,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3279496729373932,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3510107398033142,
+      "rewards/margins": 3.936466693878174,
+      "rewards/rejected": -3.585455894470215,
+      "step": 264,
+      "train_speed(iter/s)": 0.005625
+    },
+    {
+      "epoch": 1.1834451901565997,
+      "grad_norm": 2.2550525665283203,
+      "learning_rate": 7.754107902168019e-05,
+      "logits/chosen": -0.40471333265304565,
+      "logits/rejected": -0.6776788234710693,
+      "logps/chosen": -6.384232044219971,
+      "logps/rejected": -54.759212493896484,
+      "loss": 0.4860294759273529,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.394418865442276,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5021213293075562,
+      "rewards/margins": 4.259507656097412,
+      "rewards/rejected": -3.7573862075805664,
+      "step": 265,
+      "train_speed(iter/s)": 0.005625
+    },
+    {
+      "epoch": 1.1879194630872483,
+      "grad_norm": 1.6058971881866455,
+      "learning_rate": 7.681798497324716e-05,
+      "logits/chosen": -0.4261001944541931,
+      "logits/rejected": -0.6693729758262634,
+      "logps/chosen": -6.607486248016357,
+      "logps/rejected": -48.35670471191406,
+      "loss": 0.6735593676567078,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5240046381950378,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.26564663648605347,
+      "rewards/margins": 3.774982452392578,
+      "rewards/rejected": -3.509335994720459,
+      "step": 266,
+      "train_speed(iter/s)": 0.005625
+    },
+    {
+      "epoch": 1.192393736017897,
+      "grad_norm": 3.3279082775115967,
+      "learning_rate": 7.609616962343675e-05,
+      "logits/chosen": -0.49879682064056396,
+      "logits/rejected": -0.7210312485694885,
+      "logps/chosen": -4.761632919311523,
+      "logps/rejected": -49.54342269897461,
+      "loss": 0.45162299275398254,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3338029980659485,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.34260720014572144,
+      "rewards/margins": 3.9146251678466797,
+      "rewards/rejected": -3.5720183849334717,
+      "step": 267,
+      "train_speed(iter/s)": 0.005625
+    },
+    {
+      "epoch": 1.196868008948546,
+      "grad_norm": 2.1273367404937744,
+      "learning_rate": 7.537567278691964e-05,
+      "logits/chosen": -0.3661321699619293,
+      "logits/rejected": -0.6873822808265686,
+      "logps/chosen": -5.206943988800049,
+      "logps/rejected": -56.860939025878906,
+      "loss": 0.4889792203903198,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.41767942905426025,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.383396714925766,
+      "rewards/margins": 4.5527167320251465,
+      "rewards/rejected": -4.169320583343506,
+      "step": 268,
+      "train_speed(iter/s)": 0.005625
+    },
+    {
+      "epoch": 1.2013422818791946,
+      "grad_norm": 4.435976505279541,
+      "learning_rate": 7.465653420563845e-05,
+      "logits/chosen": -0.42771661281585693,
+      "logits/rejected": -0.689077615737915,
+      "logps/chosen": -4.431490898132324,
+      "logps/rejected": -53.931156158447266,
+      "loss": 0.40404367446899414,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3060677647590637,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.30835211277008057,
+      "rewards/margins": 4.412530899047852,
+      "rewards/rejected": -4.104178428649902,
+      "step": 269,
+      "train_speed(iter/s)": 0.005625
+    },
+    {
+      "epoch": 1.2058165548098434,
+      "grad_norm": 9.26548957824707,
+      "learning_rate": 7.393879354661577e-05,
+      "logits/chosen": -0.2901456356048584,
+      "logits/rejected": -0.6541563272476196,
+      "logps/chosen": -4.378477573394775,
+      "logps/rejected": -62.90247344970703,
+      "loss": 0.44348278641700745,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.31372037529945374,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.3390202820301056,
+      "rewards/margins": 4.9046478271484375,
+      "rewards/rejected": -4.565627098083496,
+      "step": 270,
+      "train_speed(iter/s)": 0.005625
+    },
+    {
+      "epoch": 1.2102908277404922,
+      "grad_norm": 3.4660370349884033,
+      "learning_rate": 7.322249039976608e-05,
+      "logits/chosen": -0.35300642251968384,
+      "logits/rejected": -0.6308537721633911,
+      "logps/chosen": -4.3834004402160645,
+      "logps/rejected": -47.50883483886719,
+      "loss": 0.48561790585517883,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3273915648460388,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.20902855694293976,
+      "rewards/margins": 3.6998932361602783,
+      "rewards/rejected": -3.4908647537231445,
+      "step": 271,
+      "train_speed(iter/s)": 0.005624
+    },
+    {
+      "epoch": 1.2147651006711409,
+      "grad_norm": 3.6232869625091553,
+      "learning_rate": 7.2507664275712e-05,
+      "logits/chosen": -0.3756940960884094,
+      "logits/rejected": -0.6699195504188538,
+      "logps/chosen": -6.058266639709473,
+      "logps/rejected": -52.071563720703125,
+      "loss": 0.5045337677001953,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.40500137209892273,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5043036937713623,
+      "rewards/margins": 4.029274940490723,
+      "rewards/rejected": -3.5249714851379395,
+      "step": 272,
+      "train_speed(iter/s)": 0.005624
+    },
+    {
+      "epoch": 1.2192393736017897,
+      "grad_norm": 1.5248574018478394,
+      "learning_rate": 7.179435460360491e-05,
+      "logits/chosen": -0.467898964881897,
+      "logits/rejected": -0.6951407194137573,
+      "logps/chosen": -4.229708194732666,
+      "logps/rejected": -45.6329345703125,
+      "loss": 0.46140700578689575,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.31180113554000854,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2676628828048706,
+      "rewards/margins": 3.5363547801971436,
+      "rewards/rejected": -3.2686920166015625,
+      "step": 273,
+      "train_speed(iter/s)": 0.005624
+    },
+    {
+      "epoch": 1.2237136465324385,
+      "grad_norm": 2.608729839324951,
+      "learning_rate": 7.108260072895013e-05,
+      "logits/chosen": -0.4148178696632385,
+      "logits/rejected": -0.63880455493927,
+      "logps/chosen": -4.667982578277588,
+      "logps/rejected": -43.82627868652344,
+      "loss": 0.4676215350627899,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.32163432240486145,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3947795331478119,
+      "rewards/margins": 3.6294405460357666,
+      "rewards/rejected": -3.2346608638763428,
+      "step": 274,
+      "train_speed(iter/s)": 0.005624
+    },
+    {
+      "epoch": 1.2281879194630871,
+      "grad_norm": 2.44980525970459,
+      "learning_rate": 7.037244191143661e-05,
+      "logits/chosen": -0.35544446110725403,
+      "logits/rejected": -0.6155003309249878,
+      "logps/chosen": -3.1806983947753906,
+      "logps/rejected": -42.958126068115234,
+      "loss": 0.42853832244873047,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.26983052492141724,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.21684399247169495,
+      "rewards/margins": 3.123112678527832,
+      "rewards/rejected": -2.906268358230591,
+      "step": 275,
+      "train_speed(iter/s)": 0.005624
+    },
+    {
+      "epoch": 1.232662192393736,
+      "grad_norm": 3.2948148250579834,
+      "learning_rate": 6.966391732277143e-05,
+      "logits/chosen": -0.25492265820503235,
+      "logits/rejected": -0.6005708575248718,
+      "logps/chosen": -5.198720932006836,
+      "logps/rejected": -50.45551300048828,
+      "loss": 0.4813903570175171,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3565835952758789,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.27139896154403687,
+      "rewards/margins": 3.685420036315918,
+      "rewards/rejected": -3.4140210151672363,
+      "step": 276,
+      "train_speed(iter/s)": 0.005624
+    },
+    {
+      "epoch": 1.2371364653243848,
+      "grad_norm": 2.477715253829956,
+      "learning_rate": 6.895706604451905e-05,
+      "logits/chosen": -0.33635056018829346,
+      "logits/rejected": -0.6376497149467468,
+      "logps/chosen": -5.271233081817627,
+      "logps/rejected": -45.175846099853516,
+      "loss": 0.5958554744720459,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.43258097767829895,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.41053637862205505,
+      "rewards/margins": 3.3894155025482178,
+      "rewards/rejected": -2.97887921333313,
+      "step": 277,
+      "train_speed(iter/s)": 0.005624
+    },
+    {
+      "epoch": 1.2416107382550337,
+      "grad_norm": 3.417055606842041,
+      "learning_rate": 6.825192706594575e-05,
+      "logits/chosen": -0.4261041283607483,
+      "logits/rejected": -0.7041194438934326,
+      "logps/chosen": -3.475921154022217,
+      "logps/rejected": -47.301570892333984,
+      "loss": 0.4375145137310028,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.26725247502326965,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3121051788330078,
+      "rewards/margins": 3.574404239654541,
+      "rewards/rejected": -3.262299060821533,
+      "step": 278,
+      "train_speed(iter/s)": 0.005624
+    },
+    {
+      "epoch": 1.2460850111856823,
+      "grad_norm": 2.0796444416046143,
+      "learning_rate": 6.75485392818689e-05,
+      "logits/chosen": -0.29555225372314453,
+      "logits/rejected": -0.5412930250167847,
+      "logps/chosen": -6.336879730224609,
+      "logps/rejected": -48.47580337524414,
+      "loss": 0.5192568898200989,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.36876535415649414,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.18509003520011902,
+      "rewards/margins": 3.489081382751465,
+      "rewards/rejected": -3.3039915561676025,
+      "step": 279,
+      "train_speed(iter/s)": 0.005624
+    },
+    {
+      "epoch": 1.250559284116331,
+      "grad_norm": 4.371717929840088,
+      "learning_rate": 6.684694149051156e-05,
+      "logits/chosen": -0.3555789589881897,
+      "logits/rejected": -0.6074813008308411,
+      "logps/chosen": -6.713379383087158,
+      "logps/rejected": -49.20836639404297,
+      "loss": 0.49521225690841675,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3682730793952942,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.340753436088562,
+      "rewards/margins": 3.6066648960113525,
+      "rewards/rejected": -3.265911340713501,
+      "step": 280,
+      "train_speed(iter/s)": 0.005623
+    },
+    {
+      "epoch": 1.25503355704698,
+      "grad_norm": 1.6855332851409912,
+      "learning_rate": 6.614717239136246e-05,
+      "logits/chosen": -0.36132338643074036,
+      "logits/rejected": -0.6780315041542053,
+      "logps/chosen": -5.581424236297607,
+      "logps/rejected": -60.914573669433594,
+      "loss": 0.45032861828804016,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.35733604431152344,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.3296295404434204,
+      "rewards/margins": 4.67379093170166,
+      "rewards/rejected": -4.344161510467529,
+      "step": 281,
+      "train_speed(iter/s)": 0.005623
+    },
+    {
+      "epoch": 1.2595078299776286,
+      "grad_norm": 3.808645009994507,
+      "learning_rate": 6.54492705830414e-05,
+      "logits/chosen": -0.35267043113708496,
+      "logits/rejected": -0.6105719208717346,
+      "logps/chosen": -5.265847206115723,
+      "logps/rejected": -49.1428337097168,
+      "loss": 0.5204509496688843,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3694938123226166,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4101027548313141,
+      "rewards/margins": 3.825331926345825,
+      "rewards/rejected": -3.415228843688965,
+      "step": 282,
+      "train_speed(iter/s)": 0.005623
+    },
+    {
+      "epoch": 1.2639821029082774,
+      "grad_norm": 6.0424370765686035,
+      "learning_rate": 6.475327456117005e-05,
+      "logits/chosen": -0.3812498152256012,
+      "logits/rejected": -0.5841005444526672,
+      "logps/chosen": -7.5697174072265625,
+      "logps/rejected": -48.44361114501953,
+      "loss": 0.6550671458244324,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.48223644495010376,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": 0.31617283821105957,
+      "rewards/margins": 3.6579878330230713,
+      "rewards/rejected": -3.3418149948120117,
+      "step": 283,
+      "train_speed(iter/s)": 0.005623
+    },
+    {
+      "epoch": 1.2684563758389262,
+      "grad_norm": 1.3673754930496216,
+      "learning_rate": 6.405922271624874e-05,
+      "logits/chosen": -0.15991437435150146,
+      "logits/rejected": -0.6190045475959778,
+      "logps/chosen": -5.2127227783203125,
+      "logps/rejected": -69.4470443725586,
+      "loss": 0.3656357526779175,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.31946197152137756,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.46399012207984924,
+      "rewards/margins": 5.099308013916016,
+      "rewards/rejected": -4.635317802429199,
+      "step": 284,
+      "train_speed(iter/s)": 0.005623
+    },
+    {
+      "epoch": 1.272930648769575,
+      "grad_norm": 1.2922435998916626,
+      "learning_rate": 6.336715333153869e-05,
+      "logits/chosen": -0.3058968782424927,
+      "logits/rejected": -0.6286692023277283,
+      "logps/chosen": -4.774682521820068,
+      "logps/rejected": -51.09849548339844,
+      "loss": 0.4398406445980072,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.33861246705055237,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.39336544275283813,
+      "rewards/margins": 3.7960190773010254,
+      "rewards/rejected": -3.402653455734253,
+      "step": 285,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 1.2774049217002237,
+      "grad_norm": 1.9561717510223389,
+      "learning_rate": 6.267710458095053e-05,
+      "logits/chosen": -0.2568581700325012,
+      "logits/rejected": -0.6096608638763428,
+      "logps/chosen": -3.851630210876465,
+      "logps/rejected": -60.34761047363281,
+      "loss": 0.3585345447063446,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.2737765312194824,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.33957624435424805,
+      "rewards/margins": 4.589919090270996,
+      "rewards/rejected": -4.250343322753906,
+      "step": 286,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 1.2818791946308725,
+      "grad_norm": 2.467003583908081,
+      "learning_rate": 6.198911452693853e-05,
+      "logits/chosen": -0.2836165428161621,
+      "logits/rejected": -0.6040266156196594,
+      "logps/chosen": -3.4633231163024902,
+      "logps/rejected": -57.385013580322266,
+      "loss": 0.36413243412971497,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.2521785497665405,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.29489317536354065,
+      "rewards/margins": 4.329642295837402,
+      "rewards/rejected": -4.0347490310668945,
+      "step": 287,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 1.2863534675615211,
+      "grad_norm": 2.8934900760650635,
+      "learning_rate": 6.130322111840114e-05,
+      "logits/chosen": -0.33327656984329224,
+      "logits/rejected": -0.6482805013656616,
+      "logps/chosen": -5.532173156738281,
+      "logps/rejected": -54.11648178100586,
+      "loss": 0.5029698610305786,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3735355734825134,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.26377081871032715,
+      "rewards/margins": 4.123575687408447,
+      "rewards/rejected": -3.85980486869812,
+      "step": 288,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 1.29082774049217,
+      "grad_norm": 4.349852085113525,
+      "learning_rate": 6.0619462188587793e-05,
+      "logits/chosen": -0.4272937774658203,
+      "logits/rejected": -0.6839829087257385,
+      "logps/chosen": -6.46608304977417,
+      "logps/rejected": -53.94236755371094,
+      "loss": 0.5734564065933228,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.45104509592056274,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.33807680010795593,
+      "rewards/margins": 4.335888385772705,
+      "rewards/rejected": -3.9978113174438477,
+      "step": 289,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 1.2953020134228188,
+      "grad_norm": 7.4762115478515625,
+      "learning_rate": 5.993787545301204e-05,
+      "logits/chosen": -0.3716397285461426,
+      "logits/rejected": -0.7264353632926941,
+      "logps/chosen": -6.0507588386535645,
+      "logps/rejected": -65.62486267089844,
+      "loss": 0.5227332711219788,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4141680598258972,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.22040605545043945,
+      "rewards/margins": 5.065747261047363,
+      "rewards/rejected": -4.845340728759766,
+      "step": 290,
+      "train_speed(iter/s)": 0.005622
+    },
+    {
+      "epoch": 1.2997762863534676,
+      "grad_norm": 1.7472515106201172,
+      "learning_rate": 5.9258498507371194e-05,
+      "logits/chosen": -0.33567675948143005,
+      "logits/rejected": -0.672459602355957,
+      "logps/chosen": -4.622915744781494,
+      "logps/rejected": -59.684303283691406,
+      "loss": 0.49214908480644226,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3552026152610779,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.2619546055793762,
+      "rewards/margins": 4.604626655578613,
+      "rewards/rejected": -4.342671871185303,
+      "step": 291,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 1.3042505592841163,
+      "grad_norm": 3.101167917251587,
+      "learning_rate": 5.8581368825472585e-05,
+      "logits/chosen": -0.404299259185791,
+      "logits/rejected": -0.6993152499198914,
+      "logps/chosen": -3.4533581733703613,
+      "logps/rejected": -59.574981689453125,
+      "loss": 0.36029624938964844,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.2589876055717468,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.31231242418289185,
+      "rewards/margins": 4.646473407745361,
+      "rewards/rejected": -4.334160804748535,
+      "step": 292,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 1.308724832214765,
+      "grad_norm": 1.6495614051818848,
+      "learning_rate": 5.790652375716652e-05,
+      "logits/chosen": -0.34474000334739685,
+      "logits/rejected": -0.6941490173339844,
+      "logps/chosen": -4.03609037399292,
+      "logps/rejected": -66.86280822753906,
+      "loss": 0.4014691114425659,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.33410951495170593,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.22682754695415497,
+      "rewards/margins": 5.202803611755371,
+      "rewards/rejected": -4.975976467132568,
+      "step": 293,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 1.313199105145414,
+      "grad_norm": 3.4665303230285645,
+      "learning_rate": 5.7234000526286156e-05,
+      "logits/chosen": -0.3931652903556824,
+      "logits/rejected": -0.6202120780944824,
+      "logps/chosen": -3.0910351276397705,
+      "logps/rejected": -51.20895767211914,
+      "loss": 0.3959859311580658,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.2664206922054291,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.27823901176452637,
+      "rewards/margins": 4.091604709625244,
+      "rewards/rejected": -3.8133652210235596,
+      "step": 294,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 1.3176733780760626,
+      "grad_norm": 2.0953636169433594,
+      "learning_rate": 5.656383622859418e-05,
+      "logits/chosen": -0.38693806529045105,
+      "logits/rejected": -0.6490756273269653,
+      "logps/chosen": -4.278903484344482,
+      "logps/rejected": -51.5899772644043,
+      "loss": 0.41964665055274963,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.2831048369407654,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.31565162539482117,
+      "rewards/margins": 4.059335231781006,
+      "rewards/rejected": -3.7436835765838623,
+      "step": 295,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 1.3221476510067114,
+      "grad_norm": 2.943864345550537,
+      "learning_rate": 5.589606782973683e-05,
+      "logits/chosen": -0.3606775104999542,
+      "logits/rejected": -0.5533517599105835,
+      "logps/chosen": -3.7958004474639893,
+      "logps/rejected": -53.00364685058594,
+      "loss": 0.39812713861465454,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.2592059075832367,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3119865357875824,
+      "rewards/margins": 4.129364490509033,
+      "rewards/rejected": -3.817378044128418,
+      "step": 296,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 1.3266219239373602,
+      "grad_norm": 3.774707794189453,
+      "learning_rate": 5.5230732163204615e-05,
+      "logits/chosen": -0.34767642617225647,
+      "logits/rejected": -0.6499403715133667,
+      "logps/chosen": -3.6022303104400635,
+      "logps/rejected": -54.93785095214844,
+      "loss": 0.40875381231307983,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.2905837595462799,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.36702224612236023,
+      "rewards/margins": 4.16538667678833,
+      "rewards/rejected": -3.7983648777008057,
+      "step": 297,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 1.331096196868009,
+      "grad_norm": 1.5885363817214966,
+      "learning_rate": 5.456786592830083e-05,
+      "logits/chosen": -0.3518737554550171,
+      "logits/rejected": -0.6097044944763184,
+      "logps/chosen": -4.608407020568848,
+      "logps/rejected": -53.15043258666992,
+      "loss": 0.40737074613571167,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.31507447361946106,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.35934901237487793,
+      "rewards/margins": 4.229493618011475,
+      "rewards/rejected": -3.870144844055176,
+      "step": 298,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 1.3355704697986577,
+      "grad_norm": 4.016380786895752,
+      "learning_rate": 5.39075056881172e-05,
+      "logits/chosen": -0.4338231086730957,
+      "logits/rejected": -0.659954309463501,
+      "logps/chosen": -5.361896514892578,
+      "logps/rejected": -52.52497863769531,
+      "loss": 0.4883747100830078,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.40210235118865967,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.23951874673366547,
+      "rewards/margins": 4.1094865798950195,
+      "rewards/rejected": -3.8699681758880615,
+      "step": 299,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 1.3400447427293065,
+      "grad_norm": 3.0947515964508057,
+      "learning_rate": 5.3249687867517095e-05,
+      "logits/chosen": -0.3721660077571869,
+      "logits/rejected": -0.6082777380943298,
+      "logps/chosen": -5.387287139892578,
+      "logps/rejected": -47.353939056396484,
+      "loss": 0.649885892868042,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4394390881061554,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.209368497133255,
+      "rewards/margins": 3.4814958572387695,
+      "rewards/rejected": -3.272127628326416,
+      "step": 300,
+      "train_speed(iter/s)": 0.005621
+    },
+    {
+      "epoch": 1.3400447427293065,
+      "eval_logits/chosen": -0.3397250771522522,
+      "eval_logits/rejected": -0.614118754863739,
+      "eval_logps/chosen": -7.453134536743164,
+      "eval_logps/rejected": -45.51069641113281,
+      "eval_loss": 0.7943270206451416,
+      "eval_nll_loss": 0.5665009617805481,
+      "eval_rewards/accuracies": 0.9027777910232544,
+      "eval_rewards/chosen": 0.16647693514823914,
+      "eval_rewards/margins": 3.2694599628448486,
+      "eval_rewards/rejected": -3.102982997894287,
+      "eval_runtime": 158.2199,
+      "eval_samples_per_second": 0.91,
+      "eval_steps_per_second": 0.228,
+      "step": 300
+    },
+    {
+      "epoch": 1.3445190156599551,
+      "grad_norm": 1.8513437509536743,
+      "learning_rate": 5.259444875112624e-05,
+      "logits/chosen": -0.35166898369789124,
+      "logits/rejected": -0.5892441868782043,
+      "logps/chosen": -6.206982612609863,
+      "logps/rejected": -53.98065185546875,
+      "loss": 0.522982120513916,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.41661083698272705,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.4109624922275543,
+      "rewards/margins": 4.339663028717041,
+      "rewards/rejected": -3.9287004470825195,
+      "step": 301,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.348993288590604,
+      "grad_norm": 2.262685537338257,
+      "learning_rate": 5.1941824481331626e-05,
+      "logits/chosen": -0.27198654413223267,
+      "logits/rejected": -0.5579496026039124,
+      "logps/chosen": -5.262180328369141,
+      "logps/rejected": -52.79492950439453,
+      "loss": 0.5066659450531006,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.40973570942878723,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.36156827211380005,
+      "rewards/margins": 4.104053497314453,
+      "rewards/rejected": -3.742485523223877,
+      "step": 302,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.3534675615212528,
+      "grad_norm": 2.9432220458984375,
+      "learning_rate": 5.129185105628756e-05,
+      "logits/chosen": -0.33754950761795044,
+      "logits/rejected": -0.6656984686851501,
+      "logps/chosen": -5.836930274963379,
+      "logps/rejected": -57.983577728271484,
+      "loss": 0.5136628150939941,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.411702424287796,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.2325628250837326,
+      "rewards/margins": 4.218040466308594,
+      "rewards/rejected": -3.9854774475097656,
+      "step": 303,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.3579418344519016,
+      "grad_norm": 1.5537747144699097,
+      "learning_rate": 5.064456432793019e-05,
+      "logits/chosen": -0.3143869936466217,
+      "logits/rejected": -0.6705507040023804,
+      "logps/chosen": -4.21674108505249,
+      "logps/rejected": -56.43550491333008,
+      "loss": 0.37259286642074585,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.2988947033882141,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.32924148440361023,
+      "rewards/margins": 4.115330696105957,
+      "rewards/rejected": -3.7860894203186035,
+      "step": 304,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.3624161073825503,
+      "grad_norm": 5.257437705993652,
+      "learning_rate": 5.000000000000002e-05,
+      "logits/chosen": -0.2882038950920105,
+      "logits/rejected": -0.57389897108078,
+      "logps/chosen": -4.055515289306641,
+      "logps/rejected": -53.850440979003906,
+      "loss": 0.3756740987300873,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.2755494713783264,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.2226015329360962,
+      "rewards/margins": 4.168307304382324,
+      "rewards/rejected": -3.9457051753997803,
+      "step": 305,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.366890380313199,
+      "grad_norm": 2.4108705520629883,
+      "learning_rate": 4.93581936260724e-05,
+      "logits/chosen": -0.3027994632720947,
+      "logits/rejected": -0.5452630519866943,
+      "logps/chosen": -5.911576271057129,
+      "logps/rejected": -46.840003967285156,
+      "loss": 0.6861900091171265,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.46356502175331116,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.27359738945961,
+      "rewards/margins": 3.5590786933898926,
+      "rewards/rejected": -3.2854814529418945,
+      "step": 306,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.371364653243848,
+      "grad_norm": 3.5625786781311035,
+      "learning_rate": 4.8719180607596484e-05,
+      "logits/chosen": -0.3174867033958435,
+      "logits/rejected": -0.5927554368972778,
+      "logps/chosen": -5.7471771240234375,
+      "logps/rejected": -49.149932861328125,
+      "loss": 0.6849573254585266,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5026857256889343,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": 0.1782091110944748,
+      "rewards/margins": 3.6787338256835938,
+      "rewards/rejected": -3.5005247592926025,
+      "step": 307,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.3758389261744965,
+      "grad_norm": 2.4612863063812256,
+      "learning_rate": 4.808299619194251e-05,
+      "logits/chosen": -0.29022541642189026,
+      "logits/rejected": -0.5778020620346069,
+      "logps/chosen": -4.631112098693848,
+      "logps/rejected": -52.577415466308594,
+      "loss": 0.4348449110984802,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.30078357458114624,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.29566630721092224,
+      "rewards/margins": 3.845643997192383,
+      "rewards/rejected": -3.5499777793884277,
+      "step": 308,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.3803131991051454,
+      "grad_norm": 1.6603471040725708,
+      "learning_rate": 4.744967547045754e-05,
+      "logits/chosen": -0.4165666103363037,
+      "logits/rejected": -0.6750180125236511,
+      "logps/chosen": -4.548413276672363,
+      "logps/rejected": -48.195499420166016,
+      "loss": 0.46350663900375366,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.36306896805763245,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.3399573862552643,
+      "rewards/margins": 3.9113879203796387,
+      "rewards/rejected": -3.5714306831359863,
+      "step": 309,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.3847874720357942,
+      "grad_norm": 6.359286308288574,
+      "learning_rate": 4.681925337653006e-05,
+      "logits/chosen": -0.4898450970649719,
+      "logits/rejected": -0.6976213455200195,
+      "logps/chosen": -2.3535208702087402,
+      "logps/rejected": -46.78605270385742,
+      "loss": 0.345736563205719,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.21656113862991333,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.34265750646591187,
+      "rewards/margins": 3.7292532920837402,
+      "rewards/rejected": -3.3865957260131836,
+      "step": 310,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.389261744966443,
+      "grad_norm": 1.8393584489822388,
+      "learning_rate": 4.6191764683662744e-05,
+      "logits/chosen": -0.3028990924358368,
+      "logits/rejected": -0.6557048559188843,
+      "logps/chosen": -3.447323799133301,
+      "logps/rejected": -54.03047180175781,
+      "loss": 0.34364035725593567,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.25607097148895264,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.3466619849205017,
+      "rewards/margins": 4.0746564865112305,
+      "rewards/rejected": -3.727994918823242,
+      "step": 311,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.3937360178970917,
+      "grad_norm": 1.5197970867156982,
+      "learning_rate": 4.5567244003554645e-05,
+      "logits/chosen": -0.3376431465148926,
+      "logits/rejected": -0.6943186521530151,
+      "logps/chosen": -5.426862716674805,
+      "logps/rejected": -52.74493408203125,
+      "loss": 0.5423266291618347,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4564194083213806,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.417688250541687,
+      "rewards/margins": 4.0157670974731445,
+      "rewards/rejected": -3.598078727722168,
+      "step": 312,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.3982102908277405,
+      "grad_norm": 1.6577972173690796,
+      "learning_rate": 4.494572578419194e-05,
+      "logits/chosen": -0.44612032175064087,
+      "logits/rejected": -0.5799017548561096,
+      "logps/chosen": -3.894256830215454,
+      "logps/rejected": -40.50112533569336,
+      "loss": 0.4608542323112488,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3280629813671112,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.35617274045944214,
+      "rewards/margins": 3.3939948081970215,
+      "rewards/rejected": -3.0378222465515137,
+      "step": 313,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.4026845637583891,
+      "grad_norm": 2.6486778259277344,
+      "learning_rate": 4.432724430794786e-05,
+      "logits/chosen": -0.33609506487846375,
+      "logits/rejected": -0.6439554691314697,
+      "logps/chosen": -5.650619029998779,
+      "logps/rejected": -49.782135009765625,
+      "loss": 0.53228360414505,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3895391523838043,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.3346465826034546,
+      "rewards/margins": 3.677081346511841,
+      "rewards/rejected": -3.3424346446990967,
+      "step": 314,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.407158836689038,
+      "grad_norm": 7.691840648651123,
+      "learning_rate": 4.371183368969165e-05,
+      "logits/chosen": -0.4097428023815155,
+      "logits/rejected": -0.7250416278839111,
+      "logps/chosen": -3.7532975673675537,
+      "logps/rejected": -60.11164093017578,
+      "loss": 0.38747715950012207,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.31893986463546753,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.16259630024433136,
+      "rewards/margins": 4.627762317657471,
+      "rewards/rejected": -4.465166091918945,
+      "step": 315,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.4116331096196868,
+      "grad_norm": 1.6380422115325928,
+      "learning_rate": 4.309952787490689e-05,
+      "logits/chosen": -0.41353487968444824,
+      "logits/rejected": -0.656099796295166,
+      "logps/chosen": -7.336640357971191,
+      "logps/rejected": -49.86144256591797,
+      "loss": 0.6947516798973083,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5703275799751282,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5092569589614868,
+      "rewards/margins": 4.116418361663818,
+      "rewards/rejected": -3.607161283493042,
+      "step": 316,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.4161073825503356,
+      "grad_norm": 6.258267402648926,
+      "learning_rate": 4.249036063781896e-05,
+      "logits/chosen": -0.3748295307159424,
+      "logits/rejected": -0.6187299489974976,
+      "logps/chosen": -3.833024024963379,
+      "logps/rejected": -52.662418365478516,
+      "loss": 0.3898906409740448,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.2913508415222168,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.3392498195171356,
+      "rewards/margins": 4.161679267883301,
+      "rewards/rejected": -3.8224291801452637,
+      "step": 317,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.4205816554809845,
+      "grad_norm": 1.4691777229309082,
+      "learning_rate": 4.1884365579532346e-05,
+      "logits/chosen": -0.3068940043449402,
+      "logits/rejected": -0.69428551197052,
+      "logps/chosen": -4.452873229980469,
+      "logps/rejected": -57.829078674316406,
+      "loss": 0.3776450753211975,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.28578072786331177,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.3617071211338043,
+      "rewards/margins": 4.398006439208984,
+      "rewards/rejected": -4.036299228668213,
+      "step": 318,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.425055928411633,
+      "grad_norm": 2.0084006786346436,
+      "learning_rate": 4.128157612617696e-05,
+      "logits/chosen": -0.41073668003082275,
+      "logits/rejected": -0.6919273734092712,
+      "logps/chosen": -7.601768493652344,
+      "logps/rejected": -57.00600051879883,
+      "loss": 0.7179478406906128,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5513160824775696,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.22752675414085388,
+      "rewards/margins": 4.380159378051758,
+      "rewards/rejected": -4.152632236480713,
+      "step": 319,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.429530201342282,
+      "grad_norm": 4.723047733306885,
+      "learning_rate": 4.0682025527064486e-05,
+      "logits/chosen": -0.14715087413787842,
+      "logits/rejected": -0.5763663053512573,
+      "logps/chosen": -5.163531303405762,
+      "logps/rejected": -71.21774291992188,
+      "loss": 0.5922179818153381,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4672166705131531,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.12609480321407318,
+      "rewards/margins": 4.96981954574585,
+      "rewards/rejected": -4.843725204467773,
+      "step": 320,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.4340044742729305,
+      "grad_norm": 3.720151424407959,
+      "learning_rate": 4.008574685285442e-05,
+      "logits/chosen": -0.34143364429473877,
+      "logits/rejected": -0.6579993963241577,
+      "logps/chosen": -5.25741720199585,
+      "logps/rejected": -58.68061447143555,
+      "loss": 0.467029333114624,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.320791095495224,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.32364290952682495,
+      "rewards/margins": 4.543745040893555,
+      "rewards/rejected": -4.220102310180664,
+      "step": 321,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.4384787472035794,
+      "grad_norm": 2.8483550548553467,
+      "learning_rate": 3.94927729937299e-05,
+      "logits/chosen": -0.3293909430503845,
+      "logits/rejected": -0.5969005823135376,
+      "logps/chosen": -6.698765277862549,
+      "logps/rejected": -58.93670654296875,
+      "loss": 0.5324967503547668,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3875662386417389,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.3495822548866272,
+      "rewards/margins": 4.754834175109863,
+      "rewards/rejected": -4.405251979827881,
+      "step": 322,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.4429530201342282,
+      "grad_norm": 1.6832650899887085,
+      "learning_rate": 3.890313665758348e-05,
+      "logits/chosen": -0.4338662624359131,
+      "logits/rejected": -0.7030037641525269,
+      "logps/chosen": -5.404772758483887,
+      "logps/rejected": -54.58758544921875,
+      "loss": 0.5070828795433044,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4381164610385895,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.32039517164230347,
+      "rewards/margins": 4.310638427734375,
+      "rewards/rejected": -3.9902429580688477,
+      "step": 323,
+      "train_speed(iter/s)": 0.005602
+    },
+    {
+      "epoch": 1.447427293064877,
+      "grad_norm": 1.7722690105438232,
+      "learning_rate": 3.8316870368213e-05,
+      "logits/chosen": -0.3775503933429718,
+      "logits/rejected": -0.5909088850021362,
+      "logps/chosen": -6.948042392730713,
+      "logps/rejected": -53.3364143371582,
+      "loss": 0.513221025466919,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.4226687550544739,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.4537815451622009,
+      "rewards/margins": 4.239094257354736,
+      "rewards/rejected": -3.785313129425049,
+      "step": 324,
+      "train_speed(iter/s)": 0.005601
+    },
+    {
+      "epoch": 1.4519015659955257,
+      "grad_norm": 3.676236629486084,
+      "learning_rate": 3.773400646352769e-05,
+      "logits/chosen": -0.36781182885169983,
+      "logits/rejected": -0.6920539140701294,
+      "logps/chosen": -3.714851140975952,
+      "logps/rejected": -61.90922546386719,
+      "loss": 0.40670472383499146,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3368794023990631,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34268397092819214,
+      "rewards/margins": 4.873111724853516,
+      "rewards/rejected": -4.530427932739258,
+      "step": 325,
+      "train_speed(iter/s)": 0.005601
+    },
+    {
+      "epoch": 1.4563758389261745,
+      "grad_norm": 1.8188880681991577,
+      "learning_rate": 3.7154577093764334e-05,
+      "logits/chosen": -0.395844966173172,
+      "logits/rejected": -0.6281816363334656,
+      "logps/chosen": -7.8523454666137695,
+      "logps/rejected": -50.93449020385742,
+      "loss": 0.486368328332901,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.382522851228714,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4690604507923126,
+      "rewards/margins": 4.0102763175964355,
+      "rewards/rejected": -3.541215658187866,
+      "step": 326,
+      "train_speed(iter/s)": 0.005601
+    },
+    {
+      "epoch": 1.4608501118568233,
+      "grad_norm": 1.347862958908081,
+      "learning_rate": 3.657861421971388e-05,
+      "logits/chosen": -0.3429858386516571,
+      "logits/rejected": -0.6542443037033081,
+      "logps/chosen": -3.7786176204681396,
+      "logps/rejected": -60.83111572265625,
+      "loss": 0.41075778007507324,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.31700214743614197,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.2717721462249756,
+      "rewards/margins": 4.375649929046631,
+      "rewards/rejected": -4.103877544403076,
+      "step": 327,
+      "train_speed(iter/s)": 0.005601
+    },
+    {
+      "epoch": 1.465324384787472,
+      "grad_norm": 2.8427813053131104,
+      "learning_rate": 3.6006149610958625e-05,
+      "logits/chosen": -0.3848859965801239,
+      "logits/rejected": -0.5358583331108093,
+      "logps/chosen": -4.8809051513671875,
+      "logps/rejected": -46.17042541503906,
+      "loss": 0.45546215772628784,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3384712338447571,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4213294982910156,
+      "rewards/margins": 3.781106472015381,
+      "rewards/rejected": -3.3597772121429443,
+      "step": 328,
+      "train_speed(iter/s)": 0.005601
+    },
+    {
+      "epoch": 1.4697986577181208,
+      "grad_norm": 3.1482083797454834,
+      "learning_rate": 3.543721484411976e-05,
+      "logits/chosen": -0.2849827706813812,
+      "logits/rejected": -0.6585791707038879,
+      "logps/chosen": -4.360767364501953,
+      "logps/rejected": -55.06772994995117,
+      "loss": 0.46504268050193787,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.36682000756263733,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.38763877749443054,
+      "rewards/margins": 4.205320835113525,
+      "rewards/rejected": -3.8176825046539307,
+      "step": 329,
+      "train_speed(iter/s)": 0.005601
+    },
+    {
+      "epoch": 1.4742729306487696,
+      "grad_norm": 3.2574269771575928,
+      "learning_rate": 3.487184130111562e-05,
+      "logits/chosen": -0.3076302111148834,
+      "logits/rejected": -0.6256691217422485,
+      "logps/chosen": -4.316102981567383,
+      "logps/rejected": -58.86445617675781,
+      "loss": 0.4626595377922058,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.335349440574646,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.3723066747188568,
+      "rewards/margins": 4.587831020355225,
+      "rewards/rejected": -4.215524196624756,
+      "step": 330,
+      "train_speed(iter/s)": 0.005601
+    },
+    {
+      "epoch": 1.4787472035794185,
+      "grad_norm": 1.6450964212417603,
+      "learning_rate": 3.4310060167430725e-05,
+      "logits/chosen": -0.32500436902046204,
+      "logits/rejected": -0.624412477016449,
+      "logps/chosen": -4.937263488769531,
+      "logps/rejected": -59.63166809082031,
+      "loss": 0.39044544100761414,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.30855754017829895,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2391117811203003,
+      "rewards/margins": 4.519699573516846,
+      "rewards/rejected": -4.280588150024414,
+      "step": 331,
+      "train_speed(iter/s)": 0.005601
+    },
+    {
+      "epoch": 1.483221476510067,
+      "grad_norm": 1.7565689086914062,
+      "learning_rate": 3.375190243039556e-05,
+      "logits/chosen": -0.3314094543457031,
+      "logits/rejected": -0.6345155239105225,
+      "logps/chosen": -4.823784828186035,
+      "logps/rejected": -51.84589767456055,
+      "loss": 0.4534035921096802,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3457861542701721,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.22532111406326294,
+      "rewards/margins": 3.867464542388916,
+      "rewards/rejected": -3.642143726348877,
+      "step": 332,
+      "train_speed(iter/s)": 0.005601
+    },
+    {
+      "epoch": 1.487695749440716,
+      "grad_norm": 4.223317623138428,
+      "learning_rate": 3.319739887747752e-05,
+      "logits/chosen": -0.5204321146011353,
+      "logits/rejected": -0.7036163210868835,
+      "logps/chosen": -4.016556262969971,
+      "logps/rejected": -48.14368438720703,
+      "loss": 0.4340854287147522,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.31488320231437683,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.32344233989715576,
+      "rewards/margins": 3.877997875213623,
+      "rewards/rejected": -3.5545554161071777,
+      "step": 333,
+      "train_speed(iter/s)": 0.005601
+    },
+    {
+      "epoch": 1.4921700223713645,
+      "grad_norm": 1.824302077293396,
+      "learning_rate": 3.264658009458239e-05,
+      "logits/chosen": -0.3357248306274414,
+      "logits/rejected": -0.5842665433883667,
+      "logps/chosen": -8.244744300842285,
+      "logps/rejected": -52.15079116821289,
+      "loss": 0.6133258938789368,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.5364583730697632,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4413433074951172,
+      "rewards/margins": 4.141359806060791,
+      "rewards/rejected": -3.700016498565674,
+      "step": 334,
+      "train_speed(iter/s)": 0.005601
+    },
+    {
+      "epoch": 1.4966442953020134,
+      "grad_norm": 2.2434675693511963,
+      "learning_rate": 3.209947646436752e-05,
+      "logits/chosen": -0.2935329079627991,
+      "logits/rejected": -0.5861437916755676,
+      "logps/chosen": -5.096604824066162,
+      "logps/rejected": -55.1965446472168,
+      "loss": 0.5088061094284058,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.37970277667045593,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2670452892780304,
+      "rewards/margins": 4.1344499588012695,
+      "rewards/rejected": -3.8674044609069824,
+      "step": 335,
+      "train_speed(iter/s)": 0.005601
+    },
+    {
+      "epoch": 1.5011185682326622,
+      "grad_norm": 5.45697546005249,
+      "learning_rate": 3.155611816456586e-05,
+      "logits/chosen": -0.39318567514419556,
+      "logits/rejected": -0.5622078776359558,
+      "logps/chosen": -6.846230506896973,
+      "logps/rejected": -50.98385238647461,
+      "loss": 0.49888622760772705,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.38763007521629333,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.3062732219696045,
+      "rewards/margins": 4.060297012329102,
+      "rewards/rejected": -3.754023551940918,
+      "step": 336,
+      "train_speed(iter/s)": 0.005601
+    },
+    {
+      "epoch": 1.505592841163311,
+      "grad_norm": 1.7538950443267822,
+      "learning_rate": 3.1016535166321356e-05,
+      "logits/chosen": -0.3007366359233856,
+      "logits/rejected": -0.5629429817199707,
+      "logps/chosen": -4.308128833770752,
+      "logps/rejected": -55.02139663696289,
+      "loss": 0.3758111298084259,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.2836534082889557,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3867635428905487,
+      "rewards/margins": 4.165569305419922,
+      "rewards/rejected": -3.778805732727051,
+      "step": 337,
+      "train_speed(iter/s)": 0.005601
+    },
+    {
+      "epoch": 1.5100671140939599,
+      "grad_norm": 1.6262391805648804,
+      "learning_rate": 3.0480757232535772e-05,
+      "logits/chosen": -0.3634895980358124,
+      "logits/rejected": -0.6413396596908569,
+      "logps/chosen": -5.209499359130859,
+      "logps/rejected": -53.341461181640625,
+      "loss": 0.46990200877189636,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3806995153427124,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.3192530572414398,
+      "rewards/margins": 4.083569526672363,
+      "rewards/rejected": -3.7643163204193115,
+      "step": 338,
+      "train_speed(iter/s)": 0.005601
+    },
+    {
+      "epoch": 1.5145413870246085,
+      "grad_norm": 5.5195088386535645,
+      "learning_rate": 2.9948813916227115e-05,
+      "logits/chosen": -0.3451216220855713,
+      "logits/rejected": -0.6123970150947571,
+      "logps/chosen": -5.359556198120117,
+      "logps/rejected": -54.252960205078125,
+      "loss": 0.46342599391937256,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3530768156051636,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.24546122550964355,
+      "rewards/margins": 4.225433826446533,
+      "rewards/rejected": -3.9799728393554688,
+      "step": 339,
+      "train_speed(iter/s)": 0.005601
+    },
+    {
+      "epoch": 1.5190156599552571,
+      "grad_norm": 3.048055648803711,
+      "learning_rate": 2.9420734558899322e-05,
+      "logits/chosen": -0.38990306854248047,
+      "logits/rejected": -0.6969528198242188,
+      "logps/chosen": -5.715635299682617,
+      "logps/rejected": -51.79830551147461,
+      "loss": 0.4567548930644989,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3587338328361511,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.3831934928894043,
+      "rewards/margins": 4.281648635864258,
+      "rewards/rejected": -3.8984546661376953,
+      "step": 340,
+      "train_speed(iter/s)": 0.005601
+    },
+    {
+      "epoch": 1.523489932885906,
+      "grad_norm": 2.501689910888672,
+      "learning_rate": 2.889654828892393e-05,
+      "logits/chosen": -0.24420952796936035,
+      "logits/rejected": -0.578873336315155,
+      "logps/chosen": -5.0676751136779785,
+      "logps/rejected": -59.5233154296875,
+      "loss": 0.37155669927597046,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.31278476119041443,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4591803550720215,
+      "rewards/margins": 4.813053607940674,
+      "rewards/rejected": -4.353872776031494,
+      "step": 341,
+      "train_speed(iter/s)": 0.0056
+    },
+    {
+      "epoch": 1.5279642058165548,
+      "grad_norm": 1.5354278087615967,
+      "learning_rate": 2.8376284019933373e-05,
+      "logits/chosen": -0.4540112018585205,
+      "logits/rejected": -0.7229375243186951,
+      "logps/chosen": -4.83169412612915,
+      "logps/rejected": -54.2523193359375,
+      "loss": 0.45802274346351624,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3309043347835541,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.35657066106796265,
+      "rewards/margins": 4.4327545166015625,
+      "rewards/rejected": -4.076183795928955,
+      "step": 342,
+      "train_speed(iter/s)": 0.0056
+    },
+    {
+      "epoch": 1.5324384787472036,
+      "grad_norm": 2.8110909461975098,
+      "learning_rate": 2.7859970449226104e-05,
+      "logits/chosen": -0.3509393334388733,
+      "logits/rejected": -0.6647713780403137,
+      "logps/chosen": -2.810865640640259,
+      "logps/rejected": -52.086273193359375,
+      "loss": 0.3939270079135895,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.2697364091873169,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.3155217170715332,
+      "rewards/margins": 3.9285857677459717,
+      "rewards/rejected": -3.6130642890930176,
+      "step": 343,
+      "train_speed(iter/s)": 0.005601
+    },
+    {
+      "epoch": 1.5369127516778525,
+      "grad_norm": 1.7518740892410278,
+      "learning_rate": 2.73476360561837e-05,
+      "logits/chosen": -0.2257290929555893,
+      "logits/rejected": -0.5468841791152954,
+      "logps/chosen": -3.983034133911133,
+      "logps/rejected": -57.0892219543457,
+      "loss": 0.4137822091579437,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.30985069274902344,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4407218098640442,
+      "rewards/margins": 4.506251335144043,
+      "rewards/rejected": -4.0655293464660645,
+      "step": 344,
+      "train_speed(iter/s)": 0.0056
+    },
+    {
+      "epoch": 1.541387024608501,
+      "grad_norm": 1.7124443054199219,
+      "learning_rate": 2.6839309100699973e-05,
+      "logits/chosen": -0.3101522922515869,
+      "logits/rejected": -0.6017225384712219,
+      "logps/chosen": -4.0843119621276855,
+      "logps/rejected": -56.81201934814453,
+      "loss": 0.4287090301513672,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3240818381309509,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2648446261882782,
+      "rewards/margins": 4.452305316925049,
+      "rewards/rejected": -4.187460899353027,
+      "step": 345,
+      "train_speed(iter/s)": 0.0056
+    },
+    {
+      "epoch": 1.54586129753915,
+      "grad_norm": 2.6213417053222656,
+      "learning_rate": 2.6335017621622116e-05,
+      "logits/chosen": -0.40039491653442383,
+      "logits/rejected": -0.699411153793335,
+      "logps/chosen": -4.307460308074951,
+      "logps/rejected": -52.542236328125,
+      "loss": 0.5415531992912292,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.35593974590301514,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2844778001308441,
+      "rewards/margins": 3.960409641265869,
+      "rewards/rejected": -3.675931453704834,
+      "step": 346,
+      "train_speed(iter/s)": 0.0056
+    },
+    {
+      "epoch": 1.5503355704697985,
+      "grad_norm": 2.4221184253692627,
+      "learning_rate": 2.5834789435204243e-05,
+      "logits/chosen": -0.28662967681884766,
+      "logits/rejected": -0.5893158912658691,
+      "logps/chosen": -5.251837730407715,
+      "logps/rejected": -55.46297836303711,
+      "loss": 0.46211305260658264,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3805118203163147,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.30533987283706665,
+      "rewards/margins": 4.2420220375061035,
+      "rewards/rejected": -3.9366822242736816,
+      "step": 347,
+      "train_speed(iter/s)": 0.0056
+    },
+    {
+      "epoch": 1.5548098434004474,
+      "grad_norm": 3.0144851207733154,
+      "learning_rate": 2.5338652133572915e-05,
+      "logits/chosen": -0.3970368504524231,
+      "logits/rejected": -0.5789538621902466,
+      "logps/chosen": -5.7425737380981445,
+      "logps/rejected": -50.25794219970703,
+      "loss": 0.4803181290626526,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3716895580291748,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.3145495057106018,
+      "rewards/margins": 4.102305889129639,
+      "rewards/rejected": -3.7877564430236816,
+      "step": 348,
+      "train_speed(iter/s)": 0.0056
+    },
+    {
+      "epoch": 1.5592841163310962,
+      "grad_norm": 4.8794660568237305,
+      "learning_rate": 2.4846633083205263e-05,
+      "logits/chosen": -0.3288257420063019,
+      "logits/rejected": -0.6307476758956909,
+      "logps/chosen": -4.90744161605835,
+      "logps/rejected": -60.52650451660156,
+      "loss": 0.5285750031471252,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.3855237364768982,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.20134314894676208,
+      "rewards/margins": 4.785792827606201,
+      "rewards/rejected": -4.584449291229248,
+      "step": 349,
+      "train_speed(iter/s)": 0.0056
+    },
+    {
+      "epoch": 1.563758389261745,
+      "grad_norm": 2.8724992275238037,
+      "learning_rate": 2.4358759423419474e-05,
+      "logits/chosen": -0.30530738830566406,
+      "logits/rejected": -0.5869967937469482,
+      "logps/chosen": -5.618663311004639,
+      "logps/rejected": -61.05699157714844,
+      "loss": 0.4239829182624817,
+      "memory(GiB)": 46.57,
+      "nll_loss": 0.33604365587234497,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5380984544754028,
+      "rewards/margins": 4.721798896789551,
+      "rewards/rejected": -4.183700084686279,
+      "step": 350,
+      "train_speed(iter/s)": 0.0056
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 446,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.0847082159026995e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}