diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,10868 @@
+{
+  "best_global_step": 600,
+  "best_metric": 0.58136773,
+  "best_model_checkpoint": "/data2/kdd/crag/cjz/output/task3_dpo_gt6_task2_gt30.3+task3gt5_0.1vp_idk0614.json/v0-20250614-165413/checkpoint-600",
+  "epoch": 0.48410277098409016,
+  "eval_steps": 300,
+  "global_step": 600,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0008068379516401503,
+      "grad_norm": 9.261910438537598,
+      "learning_rate": 3.225806451612903e-06,
+      "logits/chosen": -0.6039718985557556,
+      "logits/rejected": -0.6058337688446045,
+      "logps/chosen": -7.412725925445557,
+      "logps/rejected": -11.413676261901855,
+      "loss": 1.2381761074066162,
+      "memory(GiB)": 46.32,
+      "nll_loss": 0.5450288653373718,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1,
+      "train_speed(iter/s)": 0.005524
+    },
+    {
+      "epoch": 0.0016136759032803006,
+      "grad_norm": 5.391567230224609,
+      "learning_rate": 6.451612903225806e-06,
+      "logits/chosen": -0.6199994087219238,
+      "logits/rejected": -0.6217825412750244,
+      "logps/chosen": -7.599736213684082,
+      "logps/rejected": -15.575040817260742,
+      "loss": 1.1066936254501343,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.41354650259017944,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 2,
+      "train_speed(iter/s)": 0.005546
+    },
+    {
+      "epoch": 0.002420513854920451,
+      "grad_norm": 6.5534772872924805,
+      "learning_rate": 9.67741935483871e-06,
+      "logits/chosen": -0.6068329811096191,
+      "logits/rejected": -0.6103144288063049,
+      "logps/chosen": -11.938759803771973,
+      "logps/rejected": -14.32825756072998,
+      "loss": 1.267569661140442,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5742746591567993,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0005358309717848897,
+      "rewards/margins": -0.00011114418157376349,
+      "rewards/rejected": 0.0006469750078395009,
+      "step": 3,
+      "train_speed(iter/s)": 0.005487
+    },
+    {
+      "epoch": 0.003227351806560601,
+      "grad_norm": 3.4633655548095703,
+      "learning_rate": 1.2903225806451613e-05,
+      "logits/chosen": -0.6417700052261353,
+      "logits/rejected": -0.638687789440155,
+      "logps/chosen": -10.244372367858887,
+      "logps/rejected": -8.448144912719727,
+      "loss": 1.1101956367492676,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4145403802394867,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.002526381518691778,
+      "rewards/margins": -0.004837517160922289,
+      "rewards/rejected": 0.0023111351765692234,
+      "step": 4,
+      "train_speed(iter/s)": 0.005482
+    },
+    {
+      "epoch": 0.004034189758200751,
+      "grad_norm": 9.884061813354492,
+      "learning_rate": 1.6129032258064517e-05,
+      "logits/chosen": -0.6263395547866821,
+      "logits/rejected": -0.6316207647323608,
+      "logps/chosen": -8.768255233764648,
+      "logps/rejected": -12.074126243591309,
+      "loss": 1.2384487390518188,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5502734780311584,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.011151458136737347,
+      "rewards/margins": 0.010235130786895752,
+      "rewards/rejected": 0.0009163276990875602,
+      "step": 5,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.004841027709840902,
+      "grad_norm": 9.879764556884766,
+      "learning_rate": 1.935483870967742e-05,
+      "logits/chosen": -0.6110924482345581,
+      "logits/rejected": -0.6128084063529968,
+      "logps/chosen": -11.146334648132324,
+      "logps/rejected": -12.724503517150879,
+      "loss": 1.246738314628601,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5631918907165527,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.02471097745001316,
+      "rewards/margins": 0.02056196704506874,
+      "rewards/rejected": 0.004149014595896006,
+      "step": 6,
+      "train_speed(iter/s)": 0.005505
+    },
+    {
+      "epoch": 0.0056478656614810515,
+      "grad_norm": 3.7321648597717285,
+      "learning_rate": 2.258064516129032e-05,
+      "logits/chosen": -0.6434396505355835,
+      "logits/rejected": -0.6456248164176941,
+      "logps/chosen": -12.346940994262695,
+      "logps/rejected": -13.585854530334473,
+      "loss": 1.25950026512146,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5602214932441711,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 0.01709746941924095,
+      "rewards/margins": -0.004833616316318512,
+      "rewards/rejected": 0.021931089460849762,
+      "step": 7,
+      "train_speed(iter/s)": 0.005511
+    },
+    {
+      "epoch": 0.006454703613121202,
+      "grad_norm": 6.577890872955322,
+      "learning_rate": 2.5806451612903226e-05,
+      "logits/chosen": -0.6166916489601135,
+      "logits/rejected": -0.6146227717399597,
+      "logps/chosen": -16.807830810546875,
+      "logps/rejected": -11.38271427154541,
+      "loss": 1.2828420400619507,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5101535320281982,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.04314221069216728,
+      "rewards/margins": -0.13309986889362335,
+      "rewards/rejected": 0.08995765447616577,
+      "step": 8,
+      "train_speed(iter/s)": 0.005502
+    },
+    {
+      "epoch": 0.007261541564761352,
+      "grad_norm": 3.5513713359832764,
+      "learning_rate": 2.9032258064516133e-05,
+      "logits/chosen": -0.5994153022766113,
+      "logits/rejected": -0.6015004515647888,
+      "logps/chosen": -11.679065704345703,
+      "logps/rejected": -12.411300659179688,
+      "loss": 1.171980857849121,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4982386827468872,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.035934723913669586,
+      "rewards/margins": 0.08020534366369247,
+      "rewards/rejected": -0.04427062347531319,
+      "step": 9,
+      "train_speed(iter/s)": 0.005483
+    },
+    {
+      "epoch": 0.008068379516401502,
+      "grad_norm": 7.633578777313232,
+      "learning_rate": 3.2258064516129034e-05,
+      "logits/chosen": -0.6122820377349854,
+      "logits/rejected": -0.6125556826591492,
+      "logps/chosen": -12.382171630859375,
+      "logps/rejected": -10.675874710083008,
+      "loss": 1.3903008699417114,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6006877422332764,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.08313174545764923,
+      "rewards/margins": -0.13078533113002777,
+      "rewards/rejected": 0.04765357822179794,
+      "step": 10,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.008875217468041654,
+      "grad_norm": 4.818623065948486,
+      "learning_rate": 3.548387096774194e-05,
+      "logits/chosen": -0.6364253759384155,
+      "logits/rejected": -0.6395613551139832,
+      "logps/chosen": -7.883171558380127,
+      "logps/rejected": -15.457681655883789,
+      "loss": 1.1344804763793945,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.46768438816070557,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.008294675499200821,
+      "rewards/margins": 0.10872506350278854,
+      "rewards/rejected": -0.10043041408061981,
+      "step": 11,
+      "train_speed(iter/s)": 0.00549
+    },
+    {
+      "epoch": 0.009682055419681803,
+      "grad_norm": 3.8297243118286133,
+      "learning_rate": 3.870967741935484e-05,
+      "logits/chosen": -0.6082768440246582,
+      "logits/rejected": -0.6128782629966736,
+      "logps/chosen": -10.993257522583008,
+      "logps/rejected": -14.49487018585205,
+      "loss": 1.1111013889312744,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.42791634798049927,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.013819348067045212,
+      "rewards/margins": 0.08211039006710052,
+      "rewards/rejected": -0.09592973440885544,
+      "step": 12,
+      "train_speed(iter/s)": 0.005476
+    },
+    {
+      "epoch": 0.010488893371321953,
+      "grad_norm": 3.1733834743499756,
+      "learning_rate": 4.1935483870967746e-05,
+      "logits/chosen": -0.597683846950531,
+      "logits/rejected": -0.6003975868225098,
+      "logps/chosen": -7.611721038818359,
+      "logps/rejected": -13.589990615844727,
+      "loss": 0.9966970682144165,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3590569496154785,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.09378139674663544,
+      "rewards/margins": 0.16571055352687836,
+      "rewards/rejected": -0.07192917168140411,
+      "step": 13,
+      "train_speed(iter/s)": 0.005488
+    },
+    {
+      "epoch": 0.011295731322962103,
+      "grad_norm": 5.364157676696777,
+      "learning_rate": 4.516129032258064e-05,
+      "logits/chosen": -0.5847792029380798,
+      "logits/rejected": -0.5856592655181885,
+      "logps/chosen": -11.617069244384766,
+      "logps/rejected": -11.611367225646973,
+      "loss": 1.2788419723510742,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5410674214363098,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 0.037577953189611435,
+      "rewards/margins": -0.04553454369306564,
+      "rewards/rejected": 0.08311249315738678,
+      "step": 14,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.012102569274602255,
+      "grad_norm": 3.436879873275757,
+      "learning_rate": 4.8387096774193554e-05,
+      "logits/chosen": -0.5832479596138,
+      "logits/rejected": -0.5832363367080688,
+      "logps/chosen": -9.650967597961426,
+      "logps/rejected": -11.376352310180664,
+      "loss": 1.0634793043136597,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.34024038910865784,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.05362201854586601,
+      "rewards/margins": -0.029222950339317322,
+      "rewards/rejected": 0.08284495770931244,
+      "step": 15,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.012909407226242404,
+      "grad_norm": 3.2199597358703613,
+      "learning_rate": 5.161290322580645e-05,
+      "logits/chosen": -0.6036816239356995,
+      "logits/rejected": -0.606670618057251,
+      "logps/chosen": -10.059551239013672,
+      "logps/rejected": -16.143247604370117,
+      "loss": 1.0622740983963013,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3808459937572479,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.11249684542417526,
+      "rewards/margins": 0.040876831859350204,
+      "rewards/rejected": 0.07162001729011536,
+      "step": 16,
+      "train_speed(iter/s)": 0.00549
+    },
+    {
+      "epoch": 0.013716245177882554,
+      "grad_norm": 3.376516819000244,
+      "learning_rate": 5.4838709677419355e-05,
+      "logits/chosen": -0.5811513662338257,
+      "logits/rejected": -0.5822936296463013,
+      "logps/chosen": -10.964933395385742,
+      "logps/rejected": -13.541594505310059,
+      "loss": 1.1284891366958618,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.43761852383613586,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.11780606210231781,
+      "rewards/margins": 0.02046366035938263,
+      "rewards/rejected": 0.09734240174293518,
+      "step": 17,
+      "train_speed(iter/s)": 0.005493
+    },
+    {
+      "epoch": 0.014523083129522704,
+      "grad_norm": 3.292140245437622,
+      "learning_rate": 5.8064516129032266e-05,
+      "logits/chosen": -0.5822582840919495,
+      "logits/rejected": -0.5867500305175781,
+      "logps/chosen": -8.11985969543457,
+      "logps/rejected": -14.659163475036621,
+      "loss": 1.0471278429031372,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.40266352891921997,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.14493001997470856,
+      "rewards/margins": 0.1153046265244484,
+      "rewards/rejected": 0.029625393450260162,
+      "step": 18,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.015329921081162856,
+      "grad_norm": 3.320802688598633,
+      "learning_rate": 6.129032258064517e-05,
+      "logits/chosen": -0.5760956406593323,
+      "logits/rejected": -0.5776647925376892,
+      "logps/chosen": -9.478979110717773,
+      "logps/rejected": -11.301219940185547,
+      "loss": 1.1243548393249512,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.48838871717453003,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.20540697872638702,
+      "rewards/margins": 0.14464649558067322,
+      "rewards/rejected": 0.06076047942042351,
+      "step": 19,
+      "train_speed(iter/s)": 0.005493
+    },
+    {
+      "epoch": 0.016136759032803004,
+      "grad_norm": 2.7910656929016113,
+      "learning_rate": 6.451612903225807e-05,
+      "logits/chosen": -0.5840404033660889,
+      "logits/rejected": -0.582589864730835,
+      "logps/chosen": -13.635812759399414,
+      "logps/rejected": -10.431557655334473,
+      "loss": 1.2548669576644897,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.55869460105896,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.13213564455509186,
+      "rewards/margins": 0.044300854206085205,
+      "rewards/rejected": 0.08783479034900665,
+      "step": 20,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.016943596984443157,
+      "grad_norm": 2.2573604583740234,
+      "learning_rate": 6.774193548387096e-05,
+      "logits/chosen": -0.5839393734931946,
+      "logits/rejected": -0.5850896835327148,
+      "logps/chosen": -7.3839006423950195,
+      "logps/rejected": -9.574810028076172,
+      "loss": 0.9415242075920105,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.372197687625885,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.2435840666294098,
+      "rewards/margins": 0.30735281109809875,
+      "rewards/rejected": -0.06376874446868896,
+      "step": 21,
+      "train_speed(iter/s)": 0.005504
+    },
+    {
+      "epoch": 0.017750434936083307,
+      "grad_norm": 2.90157413482666,
+      "learning_rate": 7.096774193548388e-05,
+      "logits/chosen": -0.6114062070846558,
+      "logits/rejected": -0.6119735240936279,
+      "logps/chosen": -8.486170768737793,
+      "logps/rejected": -12.086725234985352,
+      "loss": 1.0081067085266113,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.34981024265289307,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.10258002579212189,
+      "rewards/margins": 0.12462681531906128,
+      "rewards/rejected": -0.02204679138958454,
+      "step": 22,
+      "train_speed(iter/s)": 0.005501
+    },
+    {
+      "epoch": 0.018557272887723457,
+      "grad_norm": 2.4457335472106934,
+      "learning_rate": 7.419354838709677e-05,
+      "logits/chosen": -0.5648061037063599,
+      "logits/rejected": -0.5674125552177429,
+      "logps/chosen": -12.962445259094238,
+      "logps/rejected": -15.476969718933105,
+      "loss": 1.0199884176254272,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4421077072620392,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.13948869705200195,
+      "rewards/margins": 0.31703731417655945,
+      "rewards/rejected": -0.17754864692687988,
+      "step": 23,
+      "train_speed(iter/s)": 0.005504
+    },
+    {
+      "epoch": 0.019364110839363607,
+      "grad_norm": 2.250957489013672,
+      "learning_rate": 7.741935483870968e-05,
+      "logits/chosen": -0.5476934313774109,
+      "logits/rejected": -0.5497113466262817,
+      "logps/chosen": -6.839375972747803,
+      "logps/rejected": -12.336576461791992,
+      "loss": 0.9127413630485535,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3841688632965088,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.2666690945625305,
+      "rewards/margins": 0.42246347665786743,
+      "rewards/rejected": -0.1557943969964981,
+      "step": 24,
+      "train_speed(iter/s)": 0.005505
+    },
+    {
+      "epoch": 0.020170948791003757,
+      "grad_norm": 3.6742801666259766,
+      "learning_rate": 8.064516129032258e-05,
+      "logits/chosen": -0.5886849164962769,
+      "logits/rejected": -0.5897050499916077,
+      "logps/chosen": -8.534127235412598,
+      "logps/rejected": -11.815479278564453,
+      "loss": 1.0272297859191895,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.41876962780952454,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.13335689902305603,
+      "rewards/margins": 0.27482476830482483,
+      "rewards/rejected": -0.1414678543806076,
+      "step": 25,
+      "train_speed(iter/s)": 0.005513
+    },
+    {
+      "epoch": 0.020977786742643906,
+      "grad_norm": 2.5902981758117676,
+      "learning_rate": 8.387096774193549e-05,
+      "logits/chosen": -0.5303969979286194,
+      "logits/rejected": -0.5350589752197266,
+      "logps/chosen": -9.592369079589844,
+      "logps/rejected": -17.837379455566406,
+      "loss": 0.9073010087013245,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3275393545627594,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1778128743171692,
+      "rewards/margins": 0.31808584928512573,
+      "rewards/rejected": -0.14027300477027893,
+      "step": 26,
+      "train_speed(iter/s)": 0.005509
+    },
+    {
+      "epoch": 0.021784624694284056,
+      "grad_norm": 3.656003952026367,
+      "learning_rate": 8.709677419354839e-05,
+      "logits/chosen": -0.5741274356842041,
+      "logits/rejected": -0.5759863257408142,
+      "logps/chosen": -7.401767253875732,
+      "logps/rejected": -9.595519065856934,
+      "loss": 0.9837043881416321,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.34733644127845764,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.16637372970581055,
+      "rewards/margins": 0.22424587607383728,
+      "rewards/rejected": -0.05787213519215584,
+      "step": 27,
+      "train_speed(iter/s)": 0.00551
+    },
+    {
+      "epoch": 0.022591462645924206,
+      "grad_norm": 3.6425983905792236,
+      "learning_rate": 9.032258064516129e-05,
+      "logits/chosen": -0.5271302461624146,
+      "logits/rejected": -0.5281438231468201,
+      "logps/chosen": -10.050779342651367,
+      "logps/rejected": -13.247575759887695,
+      "loss": 1.0190500020980835,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4030058681964874,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.022431861609220505,
+      "rewards/margins": 0.22834232449531555,
+      "rewards/rejected": -0.20591047406196594,
+      "step": 28,
+      "train_speed(iter/s)": 0.005522
+    },
+    {
+      "epoch": 0.02339830059756436,
+      "grad_norm": 4.539236545562744,
+      "learning_rate": 9.35483870967742e-05,
+      "logits/chosen": -0.574794352054596,
+      "logits/rejected": -0.5771020650863647,
+      "logps/chosen": -7.9426727294921875,
+      "logps/rejected": -11.816813468933105,
+      "loss": 1.0435985326766968,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4821586310863495,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.17023718357086182,
+      "rewards/margins": 0.36495453119277954,
+      "rewards/rejected": -0.19471733272075653,
+      "step": 29,
+      "train_speed(iter/s)": 0.005525
+    },
+    {
+      "epoch": 0.02420513854920451,
+      "grad_norm": 3.463172674179077,
+      "learning_rate": 9.677419354838711e-05,
+      "logits/chosen": -0.5663347244262695,
+      "logits/rejected": -0.5671164393424988,
+      "logps/chosen": -8.761878967285156,
+      "logps/rejected": -16.922607421875,
+      "loss": 0.8841907978057861,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.37557756900787354,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.23288068175315857,
+      "rewards/margins": 0.47043219208717346,
+      "rewards/rejected": -0.2375514954328537,
+      "step": 30,
+      "train_speed(iter/s)": 0.005522
+    },
+    {
+      "epoch": 0.02501197650084466,
+      "grad_norm": 7.054537296295166,
+      "learning_rate": 0.0001,
+      "logits/chosen": -0.5485356450080872,
+      "logits/rejected": -0.5510388612747192,
+      "logps/chosen": -8.1714506149292,
+      "logps/rejected": -15.964082717895508,
+      "loss": 1.0905145406723022,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.48109474778175354,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.14875195920467377,
+      "rewards/margins": 0.23288127779960632,
+      "rewards/rejected": -0.08412933349609375,
+      "step": 31,
+      "train_speed(iter/s)": 0.005519
+    },
+    {
+      "epoch": 0.02581881445248481,
+      "grad_norm": 2.8915627002716064,
+      "learning_rate": 0.0001032258064516129,
+      "logits/chosen": -0.5415224432945251,
+      "logits/rejected": -0.5406837463378906,
+      "logps/chosen": -12.312981605529785,
+      "logps/rejected": -14.444657325744629,
+      "loss": 0.9778124094009399,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4027611017227173,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.11852823942899704,
+      "rewards/margins": 0.3213120102882385,
+      "rewards/rejected": -0.2027837634086609,
+      "step": 32,
+      "train_speed(iter/s)": 0.005515
+    },
+    {
+      "epoch": 0.02662565240412496,
+      "grad_norm": 4.721103191375732,
+      "learning_rate": 0.0001064516129032258,
+      "logits/chosen": -0.5442328453063965,
+      "logits/rejected": -0.5437616109848022,
+      "logps/chosen": -9.491182327270508,
+      "logps/rejected": -18.31528663635254,
+      "loss": 1.1582751274108887,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5736849904060364,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.03584306687116623,
+      "rewards/margins": 0.35881760716438293,
+      "rewards/rejected": -0.3229745626449585,
+      "step": 33,
+      "train_speed(iter/s)": 0.005515
+    },
+    {
+      "epoch": 0.02743249035576511,
+      "grad_norm": 4.470694065093994,
+      "learning_rate": 0.00010967741935483871,
+      "logits/chosen": -0.49901843070983887,
+      "logits/rejected": -0.5027262568473816,
+      "logps/chosen": -6.029642105102539,
+      "logps/rejected": -17.065149307250977,
+      "loss": 0.7831717133522034,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2735077142715454,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.172469824552536,
+      "rewards/margins": 0.6490951180458069,
+      "rewards/rejected": -0.4766252934932709,
+      "step": 34,
+      "train_speed(iter/s)": 0.005511
+    },
+    {
+      "epoch": 0.02823932830740526,
+      "grad_norm": 4.086674690246582,
+      "learning_rate": 0.00011290322580645163,
+      "logits/chosen": -0.5251173377037048,
+      "logits/rejected": -0.5295683145523071,
+      "logps/chosen": -10.127384185791016,
+      "logps/rejected": -19.261646270751953,
+      "loss": 0.90782630443573,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.37643295526504517,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.059490665793418884,
+      "rewards/margins": 0.5788955688476562,
+      "rewards/rejected": -0.5194048881530762,
+      "step": 35,
+      "train_speed(iter/s)": 0.005516
+    },
+    {
+      "epoch": 0.02904616625904541,
+      "grad_norm": 3.153599977493286,
+      "learning_rate": 0.00011612903225806453,
+      "logits/chosen": -0.5580551624298096,
+      "logits/rejected": -0.561299741268158,
+      "logps/chosen": -7.748661518096924,
+      "logps/rejected": -18.775835037231445,
+      "loss": 0.8323644995689392,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.30874526500701904,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.17774799466133118,
+      "rewards/margins": 0.58444744348526,
+      "rewards/rejected": -0.4066994786262512,
+      "step": 36,
+      "train_speed(iter/s)": 0.005518
+    },
+    {
+      "epoch": 0.02985300421068556,
+      "grad_norm": 5.963130950927734,
+      "learning_rate": 0.00011935483870967743,
+      "logits/chosen": -0.5691896080970764,
+      "logits/rejected": -0.5704917907714844,
+      "logps/chosen": -5.695924282073975,
+      "logps/rejected": -15.709531784057617,
+      "loss": 0.8581907153129578,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.32061102986335754,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.13813850283622742,
+      "rewards/margins": 0.42923206090927124,
+      "rewards/rejected": -0.2910935878753662,
+      "step": 37,
+      "train_speed(iter/s)": 0.005516
+    },
+    {
+      "epoch": 0.03065984216232571,
+      "grad_norm": 3.532068967819214,
+      "learning_rate": 0.00012258064516129034,
+      "logits/chosen": -0.5251184105873108,
+      "logits/rejected": -0.5261702537536621,
+      "logps/chosen": -8.303949356079102,
+      "logps/rejected": -13.649654388427734,
+      "loss": 0.9825394153594971,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.39365747570991516,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.07552488148212433,
+      "rewards/margins": 0.2995148003101349,
+      "rewards/rejected": -0.22398996353149414,
+      "step": 38,
+      "train_speed(iter/s)": 0.005515
+    },
+    {
+      "epoch": 0.03146668011396586,
+      "grad_norm": 2.9115569591522217,
+      "learning_rate": 0.00012580645161290322,
+      "logits/chosen": -0.5716837048530579,
+      "logits/rejected": -0.5749698281288147,
+      "logps/chosen": -7.206816673278809,
+      "logps/rejected": -15.77934741973877,
+      "loss": 0.7430101633071899,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.27266308665275574,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.20229285955429077,
+      "rewards/margins": 0.6254266500473022,
+      "rewards/rejected": -0.42313385009765625,
+      "step": 39,
+      "train_speed(iter/s)": 0.005515
+    },
+    {
+      "epoch": 0.03227351806560601,
+      "grad_norm": 4.647506237030029,
+      "learning_rate": 0.00012903225806451613,
+      "logits/chosen": -0.5296156406402588,
+      "logits/rejected": -0.5295798778533936,
+      "logps/chosen": -10.529404640197754,
+      "logps/rejected": -16.44548988342285,
+      "loss": 0.900071918964386,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3727172613143921,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.08702702820301056,
+      "rewards/margins": 0.4926542341709137,
+      "rewards/rejected": -0.40562722086906433,
+      "step": 40,
+      "train_speed(iter/s)": 0.005514
+    },
+    {
+      "epoch": 0.03308035601724616,
+      "grad_norm": 3.4815561771392822,
+      "learning_rate": 0.00013225806451612905,
+      "logits/chosen": -0.5586342811584473,
+      "logits/rejected": -0.5603877305984497,
+      "logps/chosen": -7.803877830505371,
+      "logps/rejected": -18.412734985351562,
+      "loss": 0.8854644298553467,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.37826117873191833,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.08443664014339447,
+      "rewards/margins": 0.6363339424133301,
+      "rewards/rejected": -0.5518972873687744,
+      "step": 41,
+      "train_speed(iter/s)": 0.005514
+    },
+    {
+      "epoch": 0.033887193968886314,
+      "grad_norm": 3.6079483032226562,
+      "learning_rate": 0.00013548387096774193,
+      "logits/chosen": -0.5233692526817322,
+      "logits/rejected": -0.5257354378700256,
+      "logps/chosen": -11.989986419677734,
+      "logps/rejected": -17.48329734802246,
+      "loss": 0.9667509198188782,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4528670608997345,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.14738641679286957,
+      "rewards/margins": 0.5997685194015503,
+      "rewards/rejected": -0.4523821473121643,
+      "step": 42,
+      "train_speed(iter/s)": 0.005512
+    },
+    {
+      "epoch": 0.03469403192052646,
+      "grad_norm": 5.491096019744873,
+      "learning_rate": 0.00013870967741935487,
+      "logits/chosen": -0.5798231363296509,
+      "logits/rejected": -0.5843824148178101,
+      "logps/chosen": -8.180625915527344,
+      "logps/rejected": -19.614505767822266,
+      "loss": 1.0811830759048462,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5651165246963501,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.018763888627290726,
+      "rewards/margins": 0.5782932043075562,
+      "rewards/rejected": -0.5595293641090393,
+      "step": 43,
+      "train_speed(iter/s)": 0.005507
+    },
+    {
+      "epoch": 0.035500869872166614,
+      "grad_norm": 5.715328216552734,
+      "learning_rate": 0.00014193548387096775,
+      "logits/chosen": -0.5562050342559814,
+      "logits/rejected": -0.5585284233093262,
+      "logps/chosen": -9.799619674682617,
+      "logps/rejected": -13.962825775146484,
+      "loss": 1.046175479888916,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5247361660003662,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.0660993829369545,
+      "rewards/margins": 0.557608425617218,
+      "rewards/rejected": -0.4915090799331665,
+      "step": 44,
+      "train_speed(iter/s)": 0.005509
+    },
+    {
+      "epoch": 0.03630770782380676,
+      "grad_norm": 4.065413951873779,
+      "learning_rate": 0.00014516129032258066,
+      "logits/chosen": -0.5717958807945251,
+      "logits/rejected": -0.5754877924919128,
+      "logps/chosen": -10.432125091552734,
+      "logps/rejected": -17.75714111328125,
+      "loss": 0.9775079488754272,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4804929792881012,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.16446207463741302,
+      "rewards/margins": 0.5844215154647827,
+      "rewards/rejected": -0.4199594557285309,
+      "step": 45,
+      "train_speed(iter/s)": 0.005513
+    },
+    {
+      "epoch": 0.037114545775446914,
+      "grad_norm": 4.507160663604736,
+      "learning_rate": 0.00014838709677419355,
+      "logits/chosen": -0.5544827580451965,
+      "logits/rejected": -0.5517963171005249,
+      "logps/chosen": -10.039278030395508,
+      "logps/rejected": -16.44822120666504,
+      "loss": 0.9198786020278931,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3649781346321106,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.11493994295597076,
+      "rewards/margins": 0.5548471808433533,
+      "rewards/rejected": -0.4399072825908661,
+      "step": 46,
+      "train_speed(iter/s)": 0.005512
+    },
+    {
+      "epoch": 0.03792138372708706,
+      "grad_norm": 4.040234565734863,
+      "learning_rate": 0.00015161290322580646,
+      "logits/chosen": -0.5704259276390076,
+      "logits/rejected": -0.5716680884361267,
+      "logps/chosen": -7.400537967681885,
+      "logps/rejected": -17.6054744720459,
+      "loss": 0.9134971499443054,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3412809669971466,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.03388240188360214,
+      "rewards/margins": 0.4440326392650604,
+      "rewards/rejected": -0.4101502001285553,
+      "step": 47,
+      "train_speed(iter/s)": 0.005512
+    },
+    {
+      "epoch": 0.038728221678727213,
+      "grad_norm": 2.9642419815063477,
+      "learning_rate": 0.00015483870967741937,
+      "logits/chosen": -0.5867742300033569,
+      "logits/rejected": -0.5912387371063232,
+      "logps/chosen": -9.146940231323242,
+      "logps/rejected": -16.374706268310547,
+      "loss": 0.9059248566627502,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3897300958633423,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.1047089472413063,
+      "rewards/margins": 0.5065208673477173,
+      "rewards/rejected": -0.4018118679523468,
+      "step": 48,
+      "train_speed(iter/s)": 0.005514
+    },
+    {
+      "epoch": 0.03953505963036737,
+      "grad_norm": 3.087217330932617,
+      "learning_rate": 0.00015806451612903225,
+      "logits/chosen": -0.5874481201171875,
+      "logits/rejected": -0.5872229337692261,
+      "logps/chosen": -6.086337566375732,
+      "logps/rejected": -10.504037857055664,
+      "loss": 0.8798851370811462,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.33263421058654785,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.20601165294647217,
+      "rewards/margins": 0.4658709466457367,
+      "rewards/rejected": -0.2598593533039093,
+      "step": 49,
+      "train_speed(iter/s)": 0.005513
+    },
+    {
+      "epoch": 0.04034189758200751,
+      "grad_norm": 3.16754150390625,
+      "learning_rate": 0.00016129032258064516,
+      "logits/chosen": -0.5708128809928894,
+      "logits/rejected": -0.5798742175102234,
+      "logps/chosen": -6.393083572387695,
+      "logps/rejected": -19.668724060058594,
+      "loss": 0.8404853343963623,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.34996694326400757,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.21206828951835632,
+      "rewards/margins": 0.5756057500839233,
+      "rewards/rejected": -0.363537460565567,
+      "step": 50,
+      "train_speed(iter/s)": 0.005516
+    },
+    {
+      "epoch": 0.041148735533647667,
+      "grad_norm": 2.4009766578674316,
+      "learning_rate": 0.00016451612903225807,
+      "logits/chosen": -0.5599406361579895,
+      "logits/rejected": -0.5625043511390686,
+      "logps/chosen": -7.262391090393066,
+      "logps/rejected": -15.73720932006836,
+      "loss": 0.8113247752189636,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.28076890110969543,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.14269490540027618,
+      "rewards/margins": 0.47564250230789185,
+      "rewards/rejected": -0.33294758200645447,
+      "step": 51,
+      "train_speed(iter/s)": 0.005506
+    },
+    {
+      "epoch": 0.04195557348528781,
+      "grad_norm": 3.129634380340576,
+      "learning_rate": 0.00016774193548387098,
+      "logits/chosen": -0.5937235355377197,
+      "logits/rejected": -0.5938248038291931,
+      "logps/chosen": -5.393199920654297,
+      "logps/rejected": -15.786672592163086,
+      "loss": 0.7831435799598694,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2872075140476227,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.19873687624931335,
+      "rewards/margins": 0.5569283366203308,
+      "rewards/rejected": -0.35819149017333984,
+      "step": 52,
+      "train_speed(iter/s)": 0.005507
+    },
+    {
+      "epoch": 0.042762411436927966,
+      "grad_norm": 3.2887256145477295,
+      "learning_rate": 0.0001709677419354839,
+      "logits/chosen": -0.5820189118385315,
+      "logits/rejected": -0.5850119590759277,
+      "logps/chosen": -7.764882564544678,
+      "logps/rejected": -16.894685745239258,
+      "loss": 0.8994213342666626,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3770924210548401,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.20706558227539062,
+      "rewards/margins": 0.5472366213798523,
+      "rewards/rejected": -0.34017103910446167,
+      "step": 53,
+      "train_speed(iter/s)": 0.005506
+    },
+    {
+      "epoch": 0.04356924938856811,
+      "grad_norm": 3.7313735485076904,
+      "learning_rate": 0.00017419354838709678,
+      "logits/chosen": -0.5833962559700012,
+      "logits/rejected": -0.5876143574714661,
+      "logps/chosen": -8.130949020385742,
+      "logps/rejected": -20.79953384399414,
+      "loss": 0.9545141458511353,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4539381265640259,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.10515612363815308,
+      "rewards/margins": 0.612572968006134,
+      "rewards/rejected": -0.5074167847633362,
+      "step": 54,
+      "train_speed(iter/s)": 0.005507
+    },
+    {
+      "epoch": 0.044376087340208266,
+      "grad_norm": 5.652868747711182,
+      "learning_rate": 0.0001774193548387097,
+      "logits/chosen": -0.5960416793823242,
+      "logits/rejected": -0.5957224369049072,
+      "logps/chosen": -12.674562454223633,
+      "logps/rejected": -15.591826438903809,
+      "loss": 1.1736092567443848,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5310375690460205,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.11839848756790161,
+      "rewards/margins": 0.40016603469848633,
+      "rewards/rejected": -0.5185645222663879,
+      "step": 55,
+      "train_speed(iter/s)": 0.005509
+    },
+    {
+      "epoch": 0.04518292529184841,
+      "grad_norm": 4.002370834350586,
+      "learning_rate": 0.00018064516129032257,
+      "logits/chosen": -0.5776931643486023,
+      "logits/rejected": -0.5789185166358948,
+      "logps/chosen": -12.67449951171875,
+      "logps/rejected": -16.919376373291016,
+      "loss": 1.087173342704773,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4873568117618561,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.15739397704601288,
+      "rewards/margins": 0.39120250940322876,
+      "rewards/rejected": -0.5485965013504028,
+      "step": 56,
+      "train_speed(iter/s)": 0.005507
+    },
+    {
+      "epoch": 0.045989763243488566,
+      "grad_norm": 3.5529890060424805,
+      "learning_rate": 0.00018387096774193548,
+      "logits/chosen": -0.5732367634773254,
+      "logits/rejected": -0.5760124921798706,
+      "logps/chosen": -4.22891902923584,
+      "logps/rejected": -18.773733139038086,
+      "loss": 0.6395769119262695,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.23632453382015228,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.12576641142368317,
+      "rewards/margins": 0.908116340637207,
+      "rewards/rejected": -0.7823498845100403,
+      "step": 57,
+      "train_speed(iter/s)": 0.005505
+    },
+    {
+      "epoch": 0.04679660119512872,
+      "grad_norm": 2.638253927230835,
+      "learning_rate": 0.0001870967741935484,
+      "logits/chosen": -0.5618959069252014,
+      "logits/rejected": -0.5671476721763611,
+      "logps/chosen": -8.717156410217285,
+      "logps/rejected": -22.3723087310791,
+      "loss": 0.8119103312492371,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4192086458206177,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.18826748430728912,
+      "rewards/margins": 0.8515651226043701,
+      "rewards/rejected": -0.6632976531982422,
+      "step": 58,
+      "train_speed(iter/s)": 0.005503
+    },
+    {
+      "epoch": 0.047603439146768865,
+      "grad_norm": 4.416356086730957,
+      "learning_rate": 0.0001903225806451613,
+      "logits/chosen": -0.5946109294891357,
+      "logits/rejected": -0.5949057340621948,
+      "logps/chosen": -10.825102806091309,
+      "logps/rejected": -15.580466270446777,
+      "loss": 0.9124961495399475,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.43738603591918945,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.1081230640411377,
+      "rewards/margins": 0.6975550651550293,
+      "rewards/rejected": -0.5894321203231812,
+      "step": 59,
+      "train_speed(iter/s)": 0.005503
+    },
+    {
+      "epoch": 0.04841027709840902,
+      "grad_norm": 3.548480272293091,
+      "learning_rate": 0.00019354838709677422,
+      "logits/chosen": -0.576353907585144,
+      "logits/rejected": -0.5807583928108215,
+      "logps/chosen": -10.603282928466797,
+      "logps/rejected": -20.904390335083008,
+      "loss": 1.033903956413269,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5458623170852661,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.10141083598136902,
+      "rewards/margins": 0.9133819341659546,
+      "rewards/rejected": -0.8119711875915527,
+      "step": 60,
+      "train_speed(iter/s)": 0.005505
+    },
+    {
+      "epoch": 0.049217115050049165,
+      "grad_norm": 4.239978790283203,
+      "learning_rate": 0.0001967741935483871,
+      "logits/chosen": -0.6159774661064148,
+      "logits/rejected": -0.6157845258712769,
+      "logps/chosen": -13.034204483032227,
+      "logps/rejected": -21.466520309448242,
+      "loss": 0.9954748153686523,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5168237090110779,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.15371613204479218,
+      "rewards/margins": 0.9087806940078735,
+      "rewards/rejected": -1.0624967813491821,
+      "step": 61,
+      "train_speed(iter/s)": 0.005508
+    },
+    {
+      "epoch": 0.05002395300168932,
+      "grad_norm": 4.418153285980225,
+      "learning_rate": 0.0002,
+      "logits/chosen": -0.586484968662262,
+      "logits/rejected": -0.5885915756225586,
+      "logps/chosen": -6.359810829162598,
+      "logps/rejected": -18.686431884765625,
+      "loss": 0.7993831634521484,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3223803639411926,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.1565569043159485,
+      "rewards/margins": 0.8297788500785828,
+      "rewards/rejected": -0.6732219457626343,
+      "step": 62,
+      "train_speed(iter/s)": 0.005508
+    },
+    {
+      "epoch": 0.050830790953329465,
+      "grad_norm": 3.898704767227173,
+      "learning_rate": 0.00019999964378142156,
+      "logits/chosen": -0.5732013583183289,
+      "logits/rejected": -0.5774887800216675,
+      "logps/chosen": -12.98630142211914,
+      "logps/rejected": -19.34124755859375,
+      "loss": 0.9075384140014648,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.39192017912864685,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.19289077818393707,
+      "rewards/margins": 0.6732712388038635,
+      "rewards/rejected": -0.48038047552108765,
+      "step": 63,
+      "train_speed(iter/s)": 0.005507
+    },
+    {
+      "epoch": 0.05163762890496962,
+      "grad_norm": 6.278528213500977,
+      "learning_rate": 0.00019999857512822402,
+      "logits/chosen": -0.6075332760810852,
+      "logits/rejected": -0.6084203124046326,
+      "logps/chosen": -14.246075630187988,
+      "logps/rejected": -20.948959350585938,
+      "loss": 1.0685752630233765,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5815572142601013,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.03303433582186699,
+      "rewards/margins": 0.8266364336013794,
+      "rewards/rejected": -0.8596707582473755,
+      "step": 64,
+      "train_speed(iter/s)": 0.005508
+    },
+    {
+      "epoch": 0.052444466856609764,
+      "grad_norm": 4.589393615722656,
+      "learning_rate": 0.00019999679404802089,
+      "logits/chosen": -0.596097469329834,
+      "logits/rejected": -0.5986034870147705,
+      "logps/chosen": -8.450502395629883,
+      "logps/rejected": -21.070066452026367,
+      "loss": 0.7867013812065125,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.31530463695526123,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.13920006155967712,
+      "rewards/margins": 0.9423296451568604,
+      "rewards/rejected": -0.8031294941902161,
+      "step": 65,
+      "train_speed(iter/s)": 0.005508
+    },
+    {
+      "epoch": 0.05325130480824992,
+      "grad_norm": 4.376217365264893,
+      "learning_rate": 0.00019999430055350122,
+      "logits/chosen": -0.5758836269378662,
+      "logits/rejected": -0.5734565854072571,
+      "logps/chosen": -10.774827003479004,
+      "logps/rejected": -19.021907806396484,
+      "loss": 0.7909982800483704,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3997192680835724,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.23606787621974945,
+      "rewards/margins": 1.0521141290664673,
+      "rewards/rejected": -0.8160461783409119,
+      "step": 66,
+      "train_speed(iter/s)": 0.005507
+    },
+    {
+      "epoch": 0.05405814275989007,
+      "grad_norm": 6.438971042633057,
+      "learning_rate": 0.00019999109466242962,
+      "logits/chosen": -0.5718088150024414,
+      "logits/rejected": -0.5745964050292969,
+      "logps/chosen": -6.671931266784668,
+      "logps/rejected": -20.429231643676758,
+      "loss": 0.8954067230224609,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.45071539282798767,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.10935470461845398,
+      "rewards/margins": 0.8583554029464722,
+      "rewards/rejected": -0.7490006685256958,
+      "step": 67,
+      "train_speed(iter/s)": 0.005509
+    },
+    {
+      "epoch": 0.05486498071153022,
+      "grad_norm": 11.16286849975586,
+      "learning_rate": 0.00019998717639764602,
+      "logits/chosen": -0.5820534229278564,
+      "logits/rejected": -0.5849663019180298,
+      "logps/chosen": -9.799468040466309,
+      "logps/rejected": -18.977890014648438,
+      "loss": 1.29269278049469,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.7049438953399658,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.04825415089726448,
+      "rewards/margins": 0.5465254783630371,
+      "rewards/rejected": -0.5947796702384949,
+      "step": 68,
+      "train_speed(iter/s)": 0.005506
+    },
+    {
+      "epoch": 0.05567181866317037,
+      "grad_norm": 9.413622856140137,
+      "learning_rate": 0.00019998254578706563,
+      "logits/chosen": -0.5850585699081421,
+      "logits/rejected": -0.5874402523040771,
+      "logps/chosen": -8.903966903686523,
+      "logps/rejected": -14.843938827514648,
+      "loss": 1.3027069568634033,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.744307279586792,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.0512365885078907,
+      "rewards/margins": 0.5854336619377136,
+      "rewards/rejected": -0.6366702318191528,
+      "step": 69,
+      "train_speed(iter/s)": 0.005505
+    },
+    {
+      "epoch": 0.05647865661481052,
+      "grad_norm": 7.333382606506348,
+      "learning_rate": 0.00019997720286367863,
+      "logits/chosen": -0.545200765132904,
+      "logits/rejected": -0.5484716296195984,
+      "logps/chosen": -7.334113121032715,
+      "logps/rejected": -22.56316375732422,
+      "loss": 1.0189334154129028,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5383400321006775,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.061847541481256485,
+      "rewards/margins": 0.6559779644012451,
+      "rewards/rejected": -0.5941305160522461,
+      "step": 70,
+      "train_speed(iter/s)": 0.005506
+    },
+    {
+      "epoch": 0.05728549456645067,
+      "grad_norm": 2.8849799633026123,
+      "learning_rate": 0.00019997114766554997,
+      "logits/chosen": -0.5629354119300842,
+      "logits/rejected": -0.5669021606445312,
+      "logps/chosen": -8.698752403259277,
+      "logps/rejected": -21.85867691040039,
+      "loss": 0.8375651836395264,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3549819886684418,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.09854352474212646,
+      "rewards/margins": 0.7027413845062256,
+      "rewards/rejected": -0.6041979193687439,
+      "step": 71,
+      "train_speed(iter/s)": 0.005507
+    },
+    {
+      "epoch": 0.05809233251809082,
+      "grad_norm": 2.6733391284942627,
+      "learning_rate": 0.00019996438023581913,
+      "logits/chosen": -0.5319232940673828,
+      "logits/rejected": -0.5331096649169922,
+      "logps/chosen": -10.13166332244873,
+      "logps/rejected": -13.18635368347168,
+      "loss": 1.014970302581787,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4217461347579956,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.14793364703655243,
+      "rewards/margins": 0.3788456618785858,
+      "rewards/rejected": -0.2309119999408722,
+      "step": 72,
+      "train_speed(iter/s)": 0.005505
+    },
+    {
+      "epoch": 0.05889917046973097,
+      "grad_norm": 2.497257947921753,
+      "learning_rate": 0.00019995690062269984,
+      "logits/chosen": -0.5789859294891357,
+      "logits/rejected": -0.5805359482765198,
+      "logps/chosen": -8.139568328857422,
+      "logps/rejected": -10.85029411315918,
+      "loss": 0.9546566605567932,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4216194450855255,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.24104151129722595,
+      "rewards/margins": 0.5254760980606079,
+      "rewards/rejected": -0.28443461656570435,
+      "step": 73,
+      "train_speed(iter/s)": 0.005506
+    },
+    {
+      "epoch": 0.05970600842137112,
+      "grad_norm": 2.8535268306732178,
+      "learning_rate": 0.00019994870887947962,
+      "logits/chosen": -0.5778383612632751,
+      "logits/rejected": -0.5768096446990967,
+      "logps/chosen": -8.49053955078125,
+      "logps/rejected": -12.854878425598145,
+      "loss": 1.0000758171081543,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.37848562002182007,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.1054978221654892,
+      "rewards/margins": 0.3462926149368286,
+      "rewards/rejected": -0.24079479277133942,
+      "step": 74,
+      "train_speed(iter/s)": 0.005506
+    },
+    {
+      "epoch": 0.06051284637301127,
+      "grad_norm": 2.198607921600342,
+      "learning_rate": 0.00019993980506451947,
+      "logits/chosen": -0.530400276184082,
+      "logits/rejected": -0.530858039855957,
+      "logps/chosen": -9.152666091918945,
+      "logps/rejected": -11.464343070983887,
+      "loss": 0.9745745658874512,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.38881367444992065,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1783803254365921,
+      "rewards/margins": 0.3315035402774811,
+      "rewards/rejected": -0.1531231850385666,
+      "step": 75,
+      "train_speed(iter/s)": 0.005504
+    },
+    {
+      "epoch": 0.06131968432465142,
+      "grad_norm": 2.461601972579956,
+      "learning_rate": 0.0001999301892412535,
+      "logits/chosen": -0.5405219197273254,
+      "logits/rejected": -0.5390282869338989,
+      "logps/chosen": -10.537997245788574,
+      "logps/rejected": -15.066913604736328,
+      "loss": 0.966386079788208,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.38977545499801636,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.09291838109493256,
+      "rewards/margins": 0.3533533811569214,
+      "rewards/rejected": -0.26043495535850525,
+      "step": 76,
+      "train_speed(iter/s)": 0.005503
+    },
+    {
+      "epoch": 0.06212652227629157,
+      "grad_norm": 2.0999796390533447,
+      "learning_rate": 0.00019991986147818838,
+      "logits/chosen": -0.5443933010101318,
+      "logits/rejected": -0.5435047149658203,
+      "logps/chosen": -12.143755912780762,
+      "logps/rejected": -15.766691207885742,
+      "loss": 0.8939065337181091,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.415660560131073,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.23248028755187988,
+      "rewards/margins": 0.5352811217308044,
+      "rewards/rejected": -0.30280083417892456,
+      "step": 77,
+      "train_speed(iter/s)": 0.005502
+    },
+    {
+      "epoch": 0.06293336022793172,
+      "grad_norm": 2.9244306087493896,
+      "learning_rate": 0.00019990882184890295,
+      "logits/chosen": -0.5794655680656433,
+      "logits/rejected": -0.5853484869003296,
+      "logps/chosen": -7.884565353393555,
+      "logps/rejected": -20.136926651000977,
+      "loss": 0.8564450740814209,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.34705159068107605,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.14532405138015747,
+      "rewards/margins": 0.5069437623023987,
+      "rewards/rejected": -0.3616198003292084,
+      "step": 78,
+      "train_speed(iter/s)": 0.005502
+    },
+    {
+      "epoch": 0.06374019817957187,
+      "grad_norm": 3.461670160293579,
+      "learning_rate": 0.00019989707043204763,
+      "logits/chosen": -0.5491142868995667,
+      "logits/rejected": -0.553898811340332,
+      "logps/chosen": -7.547114372253418,
+      "logps/rejected": -18.740121841430664,
+      "loss": 0.9405392408370972,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.48508715629577637,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.128400981426239,
+      "rewards/margins": 0.6999501585960388,
+      "rewards/rejected": -0.5715491771697998,
+      "step": 79,
+      "train_speed(iter/s)": 0.005502
+    },
+    {
+      "epoch": 0.06454703613121202,
+      "grad_norm": 4.165892601013184,
+      "learning_rate": 0.0001998846073113439,
+      "logits/chosen": -0.6084208488464355,
+      "logits/rejected": -0.6092680096626282,
+      "logps/chosen": -12.750446319580078,
+      "logps/rejected": -19.53608512878418,
+      "loss": 1.0978946685791016,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5877176523208618,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1369042694568634,
+      "rewards/margins": 0.6248282194137573,
+      "rewards/rejected": -0.48792392015457153,
+      "step": 80,
+      "train_speed(iter/s)": 0.005503
+    },
+    {
+      "epoch": 0.06535387408285218,
+      "grad_norm": 2.287846565246582,
+      "learning_rate": 0.00019987143257558365,
+      "logits/chosen": -0.59499192237854,
+      "logits/rejected": -0.5972949862480164,
+      "logps/chosen": -4.519835948944092,
+      "logps/rejected": -19.02901840209961,
+      "loss": 0.58124840259552,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.15051494538784027,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.19695766270160675,
+      "rewards/margins": 0.9730147123336792,
+      "rewards/rejected": -0.7760570049285889,
+      "step": 81,
+      "train_speed(iter/s)": 0.005506
+    },
+    {
+      "epoch": 0.06616071203449232,
+      "grad_norm": 7.103982448577881,
+      "learning_rate": 0.00019985754631862855,
+      "logits/chosen": -0.5857492089271545,
+      "logits/rejected": -0.5895102024078369,
+      "logps/chosen": -8.903070449829102,
+      "logps/rejected": -26.571044921875,
+      "loss": 0.7599916458129883,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3342111110687256,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.14196240901947021,
+      "rewards/margins": 0.8281382322311401,
+      "rewards/rejected": -0.6861758232116699,
+      "step": 82,
+      "train_speed(iter/s)": 0.005504
+    },
+    {
+      "epoch": 0.06696754998613247,
+      "grad_norm": 3.764488697052002,
+      "learning_rate": 0.00019984294863940955,
+      "logits/chosen": -0.6067830324172974,
+      "logits/rejected": -0.6086009740829468,
+      "logps/chosen": -9.9881591796875,
+      "logps/rejected": -19.09541893005371,
+      "loss": 0.9752427339553833,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5045859217643738,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.14939813315868378,
+      "rewards/margins": 0.8939868211746216,
+      "rewards/rejected": -0.7445887327194214,
+      "step": 83,
+      "train_speed(iter/s)": 0.005503
+    },
+    {
+      "epoch": 0.06777438793777263,
+      "grad_norm": 3.3293344974517822,
+      "learning_rate": 0.00019982763964192585,
+      "logits/chosen": -0.6330868601799011,
+      "logits/rejected": -0.6393448710441589,
+      "logps/chosen": -4.902453899383545,
+      "logps/rejected": -24.94113540649414,
+      "loss": 0.7244248390197754,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3735058009624481,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.17604736983776093,
+      "rewards/margins": 1.2982165813446045,
+      "rewards/rejected": -1.1221693754196167,
+      "step": 84,
+      "train_speed(iter/s)": 0.005503
+    },
+    {
+      "epoch": 0.06858122588941278,
+      "grad_norm": 5.892284393310547,
+      "learning_rate": 0.00019981161943524443,
+      "logits/chosen": -0.6240598559379578,
+      "logits/rejected": -0.620481550693512,
+      "logps/chosen": -14.60393238067627,
+      "logps/rejected": -13.365829467773438,
+      "loss": 1.1541650295257568,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.48222166299819946,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.09302416443824768,
+      "rewards/margins": 0.41544342041015625,
+      "rewards/rejected": -0.5084675550460815,
+      "step": 85,
+      "train_speed(iter/s)": 0.0055
+    },
+    {
+      "epoch": 0.06938806384105292,
+      "grad_norm": 4.056751251220703,
+      "learning_rate": 0.00019979488813349933,
+      "logits/chosen": -0.6613398194313049,
+      "logits/rejected": -0.6640070080757141,
+      "logps/chosen": -9.50975227355957,
+      "logps/rejected": -21.119707107543945,
+      "loss": 0.8987840414047241,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.52927565574646,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.1999521553516388,
+      "rewards/margins": 1.1560636758804321,
+      "rewards/rejected": -0.956111490726471,
+      "step": 86,
+      "train_speed(iter/s)": 0.005499
+    },
+    {
+      "epoch": 0.07019490179269307,
+      "grad_norm": 4.351650238037109,
+      "learning_rate": 0.0001997774458558904,
+      "logits/chosen": -0.6547818779945374,
+      "logits/rejected": -0.6571143269538879,
+      "logps/chosen": -8.668737411499023,
+      "logps/rejected": -20.79534149169922,
+      "loss": 0.9775034785270691,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4770965874195099,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.02099284902215004,
+      "rewards/margins": 0.8633284568786621,
+      "rewards/rejected": -0.84233558177948,
+      "step": 87,
+      "train_speed(iter/s)": 0.005498
+    },
+    {
+      "epoch": 0.07100173974433323,
+      "grad_norm": 2.8774800300598145,
+      "learning_rate": 0.00019975929272668296,
+      "logits/chosen": -0.6759516000747681,
+      "logits/rejected": -0.6824960708618164,
+      "logps/chosen": -4.7266716957092285,
+      "logps/rejected": -26.10918426513672,
+      "loss": 0.5549903512001038,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.271104097366333,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.25939324498176575,
+      "rewards/margins": 1.6003111600875854,
+      "rewards/rejected": -1.340917944908142,
+      "step": 88,
+      "train_speed(iter/s)": 0.005501
+    },
+    {
+      "epoch": 0.07180857769597337,
+      "grad_norm": 2.8334577083587646,
+      "learning_rate": 0.00019974042887520668,
+      "logits/chosen": -0.6535760164260864,
+      "logits/rejected": -0.6572569608688354,
+      "logps/chosen": -13.957544326782227,
+      "logps/rejected": -18.16533851623535,
+      "loss": 0.9486314654350281,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5478147268295288,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2579638957977295,
+      "rewards/margins": 0.9477146863937378,
+      "rewards/rejected": -0.6897507905960083,
+      "step": 89,
+      "train_speed(iter/s)": 0.005501
+    },
+    {
+      "epoch": 0.07261541564761352,
+      "grad_norm": 5.422155380249023,
+      "learning_rate": 0.00019972085443585463,
+      "logits/chosen": -0.6767461895942688,
+      "logits/rejected": -0.6799700260162354,
+      "logps/chosen": -7.507997512817383,
+      "logps/rejected": -20.19240951538086,
+      "loss": 0.8118898868560791,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4057508111000061,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.11586537212133408,
+      "rewards/margins": 1.0757473707199097,
+      "rewards/rejected": -0.9598820805549622,
+      "step": 90,
+      "train_speed(iter/s)": 0.005503
+    },
+    {
+      "epoch": 0.07342225359925368,
+      "grad_norm": 5.541498184204102,
+      "learning_rate": 0.0001997005695480824,
+      "logits/chosen": -0.6398351788520813,
+      "logits/rejected": -0.642162024974823,
+      "logps/chosen": -10.117485046386719,
+      "logps/rejected": -21.747928619384766,
+      "loss": 0.8823437094688416,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.47223997116088867,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.04489222913980484,
+      "rewards/margins": 1.1613836288452148,
+      "rewards/rejected": -1.2062758207321167,
+      "step": 91,
+      "train_speed(iter/s)": 0.005504
+    },
+    {
+      "epoch": 0.07422909155089383,
+      "grad_norm": 2.0107967853546143,
+      "learning_rate": 0.000199679574356407,
+      "logits/chosen": -0.6938130855560303,
+      "logits/rejected": -0.7009856104850769,
+      "logps/chosen": -6.915675163269043,
+      "logps/rejected": -28.84608268737793,
+      "loss": 0.5776476860046387,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3677330017089844,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3187498152256012,
+      "rewards/margins": 1.8727707862854004,
+      "rewards/rejected": -1.554020881652832,
+      "step": 92,
+      "train_speed(iter/s)": 0.005502
+    },
+    {
+      "epoch": 0.07503592950253397,
+      "grad_norm": 3.5870676040649414,
+      "learning_rate": 0.0001996578690104061,
+      "logits/chosen": -0.673453688621521,
+      "logits/rejected": -0.676167905330658,
+      "logps/chosen": -10.7222318649292,
+      "logps/rejected": -28.77056884765625,
+      "loss": 0.950826108455658,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5682229399681091,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.09744307398796082,
+      "rewards/margins": 1.4789444208145142,
+      "rewards/rejected": -1.381501317024231,
+      "step": 93,
+      "train_speed(iter/s)": 0.005502
+    },
+    {
+      "epoch": 0.07584276745417412,
+      "grad_norm": 4.2534661293029785,
+      "learning_rate": 0.00019963545366471653,
+      "logits/chosen": -0.6709739565849304,
+      "logits/rejected": -0.6747978925704956,
+      "logps/chosen": -8.219854354858398,
+      "logps/rejected": -29.13772201538086,
+      "loss": 0.6954976320266724,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.35414204001426697,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.10512235760688782,
+      "rewards/margins": 1.6458101272583008,
+      "rewards/rejected": -1.5406877994537354,
+      "step": 94,
+      "train_speed(iter/s)": 0.005503
+    },
+    {
+      "epoch": 0.07664960540581428,
+      "grad_norm": 7.663514614105225,
+      "learning_rate": 0.0001996123284790336,
+      "logits/chosen": -0.6834441423416138,
+      "logits/rejected": -0.6848198771476746,
+      "logps/chosen": -17.666513442993164,
+      "logps/rejected": -19.222387313842773,
+      "loss": 1.197723388671875,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6848238110542297,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.06468406319618225,
+      "rewards/margins": 0.8658308982849121,
+      "rewards/rejected": -0.9305148720741272,
+      "step": 95,
+      "train_speed(iter/s)": 0.005502
+    },
+    {
+      "epoch": 0.07745644335745443,
+      "grad_norm": 16.187767028808594,
+      "learning_rate": 0.0001995884936181097,
+      "logits/chosen": -0.7171504497528076,
+      "logits/rejected": -0.7210450172424316,
+      "logps/chosen": -4.8772172927856445,
+      "logps/rejected": -27.164329528808594,
+      "loss": 0.7958053946495056,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.437197208404541,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.04156189411878586,
+      "rewards/margins": 1.4321556091308594,
+      "rewards/rejected": -1.3905936479568481,
+      "step": 96,
+      "train_speed(iter/s)": 0.005502
+    },
+    {
+      "epoch": 0.07826328130909457,
+      "grad_norm": 4.373692989349365,
+      "learning_rate": 0.00019956394925175328,
+      "logits/chosen": -0.720712423324585,
+      "logits/rejected": -0.7263769507408142,
+      "logps/chosen": -8.969646453857422,
+      "logps/rejected": -27.35487174987793,
+      "loss": 0.8684384822845459,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4593147337436676,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.039580605924129486,
+      "rewards/margins": 1.404889702796936,
+      "rewards/rejected": -1.3653091192245483,
+      "step": 97,
+      "train_speed(iter/s)": 0.005502
+    },
+    {
+      "epoch": 0.07907011926073473,
+      "grad_norm": 3.1712353229522705,
+      "learning_rate": 0.00019953869555482752,
+      "logits/chosen": -0.6926038265228271,
+      "logits/rejected": -0.6906510591506958,
+      "logps/chosen": -10.361411094665527,
+      "logps/rejected": -19.763336181640625,
+      "loss": 1.0741034746170044,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5404018759727478,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.0025314167141914368,
+      "rewards/margins": 0.9160726070404053,
+      "rewards/rejected": -0.9135411381721497,
+      "step": 98,
+      "train_speed(iter/s)": 0.005502
+    },
+    {
+      "epoch": 0.07987695721237488,
+      "grad_norm": 5.8308796882629395,
+      "learning_rate": 0.00019951273270724906,
+      "logits/chosen": -0.6417246460914612,
+      "logits/rejected": -0.6460192203521729,
+      "logps/chosen": -10.923144340515137,
+      "logps/rejected": -23.332698822021484,
+      "loss": 1.129643201828003,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5929185748100281,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.004452264867722988,
+      "rewards/margins": 0.6838414669036865,
+      "rewards/rejected": -0.6882937550544739,
+      "step": 99,
+      "train_speed(iter/s)": 0.005503
+    },
+    {
+      "epoch": 0.08068379516401503,
+      "grad_norm": 4.135215759277344,
+      "learning_rate": 0.00019948606089398698,
+      "logits/chosen": -0.6685535907745361,
+      "logits/rejected": -0.6693455576896667,
+      "logps/chosen": -11.88547134399414,
+      "logps/rejected": -19.97165298461914,
+      "loss": 0.9190454483032227,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.39345046877861023,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.05392542481422424,
+      "rewards/margins": 0.8620406985282898,
+      "rewards/rejected": -0.8081153035163879,
+      "step": 100,
+      "train_speed(iter/s)": 0.005503
+    },
+    {
+      "epoch": 0.08149063311565517,
+      "grad_norm": 3.957230806350708,
+      "learning_rate": 0.0001994586803050611,
+      "logits/chosen": -0.6953858137130737,
+      "logits/rejected": -0.6939084529876709,
+      "logps/chosen": -11.101150512695312,
+      "logps/rejected": -19.7403621673584,
+      "loss": 1.0100737810134888,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.45983725786209106,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.061330053955316544,
+      "rewards/margins": 0.847295880317688,
+      "rewards/rejected": -0.7859657406806946,
+      "step": 101,
+      "train_speed(iter/s)": 0.005498
+    },
+    {
+      "epoch": 0.08229747106729533,
+      "grad_norm": 2.8322761058807373,
+      "learning_rate": 0.00019943059113554097,
+      "logits/chosen": -0.6899937987327576,
+      "logits/rejected": -0.6942430734634399,
+      "logps/chosen": -8.74088191986084,
+      "logps/rejected": -20.522974014282227,
+      "loss": 0.8782400488853455,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3525587320327759,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.05973733961582184,
+      "rewards/margins": 0.882931113243103,
+      "rewards/rejected": -0.8231937289237976,
+      "step": 102,
+      "train_speed(iter/s)": 0.005498
+    },
+    {
+      "epoch": 0.08310430901893548,
+      "grad_norm": 2.8728346824645996,
+      "learning_rate": 0.00019940179358554424,
+      "logits/chosen": -0.6615633964538574,
+      "logits/rejected": -0.6650048494338989,
+      "logps/chosen": -8.035341262817383,
+      "logps/rejected": -13.233177185058594,
+      "loss": 0.95281982421875,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.39267757534980774,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.12954209744930267,
+      "rewards/margins": 0.4833866357803345,
+      "rewards/rejected": -0.3538445234298706,
+      "step": 103,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.08391114697057563,
+      "grad_norm": 2.8015527725219727,
+      "learning_rate": 0.00019937228786023542,
+      "logits/chosen": -0.6321871280670166,
+      "logits/rejected": -0.6367396116256714,
+      "logps/chosen": -9.781261444091797,
+      "logps/rejected": -18.408084869384766,
+      "loss": 0.9027690887451172,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4244646430015564,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.18980520963668823,
+      "rewards/margins": 0.7434633374214172,
+      "rewards/rejected": -0.5536580681800842,
+      "step": 104,
+      "train_speed(iter/s)": 0.005498
+    },
+    {
+      "epoch": 0.08471798492221577,
+      "grad_norm": 3.581929922103882,
+      "learning_rate": 0.00019934207416982415,
+      "logits/chosen": -0.6718648076057434,
+      "logits/rejected": -0.6737468242645264,
+      "logps/chosen": -10.535398483276367,
+      "logps/rejected": -15.517889022827148,
+      "loss": 0.9467408657073975,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4431319832801819,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.24101108312606812,
+      "rewards/margins": 0.5848699808120728,
+      "rewards/rejected": -0.34385889768600464,
+      "step": 105,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.08552482287385593,
+      "grad_norm": 2.7180511951446533,
+      "learning_rate": 0.00019931115272956405,
+      "logits/chosen": -0.6360980272293091,
+      "logits/rejected": -0.6395881175994873,
+      "logps/chosen": -5.562260627746582,
+      "logps/rejected": -16.978702545166016,
+      "loss": 0.9048807621002197,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3790769577026367,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.1927444040775299,
+      "rewards/margins": 0.6174324750900269,
+      "rewards/rejected": -0.42468804121017456,
+      "step": 106,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.08633166082549608,
+      "grad_norm": 2.7166309356689453,
+      "learning_rate": 0.00019927952375975094,
+      "logits/chosen": -0.6151095628738403,
+      "logits/rejected": -0.6214062571525574,
+      "logps/chosen": -8.704106330871582,
+      "logps/rejected": -15.980009078979492,
+      "loss": 0.9614640474319458,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.41800737380981445,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.22499363124370575,
+      "rewards/margins": 0.5540027618408203,
+      "rewards/rejected": -0.32900911569595337,
+      "step": 107,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.08713849877713623,
+      "grad_norm": 3.3143904209136963,
+      "learning_rate": 0.00019924718748572136,
+      "logits/chosen": -0.6602830290794373,
+      "logits/rejected": -0.6590928435325623,
+      "logps/chosen": -14.48482608795166,
+      "logps/rejected": -13.489812850952148,
+      "loss": 1.1007349491119385,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.38938888907432556,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.005462644621729851,
+      "rewards/margins": 0.18356555700302124,
+      "rewards/rejected": -0.18902820348739624,
+      "step": 108,
+      "train_speed(iter/s)": 0.005493
+    },
+    {
+      "epoch": 0.08794533672877639,
+      "grad_norm": 3.3967649936676025,
+      "learning_rate": 0.00019921414413785095,
+      "logits/chosen": -0.6139819025993347,
+      "logits/rejected": -0.6160064935684204,
+      "logps/chosen": -10.462146759033203,
+      "logps/rejected": -13.9368896484375,
+      "loss": 1.1163440942764282,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5643093585968018,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.13518843054771423,
+      "rewards/margins": 0.5490553379058838,
+      "rewards/rejected": -0.41386690735816956,
+      "step": 109,
+      "train_speed(iter/s)": 0.005493
+    },
+    {
+      "epoch": 0.08875217468041653,
+      "grad_norm": 2.125051498413086,
+      "learning_rate": 0.00019918039395155278,
+      "logits/chosen": -0.5581845641136169,
+      "logits/rejected": -0.5609419345855713,
+      "logps/chosen": -8.664159774780273,
+      "logps/rejected": -16.96472930908203,
+      "loss": 0.8410020470619202,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3336779773235321,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.24867044389247894,
+      "rewards/margins": 0.6479009985923767,
+      "rewards/rejected": -0.3992305099964142,
+      "step": 110,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.08955901263205668,
+      "grad_norm": 2.7272698879241943,
+      "learning_rate": 0.00019914593716727574,
+      "logits/chosen": -0.640306830406189,
+      "logits/rejected": -0.6392278075218201,
+      "logps/chosen": -10.320524215698242,
+      "logps/rejected": -15.357645034790039,
+      "loss": 0.9050614237785339,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3744095265865326,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.228946715593338,
+      "rewards/margins": 0.46916335821151733,
+      "rewards/rejected": -0.24021662771701813,
+      "step": 111,
+      "train_speed(iter/s)": 0.005492
+    },
+    {
+      "epoch": 0.09036585058369682,
+      "grad_norm": 2.552009344100952,
+      "learning_rate": 0.0001991107740305027,
+      "logits/chosen": -0.6064003109931946,
+      "logits/rejected": -0.607029914855957,
+      "logps/chosen": -10.730403900146484,
+      "logps/rejected": -20.569438934326172,
+      "loss": 0.9093858003616333,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4844813346862793,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.320293128490448,
+      "rewards/margins": 0.8029977083206177,
+      "rewards/rejected": -0.48270460963249207,
+      "step": 112,
+      "train_speed(iter/s)": 0.005492
+    },
+    {
+      "epoch": 0.09117268853533698,
+      "grad_norm": 2.875828742980957,
+      "learning_rate": 0.00019907490479174902,
+      "logits/chosen": -0.6465886831283569,
+      "logits/rejected": -0.6483381986618042,
+      "logps/chosen": -6.758854389190674,
+      "logps/rejected": -20.76754379272461,
+      "loss": 0.7010505199432373,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.29428812861442566,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3071742653846741,
+      "rewards/margins": 1.1479661464691162,
+      "rewards/rejected": -0.8407919406890869,
+      "step": 113,
+      "train_speed(iter/s)": 0.005491
+    },
+    {
+      "epoch": 0.09197952648697713,
+      "grad_norm": 2.3188178539276123,
+      "learning_rate": 0.00019903832970656038,
+      "logits/chosen": -0.6288142800331116,
+      "logits/rejected": -0.6315409541130066,
+      "logps/chosen": -8.855584144592285,
+      "logps/rejected": -20.421592712402344,
+      "loss": 0.841801106929779,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.38135236501693726,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.19283419847488403,
+      "rewards/margins": 0.8724038004875183,
+      "rewards/rejected": -0.6795696020126343,
+      "step": 114,
+      "train_speed(iter/s)": 0.005492
+    },
+    {
+      "epoch": 0.09278636443861728,
+      "grad_norm": 3.3676164150238037,
+      "learning_rate": 0.00019900104903551133,
+      "logits/chosen": -0.6541582345962524,
+      "logits/rejected": -0.6580868363380432,
+      "logps/chosen": -8.595276832580566,
+      "logps/rejected": -24.14297866821289,
+      "loss": 0.7960852980613708,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4363185465335846,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.10038560628890991,
+      "rewards/margins": 1.4183127880096436,
+      "rewards/rejected": -1.3179271221160889,
+      "step": 115,
+      "train_speed(iter/s)": 0.005493
+    },
+    {
+      "epoch": 0.09359320239025744,
+      "grad_norm": 3.721660614013672,
+      "learning_rate": 0.00019896306304420324,
+      "logits/chosen": -0.6533216834068298,
+      "logits/rejected": -0.6518760919570923,
+      "logps/chosen": -9.710081100463867,
+      "logps/rejected": -20.333324432373047,
+      "loss": 0.842086911201477,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.38858363032341003,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1398862600326538,
+      "rewards/margins": 1.0419015884399414,
+      "rewards/rejected": -0.9020152688026428,
+      "step": 116,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.09440004034189758,
+      "grad_norm": 4.8206281661987305,
+      "learning_rate": 0.0001989243720032624,
+      "logits/chosen": -0.7102867364883423,
+      "logits/rejected": -0.712573230266571,
+      "logps/chosen": -14.843620300292969,
+      "logps/rejected": -23.658788681030273,
+      "loss": 1.1279860734939575,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5651426315307617,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.09467095881700516,
+      "rewards/margins": 0.9923723340034485,
+      "rewards/rejected": -1.087043285369873,
+      "step": 117,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.09520687829353773,
+      "grad_norm": 3.194643020629883,
+      "learning_rate": 0.00019888497618833815,
+      "logits/chosen": -0.6824992895126343,
+      "logits/rejected": -0.6856632232666016,
+      "logps/chosen": -7.8865065574646,
+      "logps/rejected": -22.186447143554688,
+      "loss": 0.8166104555130005,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3872619867324829,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1935729682445526,
+      "rewards/margins": 1.2844107151031494,
+      "rewards/rejected": -1.090837836265564,
+      "step": 118,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.09601371624517788,
+      "grad_norm": 5.332725524902344,
+      "learning_rate": 0.00019884487588010092,
+      "logits/chosen": -0.6963211894035339,
+      "logits/rejected": -0.695375382900238,
+      "logps/chosen": -12.522785186767578,
+      "logps/rejected": -21.899364471435547,
+      "loss": 1.200064778327942,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5619924068450928,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.2353927493095398,
+      "rewards/margins": 0.7032212018966675,
+      "rewards/rejected": -0.9386140704154968,
+      "step": 119,
+      "train_speed(iter/s)": 0.005493
+    },
+    {
+      "epoch": 0.09682055419681804,
+      "grad_norm": 3.189750909805298,
+      "learning_rate": 0.0001988040713642402,
+      "logits/chosen": -0.7008529305458069,
+      "logits/rejected": -0.7020596265792847,
+      "logps/chosen": -7.352071285247803,
+      "logps/rejected": -26.472599029541016,
+      "loss": 0.6796355843544006,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.30471816658973694,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.0883897989988327,
+      "rewards/margins": 1.43672513961792,
+      "rewards/rejected": -1.3483351469039917,
+      "step": 120,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.09762739214845818,
+      "grad_norm": 3.5756847858428955,
+      "learning_rate": 0.00019876256293146257,
+      "logits/chosen": -0.6855241060256958,
+      "logits/rejected": -0.6870216131210327,
+      "logps/chosen": -9.13197135925293,
+      "logps/rejected": -25.256893157958984,
+      "loss": 0.9965449571609497,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5001046061515808,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.036015622317790985,
+      "rewards/margins": 1.1710847616195679,
+      "rewards/rejected": -1.1350692510604858,
+      "step": 121,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.09843423010009833,
+      "grad_norm": 2.60425066947937,
+      "learning_rate": 0.0001987203508774895,
+      "logits/chosen": -0.7040928602218628,
+      "logits/rejected": -0.7042312026023865,
+      "logps/chosen": -8.471722602844238,
+      "logps/rejected": -26.018938064575195,
+      "loss": 0.8459987640380859,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4736276865005493,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.24268555641174316,
+      "rewards/margins": 1.318163514137268,
+      "rewards/rejected": -1.075477957725525,
+      "step": 122,
+      "train_speed(iter/s)": 0.005493
+    },
+    {
+      "epoch": 0.09924106805173849,
+      "grad_norm": 4.30864143371582,
+      "learning_rate": 0.0001986774355030553,
+      "logits/chosen": -0.6747640371322632,
+      "logits/rejected": -0.677899956703186,
+      "logps/chosen": -14.706789016723633,
+      "logps/rejected": -16.56774139404297,
+      "loss": 1.2364513874053955,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6511675119400024,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.08759675920009613,
+      "rewards/margins": 0.6008309721946716,
+      "rewards/rejected": -0.5132341384887695,
+      "step": 123,
+      "train_speed(iter/s)": 0.005492
+    },
+    {
+      "epoch": 0.10004790600337864,
+      "grad_norm": 6.867242813110352,
+      "learning_rate": 0.00019863381711390508,
+      "logits/chosen": -0.6869674324989319,
+      "logits/rejected": -0.6902846693992615,
+      "logps/chosen": -7.829675197601318,
+      "logps/rejected": -20.095809936523438,
+      "loss": 1.043180227279663,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.530769944190979,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.11925126612186432,
+      "rewards/margins": 0.697571337223053,
+      "rewards/rejected": -0.5783200860023499,
+      "step": 124,
+      "train_speed(iter/s)": 0.005491
+    },
+    {
+      "epoch": 0.10085474395501878,
+      "grad_norm": 3.041438579559326,
+      "learning_rate": 0.0001985894960207925,
+      "logits/chosen": -0.6505750417709351,
+      "logits/rejected": -0.6511614918708801,
+      "logps/chosen": -9.314833641052246,
+      "logps/rejected": -27.15659523010254,
+      "loss": 0.7252060174942017,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.39295029640197754,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.24510648846626282,
+      "rewards/margins": 1.2805438041687012,
+      "rewards/rejected": -1.0354374647140503,
+      "step": 125,
+      "train_speed(iter/s)": 0.005491
+    },
+    {
+      "epoch": 0.10166158190665893,
+      "grad_norm": 2.509321689605713,
+      "learning_rate": 0.00019854447253947743,
+      "logits/chosen": -0.6497823596000671,
+      "logits/rejected": -0.6492441892623901,
+      "logps/chosen": -8.02572250366211,
+      "logps/rejected": -20.3783016204834,
+      "loss": 0.7982934713363647,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.33212146162986755,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.17764505743980408,
+      "rewards/margins": 0.8625535368919373,
+      "rewards/rejected": -0.6849085092544556,
+      "step": 126,
+      "train_speed(iter/s)": 0.005489
+    },
+    {
+      "epoch": 0.10246841985829909,
+      "grad_norm": 3.6529700756073,
+      "learning_rate": 0.0001984987469907239,
+      "logits/chosen": -0.6466098427772522,
+      "logits/rejected": -0.6480077505111694,
+      "logps/chosen": -9.548334121704102,
+      "logps/rejected": -16.65062141418457,
+      "loss": 0.914322555065155,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4001502990722656,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1684979647397995,
+      "rewards/margins": 0.6629449129104614,
+      "rewards/rejected": -0.49444690346717834,
+      "step": 127,
+      "train_speed(iter/s)": 0.005491
+    },
+    {
+      "epoch": 0.10327525780993924,
+      "grad_norm": 3.3696913719177246,
+      "learning_rate": 0.00019845231970029773,
+      "logits/chosen": -0.679688572883606,
+      "logits/rejected": -0.6820145845413208,
+      "logps/chosen": -10.874924659729004,
+      "logps/rejected": -24.826372146606445,
+      "loss": 0.9800686240196228,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4543137848377228,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.048351842910051346,
+      "rewards/margins": 0.8677961230278015,
+      "rewards/rejected": -0.9161479473114014,
+      "step": 128,
+      "train_speed(iter/s)": 0.00549
+    },
+    {
+      "epoch": 0.10408209576157938,
+      "grad_norm": 2.5419607162475586,
+      "learning_rate": 0.00019840519099896414,
+      "logits/chosen": -0.6618849039077759,
+      "logits/rejected": -0.6671789288520813,
+      "logps/chosen": -8.426337242126465,
+      "logps/rejected": -25.797517776489258,
+      "loss": 0.6974806189537048,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2836017608642578,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.18303632736206055,
+      "rewards/margins": 1.1003562211990356,
+      "rewards/rejected": -0.9173198938369751,
+      "step": 129,
+      "train_speed(iter/s)": 0.005491
+    },
+    {
+      "epoch": 0.10488893371321953,
+      "grad_norm": 3.4203388690948486,
+      "learning_rate": 0.00019835736122248557,
+      "logits/chosen": -0.6606203317642212,
+      "logits/rejected": -0.6571654677391052,
+      "logps/chosen": -14.509614944458008,
+      "logps/rejected": -15.079629898071289,
+      "loss": 1.101165771484375,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5219094157218933,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.07395613193511963,
+      "rewards/margins": 0.5427431464195251,
+      "rewards/rejected": -0.46878698468208313,
+      "step": 130,
+      "train_speed(iter/s)": 0.005492
+    },
+    {
+      "epoch": 0.10569577166485969,
+      "grad_norm": 2.7750000953674316,
+      "learning_rate": 0.00019830883071161908,
+      "logits/chosen": -0.7152451276779175,
+      "logits/rejected": -0.7131951451301575,
+      "logps/chosen": -9.166630744934082,
+      "logps/rejected": -19.46819305419922,
+      "loss": 0.7933439016342163,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3753884732723236,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.26774072647094727,
+      "rewards/margins": 1.0882644653320312,
+      "rewards/rejected": -0.8205236196517944,
+      "step": 131,
+      "train_speed(iter/s)": 0.005492
+    },
+    {
+      "epoch": 0.10650260961649984,
+      "grad_norm": 3.1810927391052246,
+      "learning_rate": 0.00019825959981211407,
+      "logits/chosen": -0.714146614074707,
+      "logits/rejected": -0.7095440626144409,
+      "logps/chosen": -11.571531295776367,
+      "logps/rejected": -14.685908317565918,
+      "loss": 0.8970522880554199,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.38512641191482544,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.09674864262342453,
+      "rewards/margins": 0.5926439762115479,
+      "rewards/rejected": -0.4958953261375427,
+      "step": 132,
+      "train_speed(iter/s)": 0.005491
+    },
+    {
+      "epoch": 0.10730944756813998,
+      "grad_norm": 2.839996099472046,
+      "learning_rate": 0.00019820966887470974,
+      "logits/chosen": -0.7237828373908997,
+      "logits/rejected": -0.7276051640510559,
+      "logps/chosen": -7.241865158081055,
+      "logps/rejected": -21.9840145111084,
+      "loss": 0.7037227153778076,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.29994940757751465,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.18205267190933228,
+      "rewards/margins": 1.1148442029953003,
+      "rewards/rejected": -0.9327914714813232,
+      "step": 133,
+      "train_speed(iter/s)": 0.005492
+    },
+    {
+      "epoch": 0.10811628551978014,
+      "grad_norm": 2.8896915912628174,
+      "learning_rate": 0.0001981590382551327,
+      "logits/chosen": -0.796456515789032,
+      "logits/rejected": -0.7955406308174133,
+      "logps/chosen": -11.131935119628906,
+      "logps/rejected": -19.525293350219727,
+      "loss": 0.8993673920631409,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4546453356742859,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.21719151735305786,
+      "rewards/margins": 1.081808090209961,
+      "rewards/rejected": -0.8646165728569031,
+      "step": 134,
+      "train_speed(iter/s)": 0.00549
+    },
+    {
+      "epoch": 0.10892312347142029,
+      "grad_norm": 3.5537540912628174,
+      "learning_rate": 0.00019810770831409425,
+      "logits/chosen": -0.7755017876625061,
+      "logits/rejected": -0.7809926271438599,
+      "logps/chosen": -7.181600093841553,
+      "logps/rejected": -19.379724502563477,
+      "loss": 0.7135071754455566,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.32976531982421875,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.26019349694252014,
+      "rewards/margins": 1.110938310623169,
+      "rewards/rejected": -0.850744903087616,
+      "step": 135,
+      "train_speed(iter/s)": 0.005491
+    },
+    {
+      "epoch": 0.10972996142306043,
+      "grad_norm": 3.5955300331115723,
+      "learning_rate": 0.00019805567941728796,
+      "logits/chosen": -0.799392819404602,
+      "logits/rejected": -0.8029282689094543,
+      "logps/chosen": -9.87437629699707,
+      "logps/rejected": -23.02663230895996,
+      "loss": 0.9438092708587646,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5196709036827087,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.07922995835542679,
+      "rewards/margins": 1.070724606513977,
+      "rewards/rejected": -0.9914946556091309,
+      "step": 136,
+      "train_speed(iter/s)": 0.005492
+    },
+    {
+      "epoch": 0.11053679937470058,
+      "grad_norm": 3.7304859161376953,
+      "learning_rate": 0.00019800295193538704,
+      "logits/chosen": -0.8160181641578674,
+      "logits/rejected": -0.8219972848892212,
+      "logps/chosen": -7.596125602722168,
+      "logps/rejected": -20.225994110107422,
+      "loss": 0.8485299348831177,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.43994730710983276,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.26360729336738586,
+      "rewards/margins": 0.9658780694007874,
+      "rewards/rejected": -0.7022708058357239,
+      "step": 137,
+      "train_speed(iter/s)": 0.005491
+    },
+    {
+      "epoch": 0.11134363732634074,
+      "grad_norm": 4.919328689575195,
+      "learning_rate": 0.00019794952624404167,
+      "logits/chosen": -0.7996560335159302,
+      "logits/rejected": -0.801249623298645,
+      "logps/chosen": -9.867749214172363,
+      "logps/rejected": -22.316190719604492,
+      "loss": 0.8283427357673645,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4471050798892975,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.13889475166797638,
+      "rewards/margins": 1.224007248878479,
+      "rewards/rejected": -1.085112452507019,
+      "step": 138,
+      "train_speed(iter/s)": 0.005491
+    },
+    {
+      "epoch": 0.11215047527798089,
+      "grad_norm": 2.821730136871338,
+      "learning_rate": 0.0001978954027238763,
+      "logits/chosen": -0.818955659866333,
+      "logits/rejected": -0.8183013796806335,
+      "logps/chosen": -6.504258632659912,
+      "logps/rejected": -14.602479934692383,
+      "loss": 0.7571634650230408,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.29162290692329407,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.1784059852361679,
+      "rewards/margins": 0.8354285955429077,
+      "rewards/rejected": -0.6570225954055786,
+      "step": 139,
+      "train_speed(iter/s)": 0.005491
+    },
+    {
+      "epoch": 0.11295731322962103,
+      "grad_norm": 2.481306552886963,
+      "learning_rate": 0.000197840581760487,
+      "logits/chosen": -0.8251004219055176,
+      "logits/rejected": -0.8243045210838318,
+      "logps/chosen": -13.736132621765137,
+      "logps/rejected": -17.749977111816406,
+      "loss": 0.8052379488945007,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4064967930316925,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.3237007260322571,
+      "rewards/margins": 1.295701026916504,
+      "rewards/rejected": -0.9720003008842468,
+      "step": 140,
+      "train_speed(iter/s)": 0.005492
+    },
+    {
+      "epoch": 0.1137641511812612,
+      "grad_norm": 2.695176601409912,
+      "learning_rate": 0.00019778506374443873,
+      "logits/chosen": -0.7545077800750732,
+      "logits/rejected": -0.7596877813339233,
+      "logps/chosen": -5.598147869110107,
+      "logps/rejected": -26.760784149169922,
+      "loss": 0.6858201026916504,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3311798572540283,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.19224056601524353,
+      "rewards/margins": 1.6802706718444824,
+      "rewards/rejected": -1.488029956817627,
+      "step": 141,
+      "train_speed(iter/s)": 0.005493
+    },
+    {
+      "epoch": 0.11457098913290134,
+      "grad_norm": 2.5371341705322266,
+      "learning_rate": 0.0001977288490712624,
+      "logits/chosen": -0.8187113404273987,
+      "logits/rejected": -0.8256463408470154,
+      "logps/chosen": -9.480502128601074,
+      "logps/rejected": -26.96666717529297,
+      "loss": 0.6809464693069458,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3261586129665375,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2214924842119217,
+      "rewards/margins": 1.2070807218551636,
+      "rewards/rejected": -0.9855883121490479,
+      "step": 142,
+      "train_speed(iter/s)": 0.005493
+    },
+    {
+      "epoch": 0.11537782708454149,
+      "grad_norm": 3.33156681060791,
+      "learning_rate": 0.00019767193814145226,
+      "logits/chosen": -0.8148999214172363,
+      "logits/rejected": -0.8154037594795227,
+      "logps/chosen": -16.6685848236084,
+      "logps/rejected": -16.219863891601562,
+      "loss": 1.1351004838943481,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6261444091796875,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.06986582279205322,
+      "rewards/margins": 0.7546349763870239,
+      "rewards/rejected": -0.6847692728042603,
+      "step": 143,
+      "train_speed(iter/s)": 0.005493
+    },
+    {
+      "epoch": 0.11618466503618163,
+      "grad_norm": 2.587169647216797,
+      "learning_rate": 0.00019761433136046295,
+      "logits/chosen": -0.7679372429847717,
+      "logits/rejected": -0.7711334824562073,
+      "logps/chosen": -6.84797477722168,
+      "logps/rejected": -25.882503509521484,
+      "loss": 0.6460772752761841,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2581600844860077,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.25230705738067627,
+      "rewards/margins": 1.5044541358947754,
+      "rewards/rejected": -1.2521469593048096,
+      "step": 144,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.1169915029878218,
+      "grad_norm": 3.816577911376953,
+      "learning_rate": 0.00019755602913870655,
+      "logits/chosen": -0.7486031651496887,
+      "logits/rejected": -0.7545804977416992,
+      "logps/chosen": -11.378320693969727,
+      "logps/rejected": -23.90165901184082,
+      "loss": 1.0280418395996094,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5588870644569397,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.10084515064954758,
+      "rewards/margins": 1.0778629779815674,
+      "rewards/rejected": -0.9770178198814392,
+      "step": 145,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.11779834093946194,
+      "grad_norm": 3.1183438301086426,
+      "learning_rate": 0.00019749703189154971,
+      "logits/chosen": -0.7663331031799316,
+      "logits/rejected": -0.7778096199035645,
+      "logps/chosen": -7.461820602416992,
+      "logps/rejected": -26.446916580200195,
+      "loss": 0.6639739274978638,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.28250494599342346,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.04964678734540939,
+      "rewards/margins": 1.1681914329528809,
+      "rewards/rejected": -1.1185446977615356,
+      "step": 146,
+      "train_speed(iter/s)": 0.005493
+    },
+    {
+      "epoch": 0.11860517889110209,
+      "grad_norm": 2.613725423812866,
+      "learning_rate": 0.00019743734003931082,
+      "logits/chosen": -0.808351993560791,
+      "logits/rejected": -0.8136962652206421,
+      "logps/chosen": -7.89247989654541,
+      "logps/rejected": -23.955371856689453,
+      "loss": 0.8511161804199219,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.39823463559150696,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.12871581315994263,
+      "rewards/margins": 1.055700659751892,
+      "rewards/rejected": -0.9269847869873047,
+      "step": 147,
+      "train_speed(iter/s)": 0.005493
+    },
+    {
+      "epoch": 0.11941201684274225,
+      "grad_norm": 3.0128729343414307,
+      "learning_rate": 0.00019737695400725677,
+      "logits/chosen": -0.7927097678184509,
+      "logits/rejected": -0.7919667959213257,
+      "logps/chosen": -9.877785682678223,
+      "logps/rejected": -23.477703094482422,
+      "loss": 0.7262363433837891,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3177878260612488,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.12437373399734497,
+      "rewards/margins": 1.1551289558410645,
+      "rewards/rejected": -1.0307551622390747,
+      "step": 148,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.1202188547943824,
+      "grad_norm": 2.5007247924804688,
+      "learning_rate": 0.0001973158742256001,
+      "logits/chosen": -0.815912663936615,
+      "logits/rejected": -0.8240575194358826,
+      "logps/chosen": -7.251335620880127,
+      "logps/rejected": -23.904563903808594,
+      "loss": 0.7742622494697571,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3948196470737457,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.16877779364585876,
+      "rewards/margins": 1.3302054405212402,
+      "rewards/rejected": -1.1614277362823486,
+      "step": 149,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.12102569274602254,
+      "grad_norm": 2.996128559112549,
+      "learning_rate": 0.0001972541011294959,
+      "logits/chosen": -0.8276554346084595,
+      "logits/rejected": -0.832274317741394,
+      "logps/chosen": -9.810141563415527,
+      "logps/rejected": -24.295639038085938,
+      "loss": 0.733322262763977,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.34396567940711975,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.03486320376396179,
+      "rewards/margins": 1.2224383354187012,
+      "rewards/rejected": -1.1875752210617065,
+      "step": 150,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.12183253069766269,
+      "grad_norm": 3.6684677600860596,
+      "learning_rate": 0.0001971916351590386,
+      "logits/chosen": -0.8309310674667358,
+      "logits/rejected": -0.8319404721260071,
+      "logps/chosen": -13.512450218200684,
+      "logps/rejected": -28.13376808166504,
+      "loss": 0.8970531821250916,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5336953997612,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.07107365131378174,
+      "rewards/margins": 1.3276447057724,
+      "rewards/rejected": -1.3987183570861816,
+      "step": 151,
+      "train_speed(iter/s)": 0.005492
+    },
+    {
+      "epoch": 0.12263936864930285,
+      "grad_norm": 3.4376490116119385,
+      "learning_rate": 0.00019712847675925903,
+      "logits/chosen": -0.8819084763526917,
+      "logits/rejected": -0.8899849653244019,
+      "logps/chosen": -9.134736061096191,
+      "logps/rejected": -25.074691772460938,
+      "loss": 0.7725754976272583,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.44169238209724426,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.20537638664245605,
+      "rewards/margins": 1.4855870008468628,
+      "rewards/rejected": -1.2802107334136963,
+      "step": 152,
+      "train_speed(iter/s)": 0.005492
+    },
+    {
+      "epoch": 0.12344620660094299,
+      "grad_norm": 5.506346225738525,
+      "learning_rate": 0.00019706462638012105,
+      "logits/chosen": -0.9026679992675781,
+      "logits/rejected": -0.908703088760376,
+      "logps/chosen": -13.740835189819336,
+      "logps/rejected": -25.73556900024414,
+      "loss": 1.2033629417419434,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6762228608131409,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.11278621852397919,
+      "rewards/margins": 1.2870286703109741,
+      "rewards/rejected": -1.3998148441314697,
+      "step": 153,
+      "train_speed(iter/s)": 0.005492
+    },
+    {
+      "epoch": 0.12425304455258314,
+      "grad_norm": 4.886820316314697,
+      "learning_rate": 0.00019700008447651854,
+      "logits/chosen": -0.8641173839569092,
+      "logits/rejected": -0.859977662563324,
+      "logps/chosen": -13.57033634185791,
+      "logps/rejected": -26.01408576965332,
+      "loss": 1.2000961303710938,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5635690093040466,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.19201046228408813,
+      "rewards/margins": 1.3229352235794067,
+      "rewards/rejected": -1.51494562625885,
+      "step": 154,
+      "train_speed(iter/s)": 0.005493
+    },
+    {
+      "epoch": 0.12505988250422329,
+      "grad_norm": 3.3179073333740234,
+      "learning_rate": 0.00019693485150827197,
+      "logits/chosen": -0.8760464191436768,
+      "logits/rejected": -0.8815669417381287,
+      "logps/chosen": -9.239191055297852,
+      "logps/rejected": -28.241233825683594,
+      "loss": 0.6995830535888672,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3207743167877197,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.1987459361553192,
+      "rewards/margins": 1.7694886922836304,
+      "rewards/rejected": -1.5707426071166992,
+      "step": 155,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.12586672045586345,
+      "grad_norm": 5.6134161949157715,
+      "learning_rate": 0.0001968689279401253,
+      "logits/chosen": -0.8809852600097656,
+      "logits/rejected": -0.8887759447097778,
+      "logps/chosen": -7.534684181213379,
+      "logps/rejected": -30.649181365966797,
+      "loss": 1.0883609056472778,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.7660683393478394,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2163163125514984,
+      "rewards/margins": 1.8669637441635132,
+      "rewards/rejected": -1.6506474018096924,
+      "step": 156,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.1266735584075036,
+      "grad_norm": 2.5959808826446533,
+      "learning_rate": 0.0001968023142417424,
+      "logits/chosen": -0.8142418265342712,
+      "logits/rejected": -0.8186179995536804,
+      "logps/chosen": -9.635229110717773,
+      "logps/rejected": -32.11575698852539,
+      "loss": 0.6578619480133057,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3367811441421509,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.21614979207515717,
+      "rewards/margins": 2.128612518310547,
+      "rewards/rejected": -1.912462592124939,
+      "step": 157,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.12748039635914374,
+      "grad_norm": 3.154606342315674,
+      "learning_rate": 0.00019673501088770414,
+      "logits/chosen": -0.8146523237228394,
+      "logits/rejected": -0.8199130892753601,
+      "logps/chosen": -12.74015998840332,
+      "logps/rejected": -16.670421600341797,
+      "loss": 0.9762111306190491,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5080199837684631,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.18415355682373047,
+      "rewards/margins": 0.8309141397476196,
+      "rewards/rejected": -0.6467605233192444,
+      "step": 158,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.1282872343107839,
+      "grad_norm": 6.67454719543457,
+      "learning_rate": 0.00019666701835750457,
+      "logits/chosen": -0.8286721110343933,
+      "logits/rejected": -0.8334258794784546,
+      "logps/chosen": -8.225419044494629,
+      "logps/rejected": -17.38351821899414,
+      "loss": 0.9775789380073547,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5453903079032898,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.15067297220230103,
+      "rewards/margins": 0.9060754776000977,
+      "rewards/rejected": -0.7554025650024414,
+      "step": 159,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.12909407226242403,
+      "grad_norm": 3.848874092102051,
+      "learning_rate": 0.00019659833713554773,
+      "logits/chosen": -0.7873284816741943,
+      "logits/rejected": -0.7915123105049133,
+      "logps/chosen": -10.267017364501953,
+      "logps/rejected": -25.254423141479492,
+      "loss": 0.8222072124481201,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.43752118945121765,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.18684861063957214,
+      "rewards/margins": 1.191467523574829,
+      "rewards/rejected": -1.0046190023422241,
+      "step": 160,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.1299009102140642,
+      "grad_norm": 8.467432975769043,
+      "learning_rate": 0.00019652896771114414,
+      "logits/chosen": -0.7907496094703674,
+      "logits/rejected": -0.7956827878952026,
+      "logps/chosen": -12.749449729919434,
+      "logps/rejected": -20.143428802490234,
+      "loss": 1.2200610637664795,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.7363396883010864,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.01990276202559471,
+      "rewards/margins": 0.667390763759613,
+      "rewards/rejected": -0.647487998008728,
+      "step": 161,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.13070774816570435,
+      "grad_norm": 3.0088865756988525,
+      "learning_rate": 0.00019645891057850742,
+      "logits/chosen": -0.7298374176025391,
+      "logits/rejected": -0.732286274433136,
+      "logps/chosen": -15.880043029785156,
+      "logps/rejected": -24.578083038330078,
+      "loss": 0.8190960884094238,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4379112124443054,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.2654583156108856,
+      "rewards/margins": 1.3646464347839355,
+      "rewards/rejected": -1.099188208580017,
+      "step": 162,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.13151458611734448,
+      "grad_norm": 5.012711524963379,
+      "learning_rate": 0.00019638816623675058,
+      "logits/chosen": -0.7988470196723938,
+      "logits/rejected": -0.8017285466194153,
+      "logps/chosen": -9.470846176147461,
+      "logps/rejected": -25.236000061035156,
+      "loss": 0.798559844493866,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.32053616642951965,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.05605325102806091,
+      "rewards/margins": 1.3247337341308594,
+      "rewards/rejected": -1.268680453300476,
+      "step": 163,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.13232142406898464,
+      "grad_norm": 5.894308567047119,
+      "learning_rate": 0.00019631673518988261,
+      "logits/chosen": -0.7558510303497314,
+      "logits/rejected": -0.7591589689254761,
+      "logps/chosen": -8.896431922912598,
+      "logps/rejected": -26.477436065673828,
+      "loss": 0.7629649043083191,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.35155194997787476,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.04630519449710846,
+      "rewards/margins": 1.3866008520126343,
+      "rewards/rejected": -1.340295672416687,
+      "step": 164,
+      "train_speed(iter/s)": 0.005493
+    },
+    {
+      "epoch": 0.1331282620206248,
+      "grad_norm": 4.07641077041626,
+      "learning_rate": 0.0001962446179468048,
+      "logits/chosen": -0.7340226173400879,
+      "logits/rejected": -0.7390322089195251,
+      "logps/chosen": -7.206872463226318,
+      "logps/rejected": -30.465621948242188,
+      "loss": 0.7621070742607117,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3923991620540619,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.12836496531963348,
+      "rewards/margins": 1.729160189628601,
+      "rewards/rejected": -1.6007951498031616,
+      "step": 165,
+      "train_speed(iter/s)": 0.005493
+    },
+    {
+      "epoch": 0.13393509997226494,
+      "grad_norm": 4.966350078582764,
+      "learning_rate": 0.00019617181502130723,
+      "logits/chosen": -0.7315192222595215,
+      "logits/rejected": -0.7316710352897644,
+      "logps/chosen": -9.406560897827148,
+      "logps/rejected": -20.61480712890625,
+      "loss": 0.9099687337875366,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3969041705131531,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.084879070520401,
+      "rewards/margins": 0.9290111064910889,
+      "rewards/rejected": -0.844132125377655,
+      "step": 166,
+      "train_speed(iter/s)": 0.005493
+    },
+    {
+      "epoch": 0.1347419379239051,
+      "grad_norm": 2.8620102405548096,
+      "learning_rate": 0.00019609832693206494,
+      "logits/chosen": -0.6981790065765381,
+      "logits/rejected": -0.6956609487533569,
+      "logps/chosen": -11.53895092010498,
+      "logps/rejected": -17.151912689208984,
+      "loss": 0.7675348520278931,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.364923357963562,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.22551363706588745,
+      "rewards/margins": 0.959013044834137,
+      "rewards/rejected": -0.73349928855896,
+      "step": 167,
+      "train_speed(iter/s)": 0.005493
+    },
+    {
+      "epoch": 0.13554877587554526,
+      "grad_norm": 3.910537004470825,
+      "learning_rate": 0.00019602415420263443,
+      "logits/chosen": -0.7344207763671875,
+      "logits/rejected": -0.7377809286117554,
+      "logps/chosen": -9.725975036621094,
+      "logps/rejected": -26.63901138305664,
+      "loss": 0.9027658104896545,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4818994700908661,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.0954769030213356,
+      "rewards/margins": 1.3119255304336548,
+      "rewards/rejected": -1.2164486646652222,
+      "step": 168,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.1363556138271854,
+      "grad_norm": 2.7541050910949707,
+      "learning_rate": 0.00019594929736144976,
+      "logits/chosen": -0.7230939865112305,
+      "logits/rejected": -0.7268710732460022,
+      "logps/chosen": -6.102090835571289,
+      "logps/rejected": -22.030675888061523,
+      "loss": 0.8178823590278625,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4109593629837036,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2232162207365036,
+      "rewards/margins": 1.0571808815002441,
+      "rewards/rejected": -0.8339645862579346,
+      "step": 169,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.13716245177882555,
+      "grad_norm": 2.1247079372406006,
+      "learning_rate": 0.00019587375694181888,
+      "logits/chosen": -0.7419747114181519,
+      "logits/rejected": -0.7435666918754578,
+      "logps/chosen": -10.772944450378418,
+      "logps/rejected": -17.87366485595703,
+      "loss": 0.7673830389976501,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3707871735095978,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.2476326823234558,
+      "rewards/margins": 0.9303527474403381,
+      "rewards/rejected": -0.6827200055122375,
+      "step": 170,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.1379692897304657,
+      "grad_norm": 3.058112382888794,
+      "learning_rate": 0.00019579753348191982,
+      "logits/chosen": -0.7581822276115417,
+      "logits/rejected": -0.7616063356399536,
+      "logps/chosen": -10.94616413116455,
+      "logps/rejected": -22.71567153930664,
+      "loss": 0.9573794603347778,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4561714828014374,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.10372941941022873,
+      "rewards/margins": 0.9039929509162903,
+      "rewards/rejected": -0.8002634644508362,
+      "step": 171,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.13877612768210584,
+      "grad_norm": 5.104068756103516,
+      "learning_rate": 0.00019572062752479683,
+      "logits/chosen": -0.7440593242645264,
+      "logits/rejected": -0.7444762587547302,
+      "logps/chosen": -13.373594284057617,
+      "logps/rejected": -21.11050033569336,
+      "loss": 0.9622430801391602,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6134111881256104,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.17188113927841187,
+      "rewards/margins": 1.2960879802703857,
+      "rewards/rejected": -1.1242069005966187,
+      "step": 172,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.139582965633746,
+      "grad_norm": 3.6643261909484863,
+      "learning_rate": 0.00019564303961835654,
+      "logits/chosen": -0.7351651787757874,
+      "logits/rejected": -0.7340760827064514,
+      "logps/chosen": -13.493339538574219,
+      "logps/rejected": -29.00867462158203,
+      "loss": 0.8059775233268738,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4561418890953064,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.07504750788211823,
+      "rewards/margins": 1.5861859321594238,
+      "rewards/rejected": -1.5111385583877563,
+      "step": 173,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.14038980358538614,
+      "grad_norm": 3.840665102005005,
+      "learning_rate": 0.00019556477031536397,
+      "logits/chosen": -0.7632665634155273,
+      "logits/rejected": -0.7688808441162109,
+      "logps/chosen": -7.488624572753906,
+      "logps/rejected": -27.777238845825195,
+      "loss": 0.7469439506530762,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.378052294254303,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.07338213175535202,
+      "rewards/margins": 1.5290367603302002,
+      "rewards/rejected": -1.4556546211242676,
+      "step": 174,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.1411966415370263,
+      "grad_norm": 2.548142671585083,
+      "learning_rate": 0.00019548582017343878,
+      "logits/chosen": -0.7821887731552124,
+      "logits/rejected": -0.7840255498886108,
+      "logps/chosen": -9.051867485046387,
+      "logps/rejected": -26.980369567871094,
+      "loss": 0.6677172183990479,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3486889600753784,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.10505867004394531,
+      "rewards/margins": 1.3984501361846924,
+      "rewards/rejected": -1.2933913469314575,
+      "step": 175,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.14200347948866646,
+      "grad_norm": 3.0442428588867188,
+      "learning_rate": 0.00019540618975505107,
+      "logits/chosen": -0.7756679058074951,
+      "logits/rejected": -0.7768942713737488,
+      "logps/chosen": -9.436080932617188,
+      "logps/rejected": -24.625957489013672,
+      "loss": 0.7269696593284607,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3957803249359131,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.18510255217552185,
+      "rewards/margins": 1.5727849006652832,
+      "rewards/rejected": -1.3876824378967285,
+      "step": 176,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.1428103174403066,
+      "grad_norm": 4.40380859375,
+      "learning_rate": 0.00019532587962751757,
+      "logits/chosen": -0.7868165969848633,
+      "logits/rejected": -0.7917428016662598,
+      "logps/chosen": -12.063282012939453,
+      "logps/rejected": -26.304283142089844,
+      "loss": 0.8930169343948364,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5761467814445496,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2778218984603882,
+      "rewards/margins": 1.7966634035110474,
+      "rewards/rejected": -1.5188415050506592,
+      "step": 177,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.14361715539194675,
+      "grad_norm": 6.159250259399414,
+      "learning_rate": 0.0001952448903629974,
+      "logits/chosen": -0.8135851621627808,
+      "logits/rejected": -0.8196555972099304,
+      "logps/chosen": -16.506568908691406,
+      "logps/rejected": -26.224693298339844,
+      "loss": 1.118604302406311,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5594882965087891,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.20946675539016724,
+      "rewards/margins": 1.0000211000442505,
+      "rewards/rejected": -1.209487795829773,
+      "step": 178,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.1444239933435869,
+      "grad_norm": 3.561173439025879,
+      "learning_rate": 0.00019516322253848828,
+      "logits/chosen": -0.8224231600761414,
+      "logits/rejected": -0.8233227729797363,
+      "logps/chosen": -8.352205276489258,
+      "logps/rejected": -23.940982818603516,
+      "loss": 0.7962292432785034,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4265138804912567,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.2205028533935547,
+      "rewards/margins": 1.6108112335205078,
+      "rewards/rejected": -1.3903082609176636,
+      "step": 179,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.14523083129522704,
+      "grad_norm": 3.16656756401062,
+      "learning_rate": 0.00019508087673582205,
+      "logits/chosen": -0.8061304688453674,
+      "logits/rejected": -0.8118202090263367,
+      "logps/chosen": -8.317770004272461,
+      "logps/rejected": -26.48977279663086,
+      "loss": 0.7722903490066528,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.37005698680877686,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.028686638921499252,
+      "rewards/margins": 1.5126442909240723,
+      "rewards/rejected": -1.4839575290679932,
+      "step": 180,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.1460376692468672,
+      "grad_norm": 3.15301513671875,
+      "learning_rate": 0.00019499785354166083,
+      "logits/chosen": -0.7916308641433716,
+      "logits/rejected": -0.7952077984809875,
+      "logps/chosen": -7.383344650268555,
+      "logps/rejected": -32.59227752685547,
+      "loss": 0.7744749784469604,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4444514214992523,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.0771065428853035,
+      "rewards/margins": 2.1048974990844727,
+      "rewards/rejected": -2.0277910232543945,
+      "step": 181,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.14684450719850736,
+      "grad_norm": 2.6223182678222656,
+      "learning_rate": 0.00019491415354749272,
+      "logits/chosen": -0.7739111185073853,
+      "logits/rejected": -0.7772398591041565,
+      "logps/chosen": -6.630223274230957,
+      "logps/rejected": -26.15964126586914,
+      "loss": 0.693659245967865,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3340071141719818,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1530786156654358,
+      "rewards/margins": 1.5430837869644165,
+      "rewards/rejected": -1.3900052309036255,
+      "step": 182,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.1476513451501475,
+      "grad_norm": 2.088421106338501,
+      "learning_rate": 0.00019482977734962753,
+      "logits/chosen": -0.7808999419212341,
+      "logits/rejected": -0.7902572154998779,
+      "logps/chosen": -8.449446678161621,
+      "logps/rejected": -31.33694076538086,
+      "loss": 0.5712997913360596,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.31176620721817017,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.27892935276031494,
+      "rewards/margins": 2.0656492710113525,
+      "rewards/rejected": -1.7867200374603271,
+      "step": 183,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.14845818310178766,
+      "grad_norm": 4.234419345855713,
+      "learning_rate": 0.00019474472554919272,
+      "logits/chosen": -0.8170475959777832,
+      "logits/rejected": -0.825502872467041,
+      "logps/chosen": -9.795135498046875,
+      "logps/rejected": -31.384075164794922,
+      "loss": 0.7613832354545593,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3808598518371582,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.14184550940990448,
+      "rewards/margins": 1.434211254119873,
+      "rewards/rejected": -1.2923656702041626,
+      "step": 184,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.1492650210534278,
+      "grad_norm": 2.047487735748291,
+      "learning_rate": 0.00019465899875212888,
+      "logits/chosen": -0.794952929019928,
+      "logits/rejected": -0.8015525937080383,
+      "logps/chosen": -6.017518043518066,
+      "logps/rejected": -19.722206115722656,
+      "loss": 0.6337971091270447,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.28807568550109863,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.24735234677791595,
+      "rewards/margins": 1.255548119544983,
+      "rewards/rejected": -1.0081957578659058,
+      "step": 185,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.15007185900506795,
+      "grad_norm": 3.8246495723724365,
+      "learning_rate": 0.00019457259756918552,
+      "logits/chosen": -0.8133244514465332,
+      "logits/rejected": -0.8167756199836731,
+      "logps/chosen": -10.089277267456055,
+      "logps/rejected": -26.579971313476562,
+      "loss": 0.7682158946990967,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3842141926288605,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.10676907747983932,
+      "rewards/margins": 1.6904234886169434,
+      "rewards/rejected": -1.5836546421051025,
+      "step": 186,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.1508786969567081,
+      "grad_norm": 4.293128490447998,
+      "learning_rate": 0.00019448552261591687,
+      "logits/chosen": -0.81524658203125,
+      "logits/rejected": -0.8198854327201843,
+      "logps/chosen": -12.067436218261719,
+      "logps/rejected": -23.448137283325195,
+      "loss": 0.9188786149024963,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5105058550834656,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.23126842081546783,
+      "rewards/margins": 1.3146940469741821,
+      "rewards/rejected": -1.083425760269165,
+      "step": 187,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.15168553490834824,
+      "grad_norm": 5.430900573730469,
+      "learning_rate": 0.00019439777451267718,
+      "logits/chosen": -0.8615626096725464,
+      "logits/rejected": -0.8717396855354309,
+      "logps/chosen": -12.81668472290039,
+      "logps/rejected": -33.20039367675781,
+      "loss": 0.8604999780654907,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5934706926345825,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.07574808597564697,
+      "rewards/margins": 2.146501064300537,
+      "rewards/rejected": -2.0707528591156006,
+      "step": 188,
+      "train_speed(iter/s)": 0.005498
+    },
+    {
+      "epoch": 0.1524923728599884,
+      "grad_norm": 5.0244140625,
+      "learning_rate": 0.00019430935388461656,
+      "logits/chosen": -0.8756387233734131,
+      "logits/rejected": -0.8824178576469421,
+      "logps/chosen": -9.554489135742188,
+      "logps/rejected": -33.99430847167969,
+      "loss": 0.7015911936759949,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4163678288459778,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.21942490339279175,
+      "rewards/margins": 2.339784622192383,
+      "rewards/rejected": -2.1203596591949463,
+      "step": 189,
+      "train_speed(iter/s)": 0.005498
+    },
+    {
+      "epoch": 0.15329921081162856,
+      "grad_norm": 5.72052526473999,
+      "learning_rate": 0.0001942202613616764,
+      "logits/chosen": -0.8696508407592773,
+      "logits/rejected": -0.8797045350074768,
+      "logps/chosen": -8.161410331726074,
+      "logps/rejected": -31.448095321655273,
+      "loss": 0.6777075529098511,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.36822211742401123,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.29191264510154724,
+      "rewards/margins": 1.8597280979156494,
+      "rewards/rejected": -1.5678153038024902,
+      "step": 190,
+      "train_speed(iter/s)": 0.005498
+    },
+    {
+      "epoch": 0.1541060487632687,
+      "grad_norm": 9.510895729064941,
+      "learning_rate": 0.00019413049757858496,
+      "logits/chosen": -0.8756559491157532,
+      "logits/rejected": -0.8811758756637573,
+      "logps/chosen": -9.792702674865723,
+      "logps/rejected": -34.0648078918457,
+      "loss": 0.8311678767204285,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.47524788975715637,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.09086283296346664,
+      "rewards/margins": 2.250915050506592,
+      "rewards/rejected": -2.1600520610809326,
+      "step": 191,
+      "train_speed(iter/s)": 0.005498
+    },
+    {
+      "epoch": 0.15491288671490885,
+      "grad_norm": 2.9663641452789307,
+      "learning_rate": 0.0001940400631748528,
+      "logits/chosen": -0.8579489588737488,
+      "logits/rejected": -0.8665011525154114,
+      "logps/chosen": -8.948463439941406,
+      "logps/rejected": -39.376953125,
+      "loss": 0.6801427602767944,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4235433042049408,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.021937131881713867,
+      "rewards/margins": 2.60715913772583,
+      "rewards/rejected": -2.585222005844116,
+      "step": 192,
+      "train_speed(iter/s)": 0.005499
+    },
+    {
+      "epoch": 0.15571972466654901,
+      "grad_norm": 4.202517509460449,
+      "learning_rate": 0.0001939489587947682,
+      "logits/chosen": -0.8559134006500244,
+      "logits/rejected": -0.8664484620094299,
+      "logps/chosen": -6.8310418128967285,
+      "logps/rejected": -31.699092864990234,
+      "loss": 0.6303547024726868,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.32207247614860535,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2676079273223877,
+      "rewards/margins": 2.117737293243408,
+      "rewards/rejected": -1.8501293659210205,
+      "step": 193,
+      "train_speed(iter/s)": 0.005499
+    },
+    {
+      "epoch": 0.15652656261818915,
+      "grad_norm": 2.568023681640625,
+      "learning_rate": 0.00019385718508739262,
+      "logits/chosen": -0.8656299710273743,
+      "logits/rejected": -0.8692151308059692,
+      "logps/chosen": -8.725165367126465,
+      "logps/rejected": -33.851287841796875,
+      "loss": 0.7019908428192139,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.41890949010849,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.18432064354419708,
+      "rewards/margins": 2.251908302307129,
+      "rewards/rejected": -2.0675878524780273,
+      "step": 194,
+      "train_speed(iter/s)": 0.005499
+    },
+    {
+      "epoch": 0.1573334005698293,
+      "grad_norm": 8.88502025604248,
+      "learning_rate": 0.000193764742706556,
+      "logits/chosen": -0.8621362447738647,
+      "logits/rejected": -0.8639461994171143,
+      "logps/chosen": -11.037799835205078,
+      "logps/rejected": -21.885807037353516,
+      "loss": 1.035823941230774,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5726286768913269,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.020187322050333023,
+      "rewards/margins": 1.3119888305664062,
+      "rewards/rejected": -1.2918013334274292,
+      "step": 195,
+      "train_speed(iter/s)": 0.005499
+    },
+    {
+      "epoch": 0.15814023852146947,
+      "grad_norm": 5.145761489868164,
+      "learning_rate": 0.00019367163231085227,
+      "logits/chosen": -0.8333749771118164,
+      "logits/rejected": -0.8401883840560913,
+      "logps/chosen": -12.062631607055664,
+      "logps/rejected": -28.78449249267578,
+      "loss": 0.7770197987556458,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.41585007309913635,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.08489862084388733,
+      "rewards/margins": 1.6114108562469482,
+      "rewards/rejected": -1.5265123844146729,
+      "step": 196,
+      "train_speed(iter/s)": 0.005499
+    },
+    {
+      "epoch": 0.1589470764731096,
+      "grad_norm": 9.907797813415527,
+      "learning_rate": 0.00019357785456363452,
+      "logits/chosen": -0.8024821281433105,
+      "logits/rejected": -0.809442937374115,
+      "logps/chosen": -10.157941818237305,
+      "logps/rejected": -25.024654388427734,
+      "loss": 1.1124495267868042,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5751543045043945,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.056628137826919556,
+      "rewards/margins": 1.3617597818374634,
+      "rewards/rejected": -1.4183878898620605,
+      "step": 197,
+      "train_speed(iter/s)": 0.005499
+    },
+    {
+      "epoch": 0.15975391442474976,
+      "grad_norm": 5.390470027923584,
+      "learning_rate": 0.00019348341013301014,
+      "logits/chosen": -0.8071514368057251,
+      "logits/rejected": -0.8174934387207031,
+      "logps/chosen": -8.75635814666748,
+      "logps/rejected": -33.552799224853516,
+      "loss": 0.6396787166595459,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.39103907346725464,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2289201021194458,
+      "rewards/margins": 1.8501532077789307,
+      "rewards/rejected": -1.6212331056594849,
+      "step": 198,
+      "train_speed(iter/s)": 0.005499
+    },
+    {
+      "epoch": 0.1605607523763899,
+      "grad_norm": 2.6979644298553467,
+      "learning_rate": 0.00019338829969183644,
+      "logits/chosen": -0.810739278793335,
+      "logits/rejected": -0.8188509941101074,
+      "logps/chosen": -7.122466087341309,
+      "logps/rejected": -26.984783172607422,
+      "loss": 0.7288452386856079,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3455110192298889,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.12697723507881165,
+      "rewards/margins": 1.6645798683166504,
+      "rewards/rejected": -1.5376026630401611,
+      "step": 199,
+      "train_speed(iter/s)": 0.005499
+    },
+    {
+      "epoch": 0.16136759032803005,
+      "grad_norm": 3.698697805404663,
+      "learning_rate": 0.00019329252391771555,
+      "logits/chosen": -0.7774844765663147,
+      "logits/rejected": -0.7780304551124573,
+      "logps/chosen": -12.480424880981445,
+      "logps/rejected": -22.409971237182617,
+      "loss": 0.9710755944252014,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5477850437164307,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.16317154467105865,
+      "rewards/margins": 1.5441625118255615,
+      "rewards/rejected": -1.380990982055664,
+      "step": 200,
+      "train_speed(iter/s)": 0.005498
+    },
+    {
+      "epoch": 0.1621744282796702,
+      "grad_norm": 2.8351235389709473,
+      "learning_rate": 0.00019319608349298962,
+      "logits/chosen": -0.8196174502372742,
+      "logits/rejected": -0.8330628275871277,
+      "logps/chosen": -7.196435928344727,
+      "logps/rejected": -30.938335418701172,
+      "loss": 0.6695919632911682,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.31925174593925476,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.21494938433170319,
+      "rewards/margins": 1.952761173248291,
+      "rewards/rejected": -1.737811803817749,
+      "step": 201,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.16298126623131035,
+      "grad_norm": 5.542436599731445,
+      "learning_rate": 0.0001930989791047361,
+      "logits/chosen": -0.7954035997390747,
+      "logits/rejected": -0.7969664335250854,
+      "logps/chosen": -10.17538070678711,
+      "logps/rejected": -20.19991683959961,
+      "loss": 0.8598300218582153,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.46379411220550537,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1660608947277069,
+      "rewards/margins": 1.2763766050338745,
+      "rewards/rejected": -1.1103156805038452,
+      "step": 202,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.1637881041829505,
+      "grad_norm": 5.940423011779785,
+      "learning_rate": 0.00019300121144476276,
+      "logits/chosen": -0.8003661632537842,
+      "logits/rejected": -0.8022105097770691,
+      "logps/chosen": -7.63640022277832,
+      "logps/rejected": -24.617435455322266,
+      "loss": 0.8151254057884216,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3810645043849945,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.07057296484708786,
+      "rewards/margins": 1.2225996255874634,
+      "rewards/rejected": -1.1520267724990845,
+      "step": 203,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.16459494213459067,
+      "grad_norm": 2.6105964183807373,
+      "learning_rate": 0.00019290278120960265,
+      "logits/chosen": -0.7997848391532898,
+      "logits/rejected": -0.805422306060791,
+      "logps/chosen": -14.606799125671387,
+      "logps/rejected": -31.826797485351562,
+      "loss": 0.77715665102005,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.43484237790107727,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.14822271466255188,
+      "rewards/margins": 1.6973564624786377,
+      "rewards/rejected": -1.5491337776184082,
+      "step": 204,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.1654017800862308,
+      "grad_norm": 2.472439765930176,
+      "learning_rate": 0.00019280368910050942,
+      "logits/chosen": -0.8194397687911987,
+      "logits/rejected": -0.8225022554397583,
+      "logps/chosen": -11.139198303222656,
+      "logps/rejected": -29.919803619384766,
+      "loss": 0.6480454206466675,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3610724210739136,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.27185019850730896,
+      "rewards/margins": 1.6965210437774658,
+      "rewards/rejected": -1.4246705770492554,
+      "step": 205,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.16620861803787096,
+      "grad_norm": 2.494168758392334,
+      "learning_rate": 0.00019270393582345206,
+      "logits/chosen": -0.7520598769187927,
+      "logits/rejected": -0.7561833262443542,
+      "logps/chosen": -6.9641194343566895,
+      "logps/rejected": -28.728130340576172,
+      "loss": 0.5702533721923828,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.29727238416671753,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.28327614068984985,
+      "rewards/margins": 1.785872220993042,
+      "rewards/rejected": -1.5025960206985474,
+      "step": 206,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.16701545598951112,
+      "grad_norm": 4.714883327484131,
+      "learning_rate": 0.00019260352208910997,
+      "logits/chosen": -0.8163325786590576,
+      "logits/rejected": -0.8220417499542236,
+      "logps/chosen": -9.346306800842285,
+      "logps/rejected": -24.412797927856445,
+      "loss": 0.8744742274284363,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.45610249042510986,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.13759079575538635,
+      "rewards/margins": 1.5552394390106201,
+      "rewards/rejected": -1.4176486730575562,
+      "step": 207,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.16782229394115125,
+      "grad_norm": 4.433586597442627,
+      "learning_rate": 0.0001925024486128679,
+      "logits/chosen": -0.7977516055107117,
+      "logits/rejected": -0.7999525666236877,
+      "logps/chosen": -6.685362339019775,
+      "logps/rejected": -30.688045501708984,
+      "loss": 0.7153071165084839,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4366532862186432,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.29755526781082153,
+      "rewards/margins": 2.2702927589416504,
+      "rewards/rejected": -1.972737431526184,
+      "step": 208,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.1686291318927914,
+      "grad_norm": 4.401594161987305,
+      "learning_rate": 0.00019240071611481086,
+      "logits/chosen": -0.8025681376457214,
+      "logits/rejected": -0.8077267408370972,
+      "logps/chosen": -13.663830757141113,
+      "logps/rejected": -24.164400100708008,
+      "loss": 1.104232668876648,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6018862724304199,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.03685776889324188,
+      "rewards/margins": 1.151037573814392,
+      "rewards/rejected": -1.1141799688339233,
+      "step": 209,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.16943596984443154,
+      "grad_norm": 4.337040424346924,
+      "learning_rate": 0.00019229832531971895,
+      "logits/chosen": -0.7619237899780273,
+      "logits/rejected": -0.7677690982818604,
+      "logps/chosen": -9.870525360107422,
+      "logps/rejected": -29.796783447265625,
+      "loss": 0.7632872462272644,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.44924822449684143,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.225393608212471,
+      "rewards/margins": 1.9376754760742188,
+      "rewards/rejected": -1.7122818231582642,
+      "step": 210,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.1702428077960717,
+      "grad_norm": 3.420403480529785,
+      "learning_rate": 0.00019219527695706225,
+      "logits/chosen": -0.8083672523498535,
+      "logits/rejected": -0.8105168342590332,
+      "logps/chosen": -11.423739433288574,
+      "logps/rejected": -28.65767478942871,
+      "loss": 0.821802020072937,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4434516131877899,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.02067011035978794,
+      "rewards/margins": 1.7222641706466675,
+      "rewards/rejected": -1.742934226989746,
+      "step": 211,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.17104964574771186,
+      "grad_norm": 3.6967742443084717,
+      "learning_rate": 0.0001920915717609956,
+      "logits/chosen": -0.7133735418319702,
+      "logits/rejected": -0.7154325246810913,
+      "logps/chosen": -10.63201904296875,
+      "logps/rejected": -32.25654220581055,
+      "loss": 0.787774384021759,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4298105239868164,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.11374176293611526,
+      "rewards/margins": 1.7497272491455078,
+      "rewards/rejected": -1.6359853744506836,
+      "step": 212,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.171856483699352,
+      "grad_norm": 2.264280319213867,
+      "learning_rate": 0.00019198721047035332,
+      "logits/chosen": -0.7573313117027283,
+      "logits/rejected": -0.7648686766624451,
+      "logps/chosen": -6.39276123046875,
+      "logps/rejected": -36.62308883666992,
+      "loss": 0.5318189859390259,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.290658563375473,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.29253923892974854,
+      "rewards/margins": 2.4327423572540283,
+      "rewards/rejected": -2.1402029991149902,
+      "step": 213,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.17266332165099216,
+      "grad_norm": 5.493023872375488,
+      "learning_rate": 0.00019188219382864404,
+      "logits/chosen": -0.7108193039894104,
+      "logits/rejected": -0.7183860540390015,
+      "logps/chosen": -9.189270973205566,
+      "logps/rejected": -31.597461700439453,
+      "loss": 0.7931143045425415,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.416747510433197,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.12628071010112762,
+      "rewards/margins": 2.070514440536499,
+      "rewards/rejected": -1.944233775138855,
+      "step": 214,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.17347015960263232,
+      "grad_norm": 4.492797374725342,
+      "learning_rate": 0.00019177652258404537,
+      "logits/chosen": -0.6989372968673706,
+      "logits/rejected": -0.7087696194648743,
+      "logps/chosen": -8.885594367980957,
+      "logps/rejected": -34.03559112548828,
+      "loss": 0.6859308481216431,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3817839026451111,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.28730612993240356,
+      "rewards/margins": 2.0253334045410156,
+      "rewards/rejected": -1.7380273342132568,
+      "step": 215,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.17427699755427245,
+      "grad_norm": 3.6368393898010254,
+      "learning_rate": 0.00019167019748939846,
+      "logits/chosen": -0.6883965134620667,
+      "logits/rejected": -0.6965335607528687,
+      "logps/chosen": -10.925715446472168,
+      "logps/rejected": -27.23555564880371,
+      "loss": 0.8080936074256897,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4929322898387909,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2524645924568176,
+      "rewards/margins": 1.5861448049545288,
+      "rewards/rejected": -1.333680272102356,
+      "step": 216,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.1750838355059126,
+      "grad_norm": 5.671794414520264,
+      "learning_rate": 0.00019156321930220282,
+      "logits/chosen": -0.691314160823822,
+      "logits/rejected": -0.6934138536453247,
+      "logps/chosen": -9.144570350646973,
+      "logps/rejected": -25.588308334350586,
+      "loss": 0.662991464138031,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3658694326877594,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.18018078804016113,
+      "rewards/margins": 1.5743441581726074,
+      "rewards/rejected": -1.3941633701324463,
+      "step": 217,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.17589067345755277,
+      "grad_norm": 2.1973373889923096,
+      "learning_rate": 0.00019145558878461082,
+      "logits/chosen": -0.6576846241950989,
+      "logits/rejected": -0.6600756049156189,
+      "logps/chosen": -6.376984596252441,
+      "logps/rejected": -29.26129150390625,
+      "loss": 0.6061776280403137,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2504776120185852,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.19691388309001923,
+      "rewards/margins": 1.7360740900039673,
+      "rewards/rejected": -1.5391600131988525,
+      "step": 218,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.1766975114091929,
+      "grad_norm": 2.812636137008667,
+      "learning_rate": 0.0001913473067034222,
+      "logits/chosen": -0.6517618298530579,
+      "logits/rejected": -0.6520282626152039,
+      "logps/chosen": -8.633819580078125,
+      "logps/rejected": -33.03736877441406,
+      "loss": 0.6993431448936462,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.29945075511932373,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.11881062388420105,
+      "rewards/margins": 2.108510971069336,
+      "rewards/rejected": -1.9897003173828125,
+      "step": 219,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.17750434936083306,
+      "grad_norm": 7.618241786956787,
+      "learning_rate": 0.00019123837383007883,
+      "logits/chosen": -0.644814133644104,
+      "logits/rejected": -0.6507334113121033,
+      "logps/chosen": -13.946501731872559,
+      "logps/rejected": -32.865543365478516,
+      "loss": 1.0418133735656738,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.7301790714263916,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.10093473643064499,
+      "rewards/margins": 2.085968017578125,
+      "rewards/rejected": -1.9850331544876099,
+      "step": 220,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.17831118731247322,
+      "grad_norm": 4.556211948394775,
+      "learning_rate": 0.0001911287909406589,
+      "logits/chosen": -0.6804370880126953,
+      "logits/rejected": -0.6847546696662903,
+      "logps/chosen": -12.122690200805664,
+      "logps/rejected": -36.90291976928711,
+      "loss": 0.6899932622909546,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3899839520454407,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.09739936888217926,
+      "rewards/margins": 2.288754940032959,
+      "rewards/rejected": -2.1913557052612305,
+      "step": 221,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.17911802526411336,
+      "grad_norm": 6.417980194091797,
+      "learning_rate": 0.00019101855881587166,
+      "logits/chosen": -0.7121025919914246,
+      "logits/rejected": -0.7198390364646912,
+      "logps/chosen": -10.071988105773926,
+      "logps/rejected": -30.89111328125,
+      "loss": 1.0868934392929077,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.7049968242645264,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.059290315955877304,
+      "rewards/margins": 2.18007755279541,
+      "rewards/rejected": -2.1207873821258545,
+      "step": 222,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.17992486321575352,
+      "grad_norm": 3.5655276775360107,
+      "learning_rate": 0.00019090767824105173,
+      "logits/chosen": -0.7064835429191589,
+      "logits/rejected": -0.7118909358978271,
+      "logps/chosen": -9.299367904663086,
+      "logps/rejected": -33.5615234375,
+      "loss": 0.5532280802726746,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3098145127296448,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1762467920780182,
+      "rewards/margins": 2.1976656913757324,
+      "rewards/rejected": -2.0214192867279053,
+      "step": 223,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.18073170116739365,
+      "grad_norm": 3.168497085571289,
+      "learning_rate": 0.00019079615000615352,
+      "logits/chosen": -0.7555636763572693,
+      "logits/rejected": -0.7638076543807983,
+      "logps/chosen": -6.343996524810791,
+      "logps/rejected": -35.11958694458008,
+      "loss": 0.544819712638855,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2573380768299103,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08291390538215637,
+      "rewards/margins": 2.359328269958496,
+      "rewards/rejected": -2.276414394378662,
+      "step": 224,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.1815385391190338,
+      "grad_norm": 3.8726441860198975,
+      "learning_rate": 0.00019068397490574563,
+      "logits/chosen": -0.7438031435012817,
+      "logits/rejected": -0.7529388070106506,
+      "logps/chosen": -7.913335800170898,
+      "logps/rejected": -42.34702682495117,
+      "loss": 0.6369010210037231,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.40800100564956665,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.10036337375640869,
+      "rewards/margins": 2.888615131378174,
+      "rewards/rejected": -2.7882518768310547,
+      "step": 225,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.18234537707067397,
+      "grad_norm": 3.78493595123291,
+      "learning_rate": 0.00019057115373900514,
+      "logits/chosen": -0.6967946290969849,
+      "logits/rejected": -0.7004595994949341,
+      "logps/chosen": -10.167012214660645,
+      "logps/rejected": -32.77048873901367,
+      "loss": 0.6220114231109619,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.351274311542511,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.23287636041641235,
+      "rewards/margins": 2.1248767375946045,
+      "rewards/rejected": -1.8920003175735474,
+      "step": 226,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.1831522150223141,
+      "grad_norm": 4.551224231719971,
+      "learning_rate": 0.00019045768730971196,
+      "logits/chosen": -0.7525245547294617,
+      "logits/rejected": -0.7602270841598511,
+      "logps/chosen": -7.277904987335205,
+      "logps/rejected": -39.11262130737305,
+      "loss": 0.49062812328338623,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.24764156341552734,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2196008712053299,
+      "rewards/margins": 2.6062889099121094,
+      "rewards/rejected": -2.386688232421875,
+      "step": 227,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.18395905297395426,
+      "grad_norm": 3.352492094039917,
+      "learning_rate": 0.00019034357642624313,
+      "logits/chosen": -0.7168059349060059,
+      "logits/rejected": -0.7239040732383728,
+      "logps/chosen": -14.352468490600586,
+      "logps/rejected": -43.02729415893555,
+      "loss": 0.9112517833709717,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6249459981918335,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.11119358241558075,
+      "rewards/margins": 2.6521291732788086,
+      "rewards/rejected": -2.7633233070373535,
+      "step": 228,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.18476589092559442,
+      "grad_norm": 9.10729694366455,
+      "learning_rate": 0.00019022882190156693,
+      "logits/chosen": -0.7800168991088867,
+      "logits/rejected": -0.7834780216217041,
+      "logps/chosen": -6.645830154418945,
+      "logps/rejected": -37.247894287109375,
+      "loss": 0.6339117288589478,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3112393617630005,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.1333404928445816,
+      "rewards/margins": 2.6678478717803955,
+      "rewards/rejected": -2.5345070362091064,
+      "step": 229,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.18557272887723456,
+      "grad_norm": 8.036460876464844,
+      "learning_rate": 0.0001901134245532372,
+      "logits/chosen": -0.7188634276390076,
+      "logits/rejected": -0.7256328463554382,
+      "logps/chosen": -8.977944374084473,
+      "logps/rejected": -39.439598083496094,
+      "loss": 0.7736544609069824,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.40554046630859375,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.07693760097026825,
+      "rewards/margins": 2.647231101989746,
+      "rewards/rejected": -2.570293426513672,
+      "step": 230,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.18637956682887472,
+      "grad_norm": 14.515076637268066,
+      "learning_rate": 0.00018999738520338765,
+      "logits/chosen": -0.7673274874687195,
+      "logits/rejected": -0.7723326086997986,
+      "logps/chosen": -11.787941932678223,
+      "logps/rejected": -31.827787399291992,
+      "loss": 1.2146649360656738,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.7273156642913818,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.19174721837043762,
+      "rewards/margins": 1.6821880340576172,
+      "rewards/rejected": -1.8739351034164429,
+      "step": 231,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.18718640478051488,
+      "grad_norm": 5.435638904571533,
+      "learning_rate": 0.00018988070467872565,
+      "logits/chosen": -0.6913294792175293,
+      "logits/rejected": -0.6982335448265076,
+      "logps/chosen": -8.256817817687988,
+      "logps/rejected": -33.933128356933594,
+      "loss": 0.7231890559196472,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.42718639969825745,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.2676782011985779,
+      "rewards/margins": 2.2451353073120117,
+      "rewards/rejected": -1.977457046508789,
+      "step": 232,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.187993242732155,
+      "grad_norm": 4.923380374908447,
+      "learning_rate": 0.00018976338381052662,
+      "logits/chosen": -0.6979964971542358,
+      "logits/rejected": -0.6987626552581787,
+      "logps/chosen": -9.278888702392578,
+      "logps/rejected": -31.988529205322266,
+      "loss": 0.7011741399765015,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3678920269012451,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.17272722721099854,
+      "rewards/margins": 2.0346527099609375,
+      "rewards/rejected": -1.8619252443313599,
+      "step": 233,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.18880008068379517,
+      "grad_norm": 5.102010250091553,
+      "learning_rate": 0.00018964542343462802,
+      "logits/chosen": -0.7147118449211121,
+      "logits/rejected": -0.7210678458213806,
+      "logps/chosen": -11.869325637817383,
+      "logps/rejected": -26.554332733154297,
+      "loss": 0.7859650254249573,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4002649784088135,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.16828711330890656,
+      "rewards/margins": 1.6456825733184814,
+      "rewards/rejected": -1.4773954153060913,
+      "step": 234,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.1896069186354353,
+      "grad_norm": 2.8259975910186768,
+      "learning_rate": 0.0001895268243914234,
+      "logits/chosen": -0.7181320786476135,
+      "logits/rejected": -0.7195937633514404,
+      "logps/chosen": -9.674834251403809,
+      "logps/rejected": -26.982505798339844,
+      "loss": 0.7054999470710754,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3795202672481537,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.19310441613197327,
+      "rewards/margins": 1.8784090280532837,
+      "rewards/rejected": -1.6853046417236328,
+      "step": 235,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.19041375658707546,
+      "grad_norm": 2.970991373062134,
+      "learning_rate": 0.0001894075875258564,
+      "logits/chosen": -0.7360837459564209,
+      "logits/rejected": -0.7415453195571899,
+      "logps/chosen": -7.354928493499756,
+      "logps/rejected": -26.934494018554688,
+      "loss": 0.7415454983711243,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.37382036447525024,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.24830269813537598,
+      "rewards/margins": 1.7458440065383911,
+      "rewards/rejected": -1.4975413084030151,
+      "step": 236,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.19122059453871562,
+      "grad_norm": 2.9580154418945312,
+      "learning_rate": 0.00018928771368741475,
+      "logits/chosen": -0.6901527643203735,
+      "logits/rejected": -0.6940097808837891,
+      "logps/chosen": -7.6646575927734375,
+      "logps/rejected": -28.092966079711914,
+      "loss": 0.7272118330001831,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4050924777984619,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.22855332493782043,
+      "rewards/margins": 1.7601354122161865,
+      "rewards/rejected": -1.531581997871399,
+      "step": 237,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.19202743249035575,
+      "grad_norm": 5.010228633880615,
+      "learning_rate": 0.00018916720373012426,
+      "logits/chosen": -0.7200859785079956,
+      "logits/rejected": -0.7226930260658264,
+      "logps/chosen": -10.032082557678223,
+      "logps/rejected": -19.060348510742188,
+      "loss": 0.8445019125938416,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.42176133394241333,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.18809281289577484,
+      "rewards/margins": 1.2276759147644043,
+      "rewards/rejected": -1.0395830869674683,
+      "step": 238,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.19283427044199591,
+      "grad_norm": 8.475236892700195,
+      "learning_rate": 0.0001890460585125426,
+      "logits/chosen": -0.7142471075057983,
+      "logits/rejected": -0.7235907912254333,
+      "logps/chosen": -6.3985443115234375,
+      "logps/rejected": -33.978721618652344,
+      "loss": 0.6591022610664368,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.43263474106788635,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.22361797094345093,
+      "rewards/margins": 2.1702280044555664,
+      "rewards/rejected": -1.9466102123260498,
+      "step": 239,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.19364110839363607,
+      "grad_norm": 3.6248912811279297,
+      "learning_rate": 0.00018892427889775332,
+      "logits/chosen": -0.7144255638122559,
+      "logits/rejected": -0.7168291807174683,
+      "logps/chosen": -9.34170150756836,
+      "logps/rejected": -30.304466247558594,
+      "loss": 0.6725300550460815,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.39394479990005493,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.13639676570892334,
+      "rewards/margins": 1.9869462251663208,
+      "rewards/rejected": -1.850549578666687,
+      "step": 240,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.1944479463452762,
+      "grad_norm": 6.973753452301025,
+      "learning_rate": 0.00018880186575335968,
+      "logits/chosen": -0.6703215837478638,
+      "logits/rejected": -0.6775541305541992,
+      "logps/chosen": -12.076030731201172,
+      "logps/rejected": -28.837940216064453,
+      "loss": 0.7876323461532593,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4970223605632782,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.17666694521903992,
+      "rewards/margins": 1.7176096439361572,
+      "rewards/rejected": -1.540942668914795,
+      "step": 241,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.19525478429691637,
+      "grad_norm": 3.2324650287628174,
+      "learning_rate": 0.00018867881995147836,
+      "logits/chosen": -0.7113248705863953,
+      "logits/rejected": -0.7163888216018677,
+      "logps/chosen": -9.168296813964844,
+      "logps/rejected": -34.54369354248047,
+      "loss": 0.6157009601593018,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.37985488772392273,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.28986138105392456,
+      "rewards/margins": 2.2379894256591797,
+      "rewards/rejected": -1.9481279850006104,
+      "step": 242,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.19606162224855653,
+      "grad_norm": 2.7521631717681885,
+      "learning_rate": 0.00018855514236873337,
+      "logits/chosen": -0.7281562089920044,
+      "logits/rejected": -0.734757661819458,
+      "logps/chosen": -7.001053333282471,
+      "logps/rejected": -33.900177001953125,
+      "loss": 0.5766606330871582,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3622375726699829,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.30857908725738525,
+      "rewards/margins": 2.3744540214538574,
+      "rewards/rejected": -2.0658748149871826,
+      "step": 243,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.19686846020019666,
+      "grad_norm": 3.977639675140381,
+      "learning_rate": 0.0001884308338862498,
+      "logits/chosen": -0.6999315023422241,
+      "logits/rejected": -0.7055974006652832,
+      "logps/chosen": -8.435256004333496,
+      "logps/rejected": -33.67347717285156,
+      "loss": 0.590577244758606,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.35361507534980774,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2805049419403076,
+      "rewards/margins": 2.308706283569336,
+      "rewards/rejected": -2.0282013416290283,
+      "step": 244,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.19767529815183682,
+      "grad_norm": 9.163761138916016,
+      "learning_rate": 0.00018830589538964746,
+      "logits/chosen": -0.7660690546035767,
+      "logits/rejected": -0.7758455276489258,
+      "logps/chosen": -8.95065975189209,
+      "logps/rejected": -33.142677307128906,
+      "loss": 0.8324503302574158,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.42908376455307007,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.11282955855131149,
+      "rewards/margins": 2.1978847980499268,
+      "rewards/rejected": -2.085055351257324,
+      "step": 245,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.19848213610347698,
+      "grad_norm": 4.68986701965332,
+      "learning_rate": 0.00018818032776903466,
+      "logits/chosen": -0.7814006805419922,
+      "logits/rejected": -0.7920705676078796,
+      "logps/chosen": -8.228378295898438,
+      "logps/rejected": -46.880313873291016,
+      "loss": 0.6729190349578857,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4284536838531494,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.0902179628610611,
+      "rewards/margins": 3.1966192722320557,
+      "rewards/rejected": -3.106401205062866,
+      "step": 246,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.1992889740551171,
+      "grad_norm": 5.915354251861572,
+      "learning_rate": 0.00018805413191900168,
+      "logits/chosen": -0.7995182871818542,
+      "logits/rejected": -0.805888295173645,
+      "logps/chosen": -12.739469528198242,
+      "logps/rejected": -35.06949996948242,
+      "loss": 0.7942692041397095,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.43145498633384705,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.08451483398675919,
+      "rewards/margins": 2.140747547149658,
+      "rewards/rejected": -2.22526216506958,
+      "step": 247,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.20009581200675727,
+      "grad_norm": 7.71060848236084,
+      "learning_rate": 0.00018792730873861473,
+      "logits/chosen": -0.8068017363548279,
+      "logits/rejected": -0.8110726475715637,
+      "logps/chosen": -11.049007415771484,
+      "logps/rejected": -39.38182067871094,
+      "loss": 0.8510176539421082,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4571390151977539,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.20343750715255737,
+      "rewards/margins": 2.1684627532958984,
+      "rewards/rejected": -2.3719000816345215,
+      "step": 248,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.2009026499583974,
+      "grad_norm": 9.637003898620605,
+      "learning_rate": 0.00018779985913140924,
+      "logits/chosen": -0.7676049470901489,
+      "logits/rejected": -0.7769947648048401,
+      "logps/chosen": -8.674398422241211,
+      "logps/rejected": -48.47040557861328,
+      "loss": 0.9282071590423584,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6683814525604248,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.20568305253982544,
+      "rewards/margins": 3.0397698879241943,
+      "rewards/rejected": -3.245452642440796,
+      "step": 249,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.20170948791003757,
+      "grad_norm": 8.807076454162598,
+      "learning_rate": 0.0001876717840053836,
+      "logits/chosen": -0.7096577882766724,
+      "logits/rejected": -0.7144345641136169,
+      "logps/chosen": -17.149080276489258,
+      "logps/rejected": -33.67993927001953,
+      "loss": 1.056300401687622,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6277230381965637,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.030492231249809265,
+      "rewards/margins": 2.3309168815612793,
+      "rewards/rejected": -2.3614089488983154,
+      "step": 250,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.20251632586167773,
+      "grad_norm": 4.773517608642578,
+      "learning_rate": 0.00018754308427299256,
+      "logits/chosen": -0.6819360852241516,
+      "logits/rejected": -0.6919792294502258,
+      "logps/chosen": -10.13283634185791,
+      "logps/rejected": -36.599456787109375,
+      "loss": 0.8876217007637024,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.47694316506385803,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.12443971633911133,
+      "rewards/margins": 2.1682839393615723,
+      "rewards/rejected": -2.2927236557006836,
+      "step": 251,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.20332316381331786,
+      "grad_norm": 4.1274027824401855,
+      "learning_rate": 0.00018741376085114087,
+      "logits/chosen": -0.7192296981811523,
+      "logits/rejected": -0.7263290882110596,
+      "logps/chosen": -16.484987258911133,
+      "logps/rejected": -34.811126708984375,
+      "loss": 0.8948942422866821,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5631111860275269,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.20992696285247803,
+      "rewards/margins": 2.0608201026916504,
+      "rewards/rejected": -1.8508930206298828,
+      "step": 252,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.20413000176495802,
+      "grad_norm": 4.913299560546875,
+      "learning_rate": 0.0001872838146611766,
+      "logits/chosen": -0.6601164937019348,
+      "logits/rejected": -0.6605373620986938,
+      "logps/chosen": -12.14169979095459,
+      "logps/rejected": -26.232892990112305,
+      "loss": 0.8705669045448303,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4974067509174347,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.2535248100757599,
+      "rewards/margins": 1.8131681680679321,
+      "rewards/rejected": -1.5596435070037842,
+      "step": 253,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.20493683971659818,
+      "grad_norm": 2.4414825439453125,
+      "learning_rate": 0.0001871532466288847,
+      "logits/chosen": -0.6655547618865967,
+      "logits/rejected": -0.6710434556007385,
+      "logps/chosen": -7.641450881958008,
+      "logps/rejected": -27.024456024169922,
+      "loss": 0.5410662293434143,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2969762682914734,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2998022437095642,
+      "rewards/margins": 1.8008990287780762,
+      "rewards/rejected": -1.5010967254638672,
+      "step": 254,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.2057436776682383,
+      "grad_norm": 9.875016212463379,
+      "learning_rate": 0.0001870220576844804,
+      "logits/chosen": -0.6554844379425049,
+      "logits/rejected": -0.6586723923683167,
+      "logps/chosen": -10.128920555114746,
+      "logps/rejected": -28.380104064941406,
+      "loss": 1.077211856842041,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.703222930431366,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.11552818864583969,
+      "rewards/margins": 1.8149248361587524,
+      "rewards/rejected": -1.6993966102600098,
+      "step": 255,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.20655051561987847,
+      "grad_norm": 2.7163054943084717,
+      "learning_rate": 0.0001868902487626024,
+      "logits/chosen": -0.725955605506897,
+      "logits/rejected": -0.7319348454475403,
+      "logps/chosen": -6.896210193634033,
+      "logps/rejected": -30.118925094604492,
+      "loss": 0.5430217385292053,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2342616319656372,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2470369040966034,
+      "rewards/margins": 1.7953723669052124,
+      "rewards/rejected": -1.548335313796997,
+      "step": 256,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.20735735357151863,
+      "grad_norm": 3.3832037448883057,
+      "learning_rate": 0.00018675782080230648,
+      "logits/chosen": -0.6649719476699829,
+      "logits/rejected": -0.6653549671173096,
+      "logps/chosen": -6.564362049102783,
+      "logps/rejected": -26.97052764892578,
+      "loss": 0.6336417198181152,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.33324965834617615,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.07798978686332703,
+      "rewards/margins": 1.5198760032653809,
+      "rewards/rejected": -1.4418861865997314,
+      "step": 257,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.20816419152315876,
+      "grad_norm": 4.554774761199951,
+      "learning_rate": 0.00018662477474705864,
+      "logits/chosen": -0.6891136169433594,
+      "logits/rejected": -0.7018118500709534,
+      "logps/chosen": -10.358716011047363,
+      "logps/rejected": -33.35871124267578,
+      "loss": 0.944884181022644,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.515121579170227,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.15751619637012482,
+      "rewards/margins": 1.947699785232544,
+      "rewards/rejected": -1.790183663368225,
+      "step": 258,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.20897102947479892,
+      "grad_norm": 6.9074788093566895,
+      "learning_rate": 0.00018649111154472842,
+      "logits/chosen": -0.6972513794898987,
+      "logits/rejected": -0.7002527117729187,
+      "logps/chosen": -9.99314022064209,
+      "logps/rejected": -20.69109344482422,
+      "loss": 1.1074203252792358,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6453499794006348,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.05247223377227783,
+      "rewards/margins": 1.2508971691131592,
+      "rewards/rejected": -1.1984249353408813,
+      "step": 259,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.20977786742643906,
+      "grad_norm": 6.223460674285889,
+      "learning_rate": 0.00018635683214758214,
+      "logits/chosen": -0.7319504022598267,
+      "logits/rejected": -0.7356070280075073,
+      "logps/chosen": -8.844853401184082,
+      "logps/rejected": -32.97547912597656,
+      "loss": 0.7976398468017578,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.425577849149704,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.04917633533477783,
+      "rewards/margins": 1.9350166320800781,
+      "rewards/rejected": -1.8858401775360107,
+      "step": 260,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.21058470537807922,
+      "grad_norm": 6.114218711853027,
+      "learning_rate": 0.00018622193751227607,
+      "logits/chosen": -0.7542300820350647,
+      "logits/rejected": -0.7576084733009338,
+      "logps/chosen": -5.740233898162842,
+      "logps/rejected": -24.60858917236328,
+      "loss": 0.5687559247016907,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.27638113498687744,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.23254472017288208,
+      "rewards/margins": 1.7274789810180664,
+      "rewards/rejected": -1.49493408203125,
+      "step": 261,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.21139154332971938,
+      "grad_norm": 3.313537836074829,
+      "learning_rate": 0.00018608642859984978,
+      "logits/chosen": -0.8011643886566162,
+      "logits/rejected": -0.803293764591217,
+      "logps/chosen": -9.771310806274414,
+      "logps/rejected": -28.634235382080078,
+      "loss": 0.7911385893821716,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.41166451573371887,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.265075147151947,
+      "rewards/margins": 2.063453197479248,
+      "rewards/rejected": -1.7983779907226562,
+      "step": 262,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.2121983812813595,
+      "grad_norm": 3.156975746154785,
+      "learning_rate": 0.00018595030637571906,
+      "logits/chosen": -0.7707415223121643,
+      "logits/rejected": -0.7777128219604492,
+      "logps/chosen": -10.236924171447754,
+      "logps/rejected": -30.459064483642578,
+      "loss": 0.6563097238540649,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3598516285419464,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.24525853991508484,
+      "rewards/margins": 2.071570873260498,
+      "rewards/rejected": -1.82631254196167,
+      "step": 263,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.21300521923299967,
+      "grad_norm": 4.330559730529785,
+      "learning_rate": 0.00018581357180966925,
+      "logits/chosen": -0.8070797920227051,
+      "logits/rejected": -0.8171977400779724,
+      "logps/chosen": -9.34181022644043,
+      "logps/rejected": -32.88754653930664,
+      "loss": 0.8858993053436279,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.49547457695007324,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.15189547836780548,
+      "rewards/margins": 1.979980230331421,
+      "rewards/rejected": -1.828084945678711,
+      "step": 264,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.21381205718463983,
+      "grad_norm": 5.864645957946777,
+      "learning_rate": 0.00018567622587584818,
+      "logits/chosen": -0.7768454551696777,
+      "logits/rejected": -0.781791090965271,
+      "logps/chosen": -8.989236831665039,
+      "logps/rejected": -33.998809814453125,
+      "loss": 0.748650848865509,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.46606677770614624,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.22831673920154572,
+      "rewards/margins": 2.1091978549957275,
+      "rewards/rejected": -1.8808808326721191,
+      "step": 265,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.21461889513627996,
+      "grad_norm": 8.510169982910156,
+      "learning_rate": 0.00018553826955275936,
+      "logits/chosen": -0.8200643062591553,
+      "logits/rejected": -0.8230538368225098,
+      "logps/chosen": -14.249617576599121,
+      "logps/rejected": -25.856830596923828,
+      "loss": 1.2568143606185913,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.629042387008667,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.19409911334514618,
+      "rewards/margins": 1.33782160282135,
+      "rewards/rejected": -1.5319206714630127,
+      "step": 266,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.21542573308792012,
+      "grad_norm": 3.2720420360565186,
+      "learning_rate": 0.00018539970382325482,
+      "logits/chosen": -0.8381865620613098,
+      "logits/rejected": -0.8415036201477051,
+      "logps/chosen": -7.8104567527771,
+      "logps/rejected": -31.601917266845703,
+      "loss": 0.5837196111679077,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.24857257306575775,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.14021825790405273,
+      "rewards/margins": 2.0099189281463623,
+      "rewards/rejected": -1.8697006702423096,
+      "step": 267,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.21623257103956028,
+      "grad_norm": 3.4341468811035156,
+      "learning_rate": 0.00018526052967452836,
+      "logits/chosen": -0.7830556631088257,
+      "logits/rejected": -0.7897158265113831,
+      "logps/chosen": -11.684428215026855,
+      "logps/rejected": -36.22364044189453,
+      "loss": 0.7827985882759094,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.48799192905426025,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.08203143626451492,
+      "rewards/margins": 1.9769901037216187,
+      "rewards/rejected": -1.89495849609375,
+      "step": 268,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.21703940899120042,
+      "grad_norm": 3.692969799041748,
+      "learning_rate": 0.00018512074809810833,
+      "logits/chosen": -0.7967461347579956,
+      "logits/rejected": -0.7945878505706787,
+      "logps/chosen": -10.929898262023926,
+      "logps/rejected": -23.94034194946289,
+      "loss": 0.7881439924240112,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3369988203048706,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.08296848833560944,
+      "rewards/margins": 1.621280550956726,
+      "rewards/rejected": -1.5383120775222778,
+      "step": 269,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.21784624694284058,
+      "grad_norm": 6.830929756164551,
+      "learning_rate": 0.00018498036008985058,
+      "logits/chosen": -0.746848464012146,
+      "logits/rejected": -0.7509756684303284,
+      "logps/chosen": -9.608593940734863,
+      "logps/rejected": -38.34101867675781,
+      "loss": 0.7576542496681213,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4260023534297943,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.09650997817516327,
+      "rewards/margins": 2.168517589569092,
+      "rewards/rejected": -2.072007656097412,
+      "step": 270,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.21865308489448074,
+      "grad_norm": 4.689668655395508,
+      "learning_rate": 0.0001848393666499315,
+      "logits/chosen": -0.7876875996589661,
+      "logits/rejected": -0.796357274055481,
+      "logps/chosen": -12.614479064941406,
+      "logps/rejected": -27.662067413330078,
+      "loss": 0.9168095588684082,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.52295982837677,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.0025606881827116013,
+      "rewards/margins": 1.4409575462341309,
+      "rewards/rejected": -1.43839693069458,
+      "step": 271,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.21945992284612087,
+      "grad_norm": 5.61292028427124,
+      "learning_rate": 0.00018469776878284072,
+      "logits/chosen": -0.7697591781616211,
+      "logits/rejected": -0.775164008140564,
+      "logps/chosen": -5.527304649353027,
+      "logps/rejected": -25.911758422851562,
+      "loss": 0.7468432188034058,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.32815924286842346,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.20197589695453644,
+      "rewards/margins": 1.5008738040924072,
+      "rewards/rejected": -1.2988977432250977,
+      "step": 272,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.22026676079776103,
+      "grad_norm": 2.823660373687744,
+      "learning_rate": 0.00018455556749737403,
+      "logits/chosen": -0.8068636059761047,
+      "logits/rejected": -0.8074927926063538,
+      "logps/chosen": -9.97484016418457,
+      "logps/rejected": -21.8039608001709,
+      "loss": 0.8325120210647583,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4089290499687195,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.19321191310882568,
+      "rewards/margins": 1.4057273864746094,
+      "rewards/rejected": -1.2125153541564941,
+      "step": 273,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.22107359874940116,
+      "grad_norm": 2.5285258293151855,
+      "learning_rate": 0.00018441276380662625,
+      "logits/chosen": -0.7656555771827698,
+      "logits/rejected": -0.7663451433181763,
+      "logps/chosen": -10.646867752075195,
+      "logps/rejected": -22.98876953125,
+      "loss": 0.7855734825134277,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.382598340511322,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.23073698580265045,
+      "rewards/margins": 1.3868789672851562,
+      "rewards/rejected": -1.156141996383667,
+      "step": 274,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.22188043670104132,
+      "grad_norm": 3.1718196868896484,
+      "learning_rate": 0.00018426935872798393,
+      "logits/chosen": -0.6983739137649536,
+      "logits/rejected": -0.7074112892150879,
+      "logps/chosen": -9.201760292053223,
+      "logps/rejected": -32.77637481689453,
+      "loss": 0.8582974672317505,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4306548535823822,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.09240508079528809,
+      "rewards/margins": 1.7474827766418457,
+      "rewards/rejected": -1.6550780534744263,
+      "step": 275,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.22268727465268148,
+      "grad_norm": 2.530366897583008,
+      "learning_rate": 0.00018412535328311814,
+      "logits/chosen": -0.7532445192337036,
+      "logits/rejected": -0.7571496963500977,
+      "logps/chosen": -11.776863098144531,
+      "logps/rejected": -21.776357650756836,
+      "loss": 0.7927124500274658,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3979222774505615,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3263438642024994,
+      "rewards/margins": 1.1879689693450928,
+      "rewards/rejected": -0.8616250157356262,
+      "step": 276,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.22349411260432162,
+      "grad_norm": 2.8776495456695557,
+      "learning_rate": 0.00018398074849797715,
+      "logits/chosen": -0.7302210927009583,
+      "logits/rejected": -0.7333999276161194,
+      "logps/chosen": -8.04594612121582,
+      "logps/rejected": -25.69329833984375,
+      "loss": 0.8354058861732483,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4014919400215149,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.08311442285776138,
+      "rewards/margins": 1.4473011493682861,
+      "rewards/rejected": -1.3641867637634277,
+      "step": 277,
+      "train_speed(iter/s)": 0.005493
+    },
+    {
+      "epoch": 0.22430095055596178,
+      "grad_norm": 3.1463992595672607,
+      "learning_rate": 0.00018383554540277923,
+      "logits/chosen": -0.7314996719360352,
+      "logits/rejected": -0.7379679679870605,
+      "logps/chosen": -12.349081039428711,
+      "logps/rejected": -30.154521942138672,
+      "loss": 0.7297593355178833,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4262910485267639,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.09422469139099121,
+      "rewards/margins": 1.8524787425994873,
+      "rewards/rejected": -1.758254051208496,
+      "step": 278,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.22510778850760194,
+      "grad_norm": 3.694409132003784,
+      "learning_rate": 0.00018368974503200515,
+      "logits/chosen": -0.6736661195755005,
+      "logits/rejected": -0.6801973581314087,
+      "logps/chosen": -7.8128252029418945,
+      "logps/rejected": -34.619964599609375,
+      "loss": 0.5409353971481323,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.27240046858787537,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.14602825045585632,
+      "rewards/margins": 2.1955599784851074,
+      "rewards/rejected": -2.0495314598083496,
+      "step": 279,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.22591462645924207,
+      "grad_norm": 3.1639299392700195,
+      "learning_rate": 0.00018354334842439097,
+      "logits/chosen": -0.6950223445892334,
+      "logits/rejected": -0.6963911056518555,
+      "logps/chosen": -11.003353118896484,
+      "logps/rejected": -29.378337860107422,
+      "loss": 0.6718952059745789,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.41621553897857666,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2978552281856537,
+      "rewards/margins": 2.314960479736328,
+      "rewards/rejected": -2.0171055793762207,
+      "step": 280,
+      "train_speed(iter/s)": 0.005494
+    },
+    {
+      "epoch": 0.22672146441088223,
+      "grad_norm": 10.265737533569336,
+      "learning_rate": 0.00018339635662292043,
+      "logits/chosen": -0.6680774688720703,
+      "logits/rejected": -0.6708887219429016,
+      "logps/chosen": -9.037369728088379,
+      "logps/rejected": -28.595491409301758,
+      "loss": 0.7751729488372803,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.45015332102775574,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.26366156339645386,
+      "rewards/margins": 2.183614492416382,
+      "rewards/rejected": -1.9199527502059937,
+      "step": 281,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.2275283023625224,
+      "grad_norm": 2.1682872772216797,
+      "learning_rate": 0.00018324877067481783,
+      "logits/chosen": -0.6989454627037048,
+      "logits/rejected": -0.7062041759490967,
+      "logps/chosen": -6.304581642150879,
+      "logps/rejected": -36.81526565551758,
+      "loss": 0.4870501458644867,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.27354732155799866,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.25899404287338257,
+      "rewards/margins": 2.644915819168091,
+      "rewards/rejected": -2.3859219551086426,
+      "step": 282,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.22833514031416252,
+      "grad_norm": 5.862631797790527,
+      "learning_rate": 0.00018310059163154026,
+      "logits/chosen": -0.6766980886459351,
+      "logits/rejected": -0.6787369847297668,
+      "logps/chosen": -5.963343620300293,
+      "logps/rejected": -32.693424224853516,
+      "loss": 0.6418428421020508,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.36004289984703064,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.10292582958936691,
+      "rewards/margins": 2.3059544563293457,
+      "rewards/rejected": -2.203028917312622,
+      "step": 283,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.22914197826580268,
+      "grad_norm": 12.034624099731445,
+      "learning_rate": 0.00018295182054877028,
+      "logits/chosen": -0.6584015488624573,
+      "logits/rejected": -0.6568421125411987,
+      "logps/chosen": -9.409889221191406,
+      "logps/rejected": -29.433473587036133,
+      "loss": 0.779122531414032,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4608069658279419,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.09724374115467072,
+      "rewards/margins": 2.0178637504577637,
+      "rewards/rejected": -1.920620083808899,
+      "step": 284,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.22994881621744284,
+      "grad_norm": 2.7027676105499268,
+      "learning_rate": 0.00018280245848640837,
+      "logits/chosen": -0.6325287222862244,
+      "logits/rejected": -0.638849675655365,
+      "logps/chosen": -11.803849220275879,
+      "logps/rejected": -34.92605209350586,
+      "loss": 0.6200695633888245,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3835636377334595,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3289586305618286,
+      "rewards/margins": 2.4051361083984375,
+      "rewards/rejected": -2.0761775970458984,
+      "step": 285,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.23075565416908297,
+      "grad_norm": 12.6409330368042,
+      "learning_rate": 0.0001826525065085654,
+      "logits/chosen": -0.6875444054603577,
+      "logits/rejected": -0.6914835572242737,
+      "logps/chosen": -6.616847038269043,
+      "logps/rejected": -32.69636917114258,
+      "loss": 0.7483028173446655,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.41258829832077026,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1584882140159607,
+      "rewards/margins": 2.2999954223632812,
+      "rewards/rejected": -2.141507387161255,
+      "step": 286,
+      "train_speed(iter/s)": 0.005495
+    },
+    {
+      "epoch": 0.23156249212072313,
+      "grad_norm": 11.6520357131958,
+      "learning_rate": 0.00018250196568355488,
+      "logits/chosen": -0.6649547219276428,
+      "logits/rejected": -0.6693803071975708,
+      "logps/chosen": -7.23736572265625,
+      "logps/rejected": -35.56175994873047,
+      "loss": 0.8851901888847351,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.555623471736908,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.04522712156176567,
+      "rewards/margins": 2.2976298332214355,
+      "rewards/rejected": -2.2524030208587646,
+      "step": 287,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.23236933007236327,
+      "grad_norm": 4.304396152496338,
+      "learning_rate": 0.0001823508370838857,
+      "logits/chosen": -0.6312321424484253,
+      "logits/rejected": -0.6436812281608582,
+      "logps/chosen": -5.798044204711914,
+      "logps/rejected": -41.38832092285156,
+      "loss": 0.4446811079978943,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2366604506969452,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.19939535856246948,
+      "rewards/margins": 2.506988525390625,
+      "rewards/rejected": -2.3075931072235107,
+      "step": 288,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.23317616802400343,
+      "grad_norm": 3.2178378105163574,
+      "learning_rate": 0.00018219912178625406,
+      "logits/chosen": -0.667951762676239,
+      "logits/rejected": -0.6722462773323059,
+      "logps/chosen": -8.273347854614258,
+      "logps/rejected": -38.93374252319336,
+      "loss": 0.5589238405227661,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.33250439167022705,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.24894192814826965,
+      "rewards/margins": 2.7763476371765137,
+      "rewards/rejected": -2.5274055004119873,
+      "step": 289,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.2339830059756436,
+      "grad_norm": 19.13467025756836,
+      "learning_rate": 0.0001820468208715362,
+      "logits/chosen": -0.658998429775238,
+      "logits/rejected": -0.6624220609664917,
+      "logps/chosen": -10.623708724975586,
+      "logps/rejected": -35.898441314697266,
+      "loss": 0.8776594400405884,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.504928469657898,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.03749624639749527,
+      "rewards/margins": 2.421245813369751,
+      "rewards/rejected": -2.3837497234344482,
+      "step": 290,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.23478984392728372,
+      "grad_norm": 14.0836181640625,
+      "learning_rate": 0.00018189393542478036,
+      "logits/chosen": -0.6700485348701477,
+      "logits/rejected": -0.6811757683753967,
+      "logps/chosen": -13.918525695800781,
+      "logps/rejected": -31.985313415527344,
+      "loss": 0.8883711695671082,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.494065523147583,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.11007653176784515,
+      "rewards/margins": 1.8538110256195068,
+      "rewards/rejected": -1.7437344789505005,
+      "step": 291,
+      "train_speed(iter/s)": 0.005496
+    },
+    {
+      "epoch": 0.23559668187892388,
+      "grad_norm": 1.916353464126587,
+      "learning_rate": 0.00018174046653519929,
+      "logits/chosen": -0.6249253749847412,
+      "logits/rejected": -0.6350105404853821,
+      "logps/chosen": -6.613706588745117,
+      "logps/rejected": -31.679109573364258,
+      "loss": 0.5259463787078857,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.28351810574531555,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.37387514114379883,
+      "rewards/margins": 2.260230302810669,
+      "rewards/rejected": -1.8863551616668701,
+      "step": 292,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.23640351983056404,
+      "grad_norm": 3.460676431655884,
+      "learning_rate": 0.0001815864152961624,
+      "logits/chosen": -0.7058864831924438,
+      "logits/rejected": -0.7104409337043762,
+      "logps/chosen": -9.832937240600586,
+      "logps/rejected": -33.203758239746094,
+      "loss": 0.7486201524734497,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4414381682872772,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.05328599363565445,
+      "rewards/margins": 1.9007527828216553,
+      "rewards/rejected": -1.8474667072296143,
+      "step": 293,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.23721035778220417,
+      "grad_norm": 4.965317249298096,
+      "learning_rate": 0.00018143178280518793,
+      "logits/chosen": -0.6664391756057739,
+      "logits/rejected": -0.672810971736908,
+      "logps/chosen": -9.997623443603516,
+      "logps/rejected": -40.18428039550781,
+      "loss": 0.6516451239585876,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.40531888604164124,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.053340502083301544,
+      "rewards/margins": 3.017876625061035,
+      "rewards/rejected": -2.964535713195801,
+      "step": 294,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.23801719573384433,
+      "grad_norm": 11.654240608215332,
+      "learning_rate": 0.00018127657016393523,
+      "logits/chosen": -0.6823023557662964,
+      "logits/rejected": -0.6858367919921875,
+      "logps/chosen": -13.494540214538574,
+      "logps/rejected": -26.149335861206055,
+      "loss": 1.2284287214279175,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.7871044874191284,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1301741600036621,
+      "rewards/margins": 1.65839684009552,
+      "rewards/rejected": -1.528222680091858,
+      "step": 295,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.2388240336854845,
+      "grad_norm": 6.048635005950928,
+      "learning_rate": 0.00018112077847819678,
+      "logits/chosen": -0.6762900352478027,
+      "logits/rejected": -0.6797853708267212,
+      "logps/chosen": -9.212943077087402,
+      "logps/rejected": -35.503047943115234,
+      "loss": 0.6458551287651062,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4156365990638733,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.07666485011577606,
+      "rewards/margins": 2.67659592628479,
+      "rewards/rejected": -2.599931240081787,
+      "step": 296,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.23963087163712463,
+      "grad_norm": 17.950830459594727,
+      "learning_rate": 0.00018096440885789053,
+      "logits/chosen": -0.7045325040817261,
+      "logits/rejected": -0.7085095643997192,
+      "logps/chosen": -5.067245960235596,
+      "logps/rejected": -32.400596618652344,
+      "loss": 0.6414408087730408,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.35441699624061584,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.19392728805541992,
+      "rewards/margins": 2.3720614910125732,
+      "rewards/rejected": -2.1781344413757324,
+      "step": 297,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.2404377095887648,
+      "grad_norm": 2.869894504547119,
+      "learning_rate": 0.00018080746241705168,
+      "logits/chosen": -0.679808497428894,
+      "logits/rejected": -0.681736409664154,
+      "logps/chosen": -10.83769416809082,
+      "logps/rejected": -28.55279541015625,
+      "loss": 0.7392024397850037,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.42105916142463684,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.23800711333751678,
+      "rewards/margins": 1.8557734489440918,
+      "rewards/rejected": -1.617766261100769,
+      "step": 298,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.24124454754040492,
+      "grad_norm": 3.0865039825439453,
+      "learning_rate": 0.000180649940273825,
+      "logits/chosen": -0.7064599990844727,
+      "logits/rejected": -0.7154704928398132,
+      "logps/chosen": -6.157724380493164,
+      "logps/rejected": -37.746639251708984,
+      "loss": 0.5655427575111389,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.30769801139831543,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.08631336688995361,
+      "rewards/margins": 2.3150007724761963,
+      "rewards/rejected": -2.228687286376953,
+      "step": 299,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.24205138549204508,
+      "grad_norm": 2.7285871505737305,
+      "learning_rate": 0.00018049184355045677,
+      "logits/chosen": -0.7136423587799072,
+      "logits/rejected": -0.7151377201080322,
+      "logps/chosen": -7.438178062438965,
+      "logps/rejected": -25.823518753051758,
+      "loss": 0.6484540104866028,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3245176076889038,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.27408692240715027,
+      "rewards/margins": 1.6197888851165771,
+      "rewards/rejected": -1.345702052116394,
+      "step": 300,
+      "train_speed(iter/s)": 0.005497
+    },
+    {
+      "epoch": 0.24205138549204508,
+      "eval_logits/chosen": -0.7073344588279724,
+      "eval_logits/rejected": -0.7110338807106018,
+      "eval_logps/chosen": -9.127958297729492,
+      "eval_logps/rejected": -32.3641242980957,
+      "eval_loss": 0.7329411506652832,
+      "eval_nll_loss": 0.40398380160331726,
+      "eval_rewards/accuracies": 0.8450000286102295,
+      "eval_rewards/chosen": 0.10664533823728561,
+      "eval_rewards/margins": 2.0523765087127686,
+      "eval_rewards/rejected": -1.9457309246063232,
+      "eval_runtime": 935.6049,
+      "eval_samples_per_second": 0.428,
+      "eval_steps_per_second": 0.428,
+      "step": 300
+    },
+    {
+      "epoch": 0.24285822344368524,
+      "grad_norm": 6.436818599700928,
+      "learning_rate": 0.0001803331733732868,
+      "logits/chosen": -0.6776125431060791,
+      "logits/rejected": -0.6814741492271423,
+      "logps/chosen": -10.345755577087402,
+      "logps/rejected": -31.87610626220703,
+      "loss": 1.0051960945129395,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5581401586532593,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.17682184278964996,
+      "rewards/margins": 1.9412140846252441,
+      "rewards/rejected": -2.1180360317230225,
+      "step": 301,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 0.24366506139532537,
+      "grad_norm": 6.0017781257629395,
+      "learning_rate": 0.0001801739308727404,
+      "logits/chosen": -0.7074002027511597,
+      "logits/rejected": -0.713377058506012,
+      "logps/chosen": -12.075335502624512,
+      "logps/rejected": -23.198413848876953,
+      "loss": 0.854468822479248,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5477906465530396,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.22885334491729736,
+      "rewards/margins": 1.5375834703445435,
+      "rewards/rejected": -1.308730125427246,
+      "step": 302,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 0.24447189934696553,
+      "grad_norm": 4.944755554199219,
+      "learning_rate": 0.0001800141171833203,
+      "logits/chosen": -0.6664037108421326,
+      "logits/rejected": -0.6715153455734253,
+      "logps/chosen": -10.99780559539795,
+      "logps/rejected": -40.38133239746094,
+      "loss": 1.0467002391815186,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.7734127640724182,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.23875761032104492,
+      "rewards/margins": 2.768630027770996,
+      "rewards/rejected": -2.529872417449951,
+      "step": 303,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 0.2452787372986057,
+      "grad_norm": 13.264330863952637,
+      "learning_rate": 0.0001798537334435986,
+      "logits/chosen": -0.6581575870513916,
+      "logits/rejected": -0.6663908362388611,
+      "logps/chosen": -9.642528533935547,
+      "logps/rejected": -34.64740753173828,
+      "loss": 1.2485740184783936,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.8329651355743408,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.05902397632598877,
+      "rewards/margins": 2.0006673336029053,
+      "rewards/rejected": -2.0596914291381836,
+      "step": 304,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 0.24608557525024582,
+      "grad_norm": 2.709535598754883,
+      "learning_rate": 0.00017969278079620868,
+      "logits/chosen": -0.6512877941131592,
+      "logits/rejected": -0.6544854640960693,
+      "logps/chosen": -6.917822360992432,
+      "logps/rejected": -36.45574951171875,
+      "loss": 0.436959445476532,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.23888030648231506,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.07531990855932236,
+      "rewards/margins": 2.534837007522583,
+      "rewards/rejected": -2.459517240524292,
+      "step": 305,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 0.24689241320188599,
+      "grad_norm": 4.297156810760498,
+      "learning_rate": 0.000179531260387837,
+      "logits/chosen": -0.6876344084739685,
+      "logits/rejected": -0.6926863789558411,
+      "logps/chosen": -10.314577102661133,
+      "logps/rejected": -42.0067138671875,
+      "loss": 0.6266589164733887,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.405265212059021,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.048373833298683167,
+      "rewards/margins": 2.6945550441741943,
+      "rewards/rejected": -2.646181344985962,
+      "step": 306,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 0.24769925115352615,
+      "grad_norm": 2.3430275917053223,
+      "learning_rate": 0.00017936917336921492,
+      "logits/chosen": -0.660620927810669,
+      "logits/rejected": -0.6698962450027466,
+      "logps/chosen": -10.098029136657715,
+      "logps/rejected": -38.27429962158203,
+      "loss": 0.6127811670303345,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.42024731636047363,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.12339803576469421,
+      "rewards/margins": 2.775909423828125,
+      "rewards/rejected": -2.6525111198425293,
+      "step": 307,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 0.24850608910516628,
+      "grad_norm": 7.701032638549805,
+      "learning_rate": 0.00017920652089511066,
+      "logits/chosen": -0.6981559991836548,
+      "logits/rejected": -0.7051110863685608,
+      "logps/chosen": -12.91336441040039,
+      "logps/rejected": -39.74778747558594,
+      "loss": 0.7809062600135803,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.43491145968437195,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.01711910218000412,
+      "rewards/margins": 2.6014857292175293,
+      "rewards/rejected": -2.618604898452759,
+      "step": 308,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 0.24931292705680644,
+      "grad_norm": 8.226758003234863,
+      "learning_rate": 0.00017904330412432082,
+      "logits/chosen": -0.6544554829597473,
+      "logits/rejected": -0.6588922739028931,
+      "logps/chosen": -7.886651039123535,
+      "logps/rejected": -37.24707794189453,
+      "loss": 0.6507235169410706,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.36367952823638916,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.008459217846393585,
+      "rewards/margins": 2.5813100337982178,
+      "rewards/rejected": -2.5728507041931152,
+      "step": 309,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 0.25011976500844657,
+      "grad_norm": 2.985854387283325,
+      "learning_rate": 0.00017887952421966231,
+      "logits/chosen": -0.6596094369888306,
+      "logits/rejected": -0.6650587916374207,
+      "logps/chosen": -9.925427436828613,
+      "logps/rejected": -37.92117691040039,
+      "loss": 0.589540958404541,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3470298945903778,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2002800703048706,
+      "rewards/margins": 2.7566845417022705,
+      "rewards/rejected": -2.5564048290252686,
+      "step": 310,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 0.25092660296008673,
+      "grad_norm": 8.586225509643555,
+      "learning_rate": 0.00017871518234796412,
+      "logits/chosen": -0.7069643139839172,
+      "logits/rejected": -0.7040101885795593,
+      "logps/chosen": -10.918496131896973,
+      "logps/rejected": -22.552764892578125,
+      "loss": 0.869647204875946,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.39224955439567566,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.07749204337596893,
+      "rewards/margins": 1.6755130290985107,
+      "rewards/rejected": -1.598021149635315,
+      "step": 311,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 0.2517334409117269,
+      "grad_norm": 3.319561243057251,
+      "learning_rate": 0.00017855027968005874,
+      "logits/chosen": -0.643227756023407,
+      "logits/rejected": -0.6550837755203247,
+      "logps/chosen": -6.8077073097229,
+      "logps/rejected": -39.095035552978516,
+      "loss": 0.6082978844642639,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3355411887168884,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.21203100681304932,
+      "rewards/margins": 2.4175777435302734,
+      "rewards/rejected": -2.2055468559265137,
+      "step": 312,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 0.25254027886336705,
+      "grad_norm": 3.8709287643432617,
+      "learning_rate": 0.00017838481739077404,
+      "logits/chosen": -0.6760554313659668,
+      "logits/rejected": -0.682180643081665,
+      "logps/chosen": -7.9441633224487305,
+      "logps/rejected": -27.414325714111328,
+      "loss": 0.7799100279808044,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4338618814945221,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2233610451221466,
+      "rewards/margins": 1.7509660720825195,
+      "rewards/rejected": -1.5276049375534058,
+      "step": 313,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 0.2533471168150072,
+      "grad_norm": 5.0154218673706055,
+      "learning_rate": 0.00017821879665892487,
+      "logits/chosen": -0.6456589698791504,
+      "logits/rejected": -0.649111270904541,
+      "logps/chosen": -9.17154312133789,
+      "logps/rejected": -32.664093017578125,
+      "loss": 0.6070997714996338,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.31872257590293884,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.005728209391236305,
+      "rewards/margins": 1.877259612083435,
+      "rewards/rejected": -1.8715312480926514,
+      "step": 314,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 0.2541539547666473,
+      "grad_norm": 2.769273042678833,
+      "learning_rate": 0.00017805221866730458,
+      "logits/chosen": -0.6285029649734497,
+      "logits/rejected": -0.6361050605773926,
+      "logps/chosen": -10.298542976379395,
+      "logps/rejected": -27.911968231201172,
+      "loss": 0.6473672986030579,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.37400925159454346,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.33300700783729553,
+      "rewards/margins": 1.5457414388656616,
+      "rewards/rejected": -1.2127344608306885,
+      "step": 315,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 0.2549607927182875,
+      "grad_norm": 2.675525426864624,
+      "learning_rate": 0.00017788508460267677,
+      "logits/chosen": -0.633998692035675,
+      "logits/rejected": -0.6368559002876282,
+      "logps/chosen": -8.035248756408691,
+      "logps/rejected": -33.8181266784668,
+      "loss": 0.5450462698936462,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2956991195678711,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.12976329028606415,
+      "rewards/margins": 2.3313686847686768,
+      "rewards/rejected": -2.2016055583953857,
+      "step": 316,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 0.25576763066992764,
+      "grad_norm": 2.2619502544403076,
+      "learning_rate": 0.0001777173956557665,
+      "logits/chosen": -0.6891807317733765,
+      "logits/rejected": -0.6903274059295654,
+      "logps/chosen": -10.693523406982422,
+      "logps/rejected": -31.01398468017578,
+      "loss": 0.6650394797325134,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.38576623797416687,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2987612187862396,
+      "rewards/margins": 1.9871487617492676,
+      "rewards/rejected": -1.6883876323699951,
+      "step": 317,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 0.2565744686215678,
+      "grad_norm": 7.991454124450684,
+      "learning_rate": 0.0001775491530212522,
+      "logits/chosen": -0.6753562688827515,
+      "logits/rejected": -0.6762648820877075,
+      "logps/chosen": -11.517782211303711,
+      "logps/rejected": -26.93805503845215,
+      "loss": 1.165069580078125,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6551721096038818,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.02389012649655342,
+      "rewards/margins": 1.3221609592437744,
+      "rewards/rejected": -1.2982707023620605,
+      "step": 318,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 0.25738130657320796,
+      "grad_norm": 9.860249519348145,
+      "learning_rate": 0.00017738035789775696,
+      "logits/chosen": -0.7479084730148315,
+      "logits/rejected": -0.7518092393875122,
+      "logps/chosen": -5.844160079956055,
+      "logps/rejected": -35.683162689208984,
+      "loss": 0.5556322932243347,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3325543999671936,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.10462158918380737,
+      "rewards/margins": 2.546602487564087,
+      "rewards/rejected": -2.4419808387756348,
+      "step": 319,
+      "train_speed(iter/s)": 0.005409
+    },
+    {
+      "epoch": 0.25818814452484806,
+      "grad_norm": 3.9844655990600586,
+      "learning_rate": 0.00017721101148783985,
+      "logits/chosen": -0.7617798447608948,
+      "logits/rejected": -0.7641586661338806,
+      "logps/chosen": -8.093914031982422,
+      "logps/rejected": -35.221988677978516,
+      "loss": 0.6888319849967957,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4659794867038727,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08299870789051056,
+      "rewards/margins": 2.34136700630188,
+      "rewards/rejected": -2.258368492126465,
+      "step": 320,
+      "train_speed(iter/s)": 0.005409
+    },
+    {
+      "epoch": 0.2589949824764882,
+      "grad_norm": 7.946180820465088,
+      "learning_rate": 0.00017704111499798768,
+      "logits/chosen": -0.7791930437088013,
+      "logits/rejected": -0.7833006381988525,
+      "logps/chosen": -6.748149871826172,
+      "logps/rejected": -36.56504440307617,
+      "loss": 0.8274538516998291,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5019730925559998,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.09792216122150421,
+      "rewards/margins": 2.4872283935546875,
+      "rewards/rejected": -2.3893063068389893,
+      "step": 321,
+      "train_speed(iter/s)": 0.005409
+    },
+    {
+      "epoch": 0.2598018204281284,
+      "grad_norm": 7.7827467918396,
+      "learning_rate": 0.00017687066963860615,
+      "logits/chosen": -0.7987972497940063,
+      "logits/rejected": -0.8094179034233093,
+      "logps/chosen": -3.5310564041137695,
+      "logps/rejected": -32.73319625854492,
+      "loss": 0.4403470754623413,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.1688545197248459,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.2623920440673828,
+      "rewards/margins": 2.187939405441284,
+      "rewards/rejected": -1.925547480583191,
+      "step": 322,
+      "train_speed(iter/s)": 0.00541
+    },
+    {
+      "epoch": 0.26060865837976854,
+      "grad_norm": 3.664057493209839,
+      "learning_rate": 0.0001766996766240114,
+      "logits/chosen": -0.8535178899765015,
+      "logits/rejected": -0.8564468622207642,
+      "logps/chosen": -12.827454566955566,
+      "logps/rejected": -35.108741760253906,
+      "loss": 0.7213603258132935,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.47191280126571655,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.34138235449790955,
+      "rewards/margins": 2.7680537700653076,
+      "rewards/rejected": -2.426671266555786,
+      "step": 323,
+      "train_speed(iter/s)": 0.00541
+    },
+    {
+      "epoch": 0.2614154963314087,
+      "grad_norm": 3.6671972274780273,
+      "learning_rate": 0.0001765281371724211,
+      "logits/chosen": -0.8640937209129333,
+      "logits/rejected": -0.8603441119194031,
+      "logps/chosen": -9.051156997680664,
+      "logps/rejected": -32.042171478271484,
+      "loss": 0.5764932036399841,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3475058972835541,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.24964334070682526,
+      "rewards/margins": 2.6217854022979736,
+      "rewards/rejected": -2.3721418380737305,
+      "step": 324,
+      "train_speed(iter/s)": 0.00541
+    },
+    {
+      "epoch": 0.26222233428304886,
+      "grad_norm": 4.343052864074707,
+      "learning_rate": 0.00017635605250594606,
+      "logits/chosen": -0.9237393140792847,
+      "logits/rejected": -0.9191243648529053,
+      "logps/chosen": -9.570093154907227,
+      "logps/rejected": -26.344640731811523,
+      "loss": 0.6133550405502319,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3041512668132782,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2187902331352234,
+      "rewards/margins": 1.9347566366195679,
+      "rewards/rejected": -1.7159663438796997,
+      "step": 325,
+      "train_speed(iter/s)": 0.00541
+    },
+    {
+      "epoch": 0.26302917223468897,
+      "grad_norm": 13.972990989685059,
+      "learning_rate": 0.00017618342385058145,
+      "logits/chosen": -0.9064996242523193,
+      "logits/rejected": -0.9114649295806885,
+      "logps/chosen": -10.548544883728027,
+      "logps/rejected": -39.158992767333984,
+      "loss": 0.9647412300109863,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6664921045303345,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.14826549589633942,
+      "rewards/margins": 2.8863561153411865,
+      "rewards/rejected": -2.738090753555298,
+      "step": 326,
+      "train_speed(iter/s)": 0.00541
+    },
+    {
+      "epoch": 0.26383601018632913,
+      "grad_norm": 4.962406635284424,
+      "learning_rate": 0.0001760102524361979,
+      "logits/chosen": -0.9307440519332886,
+      "logits/rejected": -0.9374938607215881,
+      "logps/chosen": -11.971577644348145,
+      "logps/rejected": -43.166099548339844,
+      "loss": 0.7845494747161865,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4260358214378357,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.014196585863828659,
+      "rewards/margins": 2.70448637008667,
+      "rewards/rejected": -2.7186827659606934,
+      "step": 327,
+      "train_speed(iter/s)": 0.00541
+    },
+    {
+      "epoch": 0.2646428481379693,
+      "grad_norm": 6.3026509284973145,
+      "learning_rate": 0.00017583653949653297,
+      "logits/chosen": -0.9104486703872681,
+      "logits/rejected": -0.9133187532424927,
+      "logps/chosen": -7.545221328735352,
+      "logps/rejected": -42.58152770996094,
+      "loss": 0.6194388270378113,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.38448214530944824,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.12115807086229324,
+      "rewards/margins": 3.0233232975006104,
+      "rewards/rejected": -2.902165412902832,
+      "step": 328,
+      "train_speed(iter/s)": 0.005411
+    },
+    {
+      "epoch": 0.26544968608960945,
+      "grad_norm": 4.22109317779541,
+      "learning_rate": 0.00017566228626918212,
+      "logits/chosen": -0.8536132574081421,
+      "logits/rejected": -0.863903284072876,
+      "logps/chosen": -9.604391098022461,
+      "logps/rejected": -38.25326919555664,
+      "loss": 0.7549124956130981,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.43309736251831055,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.11497168242931366,
+      "rewards/margins": 2.4954819679260254,
+      "rewards/rejected": -2.380510091781616,
+      "step": 329,
+      "train_speed(iter/s)": 0.005411
+    },
+    {
+      "epoch": 0.2662565240412496,
+      "grad_norm": 3.372162103652954,
+      "learning_rate": 0.0001754874939955901,
+      "logits/chosen": -0.8504335880279541,
+      "logits/rejected": -0.8582204580307007,
+      "logps/chosen": -13.342264175415039,
+      "logps/rejected": -36.4522590637207,
+      "loss": 0.9137426614761353,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6495134830474854,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.36808696389198303,
+      "rewards/margins": 2.6845757961273193,
+      "rewards/rejected": -2.316488742828369,
+      "step": 330,
+      "train_speed(iter/s)": 0.005412
+    },
+    {
+      "epoch": 0.2670633619928897,
+      "grad_norm": 2.032322883605957,
+      "learning_rate": 0.00017531216392104203,
+      "logits/chosen": -0.8323754072189331,
+      "logits/rejected": -0.8362606763839722,
+      "logps/chosen": -9.349749565124512,
+      "logps/rejected": -38.68825149536133,
+      "loss": 0.6108371019363403,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4490174949169159,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4555152654647827,
+      "rewards/margins": 3.0110297203063965,
+      "rewards/rejected": -2.555514335632324,
+      "step": 331,
+      "train_speed(iter/s)": 0.005413
+    },
+    {
+      "epoch": 0.2678701999445299,
+      "grad_norm": 2.1273860931396484,
+      "learning_rate": 0.00017513629729465455,
+      "logits/chosen": -0.8378605842590332,
+      "logits/rejected": -0.8425085544586182,
+      "logps/chosen": -7.634585380554199,
+      "logps/rejected": -34.26400375366211,
+      "loss": 0.5199471712112427,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3184734284877777,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.24010597169399261,
+      "rewards/margins": 2.4833943843841553,
+      "rewards/rejected": -2.243288516998291,
+      "step": 332,
+      "train_speed(iter/s)": 0.005414
+    },
+    {
+      "epoch": 0.26867703789617003,
+      "grad_norm": 13.832210540771484,
+      "learning_rate": 0.0001749598953693668,
+      "logits/chosen": -0.8178601264953613,
+      "logits/rejected": -0.8171352744102478,
+      "logps/chosen": -7.816558361053467,
+      "logps/rejected": -27.310321807861328,
+      "loss": 0.8487988710403442,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5664465427398682,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.13675397634506226,
+      "rewards/margins": 2.1682660579681396,
+      "rewards/rejected": -2.0315120220184326,
+      "step": 333,
+      "train_speed(iter/s)": 0.005414
+    },
+    {
+      "epoch": 0.2694838758478102,
+      "grad_norm": 4.107896327972412,
+      "learning_rate": 0.00017478295940193163,
+      "logits/chosen": -0.8216401934623718,
+      "logits/rejected": -0.8330098986625671,
+      "logps/chosen": -8.207337379455566,
+      "logps/rejected": -38.14511489868164,
+      "loss": 0.8124181032180786,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4575313925743103,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.033875416964292526,
+      "rewards/margins": 2.214970350265503,
+      "rewards/rejected": -2.1810948848724365,
+      "step": 334,
+      "train_speed(iter/s)": 0.005415
+    },
+    {
+      "epoch": 0.27029071379945036,
+      "grad_norm": 2.043121814727783,
+      "learning_rate": 0.00017460549065290666,
+      "logits/chosen": -0.7847851514816284,
+      "logits/rejected": -0.7903647422790527,
+      "logps/chosen": -11.188026428222656,
+      "logps/rejected": -31.82864761352539,
+      "loss": 0.5849981904029846,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3552479147911072,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3021475374698639,
+      "rewards/margins": 2.2114672660827637,
+      "rewards/rejected": -1.909319519996643,
+      "step": 335,
+      "train_speed(iter/s)": 0.005415
+    },
+    {
+      "epoch": 0.2710975517510905,
+      "grad_norm": 2.116966962814331,
+      "learning_rate": 0.0001744274903866452,
+      "logits/chosen": -0.770881712436676,
+      "logits/rejected": -0.7834229469299316,
+      "logps/chosen": -3.7972168922424316,
+      "logps/rejected": -37.75122833251953,
+      "loss": 0.3996497392654419,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2140352874994278,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.27353012561798096,
+      "rewards/margins": 2.583765983581543,
+      "rewards/rejected": -2.3102359771728516,
+      "step": 336,
+      "train_speed(iter/s)": 0.005415
+    },
+    {
+      "epoch": 0.2719043897027306,
+      "grad_norm": 4.1290411949157715,
+      "learning_rate": 0.00017424895987128722,
+      "logits/chosen": -0.786916971206665,
+      "logits/rejected": -0.7911162376403809,
+      "logps/chosen": -9.502747535705566,
+      "logps/rejected": -32.8641357421875,
+      "loss": 0.7896320819854736,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4624251425266266,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.12185362726449966,
+      "rewards/margins": 2.144758701324463,
+      "rewards/rejected": -2.022905111312866,
+      "step": 337,
+      "train_speed(iter/s)": 0.005415
+    },
+    {
+      "epoch": 0.2727112276543708,
+      "grad_norm": 1.9379422664642334,
+      "learning_rate": 0.0001740699003787505,
+      "logits/chosen": -0.8224512338638306,
+      "logits/rejected": -0.8290694355964661,
+      "logps/chosen": -5.654224872589111,
+      "logps/rejected": -40.40040588378906,
+      "loss": 0.32085683941841125,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.17423731088638306,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2838688790798187,
+      "rewards/margins": 2.9954781532287598,
+      "rewards/rejected": -2.711609363555908,
+      "step": 338,
+      "train_speed(iter/s)": 0.005415
+    },
+    {
+      "epoch": 0.27351806560601094,
+      "grad_norm": 2.5411887168884277,
+      "learning_rate": 0.00017389031318472143,
+      "logits/chosen": -0.8840833306312561,
+      "logits/rejected": -0.8846546411514282,
+      "logps/chosen": -14.506649017333984,
+      "logps/rejected": -35.60776901245117,
+      "loss": 0.6948885917663574,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4158177375793457,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.05735927075147629,
+      "rewards/margins": 2.4177799224853516,
+      "rewards/rejected": -2.3604207038879395,
+      "step": 339,
+      "train_speed(iter/s)": 0.005415
+    },
+    {
+      "epoch": 0.2743249035576511,
+      "grad_norm": 4.5371856689453125,
+      "learning_rate": 0.00017371019956864583,
+      "logits/chosen": -0.8519307374954224,
+      "logits/rejected": -0.8563728928565979,
+      "logps/chosen": -18.773422241210938,
+      "logps/rejected": -25.609325408935547,
+      "loss": 1.2189257144927979,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.7953273057937622,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.14804774522781372,
+      "rewards/margins": 1.684711217880249,
+      "rewards/rejected": -1.5366634130477905,
+      "step": 340,
+      "train_speed(iter/s)": 0.005416
+    },
+    {
+      "epoch": 0.27513174150929126,
+      "grad_norm": 5.0346455574035645,
+      "learning_rate": 0.00017352956081372007,
+      "logits/chosen": -0.901013970375061,
+      "logits/rejected": -0.9049715995788574,
+      "logps/chosen": -8.963022232055664,
+      "logps/rejected": -34.29716110229492,
+      "loss": 0.8155198097229004,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.40066373348236084,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.041613977402448654,
+      "rewards/margins": 2.3150718212127686,
+      "rewards/rejected": -2.3566856384277344,
+      "step": 341,
+      "train_speed(iter/s)": 0.005416
+    },
+    {
+      "epoch": 0.2759385794609314,
+      "grad_norm": 4.706645488739014,
+      "learning_rate": 0.00017334839820688176,
+      "logits/chosen": -0.8764183521270752,
+      "logits/rejected": -0.8793443441390991,
+      "logps/chosen": -11.185328483581543,
+      "logps/rejected": -27.840126037597656,
+      "loss": 0.8221145868301392,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.36857330799102783,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.029647603631019592,
+      "rewards/margins": 1.7318322658538818,
+      "rewards/rejected": -1.7021846771240234,
+      "step": 342,
+      "train_speed(iter/s)": 0.005416
+    },
+    {
+      "epoch": 0.2767454174125715,
+      "grad_norm": 3.4059722423553467,
+      "learning_rate": 0.00017316671303880065,
+      "logits/chosen": -0.8800357580184937,
+      "logits/rejected": -0.8856234550476074,
+      "logps/chosen": -10.418521881103516,
+      "logps/rejected": -44.70547866821289,
+      "loss": 0.621554970741272,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.41401058435440063,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.08430502563714981,
+      "rewards/margins": 3.2389299869537354,
+      "rewards/rejected": -3.1546249389648438,
+      "step": 343,
+      "train_speed(iter/s)": 0.005417
+    },
+    {
+      "epoch": 0.2775522553642117,
+      "grad_norm": 8.598644256591797,
+      "learning_rate": 0.00017298450660386934,
+      "logits/chosen": -0.8565375208854675,
+      "logits/rejected": -0.8581067323684692,
+      "logps/chosen": -11.926506042480469,
+      "logps/rejected": -36.14910888671875,
+      "loss": 0.7724243998527527,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.41390153765678406,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.05219612270593643,
+      "rewards/margins": 2.1874096393585205,
+      "rewards/rejected": -2.239605665206909,
+      "step": 344,
+      "train_speed(iter/s)": 0.005417
+    },
+    {
+      "epoch": 0.27835909331585185,
+      "grad_norm": 5.491206645965576,
+      "learning_rate": 0.00017280178020019418,
+      "logits/chosen": -0.8360708951950073,
+      "logits/rejected": -0.8349617123603821,
+      "logps/chosen": -9.427896499633789,
+      "logps/rejected": -29.778169631958008,
+      "loss": 0.7176328301429749,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.46471166610717773,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3919517695903778,
+      "rewards/margins": 2.2337028980255127,
+      "rewards/rejected": -1.841751217842102,
+      "step": 345,
+      "train_speed(iter/s)": 0.005417
+    },
+    {
+      "epoch": 0.279165931267492,
+      "grad_norm": 2.732307195663452,
+      "learning_rate": 0.000172618535129586,
+      "logits/chosen": -0.8330416083335876,
+      "logits/rejected": -0.8343455195426941,
+      "logps/chosen": -13.94620418548584,
+      "logps/rejected": -34.905059814453125,
+      "loss": 0.6391770243644714,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.40491989254951477,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.024340404197573662,
+      "rewards/margins": 2.191556453704834,
+      "rewards/rejected": -2.1672160625457764,
+      "step": 346,
+      "train_speed(iter/s)": 0.005417
+    },
+    {
+      "epoch": 0.27997276921913217,
+      "grad_norm": 2.748098373413086,
+      "learning_rate": 0.00017243477269755073,
+      "logits/chosen": -0.8207265138626099,
+      "logits/rejected": -0.8255487680435181,
+      "logps/chosen": -8.841146469116211,
+      "logps/rejected": -40.18748092651367,
+      "loss": 0.5847609043121338,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4391492009162903,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.18971768021583557,
+      "rewards/margins": 3.0055530071258545,
+      "rewards/rejected": -2.815835475921631,
+      "step": 347,
+      "train_speed(iter/s)": 0.005417
+    },
+    {
+      "epoch": 0.28077960717077227,
+      "grad_norm": 8.588139533996582,
+      "learning_rate": 0.00017225049421328023,
+      "logits/chosen": -0.8355762362480164,
+      "logits/rejected": -0.8387144804000854,
+      "logps/chosen": -8.67043399810791,
+      "logps/rejected": -34.789730072021484,
+      "loss": 0.5931243896484375,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.30945685505867004,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.06438732147216797,
+      "rewards/margins": 2.153982639312744,
+      "rewards/rejected": -2.089595317840576,
+      "step": 348,
+      "train_speed(iter/s)": 0.005418
+    },
+    {
+      "epoch": 0.28158644512241243,
+      "grad_norm": 2.7876644134521484,
+      "learning_rate": 0.00017206570098964293,
+      "logits/chosen": -0.8504444360733032,
+      "logits/rejected": -0.8515408635139465,
+      "logps/chosen": -10.413717269897461,
+      "logps/rejected": -31.34994888305664,
+      "loss": 0.5911897420883179,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3716551661491394,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2757214307785034,
+      "rewards/margins": 2.5438179969787598,
+      "rewards/rejected": -2.268096446990967,
+      "step": 349,
+      "train_speed(iter/s)": 0.005418
+    },
+    {
+      "epoch": 0.2823932830740526,
+      "grad_norm": 2.771085262298584,
+      "learning_rate": 0.00017188039434317437,
+      "logits/chosen": -0.7710286974906921,
+      "logits/rejected": -0.7747238874435425,
+      "logps/chosen": -11.302440643310547,
+      "logps/rejected": -31.574066162109375,
+      "loss": 0.7104746103286743,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.44341009855270386,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2658320367336273,
+      "rewards/margins": 2.1740376949310303,
+      "rewards/rejected": -1.908205509185791,
+      "step": 350,
+      "train_speed(iter/s)": 0.005418
+    },
+    {
+      "epoch": 0.28320012102569275,
+      "grad_norm": 8.733509063720703,
+      "learning_rate": 0.00017169457559406795,
+      "logits/chosen": -0.7990887761116028,
+      "logits/rejected": -0.8017781376838684,
+      "logps/chosen": -10.448628425598145,
+      "logps/rejected": -32.143943786621094,
+      "loss": 1.2729414701461792,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.8385729789733887,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.18963520228862762,
+      "rewards/margins": 1.997046947479248,
+      "rewards/rejected": -2.1866822242736816,
+      "step": 351,
+      "train_speed(iter/s)": 0.005417
+    },
+    {
+      "epoch": 0.2840069589773329,
+      "grad_norm": 3.8308520317077637,
+      "learning_rate": 0.00017150824606616553,
+      "logits/chosen": -0.7534295320510864,
+      "logits/rejected": -0.7567183971405029,
+      "logps/chosen": -7.442231178283691,
+      "logps/rejected": -32.14317321777344,
+      "loss": 0.5511249899864197,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.34539952874183655,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.26004305481910706,
+      "rewards/margins": 2.451585531234741,
+      "rewards/rejected": -2.191542387008667,
+      "step": 352,
+      "train_speed(iter/s)": 0.005417
+    },
+    {
+      "epoch": 0.2848137969289731,
+      "grad_norm": 3.919644832611084,
+      "learning_rate": 0.00017132140708694786,
+      "logits/chosen": -0.7558514475822449,
+      "logits/rejected": -0.7550539970397949,
+      "logps/chosen": -8.997418403625488,
+      "logps/rejected": -29.441905975341797,
+      "loss": 0.6320300698280334,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3772739768028259,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2652067542076111,
+      "rewards/margins": 2.186215877532959,
+      "rewards/rejected": -1.9210093021392822,
+      "step": 353,
+      "train_speed(iter/s)": 0.005418
+    },
+    {
+      "epoch": 0.2856206348806132,
+      "grad_norm": 20.330293655395508,
+      "learning_rate": 0.0001711340599875253,
+      "logits/chosen": -0.7443191409111023,
+      "logits/rejected": -0.7491686344146729,
+      "logps/chosen": -11.211926460266113,
+      "logps/rejected": -37.2029914855957,
+      "loss": 1.2416906356811523,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.7969366908073425,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.1411081850528717,
+      "rewards/margins": 2.08335542678833,
+      "rewards/rejected": -2.22446346282959,
+      "step": 354,
+      "train_speed(iter/s)": 0.005418
+    },
+    {
+      "epoch": 0.28642747283225334,
+      "grad_norm": 13.989158630371094,
+      "learning_rate": 0.00017094620610262815,
+      "logits/chosen": -0.7479000091552734,
+      "logits/rejected": -0.750941812992096,
+      "logps/chosen": -11.775254249572754,
+      "logps/rejected": -35.6055908203125,
+      "loss": 1.2802770137786865,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.7870579957962036,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.2751302123069763,
+      "rewards/margins": 1.906927227973938,
+      "rewards/rejected": -2.1820573806762695,
+      "step": 355,
+      "train_speed(iter/s)": 0.005418
+    },
+    {
+      "epoch": 0.2872343107838935,
+      "grad_norm": 7.8280930519104,
+      "learning_rate": 0.0001707578467705973,
+      "logits/chosen": -0.7759760022163391,
+      "logits/rejected": -0.7844580411911011,
+      "logps/chosen": -7.402548789978027,
+      "logps/rejected": -38.92572784423828,
+      "loss": 0.7681185603141785,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5315437912940979,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.005034878849983215,
+      "rewards/margins": 2.443704128265381,
+      "rewards/rejected": -2.438669443130493,
+      "step": 356,
+      "train_speed(iter/s)": 0.005418
+    },
+    {
+      "epoch": 0.28804114873553366,
+      "grad_norm": 24.46771812438965,
+      "learning_rate": 0.00017056898333337462,
+      "logits/chosen": -0.7455392479896545,
+      "logits/rejected": -0.7464004755020142,
+      "logps/chosen": -15.696362495422363,
+      "logps/rejected": -36.21916198730469,
+      "loss": 0.699033796787262,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.42450839281082153,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.19790178537368774,
+      "rewards/margins": 2.3564441204071045,
+      "rewards/rejected": -2.1585421562194824,
+      "step": 357,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.2888479866871738,
+      "grad_norm": 3.165498971939087,
+      "learning_rate": 0.0001703796171364934,
+      "logits/chosen": -0.7834245562553406,
+      "logits/rejected": -0.7828773260116577,
+      "logps/chosen": -8.231873512268066,
+      "logps/rejected": -30.543338775634766,
+      "loss": 0.7491443753242493,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4628988206386566,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.24111242592334747,
+      "rewards/margins": 2.3805058002471924,
+      "rewards/rejected": -2.1393933296203613,
+      "step": 358,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.2896548246388139,
+      "grad_norm": 2.6455235481262207,
+      "learning_rate": 0.00017018974952906884,
+      "logits/chosen": -0.8238362073898315,
+      "logits/rejected": -0.8289802670478821,
+      "logps/chosen": -6.384030818939209,
+      "logps/rejected": -37.180633544921875,
+      "loss": 0.5261876583099365,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.28632083535194397,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.18914461135864258,
+      "rewards/margins": 2.69685959815979,
+      "rewards/rejected": -2.5077149868011475,
+      "step": 359,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.2904616625904541,
+      "grad_norm": 2.369776964187622,
+      "learning_rate": 0.00016999938186378825,
+      "logits/chosen": -0.7982887625694275,
+      "logits/rejected": -0.8035246729850769,
+      "logps/chosen": -6.851421356201172,
+      "logps/rejected": -45.864418029785156,
+      "loss": 0.5188026428222656,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3399565815925598,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3176203966140747,
+      "rewards/margins": 3.4621381759643555,
+      "rewards/rejected": -3.144517660140991,
+      "step": 360,
+      "train_speed(iter/s)": 0.00542
+    },
+    {
+      "epoch": 0.29126850054209424,
+      "grad_norm": 3.786916971206665,
+      "learning_rate": 0.0001698085154969017,
+      "logits/chosen": -0.8295035362243652,
+      "logits/rejected": -0.8320608139038086,
+      "logps/chosen": -10.462471961975098,
+      "logps/rejected": -44.3033561706543,
+      "loss": 0.6038550138473511,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.44244384765625,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.005623971112072468,
+      "rewards/margins": 2.9000349044799805,
+      "rewards/rejected": -2.8944108486175537,
+      "step": 361,
+      "train_speed(iter/s)": 0.00542
+    },
+    {
+      "epoch": 0.2920753384937344,
+      "grad_norm": 2.3236849308013916,
+      "learning_rate": 0.000169617151788212,
+      "logits/chosen": -0.8378246426582336,
+      "logits/rejected": -0.8476194739341736,
+      "logps/chosen": -7.4671311378479,
+      "logps/rejected": -45.69831466674805,
+      "loss": 0.549799919128418,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3557121455669403,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2878333032131195,
+      "rewards/margins": 3.4509360790252686,
+      "rewards/rejected": -3.163102865219116,
+      "step": 362,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.29288217644537456,
+      "grad_norm": 4.218591213226318,
+      "learning_rate": 0.0001694252921010654,
+      "logits/chosen": -0.8441336154937744,
+      "logits/rejected": -0.854229211807251,
+      "logps/chosen": -7.4483184814453125,
+      "logps/rejected": -46.634490966796875,
+      "loss": 0.5104203224182129,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2786526679992676,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.10106715559959412,
+      "rewards/margins": 3.364898681640625,
+      "rewards/rejected": -3.263831377029419,
+      "step": 363,
+      "train_speed(iter/s)": 0.00542
+    },
+    {
+      "epoch": 0.2936890143970147,
+      "grad_norm": 6.0027055740356445,
+      "learning_rate": 0.0001692329378023416,
+      "logits/chosen": -0.8181811571121216,
+      "logits/rejected": -0.8173055648803711,
+      "logps/chosen": -11.75820541381836,
+      "logps/rejected": -28.230079650878906,
+      "loss": 0.8542529344558716,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.41755104064941406,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.016829999163746834,
+      "rewards/margins": 1.9391112327575684,
+      "rewards/rejected": -1.9222811460494995,
+      "step": 364,
+      "train_speed(iter/s)": 0.00542
+    },
+    {
+      "epoch": 0.29449585234865483,
+      "grad_norm": 2.452047109603882,
+      "learning_rate": 0.00016904009026244405,
+      "logits/chosen": -0.8576743006706238,
+      "logits/rejected": -0.8569716215133667,
+      "logps/chosen": -8.94229507446289,
+      "logps/rejected": -36.84193420410156,
+      "loss": 0.5307234525680542,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.27585676312446594,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1474345177412033,
+      "rewards/margins": 2.3831472396850586,
+      "rewards/rejected": -2.235712766647339,
+      "step": 365,
+      "train_speed(iter/s)": 0.00542
+    },
+    {
+      "epoch": 0.295302690300295,
+      "grad_norm": 4.770409107208252,
+      "learning_rate": 0.00016884675085529033,
+      "logits/chosen": -0.8674459457397461,
+      "logits/rejected": -0.8703688383102417,
+      "logps/chosen": -10.529121398925781,
+      "logps/rejected": -38.25895309448242,
+      "loss": 0.6165487766265869,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3394632339477539,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.13632023334503174,
+      "rewards/margins": 2.6175537109375,
+      "rewards/rejected": -2.4812331199645996,
+      "step": 366,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.29610952825193515,
+      "grad_norm": 2.726836681365967,
+      "learning_rate": 0.00016865292095830214,
+      "logits/chosen": -0.8154149055480957,
+      "logits/rejected": -0.8210477232933044,
+      "logps/chosen": -8.451236724853516,
+      "logps/rejected": -30.605690002441406,
+      "loss": 0.5677744150161743,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.30434367060661316,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3030344247817993,
+      "rewards/margins": 2.1608378887176514,
+      "rewards/rejected": -1.8578035831451416,
+      "step": 367,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.2969163662035753,
+      "grad_norm": 4.918789386749268,
+      "learning_rate": 0.00016845860195239574,
+      "logits/chosen": -0.8281161189079285,
+      "logits/rejected": -0.8330156207084656,
+      "logps/chosen": -11.424484252929688,
+      "logps/rejected": -32.49198913574219,
+      "loss": 0.7686675786972046,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5117014050483704,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3326207995414734,
+      "rewards/margins": 2.3775177001953125,
+      "rewards/rejected": -2.0448966026306152,
+      "step": 368,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.29772320415521547,
+      "grad_norm": 7.9901652336120605,
+      "learning_rate": 0.00016826379522197197,
+      "logits/chosen": -0.7991360425949097,
+      "logits/rejected": -0.8027262091636658,
+      "logps/chosen": -8.358216285705566,
+      "logps/rejected": -31.194568634033203,
+      "loss": 0.7576655745506287,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.49762627482414246,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.10716473311185837,
+      "rewards/margins": 2.159275531768799,
+      "rewards/rejected": -2.0521109104156494,
+      "step": 369,
+      "train_speed(iter/s)": 0.005422
+    },
+    {
+      "epoch": 0.2985300421068556,
+      "grad_norm": 7.625348091125488,
+      "learning_rate": 0.0001680685021549063,
+      "logits/chosen": -0.7968946099281311,
+      "logits/rejected": -0.8060710430145264,
+      "logps/chosen": -7.866345405578613,
+      "logps/rejected": -40.948158264160156,
+      "loss": 0.9940714240074158,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.7132853269577026,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.16774287819862366,
+      "rewards/margins": 2.4824962615966797,
+      "rewards/rejected": -2.6502389907836914,
+      "step": 370,
+      "train_speed(iter/s)": 0.005423
+    },
+    {
+      "epoch": 0.29933688005849574,
+      "grad_norm": 4.839093208312988,
+      "learning_rate": 0.0001678727241425391,
+      "logits/chosen": -0.792543888092041,
+      "logits/rejected": -0.7907209992408752,
+      "logps/chosen": -10.626989364624023,
+      "logps/rejected": -25.664470672607422,
+      "loss": 0.91594398021698,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5407547950744629,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.0037185251712799072,
+      "rewards/margins": 1.719236969947815,
+      "rewards/rejected": -1.7155182361602783,
+      "step": 371,
+      "train_speed(iter/s)": 0.005423
+    },
+    {
+      "epoch": 0.3001437180101359,
+      "grad_norm": 11.523240089416504,
+      "learning_rate": 0.00016767646257966572,
+      "logits/chosen": -0.8143973350524902,
+      "logits/rejected": -0.8215935230255127,
+      "logps/chosen": -6.086573123931885,
+      "logps/rejected": -34.30033493041992,
+      "loss": 0.6105502247810364,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.36522942781448364,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2981349527835846,
+      "rewards/margins": 2.788585662841797,
+      "rewards/rejected": -2.490450859069824,
+      "step": 372,
+      "train_speed(iter/s)": 0.005423
+    },
+    {
+      "epoch": 0.30095055596177606,
+      "grad_norm": 5.205559253692627,
+      "learning_rate": 0.00016747971886452642,
+      "logits/chosen": -0.7993229031562805,
+      "logits/rejected": -0.8019697666168213,
+      "logps/chosen": -14.90966796875,
+      "logps/rejected": -40.805274963378906,
+      "loss": 1.5332070589065552,
+      "memory(GiB)": 46.83,
+      "nll_loss": 1.2682892084121704,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.025209464132785797,
+      "rewards/margins": 2.6658260822296143,
+      "rewards/rejected": -2.6406168937683105,
+      "step": 373,
+      "train_speed(iter/s)": 0.005423
+    },
+    {
+      "epoch": 0.3017573939134162,
+      "grad_norm": 2.872535228729248,
+      "learning_rate": 0.00016728249439879655,
+      "logits/chosen": -0.7731214165687561,
+      "logits/rejected": -0.7786182165145874,
+      "logps/chosen": -5.839204788208008,
+      "logps/rejected": -37.67174530029297,
+      "loss": 0.507649838924408,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2965702712535858,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.22565633058547974,
+      "rewards/margins": 2.726480007171631,
+      "rewards/rejected": -2.500823497772217,
+      "step": 374,
+      "train_speed(iter/s)": 0.005424
+    },
+    {
+      "epoch": 0.3025642318650564,
+      "grad_norm": 9.740580558776855,
+      "learning_rate": 0.00016708479058757649,
+      "logits/chosen": -0.8201895952224731,
+      "logits/rejected": -0.8329208493232727,
+      "logps/chosen": -10.150799751281738,
+      "logps/rejected": -40.12678527832031,
+      "loss": 0.5580483078956604,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.32952913641929626,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.20641882717609406,
+      "rewards/margins": 2.5419578552246094,
+      "rewards/rejected": -2.335538625717163,
+      "step": 375,
+      "train_speed(iter/s)": 0.005424
+    },
+    {
+      "epoch": 0.3033710698166965,
+      "grad_norm": 4.967090606689453,
+      "learning_rate": 0.00016688660883938165,
+      "logits/chosen": -0.7816774845123291,
+      "logits/rejected": -0.7863184213638306,
+      "logps/chosen": -11.115227699279785,
+      "logps/rejected": -41.1477165222168,
+      "loss": 0.7776178121566772,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5047651529312134,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.12607808411121368,
+      "rewards/margins": 2.677337884902954,
+      "rewards/rejected": -2.8034157752990723,
+      "step": 376,
+      "train_speed(iter/s)": 0.005424
+    },
+    {
+      "epoch": 0.30417790776833664,
+      "grad_norm": 5.764723777770996,
+      "learning_rate": 0.00016668795056613242,
+      "logits/chosen": -0.8348323106765747,
+      "logits/rejected": -0.836425244808197,
+      "logps/chosen": -11.849743843078613,
+      "logps/rejected": -35.644893646240234,
+      "loss": 0.860490083694458,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.44726479053497314,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.01643115095794201,
+      "rewards/margins": 2.4391143321990967,
+      "rewards/rejected": -2.4226832389831543,
+      "step": 377,
+      "train_speed(iter/s)": 0.005424
+    },
+    {
+      "epoch": 0.3049847457199768,
+      "grad_norm": 6.4779510498046875,
+      "learning_rate": 0.0001664888171831442,
+      "logits/chosen": -0.8333585262298584,
+      "logits/rejected": -0.8423128128051758,
+      "logps/chosen": -8.405279159545898,
+      "logps/rejected": -37.56438446044922,
+      "loss": 0.6916205883026123,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.43494561314582825,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.054853081703186035,
+      "rewards/margins": 2.354010581970215,
+      "rewards/rejected": -2.2991573810577393,
+      "step": 378,
+      "train_speed(iter/s)": 0.005424
+    },
+    {
+      "epoch": 0.30579158367161696,
+      "grad_norm": 2.1519875526428223,
+      "learning_rate": 0.00016628921010911714,
+      "logits/chosen": -0.8162001371383667,
+      "logits/rejected": -0.8233896493911743,
+      "logps/chosen": -6.277496814727783,
+      "logps/rejected": -37.37247848510742,
+      "loss": 0.3955227732658386,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.22343112528324127,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.15012571215629578,
+      "rewards/margins": 2.704033851623535,
+      "rewards/rejected": -2.553908109664917,
+      "step": 379,
+      "train_speed(iter/s)": 0.005425
+    },
+    {
+      "epoch": 0.3065984216232571,
+      "grad_norm": 5.495427131652832,
+      "learning_rate": 0.00016608913076612627,
+      "logits/chosen": -0.7601447105407715,
+      "logits/rejected": -0.7649034261703491,
+      "logps/chosen": -11.018935203552246,
+      "logps/rejected": -31.04195785522461,
+      "loss": 0.7440771460533142,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3949829936027527,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.16132403910160065,
+      "rewards/margins": 2.152961015701294,
+      "rewards/rejected": -1.9916372299194336,
+      "step": 380,
+      "train_speed(iter/s)": 0.005425
+    },
+    {
+      "epoch": 0.3074052595748972,
+      "grad_norm": 12.617781639099121,
+      "learning_rate": 0.00016588858057961113,
+      "logits/chosen": -0.823615550994873,
+      "logits/rejected": -0.8296138048171997,
+      "logps/chosen": -11.327326774597168,
+      "logps/rejected": -38.99674606323242,
+      "loss": 0.916981041431427,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6145994067192078,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.005291827023029327,
+      "rewards/margins": 2.653188705444336,
+      "rewards/rejected": -2.658480644226074,
+      "step": 381,
+      "train_speed(iter/s)": 0.005425
+    },
+    {
+      "epoch": 0.3082120975265374,
+      "grad_norm": 10.325765609741211,
+      "learning_rate": 0.00016568756097836576,
+      "logits/chosen": -0.7539953589439392,
+      "logits/rejected": -0.7629266381263733,
+      "logps/chosen": -12.126445770263672,
+      "logps/rejected": -36.76786804199219,
+      "loss": 0.9146925210952759,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5855523347854614,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.018190251663327217,
+      "rewards/margins": 2.345966339111328,
+      "rewards/rejected": -2.3277759552001953,
+      "step": 382,
+      "train_speed(iter/s)": 0.005425
+    },
+    {
+      "epoch": 0.30901893547817755,
+      "grad_norm": 4.079949378967285,
+      "learning_rate": 0.00016548607339452853,
+      "logits/chosen": -0.7926503419876099,
+      "logits/rejected": -0.8010275363922119,
+      "logps/chosen": -10.392716407775879,
+      "logps/rejected": -33.4859733581543,
+      "loss": 0.6544159054756165,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.31039926409721375,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.19881543517112732,
+      "rewards/margins": 2.027736186981201,
+      "rewards/rejected": -1.828920841217041,
+      "step": 383,
+      "train_speed(iter/s)": 0.005425
+    },
+    {
+      "epoch": 0.3098257734298177,
+      "grad_norm": 2.981825828552246,
+      "learning_rate": 0.0001652841192635718,
+      "logits/chosen": -0.7877278923988342,
+      "logits/rejected": -0.7911107540130615,
+      "logps/chosen": -10.606431007385254,
+      "logps/rejected": -31.438617706298828,
+      "loss": 0.7501833438873291,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4839434027671814,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.36393457651138306,
+      "rewards/margins": 2.2303647994995117,
+      "rewards/rejected": -1.8664300441741943,
+      "step": 384,
+      "train_speed(iter/s)": 0.005426
+    },
+    {
+      "epoch": 0.31063261138145787,
+      "grad_norm": 5.048736572265625,
+      "learning_rate": 0.0001650817000242919,
+      "logits/chosen": -0.741836428642273,
+      "logits/rejected": -0.7429819703102112,
+      "logps/chosen": -8.188630104064941,
+      "logps/rejected": -21.78225326538086,
+      "loss": 0.7458047866821289,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3462568521499634,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1975884884595871,
+      "rewards/margins": 1.2948282957077026,
+      "rewards/rejected": -1.0972397327423096,
+      "step": 385,
+      "train_speed(iter/s)": 0.005426
+    },
+    {
+      "epoch": 0.31143944933309803,
+      "grad_norm": 3.7488629817962646,
+      "learning_rate": 0.00016487881711879868,
+      "logits/chosen": -0.7345485687255859,
+      "logits/rejected": -0.7461491227149963,
+      "logps/chosen": -8.506485939025879,
+      "logps/rejected": -31.39336395263672,
+      "loss": 0.6992615461349487,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.38303038477897644,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.12145505100488663,
+      "rewards/margins": 1.6618486642837524,
+      "rewards/rejected": -1.5403934717178345,
+      "step": 386,
+      "train_speed(iter/s)": 0.005426
+    },
+    {
+      "epoch": 0.31224628728473813,
+      "grad_norm": 4.740914344787598,
+      "learning_rate": 0.0001646754719925053,
+      "logits/chosen": -0.7271203398704529,
+      "logits/rejected": -0.7348542213439941,
+      "logps/chosen": -8.581640243530273,
+      "logps/rejected": -30.84325408935547,
+      "loss": 0.5986461639404297,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.30818265676498413,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2583742141723633,
+      "rewards/margins": 1.7723979949951172,
+      "rewards/rejected": -1.514023780822754,
+      "step": 387,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.3130531252363783,
+      "grad_norm": 2.877361536026001,
+      "learning_rate": 0.00016447166609411809,
+      "logits/chosen": -0.737002432346344,
+      "logits/rejected": -0.7362369894981384,
+      "logps/chosen": -9.167498588562012,
+      "logps/rejected": -21.97631072998047,
+      "loss": 0.7882663607597351,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3854939937591553,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.27406081557273865,
+      "rewards/margins": 1.2815303802490234,
+      "rewards/rejected": -1.007469654083252,
+      "step": 388,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.31385996318801845,
+      "grad_norm": 2.516064405441284,
+      "learning_rate": 0.00016426740087562588,
+      "logits/chosen": -0.7427570223808289,
+      "logits/rejected": -0.7460445761680603,
+      "logps/chosen": -7.309655666351318,
+      "logps/rejected": -30.33109474182129,
+      "loss": 0.5361584424972534,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.29300200939178467,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.18929171562194824,
+      "rewards/margins": 1.8147234916687012,
+      "rewards/rejected": -1.625431776046753,
+      "step": 389,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.3146668011396586,
+      "grad_norm": 5.825074195861816,
+      "learning_rate": 0.0001640626777922901,
+      "logits/chosen": -0.7519196271896362,
+      "logits/rejected": -0.7524212598800659,
+      "logps/chosen": -7.206381320953369,
+      "logps/rejected": -24.535839080810547,
+      "loss": 0.6842062473297119,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3918344974517822,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.23575374484062195,
+      "rewards/margins": 1.7519906759262085,
+      "rewards/rejected": -1.5162369012832642,
+      "step": 390,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.3154736390912988,
+      "grad_norm": 3.599740505218506,
+      "learning_rate": 0.000163857498302634,
+      "logits/chosen": -0.7379420399665833,
+      "logits/rejected": -0.7385002970695496,
+      "logps/chosen": -8.897488594055176,
+      "logps/rejected": -21.334794998168945,
+      "loss": 0.7518292665481567,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.35948920249938965,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.0675305724143982,
+      "rewards/margins": 1.2294992208480835,
+      "rewards/rejected": -1.161968469619751,
+      "step": 391,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.31628047704293893,
+      "grad_norm": 3.701690673828125,
+      "learning_rate": 0.0001636518638684325,
+      "logits/chosen": -0.7553545832633972,
+      "logits/rejected": -0.7599779367446899,
+      "logps/chosen": -11.763086318969727,
+      "logps/rejected": -29.765409469604492,
+      "loss": 0.6486645936965942,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3567640483379364,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.10855480283498764,
+      "rewards/margins": 1.6032474040985107,
+      "rewards/rejected": -1.4946926832199097,
+      "step": 392,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.31708731499457904,
+      "grad_norm": 2.8278613090515137,
+      "learning_rate": 0.00016344577595470174,
+      "logits/chosen": -0.7699616551399231,
+      "logits/rejected": -0.7747686505317688,
+      "logps/chosen": -8.457822799682617,
+      "logps/rejected": -25.99679183959961,
+      "loss": 0.7349235415458679,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.37837332487106323,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.10739874094724655,
+      "rewards/margins": 1.572762131690979,
+      "rewards/rejected": -1.4653635025024414,
+      "step": 393,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.3178941529462192,
+      "grad_norm": 3.7045748233795166,
+      "learning_rate": 0.0001632392360296886,
+      "logits/chosen": -0.690305233001709,
+      "logits/rejected": -0.6888067722320557,
+      "logps/chosen": -9.403566360473633,
+      "logps/rejected": -22.69052505493164,
+      "loss": 0.6981708407402039,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.42633530497550964,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.4319700002670288,
+      "rewards/margins": 1.8832911252975464,
+      "rewards/rejected": -1.4513212442398071,
+      "step": 394,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.31870099089785936,
+      "grad_norm": 3.8347578048706055,
+      "learning_rate": 0.00016303224556486021,
+      "logits/chosen": -0.7941313982009888,
+      "logits/rejected": -0.7985343933105469,
+      "logps/chosen": -8.82894515991211,
+      "logps/rejected": -35.2305793762207,
+      "loss": 0.6906291842460632,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4142599105834961,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.22435417771339417,
+      "rewards/margins": 2.410836696624756,
+      "rewards/rejected": -2.1864821910858154,
+      "step": 395,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.3195078288494995,
+      "grad_norm": 2.570457935333252,
+      "learning_rate": 0.00016282480603489359,
+      "logits/chosen": -0.7678409218788147,
+      "logits/rejected": -0.7748376727104187,
+      "logps/chosen": -7.719938278198242,
+      "logps/rejected": -37.97621536254883,
+      "loss": 0.5270894765853882,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.29066720604896545,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2296931892633438,
+      "rewards/margins": 2.5373897552490234,
+      "rewards/rejected": -2.307696580886841,
+      "step": 396,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.3203146668011397,
+      "grad_norm": 3.276946544647217,
+      "learning_rate": 0.000162616918917665,
+      "logits/chosen": -0.8192197680473328,
+      "logits/rejected": -0.8224659562110901,
+      "logps/chosen": -12.474966049194336,
+      "logps/rejected": -34.623626708984375,
+      "loss": 0.6273024678230286,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4210663139820099,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.15507780015468597,
+      "rewards/margins": 2.483549118041992,
+      "rewards/rejected": -2.3284709453582764,
+      "step": 397,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.3211215047527798,
+      "grad_norm": 4.271194934844971,
+      "learning_rate": 0.00016240858569423956,
+      "logits/chosen": -0.8221184611320496,
+      "logits/rejected": -0.8321431875228882,
+      "logps/chosen": -9.941547393798828,
+      "logps/rejected": -42.07545471191406,
+      "loss": 0.8007271885871887,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4452548623085022,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.05637914687395096,
+      "rewards/margins": 2.7785463333129883,
+      "rewards/rejected": -2.834925651550293,
+      "step": 398,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.32192834270441995,
+      "grad_norm": 11.448945999145508,
+      "learning_rate": 0.0001621998078488605,
+      "logits/chosen": -0.8299403190612793,
+      "logits/rejected": -0.8370727896690369,
+      "logps/chosen": -14.581025123596191,
+      "logps/rejected": -46.871543884277344,
+      "loss": 1.0272753238677979,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.7910286784172058,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08015693724155426,
+      "rewards/margins": 3.2413315773010254,
+      "rewards/rejected": -3.161174774169922,
+      "step": 399,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.3227351806560601,
+      "grad_norm": 10.813321113586426,
+      "learning_rate": 0.00016199058686893882,
+      "logits/chosen": -0.85502690076828,
+      "logits/rejected": -0.8559147715568542,
+      "logps/chosen": -10.797840118408203,
+      "logps/rejected": -27.456714630126953,
+      "loss": 1.2833585739135742,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.8083551526069641,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.05027822405099869,
+      "rewards/margins": 1.7881048917770386,
+      "rewards/rejected": -1.8383830785751343,
+      "step": 400,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.32354201860770027,
+      "grad_norm": 7.667049884796143,
+      "learning_rate": 0.00016178092424504246,
+      "logits/chosen": -0.8004090785980225,
+      "logits/rejected": -0.8061416149139404,
+      "logps/chosen": -9.709630966186523,
+      "logps/rejected": -47.190277099609375,
+      "loss": 0.6974176168441772,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5138447284698486,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.048886872828006744,
+      "rewards/margins": 3.236457109451294,
+      "rewards/rejected": -3.285344123840332,
+      "step": 401,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.3243488565593404,
+      "grad_norm": 5.483085632324219,
+      "learning_rate": 0.00016157082147088593,
+      "logits/chosen": -0.8190602660179138,
+      "logits/rejected": -0.824268639087677,
+      "logps/chosen": -14.70582103729248,
+      "logps/rejected": -41.711917877197266,
+      "loss": 1.1310678720474243,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6881631016731262,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.03247120603919029,
+      "rewards/margins": 2.756627321243286,
+      "rewards/rejected": -2.724156141281128,
+      "step": 402,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.3251556945109806,
+      "grad_norm": 4.824715614318848,
+      "learning_rate": 0.0001613602800433194,
+      "logits/chosen": -0.8436318039894104,
+      "logits/rejected": -0.8526687026023865,
+      "logps/chosen": -8.126108169555664,
+      "logps/rejected": -42.07024383544922,
+      "loss": 0.6571808457374573,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4890969395637512,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.10289221256971359,
+      "rewards/margins": 2.9235923290252686,
+      "rewards/rejected": -2.8207004070281982,
+      "step": 403,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.3259625324626207,
+      "grad_norm": 6.610788345336914,
+      "learning_rate": 0.00016114930146231827,
+      "logits/chosen": -0.8161177635192871,
+      "logits/rejected": -0.8242086172103882,
+      "logps/chosen": -12.218559265136719,
+      "logps/rejected": -44.647254943847656,
+      "loss": 0.6997122168540955,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4584552049636841,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.26513317227363586,
+      "rewards/margins": 2.8937909603118896,
+      "rewards/rejected": -3.158923864364624,
+      "step": 404,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.32676937041426085,
+      "grad_norm": 2.9718236923217773,
+      "learning_rate": 0.0001609378872309723,
+      "logits/chosen": -0.766316831111908,
+      "logits/rejected": -0.7643468379974365,
+      "logps/chosen": -6.773965835571289,
+      "logps/rejected": -29.697847366333008,
+      "loss": 0.46833112835884094,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2168104350566864,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.19758227467536926,
+      "rewards/margins": 2.3953795433044434,
+      "rewards/rejected": -2.1977975368499756,
+      "step": 405,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.327576208365901,
+      "grad_norm": 5.874072551727295,
+      "learning_rate": 0.00016072603885547508,
+      "logits/chosen": -0.78830885887146,
+      "logits/rejected": -0.7853043079376221,
+      "logps/chosen": -11.376614570617676,
+      "logps/rejected": -30.687149047851562,
+      "loss": 0.7618173956871033,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3364633023738861,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.12673895061016083,
+      "rewards/margins": 1.8598191738128662,
+      "rewards/rejected": -1.986557960510254,
+      "step": 406,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.32838304631754117,
+      "grad_norm": 3.214999198913574,
+      "learning_rate": 0.00016051375784511312,
+      "logits/chosen": -0.7452774047851562,
+      "logits/rejected": -0.7576760649681091,
+      "logps/chosen": -6.296991348266602,
+      "logps/rejected": -37.21446228027344,
+      "loss": 0.5500832796096802,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2983091473579407,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.16730837523937225,
+      "rewards/margins": 2.706336259841919,
+      "rewards/rejected": -2.539027690887451,
+      "step": 407,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.32918988426918133,
+      "grad_norm": 3.2364044189453125,
+      "learning_rate": 0.00016030104571225522,
+      "logits/chosen": -0.8141746520996094,
+      "logits/rejected": -0.8147263526916504,
+      "logps/chosen": -8.27431583404541,
+      "logps/rejected": -27.03731918334961,
+      "loss": 0.6542465686798096,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.32871556282043457,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.20524322986602783,
+      "rewards/margins": 1.7059212923049927,
+      "rewards/rejected": -1.5006780624389648,
+      "step": 408,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.32999672222082144,
+      "grad_norm": 4.386970043182373,
+      "learning_rate": 0.0001600879039723417,
+      "logits/chosen": -0.7278757095336914,
+      "logits/rejected": -0.7296465635299683,
+      "logps/chosen": -10.145583152770996,
+      "logps/rejected": -34.609405517578125,
+      "loss": 0.6105880737304688,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.36851969361305237,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.062060222029685974,
+      "rewards/margins": 2.2632784843444824,
+      "rewards/rejected": -2.2012181282043457,
+      "step": 409,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.3308035601724616,
+      "grad_norm": 3.3711960315704346,
+      "learning_rate": 0.0001598743341438734,
+      "logits/chosen": -0.69020015001297,
+      "logits/rejected": -0.6957679986953735,
+      "logps/chosen": -10.575393676757812,
+      "logps/rejected": -32.856101989746094,
+      "loss": 0.6940324306488037,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.43308204412460327,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3041924238204956,
+      "rewards/margins": 2.375181198120117,
+      "rewards/rejected": -2.070988893508911,
+      "step": 410,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.33161039812410176,
+      "grad_norm": 3.1790573596954346,
+      "learning_rate": 0.00015966033774840127,
+      "logits/chosen": -0.7445316314697266,
+      "logits/rejected": -0.748544454574585,
+      "logps/chosen": -7.364559650421143,
+      "logps/rejected": -30.644813537597656,
+      "loss": 0.5010148882865906,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2939934730529785,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2668747305870056,
+      "rewards/margins": 2.051632881164551,
+      "rewards/rejected": -1.7847580909729004,
+      "step": 411,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.3324172360757419,
+      "grad_norm": 5.062440395355225,
+      "learning_rate": 0.00015944591631051504,
+      "logits/chosen": -0.6655209064483643,
+      "logits/rejected": -0.6722307801246643,
+      "logps/chosen": -8.464452743530273,
+      "logps/rejected": -28.55743408203125,
+      "loss": 0.7590057849884033,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.41250795125961304,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.13860762119293213,
+      "rewards/margins": 1.7520344257354736,
+      "rewards/rejected": -1.613426923751831,
+      "step": 412,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.3332240740273821,
+      "grad_norm": 2.3003089427948,
+      "learning_rate": 0.00015923107135783275,
+      "logits/chosen": -0.735182523727417,
+      "logits/rejected": -0.741850733757019,
+      "logps/chosen": -4.926098346710205,
+      "logps/rejected": -27.20798110961914,
+      "loss": 0.5019189715385437,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.22264572978019714,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.19852803647518158,
+      "rewards/margins": 1.7793362140655518,
+      "rewards/rejected": -1.580808162689209,
+      "step": 413,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.33403091197902224,
+      "grad_norm": 2.761085271835327,
+      "learning_rate": 0.00015901580442098968,
+      "logits/chosen": -0.6879050135612488,
+      "logits/rejected": -0.6989721059799194,
+      "logps/chosen": -8.289688110351562,
+      "logps/rejected": -36.11437225341797,
+      "loss": 0.5515791773796082,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.359782874584198,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.35169073939323425,
+      "rewards/margins": 2.3315212726593018,
+      "rewards/rejected": -1.9798306226730347,
+      "step": 414,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.33483774993066234,
+      "grad_norm": 2.9564995765686035,
+      "learning_rate": 0.00015880011703362748,
+      "logits/chosen": -0.7736977338790894,
+      "logits/rejected": -0.7791394591331482,
+      "logps/chosen": -11.907139778137207,
+      "logps/rejected": -30.767383575439453,
+      "loss": 0.5806700587272644,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.38372740149497986,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4125119745731354,
+      "rewards/margins": 2.088625192642212,
+      "rewards/rejected": -1.6761131286621094,
+      "step": 415,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.3356445878823025,
+      "grad_norm": 2.9187235832214355,
+      "learning_rate": 0.0001585840107323832,
+      "logits/chosen": -0.7161890864372253,
+      "logits/rejected": -0.7156458497047424,
+      "logps/chosen": -8.870182037353516,
+      "logps/rejected": -30.228370666503906,
+      "loss": 0.7012996077537537,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3564344346523285,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.10954218357801437,
+      "rewards/margins": 2.018000364303589,
+      "rewards/rejected": -1.9084582328796387,
+      "step": 416,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.33645142583394266,
+      "grad_norm": 5.864604473114014,
+      "learning_rate": 0.00015836748705687841,
+      "logits/chosen": -0.7383227348327637,
+      "logits/rejected": -0.7456459999084473,
+      "logps/chosen": -7.744223594665527,
+      "logps/rejected": -38.64203643798828,
+      "loss": 0.5262930393218994,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3058444857597351,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.17473873496055603,
+      "rewards/margins": 2.6876578330993652,
+      "rewards/rejected": -2.5129189491271973,
+      "step": 417,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.3372582637855828,
+      "grad_norm": 9.024547576904297,
+      "learning_rate": 0.00015815054754970831,
+      "logits/chosen": -0.7634251117706299,
+      "logits/rejected": -0.7683747410774231,
+      "logps/chosen": -10.462536811828613,
+      "logps/rejected": -31.69692039489746,
+      "loss": 0.9514245986938477,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6364089250564575,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.21211130917072296,
+      "rewards/margins": 2.214094877243042,
+      "rewards/rejected": -2.001983642578125,
+      "step": 418,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.338065101737223,
+      "grad_norm": 8.62049388885498,
+      "learning_rate": 0.00015793319375643052,
+      "logits/chosen": -0.7581362724304199,
+      "logits/rejected": -0.7588805556297302,
+      "logps/chosen": -7.701593399047852,
+      "logps/rejected": -35.039794921875,
+      "loss": 0.46434247493743896,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.32877326011657715,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.22201837599277496,
+      "rewards/margins": 2.5157957077026367,
+      "rewards/rejected": -2.2937769889831543,
+      "step": 419,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.3388719396888631,
+      "grad_norm": 6.575305938720703,
+      "learning_rate": 0.00015771542722555423,
+      "logits/chosen": -0.7857352495193481,
+      "logits/rejected": -0.7996398210525513,
+      "logps/chosen": -10.29336166381836,
+      "logps/rejected": -46.0968017578125,
+      "loss": 0.738344669342041,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.45971018075942993,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.021582117304205894,
+      "rewards/margins": 2.711317777633667,
+      "rewards/rejected": -2.6897356510162354,
+      "step": 420,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.33967877764050325,
+      "grad_norm": 1.8250871896743774,
+      "learning_rate": 0.00015749724950852909,
+      "logits/chosen": -0.8029757738113403,
+      "logits/rejected": -0.803418755531311,
+      "logps/chosen": -8.469568252563477,
+      "logps/rejected": -41.49656295776367,
+      "loss": 0.4241314232349396,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2885726988315582,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.14934808015823364,
+      "rewards/margins": 3.204773187637329,
+      "rewards/rejected": -3.0554254055023193,
+      "step": 421,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.3404856155921434,
+      "grad_norm": 7.909032821655273,
+      "learning_rate": 0.00015727866215973425,
+      "logits/chosen": -0.8152115941047668,
+      "logits/rejected": -0.8257274627685547,
+      "logps/chosen": -8.088623046875,
+      "logps/rejected": -48.481143951416016,
+      "loss": 0.6303491592407227,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3295513391494751,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.07966871559619904,
+      "rewards/margins": 3.391120672225952,
+      "rewards/rejected": -3.3114516735076904,
+      "step": 422,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.34129245354378357,
+      "grad_norm": 5.323256015777588,
+      "learning_rate": 0.00015705966673646723,
+      "logits/chosen": -0.8351114392280579,
+      "logits/rejected": -0.8382234573364258,
+      "logps/chosen": -8.482909202575684,
+      "logps/rejected": -37.57075881958008,
+      "loss": 0.6785515546798706,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.363737553358078,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1108720526099205,
+      "rewards/margins": 2.615013837814331,
+      "rewards/rejected": -2.5041418075561523,
+      "step": 423,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.34209929149542373,
+      "grad_norm": 27.7071475982666,
+      "learning_rate": 0.00015684026479893272,
+      "logits/chosen": -0.8173145651817322,
+      "logits/rejected": -0.8212793469429016,
+      "logps/chosen": -15.243718147277832,
+      "logps/rejected": -35.84337615966797,
+      "loss": 0.9820355176925659,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5321431756019592,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.15343250334262848,
+      "rewards/margins": 2.1440441608428955,
+      "rewards/rejected": -2.2974765300750732,
+      "step": 424,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.3429061294470639,
+      "grad_norm": 2.558488368988037,
+      "learning_rate": 0.00015662045791023173,
+      "logits/chosen": -0.7981111407279968,
+      "logits/rejected": -0.804236888885498,
+      "logps/chosen": -12.458770751953125,
+      "logps/rejected": -38.74690246582031,
+      "loss": 0.5468552708625793,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.40575528144836426,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3049697279930115,
+      "rewards/margins": 2.893904685974121,
+      "rewards/rejected": -2.588935375213623,
+      "step": 425,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.343712967398704,
+      "grad_norm": 11.186711311340332,
+      "learning_rate": 0.00015640024763635015,
+      "logits/chosen": -0.8168637156486511,
+      "logits/rejected": -0.8147194385528564,
+      "logps/chosen": -18.96175193786621,
+      "logps/rejected": -30.127038955688477,
+      "loss": 1.35488760471344,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6938337683677673,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.32947003841400146,
+      "rewards/margins": 1.8565242290496826,
+      "rewards/rejected": -2.1859941482543945,
+      "step": 426,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.34451980535034415,
+      "grad_norm": 2.7602322101593018,
+      "learning_rate": 0.00015617963554614782,
+      "logits/chosen": -0.7925236225128174,
+      "logits/rejected": -0.8042585253715515,
+      "logps/chosen": -7.885419845581055,
+      "logps/rejected": -41.48836135864258,
+      "loss": 0.5029093027114868,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3265308141708374,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.19351695477962494,
+      "rewards/margins": 3.003077507019043,
+      "rewards/rejected": -2.809560537338257,
+      "step": 427,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.3453266433019843,
+      "grad_norm": 2.7690954208374023,
+      "learning_rate": 0.00015595862321134723,
+      "logits/chosen": -0.801167368888855,
+      "logits/rejected": -0.8079819083213806,
+      "logps/chosen": -11.006603240966797,
+      "logps/rejected": -47.515464782714844,
+      "loss": 0.5393511652946472,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.357557088136673,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.09760009497404099,
+      "rewards/margins": 3.3984105587005615,
+      "rewards/rejected": -3.3008105754852295,
+      "step": 428,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.3461334812536245,
+      "grad_norm": 3.378408432006836,
+      "learning_rate": 0.00015573721220652243,
+      "logits/chosen": -0.7705251574516296,
+      "logits/rejected": -0.7756682634353638,
+      "logps/chosen": -7.106504440307617,
+      "logps/rejected": -38.38644790649414,
+      "loss": 0.45007073879241943,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2939262390136719,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2970801591873169,
+      "rewards/margins": 2.9060657024383545,
+      "rewards/rejected": -2.608985424041748,
+      "step": 429,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.34694031920526464,
+      "grad_norm": 11.245018005371094,
+      "learning_rate": 0.0001555154041090876,
+      "logits/chosen": -0.7640025019645691,
+      "logits/rejected": -0.7694700360298157,
+      "logps/chosen": -7.503448486328125,
+      "logps/rejected": -29.179779052734375,
+      "loss": 0.6581162214279175,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3762749135494232,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.18421027064323425,
+      "rewards/margins": 2.1600494384765625,
+      "rewards/rejected": -1.9758392572402954,
+      "step": 430,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.34774715715690474,
+      "grad_norm": 4.24967098236084,
+      "learning_rate": 0.0001552932004992861,
+      "logits/chosen": -0.7261820435523987,
+      "logits/rejected": -0.7354370355606079,
+      "logps/chosen": -8.89859676361084,
+      "logps/rejected": -43.472408294677734,
+      "loss": 0.5466022491455078,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2866482138633728,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.21286547183990479,
+      "rewards/margins": 3.0320310592651367,
+      "rewards/rejected": -2.8191657066345215,
+      "step": 431,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.3485539951085449,
+      "grad_norm": 17.65904998779297,
+      "learning_rate": 0.00015507060296017896,
+      "logits/chosen": -0.7059640884399414,
+      "logits/rejected": -0.7113645076751709,
+      "logps/chosen": -15.478120803833008,
+      "logps/rejected": -38.02043914794922,
+      "loss": 0.9082990288734436,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5757866501808167,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.15995851159095764,
+      "rewards/margins": 2.8075268268585205,
+      "rewards/rejected": -2.6475682258605957,
+      "step": 432,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.34936083306018506,
+      "grad_norm": 13.476463317871094,
+      "learning_rate": 0.00015484761307763383,
+      "logits/chosen": -0.700376033782959,
+      "logits/rejected": -0.7011431455612183,
+      "logps/chosen": -9.81491470336914,
+      "logps/rejected": -33.46759796142578,
+      "loss": 0.7563650608062744,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5197194814682007,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.14740222692489624,
+      "rewards/margins": 2.482841730117798,
+      "rewards/rejected": -2.335439682006836,
+      "step": 433,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.3501676710118252,
+      "grad_norm": 2.2564306259155273,
+      "learning_rate": 0.00015462423244031346,
+      "logits/chosen": -0.7145711183547974,
+      "logits/rejected": -0.7142253518104553,
+      "logps/chosen": -6.049681663513184,
+      "logps/rejected": -40.261722564697266,
+      "loss": 0.39624738693237305,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2778511047363281,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3084055185317993,
+      "rewards/margins": 3.100196361541748,
+      "rewards/rejected": -2.79179048538208,
+      "step": 434,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.3509745089634654,
+      "grad_norm": 3.1633989810943604,
+      "learning_rate": 0.00015440046263966446,
+      "logits/chosen": -0.734190046787262,
+      "logits/rejected": -0.7362701296806335,
+      "logps/chosen": -7.924525737762451,
+      "logps/rejected": -30.7021484375,
+      "loss": 0.6030353307723999,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3201526999473572,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.07075662910938263,
+      "rewards/margins": 1.9832499027252197,
+      "rewards/rejected": -1.9124932289123535,
+      "step": 435,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.35178134691510554,
+      "grad_norm": 9.708440780639648,
+      "learning_rate": 0.00015417630526990615,
+      "logits/chosen": -0.7230247855186462,
+      "logits/rejected": -0.7281976342201233,
+      "logps/chosen": -7.0415544509887695,
+      "logps/rejected": -37.40678787231445,
+      "loss": 0.9090821743011475,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6137229204177856,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.017065048217773438,
+      "rewards/margins": 2.4000210762023926,
+      "rewards/rejected": -2.417086124420166,
+      "step": 436,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.35258818486674565,
+      "grad_norm": 2.7294411659240723,
+      "learning_rate": 0.00015395176192801882,
+      "logits/chosen": -0.66111159324646,
+      "logits/rejected": -0.6679624319076538,
+      "logps/chosen": -10.94334602355957,
+      "logps/rejected": -39.744171142578125,
+      "loss": 0.558108925819397,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3573530912399292,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.07710271328687668,
+      "rewards/margins": 2.2725815773010254,
+      "rewards/rejected": -2.195478916168213,
+      "step": 437,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.3533950228183858,
+      "grad_norm": 2.2957513332366943,
+      "learning_rate": 0.00015372683421373268,
+      "logits/chosen": -0.7276008129119873,
+      "logits/rejected": -0.733298659324646,
+      "logps/chosen": -9.018013954162598,
+      "logps/rejected": -39.43022155761719,
+      "loss": 0.5581925511360168,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.35958534479141235,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3142196834087372,
+      "rewards/margins": 2.4971206188201904,
+      "rewards/rejected": -2.182900905609131,
+      "step": 438,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.35420186077002597,
+      "grad_norm": 2.652552604675293,
+      "learning_rate": 0.0001535015237295164,
+      "logits/chosen": -0.7248872518539429,
+      "logits/rejected": -0.728070080280304,
+      "logps/chosen": -9.407607078552246,
+      "logps/rejected": -35.77960205078125,
+      "loss": 0.7365480065345764,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5000982880592346,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.21720585227012634,
+      "rewards/margins": 2.5895490646362305,
+      "rewards/rejected": -2.372343063354492,
+      "step": 439,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.3550086987216661,
+      "grad_norm": 16.293540954589844,
+      "learning_rate": 0.00015327583208056553,
+      "logits/chosen": -0.7425347566604614,
+      "logits/rejected": -0.7465611100196838,
+      "logps/chosen": -6.231223106384277,
+      "logps/rejected": -33.581016540527344,
+      "loss": 0.5848970413208008,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3383205533027649,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.22708339989185333,
+      "rewards/margins": 2.6124606132507324,
+      "rewards/rejected": -2.3853771686553955,
+      "step": 440,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.3558155366733063,
+      "grad_norm": 7.516906261444092,
+      "learning_rate": 0.00015304976087479128,
+      "logits/chosen": -0.7300995588302612,
+      "logits/rejected": -0.731682538986206,
+      "logps/chosen": -7.094137191772461,
+      "logps/rejected": -24.851308822631836,
+      "loss": 0.6824460625648499,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.28373995423316956,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.08390329778194427,
+      "rewards/margins": 1.636284351348877,
+      "rewards/rejected": -1.5523810386657715,
+      "step": 441,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.35662237462494645,
+      "grad_norm": 5.410120487213135,
+      "learning_rate": 0.00015282331172280887,
+      "logits/chosen": -0.7709946036338806,
+      "logits/rejected": -0.778519868850708,
+      "logps/chosen": -14.137146949768066,
+      "logps/rejected": -39.00368881225586,
+      "loss": 1.1119240522384644,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5970031023025513,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.0509246289730072,
+      "rewards/margins": 2.253941297531128,
+      "rewards/rejected": -2.304866075515747,
+      "step": 442,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.35742921257658655,
+      "grad_norm": 2.6960244178771973,
+      "learning_rate": 0.0001525964862379263,
+      "logits/chosen": -0.7315975427627563,
+      "logits/rejected": -0.7369835376739502,
+      "logps/chosen": -5.271239280700684,
+      "logps/rejected": -27.497392654418945,
+      "loss": 0.5291019082069397,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2573389708995819,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2271825522184372,
+      "rewards/margins": 2.134850263595581,
+      "rewards/rejected": -1.907667636871338,
+      "step": 443,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.3582360505282267,
+      "grad_norm": 3.0122978687286377,
+      "learning_rate": 0.0001523692860361325,
+      "logits/chosen": -0.7649153470993042,
+      "logits/rejected": -0.7727422118186951,
+      "logps/chosen": -12.25500774383545,
+      "logps/rejected": -40.75428771972656,
+      "loss": 0.712749183177948,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5094113349914551,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.25550708174705505,
+      "rewards/margins": 2.767303466796875,
+      "rewards/rejected": -2.511796474456787,
+      "step": 444,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.3590428884798669,
+      "grad_norm": 3.1875035762786865,
+      "learning_rate": 0.00015214171273608614,
+      "logits/chosen": -0.7680434584617615,
+      "logits/rejected": -0.773673951625824,
+      "logps/chosen": -10.56375503540039,
+      "logps/rejected": -46.056488037109375,
+      "loss": 0.5614004731178284,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3534352779388428,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.05404359847307205,
+      "rewards/margins": 3.072054386138916,
+      "rewards/rejected": -3.0180106163024902,
+      "step": 445,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.35984972643150703,
+      "grad_norm": 2.6271064281463623,
+      "learning_rate": 0.00015191376795910404,
+      "logits/chosen": -0.7706547975540161,
+      "logits/rejected": -0.7753493189811707,
+      "logps/chosen": -10.181999206542969,
+      "logps/rejected": -36.871089935302734,
+      "loss": 0.7368884682655334,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5453473925590515,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.20240941643714905,
+      "rewards/margins": 2.66059947013855,
+      "rewards/rejected": -2.4581899642944336,
+      "step": 446,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.3606565643831472,
+      "grad_norm": 2.9951157569885254,
+      "learning_rate": 0.0001516854533291494,
+      "logits/chosen": -0.7616828083992004,
+      "logits/rejected": -0.7694711685180664,
+      "logps/chosen": -7.686607837677002,
+      "logps/rejected": -41.72177505493164,
+      "loss": 0.5836425423622131,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.33194291591644287,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.03806695714592934,
+      "rewards/margins": 2.7966089248657227,
+      "rewards/rejected": -2.75854229927063,
+      "step": 447,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.3614634023347873,
+      "grad_norm": 6.78880500793457,
+      "learning_rate": 0.00015145677047282056,
+      "logits/chosen": -0.768774151802063,
+      "logits/rejected": -0.7738460302352905,
+      "logps/chosen": -7.29971170425415,
+      "logps/rejected": -23.89675521850586,
+      "loss": 0.7116811871528625,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.363356351852417,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2802785038948059,
+      "rewards/margins": 1.6327687501907349,
+      "rewards/rejected": -1.3524901866912842,
+      "step": 448,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.36227024028642746,
+      "grad_norm": 2.428403854370117,
+      "learning_rate": 0.00015122772101933913,
+      "logits/chosen": -0.713983416557312,
+      "logits/rejected": -0.719916582107544,
+      "logps/chosen": -9.733499526977539,
+      "logps/rejected": -38.1261100769043,
+      "loss": 0.5839467644691467,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3204602897167206,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.18539123237133026,
+      "rewards/margins": 2.5533952713012695,
+      "rewards/rejected": -2.368004083633423,
+      "step": 449,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.3630770782380676,
+      "grad_norm": 2.1015625,
+      "learning_rate": 0.00015099830660053848,
+      "logits/chosen": -0.6917126774787903,
+      "logits/rejected": -0.704102098941803,
+      "logps/chosen": -6.558714866638184,
+      "logps/rejected": -44.186180114746094,
+      "loss": 0.42745494842529297,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.25199466943740845,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.26587462425231934,
+      "rewards/margins": 3.109530210494995,
+      "rewards/rejected": -2.8436553478240967,
+      "step": 450,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.3638839161897078,
+      "grad_norm": 7.269819736480713,
+      "learning_rate": 0.00015076852885085223,
+      "logits/chosen": -0.685967206954956,
+      "logits/rejected": -0.6903418898582458,
+      "logps/chosen": -8.861746788024902,
+      "logps/rejected": -31.555381774902344,
+      "loss": 0.6940796375274658,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4090823233127594,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1664535403251648,
+      "rewards/margins": 2.2505595684051514,
+      "rewards/rejected": -2.0841057300567627,
+      "step": 451,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.36469075414134794,
+      "grad_norm": 7.895684719085693,
+      "learning_rate": 0.00015053838940730244,
+      "logits/chosen": -0.7125756740570068,
+      "logits/rejected": -0.7196484804153442,
+      "logps/chosen": -7.934318542480469,
+      "logps/rejected": -32.33711624145508,
+      "loss": 0.9927008152008057,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5683808326721191,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.047097161412239075,
+      "rewards/margins": 1.7799056768417358,
+      "rewards/rejected": -1.7328083515167236,
+      "step": 452,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.3654975920929881,
+      "grad_norm": 2.145096778869629,
+      "learning_rate": 0.00015030788990948797,
+      "logits/chosen": -0.7455975413322449,
+      "logits/rejected": -0.748679518699646,
+      "logps/chosen": -8.74243450164795,
+      "logps/rejected": -32.179813385009766,
+      "loss": 0.5566769242286682,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.35227102041244507,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.20155397057533264,
+      "rewards/margins": 2.3393735885620117,
+      "rewards/rejected": -2.137819766998291,
+      "step": 453,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.3663044300446282,
+      "grad_norm": 12.220844268798828,
+      "learning_rate": 0.00015007703199957297,
+      "logits/chosen": -0.6872383952140808,
+      "logits/rejected": -0.6934896111488342,
+      "logps/chosen": -12.1061429977417,
+      "logps/rejected": -36.80299377441406,
+      "loss": 0.761122465133667,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4497636556625366,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.06379126012325287,
+      "rewards/margins": 2.3671936988830566,
+      "rewards/rejected": -2.3034024238586426,
+      "step": 454,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.36711126799626836,
+      "grad_norm": 1.585224986076355,
+      "learning_rate": 0.00014984581732227492,
+      "logits/chosen": -0.7161681652069092,
+      "logits/rejected": -0.7274316549301147,
+      "logps/chosen": -8.679915428161621,
+      "logps/rejected": -39.74371337890625,
+      "loss": 0.45609888434410095,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3443814516067505,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.48159369826316833,
+      "rewards/margins": 3.0060887336730957,
+      "rewards/rejected": -2.5244951248168945,
+      "step": 455,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.3679181059479085,
+      "grad_norm": 5.118838310241699,
+      "learning_rate": 0.00014961424752485308,
+      "logits/chosen": -0.7437467575073242,
+      "logits/rejected": -0.754626989364624,
+      "logps/chosen": -11.635477066040039,
+      "logps/rejected": -46.820133209228516,
+      "loss": 0.9505035281181335,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6606087684631348,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.12925539910793304,
+      "rewards/margins": 2.922895908355713,
+      "rewards/rejected": -3.0521514415740967,
+      "step": 456,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.3687249438995487,
+      "grad_norm": 6.369676113128662,
+      "learning_rate": 0.00014938232425709672,
+      "logits/chosen": -0.7123354077339172,
+      "logits/rejected": -0.7202441692352295,
+      "logps/chosen": -7.666382312774658,
+      "logps/rejected": -39.06452178955078,
+      "loss": 0.5345906019210815,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.288968950510025,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.09636388719081879,
+      "rewards/margins": 2.4988255500793457,
+      "rewards/rejected": -2.4024617671966553,
+      "step": 457,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.36953178185118885,
+      "grad_norm": 4.273453712463379,
+      "learning_rate": 0.00014915004917131344,
+      "logits/chosen": -0.7618826627731323,
+      "logits/rejected": -0.758380115032196,
+      "logps/chosen": -9.670495986938477,
+      "logps/rejected": -27.89179039001465,
+      "loss": 0.6546672582626343,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.41434964537620544,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.03314019367098808,
+      "rewards/margins": 2.1142916679382324,
+      "rewards/rejected": -2.081151247024536,
+      "step": 458,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.37033861980282895,
+      "grad_norm": 1.8925552368164062,
+      "learning_rate": 0.0001489174239223172,
+      "logits/chosen": -0.7534856200218201,
+      "logits/rejected": -0.7552649974822998,
+      "logps/chosen": -7.762975215911865,
+      "logps/rejected": -33.306766510009766,
+      "loss": 0.44340604543685913,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.23383133113384247,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.27828943729400635,
+      "rewards/margins": 2.5259461402893066,
+      "rewards/rejected": -2.247657060623169,
+      "step": 459,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.3711454577544691,
+      "grad_norm": 3.5424697399139404,
+      "learning_rate": 0.00014868445016741677,
+      "logits/chosen": -0.7251965403556824,
+      "logits/rejected": -0.7256088852882385,
+      "logps/chosen": -13.685113906860352,
+      "logps/rejected": -30.76580238342285,
+      "loss": 0.8778742551803589,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.578425407409668,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.04395178705453873,
+      "rewards/margins": 2.217789649963379,
+      "rewards/rejected": -2.173837900161743,
+      "step": 460,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.37195229570610927,
+      "grad_norm": 4.210732460021973,
+      "learning_rate": 0.00014845112956640369,
+      "logits/chosen": -0.7375807166099548,
+      "logits/rejected": -0.7479276657104492,
+      "logps/chosen": -10.048831939697266,
+      "logps/rejected": -41.19210433959961,
+      "loss": 0.5977764129638672,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3318694531917572,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.19474266469478607,
+      "rewards/margins": 3.0025835037231445,
+      "rewards/rejected": -2.8078413009643555,
+      "step": 461,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.37275913365774943,
+      "grad_norm": 4.395127296447754,
+      "learning_rate": 0.00014821746378154062,
+      "logits/chosen": -0.6776711940765381,
+      "logits/rejected": -0.6841428279876709,
+      "logps/chosen": -6.95946741104126,
+      "logps/rejected": -36.13311767578125,
+      "loss": 0.4928424060344696,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.30789151787757874,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.32969141006469727,
+      "rewards/margins": 2.7516045570373535,
+      "rewards/rejected": -2.4219131469726562,
+      "step": 462,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.3735659716093896,
+      "grad_norm": 3.365333318710327,
+      "learning_rate": 0.00014798345447754948,
+      "logits/chosen": -0.7575942277908325,
+      "logits/rejected": -0.7648032903671265,
+      "logps/chosen": -8.037223815917969,
+      "logps/rejected": -43.406375885009766,
+      "loss": 0.5700507164001465,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.37956228852272034,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2864043116569519,
+      "rewards/margins": 3.18467116355896,
+      "rewards/rejected": -2.8982667922973633,
+      "step": 463,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.37437280956102975,
+      "grad_norm": 7.311818599700928,
+      "learning_rate": 0.0001477491033215994,
+      "logits/chosen": -0.6993564963340759,
+      "logits/rejected": -0.7100475430488586,
+      "logps/chosen": -11.119054794311523,
+      "logps/rejected": -40.00886917114258,
+      "loss": 1.3310893774032593,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6867990493774414,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.33331215381622314,
+      "rewards/margins": 2.3917033672332764,
+      "rewards/rejected": -2.725015640258789,
+      "step": 464,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.37517964751266986,
+      "grad_norm": 10.936890602111816,
+      "learning_rate": 0.00014751441198329523,
+      "logits/chosen": -0.6920858025550842,
+      "logits/rejected": -0.6920123100280762,
+      "logps/chosen": -14.5137939453125,
+      "logps/rejected": -37.997467041015625,
+      "loss": 0.9086753726005554,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5238281488418579,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.10487417131662369,
+      "rewards/margins": 2.618326187133789,
+      "rewards/rejected": -2.7232000827789307,
+      "step": 465,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.37598648546431,
+      "grad_norm": 15.677000045776367,
+      "learning_rate": 0.0001472793821346652,
+      "logits/chosen": -0.6896384954452515,
+      "logits/rejected": -0.6940050721168518,
+      "logps/chosen": -10.987081527709961,
+      "logps/rejected": -40.3748779296875,
+      "loss": 0.6646585464477539,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.45032551884651184,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.16333404183387756,
+      "rewards/margins": 2.996572494506836,
+      "rewards/rejected": -2.8332386016845703,
+      "step": 466,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.3767933234159502,
+      "grad_norm": 4.351585865020752,
+      "learning_rate": 0.00014704401545014925,
+      "logits/chosen": -0.7135500311851501,
+      "logits/rejected": -0.7152791023254395,
+      "logps/chosen": -9.104679107666016,
+      "logps/rejected": -35.246437072753906,
+      "loss": 0.7736812829971313,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.42992913722991943,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.11151327192783356,
+      "rewards/margins": 2.5536930561065674,
+      "rewards/rejected": -2.4421796798706055,
+      "step": 467,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.37760016136759034,
+      "grad_norm": 5.113152980804443,
+      "learning_rate": 0.00014680831360658713,
+      "logits/chosen": -0.631149172782898,
+      "logits/rejected": -0.6411234140396118,
+      "logps/chosen": -9.986248016357422,
+      "logps/rejected": -47.841514587402344,
+      "loss": 0.6793885231018066,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4218906760215759,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.06469957530498505,
+      "rewards/margins": 3.1944708824157715,
+      "rewards/rejected": -3.1297712326049805,
+      "step": 468,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.3784069993192305,
+      "grad_norm": 5.426872730255127,
+      "learning_rate": 0.00014657227828320635,
+      "logits/chosen": -0.6784893274307251,
+      "logits/rejected": -0.6818794012069702,
+      "logps/chosen": -11.335710525512695,
+      "logps/rejected": -35.59712219238281,
+      "loss": 0.751977264881134,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4554847776889801,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.10514962673187256,
+      "rewards/margins": 2.4257240295410156,
+      "rewards/rejected": -2.3205745220184326,
+      "step": 469,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.3792138372708706,
+      "grad_norm": 3.7496237754821777,
+      "learning_rate": 0.0001463359111616103,
+      "logits/chosen": -0.659194827079773,
+      "logits/rejected": -0.6669769883155823,
+      "logps/chosen": -14.53132438659668,
+      "logps/rejected": -37.95043182373047,
+      "loss": 0.8416832089424133,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.547598659992218,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.08502380549907684,
+      "rewards/margins": 2.0641098022460938,
+      "rewards/rejected": -1.9790860414505005,
+      "step": 470,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.38002067522251076,
+      "grad_norm": 2.0352225303649902,
+      "learning_rate": 0.0001460992139257661,
+      "logits/chosen": -0.6402337551116943,
+      "logits/rejected": -0.6429924964904785,
+      "logps/chosen": -5.608386039733887,
+      "logps/rejected": -33.254302978515625,
+      "loss": 0.3905201256275177,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2183467447757721,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.20369279384613037,
+      "rewards/margins": 2.4486987590789795,
+      "rewards/rejected": -2.2450060844421387,
+      "step": 471,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.3808275131741509,
+      "grad_norm": 5.74570894241333,
+      "learning_rate": 0.00014586218826199285,
+      "logits/chosen": -0.643539547920227,
+      "logits/rejected": -0.6555419564247131,
+      "logps/chosen": -10.091498374938965,
+      "logps/rejected": -36.76713562011719,
+      "loss": 0.6122481226921082,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4549962282180786,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.24730682373046875,
+      "rewards/margins": 2.464998722076416,
+      "rewards/rejected": -2.2176918983459473,
+      "step": 472,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.3816343511257911,
+      "grad_norm": 2.2472074031829834,
+      "learning_rate": 0.00014562483585894945,
+      "logits/chosen": -0.6934607028961182,
+      "logits/rejected": -0.6990819573402405,
+      "logps/chosen": -8.899802207946777,
+      "logps/rejected": -36.28021240234375,
+      "loss": 0.5312250852584839,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.31914129853248596,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.30887866020202637,
+      "rewards/margins": 2.640252113342285,
+      "rewards/rejected": -2.3313732147216797,
+      "step": 473,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.38244118907743124,
+      "grad_norm": 10.326676368713379,
+      "learning_rate": 0.0001453871584076226,
+      "logits/chosen": -0.6442930102348328,
+      "logits/rejected": -0.6484919786453247,
+      "logps/chosen": -9.037781715393066,
+      "logps/rejected": -27.98940658569336,
+      "loss": 0.7469712495803833,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4642283320426941,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.17530076205730438,
+      "rewards/margins": 1.7467072010040283,
+      "rewards/rejected": -1.571406364440918,
+      "step": 474,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.3832480270290714,
+      "grad_norm": 9.987951278686523,
+      "learning_rate": 0.00014514915760131483,
+      "logits/chosen": -0.7020606994628906,
+      "logits/rejected": -0.702151894569397,
+      "logps/chosen": -9.714710235595703,
+      "logps/rejected": -23.569639205932617,
+      "loss": 0.7522432208061218,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.40092915296554565,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1317347288131714,
+      "rewards/margins": 1.5514979362487793,
+      "rewards/rejected": -1.4197630882263184,
+      "step": 475,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.3840548649807115,
+      "grad_norm": 2.1257741451263428,
+      "learning_rate": 0.00014491083513563222,
+      "logits/chosen": -0.6923786401748657,
+      "logits/rejected": -0.6987132430076599,
+      "logps/chosen": -7.563580513000488,
+      "logps/rejected": -29.709890365600586,
+      "loss": 0.5339487195014954,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.31400057673454285,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.370953232049942,
+      "rewards/margins": 2.0969557762145996,
+      "rewards/rejected": -1.7260024547576904,
+      "step": 476,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.38486170293235167,
+      "grad_norm": 3.4022998809814453,
+      "learning_rate": 0.00014467219270847265,
+      "logits/chosen": -0.6793309450149536,
+      "logits/rejected": -0.6822490692138672,
+      "logps/chosen": -8.713945388793945,
+      "logps/rejected": -35.1626091003418,
+      "loss": 0.5065045356750488,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2830030620098114,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1324760913848877,
+      "rewards/margins": 2.0448875427246094,
+      "rewards/rejected": -1.9124114513397217,
+      "step": 477,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.38566854088399183,
+      "grad_norm": 3.169051170349121,
+      "learning_rate": 0.0001444332320200134,
+      "logits/chosen": -0.7434568405151367,
+      "logits/rejected": -0.7467195391654968,
+      "logps/chosen": -8.970917701721191,
+      "logps/rejected": -25.424684524536133,
+      "loss": 0.7533260583877563,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3945104777812958,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.08574176579713821,
+      "rewards/margins": 1.5797266960144043,
+      "rewards/rejected": -1.4939846992492676,
+      "step": 478,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.386475378835632,
+      "grad_norm": 3.456509828567505,
+      "learning_rate": 0.00014419395477269922,
+      "logits/chosen": -0.7100328207015991,
+      "logits/rejected": -0.7100369334220886,
+      "logps/chosen": -8.946069717407227,
+      "logps/rejected": -26.77371597290039,
+      "loss": 0.703390896320343,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3992156982421875,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.23448720574378967,
+      "rewards/margins": 1.9379780292510986,
+      "rewards/rejected": -1.7034906148910522,
+      "step": 479,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.38728221678727215,
+      "grad_norm": 2.8148579597473145,
+      "learning_rate": 0.00014395436267123016,
+      "logits/chosen": -0.7651592493057251,
+      "logits/rejected": -0.7705404758453369,
+      "logps/chosen": -9.789369583129883,
+      "logps/rejected": -32.379886627197266,
+      "loss": 0.6621928811073303,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.42258700728416443,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3690515160560608,
+      "rewards/margins": 2.27026629447937,
+      "rewards/rejected": -1.9012147188186646,
+      "step": 480,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.3880890547389123,
+      "grad_norm": 2.1083407402038574,
+      "learning_rate": 0.00014371445742254933,
+      "logits/chosen": -0.7674591541290283,
+      "logits/rejected": -0.7766936421394348,
+      "logps/chosen": -7.339907646179199,
+      "logps/rejected": -41.22381591796875,
+      "loss": 0.46437036991119385,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.32933878898620605,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4231037497520447,
+      "rewards/margins": 2.872163772583008,
+      "rewards/rejected": -2.4490599632263184,
+      "step": 481,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.3888958926905524,
+      "grad_norm": 9.479911804199219,
+      "learning_rate": 0.00014347424073583086,
+      "logits/chosen": -0.7989487648010254,
+      "logits/rejected": -0.8022449016571045,
+      "logps/chosen": -6.2405619621276855,
+      "logps/rejected": -35.935890197753906,
+      "loss": 0.4960899353027344,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.32181254029273987,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3059425950050354,
+      "rewards/margins": 2.816394805908203,
+      "rewards/rejected": -2.5104520320892334,
+      "step": 482,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.3897027306421926,
+      "grad_norm": 30.05691909790039,
+      "learning_rate": 0.00014323371432246773,
+      "logits/chosen": -0.7825378775596619,
+      "logits/rejected": -0.7933679819107056,
+      "logps/chosen": -7.802068710327148,
+      "logps/rejected": -29.27901840209961,
+      "loss": 0.8247991800308228,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.445316344499588,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.19992296397686005,
+      "rewards/margins": 1.4237523078918457,
+      "rewards/rejected": -1.2238292694091797,
+      "step": 483,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.39050956859383273,
+      "grad_norm": 3.872990131378174,
+      "learning_rate": 0.00014299287989605943,
+      "logits/chosen": -0.7813780903816223,
+      "logits/rejected": -0.7811882495880127,
+      "logps/chosen": -9.697345733642578,
+      "logps/rejected": -30.960369110107422,
+      "loss": 0.5493797063827515,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3628386855125427,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.29603177309036255,
+      "rewards/margins": 2.3790576457977295,
+      "rewards/rejected": -2.0830256938934326,
+      "step": 484,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.3913164065454729,
+      "grad_norm": 12.008702278137207,
+      "learning_rate": 0.00014275173917239993,
+      "logits/chosen": -0.7886897325515747,
+      "logits/rejected": -0.7941228747367859,
+      "logps/chosen": -7.900599002838135,
+      "logps/rejected": -31.16321563720703,
+      "loss": 0.7939125895500183,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.45570316910743713,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.10572385787963867,
+      "rewards/margins": 1.8910467624664307,
+      "rewards/rejected": -1.785322904586792,
+      "step": 485,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.39212324449711305,
+      "grad_norm": 5.970835208892822,
+      "learning_rate": 0.00014251029386946532,
+      "logits/chosen": -0.8634493350982666,
+      "logits/rejected": -0.8646601438522339,
+      "logps/chosen": -6.923543453216553,
+      "logps/rejected": -28.410593032836914,
+      "loss": 0.5631651282310486,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.31717023253440857,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.24359825253486633,
+      "rewards/margins": 2.0541610717773438,
+      "rewards/rejected": -1.8105626106262207,
+      "step": 486,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.39293008244875316,
+      "grad_norm": 2.754620313644409,
+      "learning_rate": 0.00014226854570740168,
+      "logits/chosen": -0.8398474454879761,
+      "logits/rejected": -0.8498659133911133,
+      "logps/chosen": -7.03415584564209,
+      "logps/rejected": -40.334163665771484,
+      "loss": 0.4721854329109192,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.34017014503479004,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.16663886606693268,
+      "rewards/margins": 2.977405309677124,
+      "rewards/rejected": -2.8107664585113525,
+      "step": 487,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.3937369204003933,
+      "grad_norm": 2.9349205493927,
+      "learning_rate": 0.00014202649640851275,
+      "logits/chosen": -0.8891080617904663,
+      "logits/rejected": -0.8925038576126099,
+      "logps/chosen": -5.047273635864258,
+      "logps/rejected": -34.245174407958984,
+      "loss": 0.3938831090927124,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.21636192500591278,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2093987911939621,
+      "rewards/margins": 2.6690750122070312,
+      "rewards/rejected": -2.4596762657165527,
+      "step": 488,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.3945437583520335,
+      "grad_norm": 6.7003173828125,
+      "learning_rate": 0.00014178414769724766,
+      "logits/chosen": -0.8764248490333557,
+      "logits/rejected": -0.876374363899231,
+      "logps/chosen": -8.440370559692383,
+      "logps/rejected": -35.263214111328125,
+      "loss": 0.5095587968826294,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.34542787075042725,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.19051702320575714,
+      "rewards/margins": 2.7904720306396484,
+      "rewards/rejected": -2.5999550819396973,
+      "step": 489,
+      "train_speed(iter/s)": 0.005438
+    },
+    {
+      "epoch": 0.39535059630367364,
+      "grad_norm": 3.259153366088867,
+      "learning_rate": 0.00014154150130018866,
+      "logits/chosen": -0.9377500414848328,
+      "logits/rejected": -0.9458149671554565,
+      "logps/chosen": -11.506027221679688,
+      "logps/rejected": -35.232669830322266,
+      "loss": 0.6641853451728821,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.46648576855659485,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3142199218273163,
+      "rewards/margins": 2.9259414672851562,
+      "rewards/rejected": -2.6117210388183594,
+      "step": 490,
+      "train_speed(iter/s)": 0.005438
+    },
+    {
+      "epoch": 0.3961574342553138,
+      "grad_norm": 2.46926212310791,
+      "learning_rate": 0.00014129855894603886,
+      "logits/chosen": -0.9563973546028137,
+      "logits/rejected": -0.9663281440734863,
+      "logps/chosen": -8.163786888122559,
+      "logps/rejected": -45.545860290527344,
+      "loss": 0.462195485830307,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.34487971663475037,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.1903153955936432,
+      "rewards/margins": 3.344923257827759,
+      "rewards/rejected": -3.1546080112457275,
+      "step": 491,
+      "train_speed(iter/s)": 0.005438
+    },
+    {
+      "epoch": 0.39696427220695396,
+      "grad_norm": 11.747800827026367,
+      "learning_rate": 0.00014105532236560983,
+      "logits/chosen": -0.9172489047050476,
+      "logits/rejected": -0.9255672693252563,
+      "logps/chosen": -8.56010627746582,
+      "logps/rejected": -44.462188720703125,
+      "loss": 0.5068323612213135,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3179493546485901,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.13865207135677338,
+      "rewards/margins": 2.8126437664031982,
+      "rewards/rejected": -2.6739919185638428,
+      "step": 492,
+      "train_speed(iter/s)": 0.005438
+    },
+    {
+      "epoch": 0.39777111015859407,
+      "grad_norm": 5.142593860626221,
+      "learning_rate": 0.00014081179329180943,
+      "logits/chosen": -0.9059480428695679,
+      "logits/rejected": -0.9175886511802673,
+      "logps/chosen": -5.26338529586792,
+      "logps/rejected": -45.20058822631836,
+      "loss": 0.40461504459381104,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.20817551016807556,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.24090781807899475,
+      "rewards/margins": 3.5246479511260986,
+      "rewards/rejected": -3.283740282058716,
+      "step": 493,
+      "train_speed(iter/s)": 0.005438
+    },
+    {
+      "epoch": 0.3985779481102342,
+      "grad_norm": 13.850090980529785,
+      "learning_rate": 0.00014056797345962915,
+      "logits/chosen": -0.97746741771698,
+      "logits/rejected": -0.9824937582015991,
+      "logps/chosen": -8.306085586547852,
+      "logps/rejected": -45.58984375,
+      "loss": 0.7906121015548706,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5601307153701782,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.0417952798306942,
+      "rewards/margins": 3.4094972610473633,
+      "rewards/rejected": -3.3677022457122803,
+      "step": 494,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.3993847860618744,
+      "grad_norm": 2.4637293815612793,
+      "learning_rate": 0.0001403238646061322,
+      "logits/chosen": -0.9754372835159302,
+      "logits/rejected": -0.9848412275314331,
+      "logps/chosen": -7.990797996520996,
+      "logps/rejected": -48.4688720703125,
+      "loss": 0.4341033101081848,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.308696985244751,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.27976250648498535,
+      "rewards/margins": 3.742406129837036,
+      "rewards/rejected": -3.462643623352051,
+      "step": 495,
+      "train_speed(iter/s)": 0.005438
+    },
+    {
+      "epoch": 0.40019162401351455,
+      "grad_norm": 3.6680634021759033,
+      "learning_rate": 0.00014007946847044065,
+      "logits/chosen": -0.951252818107605,
+      "logits/rejected": -0.9680181741714478,
+      "logps/chosen": -4.847614765167236,
+      "logps/rejected": -46.7112922668457,
+      "loss": 0.383756160736084,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.24948665499687195,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.17591829597949982,
+      "rewards/margins": 3.246157646179199,
+      "rewards/rejected": -3.070239305496216,
+      "step": 496,
+      "train_speed(iter/s)": 0.005438
+    },
+    {
+      "epoch": 0.4009984619651547,
+      "grad_norm": 3.807244062423706,
+      "learning_rate": 0.00013983478679372344,
+      "logits/chosen": -0.9732386469841003,
+      "logits/rejected": -0.9831098318099976,
+      "logps/chosen": -7.762671947479248,
+      "logps/rejected": -52.341617584228516,
+      "loss": 0.43133464455604553,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2912958860397339,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1585668921470642,
+      "rewards/margins": 3.8791747093200684,
+      "rewards/rejected": -3.7206082344055176,
+      "step": 497,
+      "train_speed(iter/s)": 0.005438
+    },
+    {
+      "epoch": 0.4018052999167948,
+      "grad_norm": 14.390305519104004,
+      "learning_rate": 0.00013958982131918372,
+      "logits/chosen": -0.9233254194259644,
+      "logits/rejected": -0.926517128944397,
+      "logps/chosen": -11.224777221679688,
+      "logps/rejected": -33.05560302734375,
+      "loss": 0.8548044562339783,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4569489359855652,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.0009491965174674988,
+      "rewards/margins": 2.3791630268096924,
+      "rewards/rejected": -2.378213882446289,
+      "step": 498,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.40261213786843497,
+      "grad_norm": 7.265214443206787,
+      "learning_rate": 0.00013934457379204658,
+      "logits/chosen": -0.9037230014801025,
+      "logits/rejected": -0.9072719216346741,
+      "logps/chosen": -13.305590629577637,
+      "logps/rejected": -42.6323127746582,
+      "loss": 0.5567822456359863,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4082874059677124,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3724799156188965,
+      "rewards/margins": 3.5428996086120605,
+      "rewards/rejected": -3.1704201698303223,
+      "step": 499,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.40341897582007513,
+      "grad_norm": 2.8691728115081787,
+      "learning_rate": 0.0001390990459595465,
+      "logits/chosen": -0.9267888069152832,
+      "logits/rejected": -0.936477780342102,
+      "logps/chosen": -9.545984268188477,
+      "logps/rejected": -42.449127197265625,
+      "loss": 0.7537754774093628,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5160152912139893,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.12287023663520813,
+      "rewards/margins": 3.151482105255127,
+      "rewards/rejected": -3.0286121368408203,
+      "step": 500,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.4042258137717153,
+      "grad_norm": 17.34552764892578,
+      "learning_rate": 0.00013885323957091503,
+      "logits/chosen": -0.8896504640579224,
+      "logits/rejected": -0.8915640115737915,
+      "logps/chosen": -9.813822746276855,
+      "logps/rejected": -36.02322769165039,
+      "loss": 1.0269454717636108,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6156859397888184,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.17760202288627625,
+      "rewards/margins": 2.4105653762817383,
+      "rewards/rejected": -2.588167667388916,
+      "step": 501,
+      "train_speed(iter/s)": 0.005438
+    },
+    {
+      "epoch": 0.40503265172335545,
+      "grad_norm": 2.611849069595337,
+      "learning_rate": 0.00013860715637736818,
+      "logits/chosen": -0.8846621513366699,
+      "logits/rejected": -0.8913875222206116,
+      "logps/chosen": -7.3572540283203125,
+      "logps/rejected": -43.120914459228516,
+      "loss": 0.5870882868766785,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.45941975712776184,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.24965612590312958,
+      "rewards/margins": 3.1028006076812744,
+      "rewards/rejected": -2.8531441688537598,
+      "step": 502,
+      "train_speed(iter/s)": 0.005438
+    },
+    {
+      "epoch": 0.4058394896749956,
+      "grad_norm": 9.19338607788086,
+      "learning_rate": 0.00013836079813209405,
+      "logits/chosen": -0.8596163988113403,
+      "logits/rejected": -0.8734834790229797,
+      "logps/chosen": -6.4730024337768555,
+      "logps/rejected": -48.3375358581543,
+      "loss": 0.5926974415779114,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.40696945786476135,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.13774055242538452,
+      "rewards/margins": 3.3298587799072266,
+      "rewards/rejected": -3.1921184062957764,
+      "step": 503,
+      "train_speed(iter/s)": 0.005438
+    },
+    {
+      "epoch": 0.4066463276266357,
+      "grad_norm": 2.5772955417633057,
+      "learning_rate": 0.00013811416659024035,
+      "logits/chosen": -0.8637710809707642,
+      "logits/rejected": -0.869887113571167,
+      "logps/chosen": -8.725384712219238,
+      "logps/rejected": -44.57448959350586,
+      "loss": 0.5925300717353821,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.48120686411857605,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.12150951474905014,
+      "rewards/margins": 3.436396598815918,
+      "rewards/rejected": -3.314887046813965,
+      "step": 504,
+      "train_speed(iter/s)": 0.005438
+    },
+    {
+      "epoch": 0.4074531655782759,
+      "grad_norm": 4.892908096313477,
+      "learning_rate": 0.00013786726350890183,
+      "logits/chosen": -0.874576210975647,
+      "logits/rejected": -0.8784814476966858,
+      "logps/chosen": -9.178255081176758,
+      "logps/rejected": -38.52834701538086,
+      "loss": 0.6706528067588806,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.49778318405151367,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.21889682114124298,
+      "rewards/margins": 3.011408805847168,
+      "rewards/rejected": -2.7925119400024414,
+      "step": 505,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.40826000352991604,
+      "grad_norm": 2.44610333442688,
+      "learning_rate": 0.00013762009064710774,
+      "logits/chosen": -0.8441749811172485,
+      "logits/rejected": -0.8464869260787964,
+      "logps/chosen": -7.5994648933410645,
+      "logps/rejected": -34.768951416015625,
+      "loss": 0.5083529353141785,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3053922653198242,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.30249646306037903,
+      "rewards/margins": 2.588782787322998,
+      "rewards/rejected": -2.2862861156463623,
+      "step": 506,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.4090668414815562,
+      "grad_norm": 6.779381275177002,
+      "learning_rate": 0.00013737264976580943,
+      "logits/chosen": -0.8816607594490051,
+      "logits/rejected": -0.8868498206138611,
+      "logps/chosen": -11.123010635375977,
+      "logps/rejected": -37.859195709228516,
+      "loss": 0.8256106376647949,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.40417546033859253,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.1513340175151825,
+      "rewards/margins": 2.473325252532959,
+      "rewards/rejected": -2.624659299850464,
+      "step": 507,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.40987367943319636,
+      "grad_norm": 3.7313573360443115,
+      "learning_rate": 0.00013712494262786766,
+      "logits/chosen": -0.8750671148300171,
+      "logits/rejected": -0.8821631669998169,
+      "logps/chosen": -10.883421897888184,
+      "logps/rejected": -42.575679779052734,
+      "loss": 0.748051643371582,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5597954988479614,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.19764858484268188,
+      "rewards/margins": 3.004587173461914,
+      "rewards/rejected": -2.806938409805298,
+      "step": 508,
+      "train_speed(iter/s)": 0.005438
+    },
+    {
+      "epoch": 0.41068051738483646,
+      "grad_norm": 2.834299087524414,
+      "learning_rate": 0.0001368769709980401,
+      "logits/chosen": -0.862756073474884,
+      "logits/rejected": -0.8707681894302368,
+      "logps/chosen": -8.581947326660156,
+      "logps/rejected": -42.503753662109375,
+      "loss": 0.3937508761882782,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.23043422400951385,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2632564604282379,
+      "rewards/margins": 2.976945400238037,
+      "rewards/rejected": -2.713688850402832,
+      "step": 509,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.4114873553364766,
+      "grad_norm": 2.886272668838501,
+      "learning_rate": 0.00013662873664296884,
+      "logits/chosen": -0.819119930267334,
+      "logits/rejected": -0.826356053352356,
+      "logps/chosen": -8.14879035949707,
+      "logps/rejected": -42.62828826904297,
+      "loss": 0.5604068636894226,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.31821465492248535,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3842453360557556,
+      "rewards/margins": 3.0441181659698486,
+      "rewards/rejected": -2.6598730087280273,
+      "step": 510,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.4122941932881168,
+      "grad_norm": 3.3949501514434814,
+      "learning_rate": 0.00013638024133116763,
+      "logits/chosen": -0.802520215511322,
+      "logits/rejected": -0.8098280429840088,
+      "logps/chosen": -12.40538215637207,
+      "logps/rejected": -38.84916305541992,
+      "loss": 0.8826647400856018,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5703054666519165,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.012079771608114243,
+      "rewards/margins": 2.588003396987915,
+      "rewards/rejected": -2.575923442840576,
+      "step": 511,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.41310103123975694,
+      "grad_norm": 6.568037509918213,
+      "learning_rate": 0.00013613148683300937,
+      "logits/chosen": -0.8070806860923767,
+      "logits/rejected": -0.813240647315979,
+      "logps/chosen": -9.467267990112305,
+      "logps/rejected": -39.797325134277344,
+      "loss": 0.5164275765419006,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.39448225498199463,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.28209254145622253,
+      "rewards/margins": 2.9220352172851562,
+      "rewards/rejected": -2.6399426460266113,
+      "step": 512,
+      "train_speed(iter/s)": 0.00544
+    },
+    {
+      "epoch": 0.4139078691913971,
+      "grad_norm": 10.136809349060059,
+      "learning_rate": 0.0001358824749207136,
+      "logits/chosen": -0.8575885891914368,
+      "logits/rejected": -0.8593306541442871,
+      "logps/chosen": -10.6421480178833,
+      "logps/rejected": -39.468055725097656,
+      "loss": 0.7010791301727295,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5033397674560547,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.156565859913826,
+      "rewards/margins": 2.7266011238098145,
+      "rewards/rejected": -2.570035219192505,
+      "step": 513,
+      "train_speed(iter/s)": 0.00544
+    },
+    {
+      "epoch": 0.41471470714303726,
+      "grad_norm": 3.576723098754883,
+      "learning_rate": 0.00013563320736833365,
+      "logits/chosen": -0.8105019330978394,
+      "logits/rejected": -0.8146560788154602,
+      "logps/chosen": -10.574302673339844,
+      "logps/rejected": -36.661590576171875,
+      "loss": 0.665181040763855,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4042365849018097,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.13927578926086426,
+      "rewards/margins": 2.6975083351135254,
+      "rewards/rejected": -2.558232545852661,
+      "step": 514,
+      "train_speed(iter/s)": 0.00544
+    },
+    {
+      "epoch": 0.41552154509467737,
+      "grad_norm": 2.363494396209717,
+      "learning_rate": 0.00013538368595174423,
+      "logits/chosen": -0.8046780228614807,
+      "logits/rejected": -0.8101540803909302,
+      "logps/chosen": -12.763677597045898,
+      "logps/rejected": -39.75190353393555,
+      "loss": 0.5567308664321899,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4450889527797699,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.285984069108963,
+      "rewards/margins": 3.129922389984131,
+      "rewards/rejected": -2.843937873840332,
+      "step": 515,
+      "train_speed(iter/s)": 0.00544
+    },
+    {
+      "epoch": 0.41632838304631753,
+      "grad_norm": 2.798708438873291,
+      "learning_rate": 0.0001351339124486286,
+      "logits/chosen": -0.823294460773468,
+      "logits/rejected": -0.8240728974342346,
+      "logps/chosen": -8.804251670837402,
+      "logps/rejected": -36.77720260620117,
+      "loss": 0.5373676419258118,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.35823625326156616,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.25987887382507324,
+      "rewards/margins": 2.7686750888824463,
+      "rewards/rejected": -2.508796453475952,
+      "step": 516,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.4171352209979577,
+      "grad_norm": 3.738819122314453,
+      "learning_rate": 0.00013488388863846598,
+      "logits/chosen": -0.8327402472496033,
+      "logits/rejected": -0.8359934091567993,
+      "logps/chosen": -12.034650802612305,
+      "logps/rejected": -33.55274200439453,
+      "loss": 0.7193576097488403,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.506946325302124,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.16285476088523865,
+      "rewards/margins": 2.3808743953704834,
+      "rewards/rejected": -2.218019485473633,
+      "step": 517,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.41794205894959785,
+      "grad_norm": 2.734147071838379,
+      "learning_rate": 0.00013463361630251897,
+      "logits/chosen": -0.8137484788894653,
+      "logits/rejected": -0.8196120858192444,
+      "logps/chosen": -16.61955451965332,
+      "logps/rejected": -33.3659553527832,
+      "loss": 0.8027633428573608,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5399143099784851,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.31846553087234497,
+      "rewards/margins": 2.49395751953125,
+      "rewards/rejected": -2.1754918098449707,
+      "step": 518,
+      "train_speed(iter/s)": 0.00544
+    },
+    {
+      "epoch": 0.418748896901238,
+      "grad_norm": 1.6672159433364868,
+      "learning_rate": 0.00013438309722382056,
+      "logits/chosen": -0.8162409663200378,
+      "logits/rejected": -0.8220106959342957,
+      "logps/chosen": -8.6817045211792,
+      "logps/rejected": -37.332298278808594,
+      "loss": 0.4413672685623169,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2940937876701355,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.28105995059013367,
+      "rewards/margins": 2.7084476947784424,
+      "rewards/rejected": -2.4273879528045654,
+      "step": 519,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.4195557348528781,
+      "grad_norm": 2.7764697074890137,
+      "learning_rate": 0.00013413233318716188,
+      "logits/chosen": -0.8464371562004089,
+      "logits/rejected": -0.8485240936279297,
+      "logps/chosen": -10.62841796875,
+      "logps/rejected": -36.70519256591797,
+      "loss": 0.7128565907478333,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.46534037590026855,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.22482553124427795,
+      "rewards/margins": 2.6670079231262207,
+      "rewards/rejected": -2.4421825408935547,
+      "step": 520,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.4203625728045183,
+      "grad_norm": 4.593116760253906,
+      "learning_rate": 0.00013388132597907904,
+      "logits/chosen": -0.8110215663909912,
+      "logits/rejected": -0.8151426315307617,
+      "logps/chosen": -10.328957557678223,
+      "logps/rejected": -38.320430755615234,
+      "loss": 0.6786235570907593,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.339311420917511,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.1592826545238495,
+      "rewards/margins": 2.4331724643707275,
+      "rewards/rejected": -2.2738895416259766,
+      "step": 521,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.42116941075615844,
+      "grad_norm": 4.485230445861816,
+      "learning_rate": 0.0001336300773878407,
+      "logits/chosen": -0.8129779696464539,
+      "logits/rejected": -0.8200620412826538,
+      "logps/chosen": -7.188672065734863,
+      "logps/rejected": -38.45001220703125,
+      "loss": 0.6729139089584351,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4256826639175415,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.16748948395252228,
+      "rewards/margins": 2.9809718132019043,
+      "rewards/rejected": -2.8134822845458984,
+      "step": 522,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.4219762487077986,
+      "grad_norm": 3.1994502544403076,
+      "learning_rate": 0.00013337858920343512,
+      "logits/chosen": -0.8458603024482727,
+      "logits/rejected": -0.852841854095459,
+      "logps/chosen": -8.571792602539062,
+      "logps/rejected": -42.51072692871094,
+      "loss": 0.4996880292892456,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2992246150970459,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.23098604381084442,
+      "rewards/margins": 3.1265108585357666,
+      "rewards/rejected": -2.895524740219116,
+      "step": 523,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.42278308665943876,
+      "grad_norm": 3.704655170440674,
+      "learning_rate": 0.00013312686321755761,
+      "logits/chosen": -0.8289926648139954,
+      "logits/rejected": -0.8301224708557129,
+      "logps/chosen": -7.220808029174805,
+      "logps/rejected": -33.644962310791016,
+      "loss": 0.4415580928325653,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2433617264032364,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.19586443901062012,
+      "rewards/margins": 2.462473154067993,
+      "rewards/rejected": -2.266608238220215,
+      "step": 524,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.4235899246110789,
+      "grad_norm": 14.6280517578125,
+      "learning_rate": 0.0001328749012235976,
+      "logits/chosen": -0.8482613563537598,
+      "logits/rejected": -0.858103334903717,
+      "logps/chosen": -6.205442905426025,
+      "logps/rejected": -37.40229797363281,
+      "loss": 0.7436372637748718,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4056238830089569,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.02807258814573288,
+      "rewards/margins": 2.8922817707061768,
+      "rewards/rejected": -2.9203548431396484,
+      "step": 525,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.424396762562719,
+      "grad_norm": 10.639749526977539,
+      "learning_rate": 0.000132622705016626,
+      "logits/chosen": -0.8543103337287903,
+      "logits/rejected": -0.8550821542739868,
+      "logps/chosen": -8.963075637817383,
+      "logps/rejected": -44.753875732421875,
+      "loss": 0.5276005864143372,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3191993534564972,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.08133304864168167,
+      "rewards/margins": 2.828303098678589,
+      "rewards/rejected": -2.9096360206604004,
+      "step": 526,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.4252036005143592,
+      "grad_norm": 5.606629848480225,
+      "learning_rate": 0.00013237027639338223,
+      "logits/chosen": -0.7930728197097778,
+      "logits/rejected": -0.8049328327178955,
+      "logps/chosen": -11.980341911315918,
+      "logps/rejected": -42.134681701660156,
+      "loss": 0.8973350524902344,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6916304230690002,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2415471076965332,
+      "rewards/margins": 2.853031635284424,
+      "rewards/rejected": -2.6114845275878906,
+      "step": 527,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.42601043846599934,
+      "grad_norm": 3.3900585174560547,
+      "learning_rate": 0.0001321176171522617,
+      "logits/chosen": -0.8453109264373779,
+      "logits/rejected": -0.8487651348114014,
+      "logps/chosen": -10.572664260864258,
+      "logps/rejected": -33.60906982421875,
+      "loss": 0.5498627424240112,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4081331193447113,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.28006234765052795,
+      "rewards/margins": 2.9145495891571045,
+      "rewards/rejected": -2.6344871520996094,
+      "step": 528,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.4268172764176395,
+      "grad_norm": 5.930169582366943,
+      "learning_rate": 0.00013186472909330264,
+      "logits/chosen": -0.8630443215370178,
+      "logits/rejected": -0.8701946139335632,
+      "logps/chosen": -7.021512985229492,
+      "logps/rejected": -36.796783447265625,
+      "loss": 0.7053890824317932,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4606189727783203,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2262631058692932,
+      "rewards/margins": 2.8637092113494873,
+      "rewards/rejected": -2.637446403503418,
+      "step": 529,
+      "train_speed(iter/s)": 0.00544
+    },
+    {
+      "epoch": 0.42762411436927966,
+      "grad_norm": 6.3762688636779785,
+      "learning_rate": 0.00013161161401817353,
+      "logits/chosen": -0.8374471068382263,
+      "logits/rejected": -0.8438255786895752,
+      "logps/chosen": -9.851973533630371,
+      "logps/rejected": -39.308502197265625,
+      "loss": 0.5261674523353577,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.33455437421798706,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.15020446479320526,
+      "rewards/margins": 2.7218151092529297,
+      "rewards/rejected": -2.571610689163208,
+      "step": 530,
+      "train_speed(iter/s)": 0.00544
+    },
+    {
+      "epoch": 0.4284309523209198,
+      "grad_norm": 6.240972995758057,
+      "learning_rate": 0.00013135827373016027,
+      "logits/chosen": -0.8955954313278198,
+      "logits/rejected": -0.9030234813690186,
+      "logps/chosen": -11.273481369018555,
+      "logps/rejected": -34.91842269897461,
+      "loss": 0.7530210614204407,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5049380660057068,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.19476205110549927,
+      "rewards/margins": 2.6257526874542236,
+      "rewards/rejected": -2.430990695953369,
+      "step": 531,
+      "train_speed(iter/s)": 0.00544
+    },
+    {
+      "epoch": 0.4292377902725599,
+      "grad_norm": 3.5795764923095703,
+      "learning_rate": 0.00013110471003415321,
+      "logits/chosen": -0.8652918338775635,
+      "logits/rejected": -0.8691776990890503,
+      "logps/chosen": -8.708220481872559,
+      "logps/rejected": -40.791812896728516,
+      "loss": 0.5590424537658691,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.29747384786605835,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.06008954346179962,
+      "rewards/margins": 2.697359085083008,
+      "rewards/rejected": -2.6372694969177246,
+      "step": 532,
+      "train_speed(iter/s)": 0.00544
+    },
+    {
+      "epoch": 0.4300446282242001,
+      "grad_norm": 3.8843157291412354,
+      "learning_rate": 0.0001308509247366343,
+      "logits/chosen": -0.7854755520820618,
+      "logits/rejected": -0.792789101600647,
+      "logps/chosen": -7.507791519165039,
+      "logps/rejected": -49.7330322265625,
+      "loss": 0.4467013478279114,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.27506744861602783,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.24458295106887817,
+      "rewards/margins": 3.935803174972534,
+      "rewards/rejected": -3.691220283508301,
+      "step": 533,
+      "train_speed(iter/s)": 0.00544
+    },
+    {
+      "epoch": 0.43085146617584025,
+      "grad_norm": 3.57249116897583,
+      "learning_rate": 0.00013059691964566433,
+      "logits/chosen": -0.8396457433700562,
+      "logits/rejected": -0.8505795001983643,
+      "logps/chosen": -8.021794319152832,
+      "logps/rejected": -43.013790130615234,
+      "loss": 0.6681239604949951,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.40499186515808105,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.14213377237319946,
+      "rewards/margins": 3.0652308464050293,
+      "rewards/rejected": -2.9230968952178955,
+      "step": 534,
+      "train_speed(iter/s)": 0.00544
+    },
+    {
+      "epoch": 0.4316583041274804,
+      "grad_norm": 5.327953338623047,
+      "learning_rate": 0.00013034269657086992,
+      "logits/chosen": -0.8565971255302429,
+      "logits/rejected": -0.8621482253074646,
+      "logps/chosen": -7.855462551116943,
+      "logps/rejected": -44.381805419921875,
+      "loss": 0.4837571680545807,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.32484912872314453,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.4074184000492096,
+      "rewards/margins": 3.6497139930725098,
+      "rewards/rejected": -3.242295265197754,
+      "step": 535,
+      "train_speed(iter/s)": 0.00544
+    },
+    {
+      "epoch": 0.43246514207912057,
+      "grad_norm": 4.380790710449219,
+      "learning_rate": 0.00013008825732343073,
+      "logits/chosen": -0.8305094838142395,
+      "logits/rejected": -0.8384683132171631,
+      "logps/chosen": -5.753545761108398,
+      "logps/rejected": -40.71387481689453,
+      "loss": 0.48914748430252075,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.27660253643989563,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2127423733472824,
+      "rewards/margins": 2.9575328826904297,
+      "rewards/rejected": -2.744790554046631,
+      "step": 536,
+      "train_speed(iter/s)": 0.00544
+    },
+    {
+      "epoch": 0.4332719800307607,
+      "grad_norm": 2.478238344192505,
+      "learning_rate": 0.0001298336037160665,
+      "logits/chosen": -0.8390322327613831,
+      "logits/rejected": -0.8421530723571777,
+      "logps/chosen": -9.2628173828125,
+      "logps/rejected": -33.14223861694336,
+      "loss": 0.6333991289138794,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.385562539100647,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.2185012400150299,
+      "rewards/margins": 2.4338998794555664,
+      "rewards/rejected": -2.2153985500335693,
+      "step": 537,
+      "train_speed(iter/s)": 0.00544
+    },
+    {
+      "epoch": 0.43407881798240083,
+      "grad_norm": 4.371514320373535,
+      "learning_rate": 0.00012957873756302415,
+      "logits/chosen": -0.7974674701690674,
+      "logits/rejected": -0.7956443428993225,
+      "logps/chosen": -11.487338066101074,
+      "logps/rejected": -37.02048110961914,
+      "loss": 0.6238091588020325,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4722408056259155,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2411145120859146,
+      "rewards/margins": 2.765226125717163,
+      "rewards/rejected": -2.52411150932312,
+      "step": 538,
+      "train_speed(iter/s)": 0.00544
+    },
+    {
+      "epoch": 0.434885655934041,
+      "grad_norm": 2.3261210918426514,
+      "learning_rate": 0.00012932366068006486,
+      "logits/chosen": -0.8332119584083557,
+      "logits/rejected": -0.8431892395019531,
+      "logps/chosen": -5.291442394256592,
+      "logps/rejected": -40.494781494140625,
+      "loss": 0.35455459356307983,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2400093972682953,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3891444504261017,
+      "rewards/margins": 3.3702661991119385,
+      "rewards/rejected": -2.981121778488159,
+      "step": 539,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.43569249388568115,
+      "grad_norm": 3.177072763442993,
+      "learning_rate": 0.00012906837488445115,
+      "logits/chosen": -0.8743947744369507,
+      "logits/rejected": -0.8769956827163696,
+      "logps/chosen": -9.416547775268555,
+      "logps/rejected": -38.16511917114258,
+      "loss": 0.5091468095779419,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.34846216440200806,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2577158510684967,
+      "rewards/margins": 2.8782832622528076,
+      "rewards/rejected": -2.620567560195923,
+      "step": 540,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.4364993318373213,
+      "grad_norm": 2.1750667095184326,
+      "learning_rate": 0.00012881288199493378,
+      "logits/chosen": -0.8983636498451233,
+      "logits/rejected": -0.9017794132232666,
+      "logps/chosen": -5.9792609214782715,
+      "logps/rejected": -39.93588638305664,
+      "loss": 0.42438769340515137,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2854105830192566,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.38626906275749207,
+      "rewards/margins": 3.287107467651367,
+      "rewards/rejected": -2.9008383750915527,
+      "step": 541,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.4373061697889615,
+      "grad_norm": 4.120799541473389,
+      "learning_rate": 0.00012855718383173915,
+      "logits/chosen": -0.8792024254798889,
+      "logits/rejected": -0.8899410367012024,
+      "logps/chosen": -8.377023696899414,
+      "logps/rejected": -31.686115264892578,
+      "loss": 0.6103077530860901,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3152506947517395,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3780965507030487,
+      "rewards/margins": 2.42580246925354,
+      "rewards/rejected": -2.047706127166748,
+      "step": 542,
+      "train_speed(iter/s)": 0.005442
+    },
+    {
+      "epoch": 0.4381130077406016,
+      "grad_norm": 2.2526559829711914,
+      "learning_rate": 0.0001283012822165559,
+      "logits/chosen": -0.9368709921836853,
+      "logits/rejected": -0.9483405947685242,
+      "logps/chosen": -6.628868103027344,
+      "logps/rejected": -39.02850341796875,
+      "loss": 0.38257747888565063,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.24682456254959106,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.35517507791519165,
+      "rewards/margins": 2.6254923343658447,
+      "rewards/rejected": -2.270317554473877,
+      "step": 543,
+      "train_speed(iter/s)": 0.005442
+    },
+    {
+      "epoch": 0.43891984569224174,
+      "grad_norm": 2.8567118644714355,
+      "learning_rate": 0.00012804517897252227,
+      "logits/chosen": -0.9481716156005859,
+      "logits/rejected": -0.9497692584991455,
+      "logps/chosen": -7.450263977050781,
+      "logps/rejected": -31.115060806274414,
+      "loss": 0.47185516357421875,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3170330822467804,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.31209883093833923,
+      "rewards/margins": 2.585287570953369,
+      "rewards/rejected": -2.273188591003418,
+      "step": 544,
+      "train_speed(iter/s)": 0.005442
+    },
+    {
+      "epoch": 0.4397266836438819,
+      "grad_norm": 3.762911319732666,
+      "learning_rate": 0.00012778887592421293,
+      "logits/chosen": -0.9280496835708618,
+      "logits/rejected": -0.9334405660629272,
+      "logps/chosen": -8.896903991699219,
+      "logps/rejected": -45.50485610961914,
+      "loss": 0.4354952573776245,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2929496467113495,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3440062999725342,
+      "rewards/margins": 3.7558441162109375,
+      "rewards/rejected": -3.4118378162384033,
+      "step": 545,
+      "train_speed(iter/s)": 0.005442
+    },
+    {
+      "epoch": 0.44053352159552206,
+      "grad_norm": 5.700692176818848,
+      "learning_rate": 0.000127532374897626,
+      "logits/chosen": -1.0263915061950684,
+      "logits/rejected": -1.0273609161376953,
+      "logps/chosen": -8.307823181152344,
+      "logps/rejected": -36.91926574707031,
+      "loss": 0.7563344836235046,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.43058276176452637,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.014373952522873878,
+      "rewards/margins": 2.57063889503479,
+      "rewards/rejected": -2.585012912750244,
+      "step": 546,
+      "train_speed(iter/s)": 0.005442
+    },
+    {
+      "epoch": 0.4413403595471622,
+      "grad_norm": 3.6434919834136963,
+      "learning_rate": 0.0001272756777201701,
+      "logits/chosen": -0.9778340458869934,
+      "logits/rejected": -0.9919235110282898,
+      "logps/chosen": -6.4957427978515625,
+      "logps/rejected": -48.30223083496094,
+      "loss": 0.48351603746414185,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.31254929304122925,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2537922263145447,
+      "rewards/margins": 3.731672525405884,
+      "rewards/rejected": -3.4778804779052734,
+      "step": 547,
+      "train_speed(iter/s)": 0.005442
+    },
+    {
+      "epoch": 0.4421471974988023,
+      "grad_norm": 15.765288352966309,
+      "learning_rate": 0.00012701878622065136,
+      "logits/chosen": -1.0241070985794067,
+      "logits/rejected": -1.036070466041565,
+      "logps/chosen": -8.980073928833008,
+      "logps/rejected": -42.19818878173828,
+      "loss": 0.648382842540741,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4570547342300415,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3133217394351959,
+      "rewards/margins": 2.865694999694824,
+      "rewards/rejected": -2.5523734092712402,
+      "step": 548,
+      "train_speed(iter/s)": 0.005442
+    },
+    {
+      "epoch": 0.4429540354504425,
+      "grad_norm": 7.8712992668151855,
+      "learning_rate": 0.00012676170222926023,
+      "logits/chosen": -1.0750609636306763,
+      "logits/rejected": -1.08124840259552,
+      "logps/chosen": -8.864872932434082,
+      "logps/rejected": -44.60932540893555,
+      "loss": 1.103929042816162,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.7646006345748901,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.1929735392332077,
+      "rewards/margins": 3.46407413482666,
+      "rewards/rejected": -3.2711007595062256,
+      "step": 549,
+      "train_speed(iter/s)": 0.005442
+    },
+    {
+      "epoch": 0.44376087340208265,
+      "grad_norm": 10.148578643798828,
+      "learning_rate": 0.00012650442757755859,
+      "logits/chosen": -1.0684211254119873,
+      "logits/rejected": -1.0741227865219116,
+      "logps/chosen": -9.018268585205078,
+      "logps/rejected": -51.82669448852539,
+      "loss": 0.7279898524284363,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.6217176914215088,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.23028400540351868,
+      "rewards/margins": 4.1028971672058105,
+      "rewards/rejected": -3.8726134300231934,
+      "step": 550,
+      "train_speed(iter/s)": 0.005442
+    },
+    {
+      "epoch": 0.4445677113537228,
+      "grad_norm": 9.017724990844727,
+      "learning_rate": 0.00012624696409846667,
+      "logits/chosen": -1.107726812362671,
+      "logits/rejected": -1.1250691413879395,
+      "logps/chosen": -10.755329132080078,
+      "logps/rejected": -48.62479782104492,
+      "loss": 0.6286925077438354,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.39722785353660583,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.28724217414855957,
+      "rewards/margins": 3.572014570236206,
+      "rewards/rejected": -3.2847723960876465,
+      "step": 551,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.44537454930536297,
+      "grad_norm": 3.4098925590515137,
+      "learning_rate": 0.0001259893136262499,
+      "logits/chosen": -1.1038252115249634,
+      "logits/rejected": -1.1076061725616455,
+      "logps/chosen": -6.588968753814697,
+      "logps/rejected": -41.67078399658203,
+      "loss": 0.39038464426994324,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2124042510986328,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3312928378582001,
+      "rewards/margins": 3.40315580368042,
+      "rewards/rejected": -3.0718629360198975,
+      "step": 552,
+      "train_speed(iter/s)": 0.005442
+    },
+    {
+      "epoch": 0.4461813872570031,
+      "grad_norm": 8.600381851196289,
+      "learning_rate": 0.0001257314779965061,
+      "logits/chosen": -1.1414737701416016,
+      "logits/rejected": -1.1534221172332764,
+      "logps/chosen": -6.600671768188477,
+      "logps/rejected": -49.37320327758789,
+      "loss": 0.683100163936615,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5235199928283691,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.23751148581504822,
+      "rewards/margins": 4.117624282836914,
+      "rewards/rejected": -3.8801121711730957,
+      "step": 553,
+      "train_speed(iter/s)": 0.005442
+    },
+    {
+      "epoch": 0.44698822520864323,
+      "grad_norm": 8.508306503295898,
+      "learning_rate": 0.00012547345904615197,
+      "logits/chosen": -1.0857504606246948,
+      "logits/rejected": -1.0946986675262451,
+      "logps/chosen": -14.657316207885742,
+      "logps/rejected": -41.80220413208008,
+      "loss": 0.6490945816040039,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.42346134781837463,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.25183647871017456,
+      "rewards/margins": 3.0182981491088867,
+      "rewards/rejected": -2.7664616107940674,
+      "step": 554,
+      "train_speed(iter/s)": 0.005442
+    },
+    {
+      "epoch": 0.4477950631602834,
+      "grad_norm": 5.108850955963135,
+      "learning_rate": 0.00012521525861341047,
+      "logits/chosen": -1.0797810554504395,
+      "logits/rejected": -1.0843287706375122,
+      "logps/chosen": -12.274191856384277,
+      "logps/rejected": -43.346046447753906,
+      "loss": 0.8347964882850647,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.45574951171875,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.06146601587533951,
+      "rewards/margins": 3.368800640106201,
+      "rewards/rejected": -3.3073341846466064,
+      "step": 555,
+      "train_speed(iter/s)": 0.005443
+    },
+    {
+      "epoch": 0.44860190111192355,
+      "grad_norm": 4.0859575271606445,
+      "learning_rate": 0.0001249568785377974,
+      "logits/chosen": -1.0649352073669434,
+      "logits/rejected": -1.0609965324401855,
+      "logps/chosen": -10.706140518188477,
+      "logps/rejected": -35.780635833740234,
+      "loss": 0.5825737118721008,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3680080473423004,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.11751089990139008,
+      "rewards/margins": 2.68560528755188,
+      "rewards/rejected": -2.568094253540039,
+      "step": 556,
+      "train_speed(iter/s)": 0.005442
+    },
+    {
+      "epoch": 0.4494087390635637,
+      "grad_norm": 4.83468770980835,
+      "learning_rate": 0.00012469832066010843,
+      "logits/chosen": -1.0108416080474854,
+      "logits/rejected": -1.0031400918960571,
+      "logps/chosen": -15.39680290222168,
+      "logps/rejected": -33.991302490234375,
+      "loss": 0.7029147148132324,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4147963523864746,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.0027669351547956467,
+      "rewards/margins": 2.234447479248047,
+      "rewards/rejected": -2.2372145652770996,
+      "step": 557,
+      "train_speed(iter/s)": 0.005443
+    },
+    {
+      "epoch": 0.45021557701520387,
+      "grad_norm": 3.3592216968536377,
+      "learning_rate": 0.00012443958682240594,
+      "logits/chosen": -0.989718496799469,
+      "logits/rejected": -0.9940285086631775,
+      "logps/chosen": -4.154476642608643,
+      "logps/rejected": -40.08197021484375,
+      "loss": 0.41042032837867737,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.21786123514175415,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.183738574385643,
+      "rewards/margins": 3.009525775909424,
+      "rewards/rejected": -2.8257875442504883,
+      "step": 558,
+      "train_speed(iter/s)": 0.005443
+    },
+    {
+      "epoch": 0.451022414966844,
+      "grad_norm": 2.1088263988494873,
+      "learning_rate": 0.0001241806788680059,
+      "logits/chosen": -0.9063556790351868,
+      "logits/rejected": -0.902630090713501,
+      "logps/chosen": -13.559588432312012,
+      "logps/rejected": -36.965415954589844,
+      "loss": 0.553245484828949,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.44025957584381104,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4070787727832794,
+      "rewards/margins": 3.4192891120910645,
+      "rewards/rejected": -3.0122101306915283,
+      "step": 559,
+      "train_speed(iter/s)": 0.005443
+    },
+    {
+      "epoch": 0.45182925291848414,
+      "grad_norm": 4.130622863769531,
+      "learning_rate": 0.00012392159864146483,
+      "logits/chosen": -0.9277795553207397,
+      "logits/rejected": -0.938519299030304,
+      "logps/chosen": -6.124217510223389,
+      "logps/rejected": -49.98640823364258,
+      "loss": 0.4337657690048218,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.281049907207489,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.27738919854164124,
+      "rewards/margins": 3.8319642543792725,
+      "rewards/rejected": -3.554574728012085,
+      "step": 560,
+      "train_speed(iter/s)": 0.005443
+    },
+    {
+      "epoch": 0.4526360908701243,
+      "grad_norm": 6.3929243087768555,
+      "learning_rate": 0.0001236623479885665,
+      "logits/chosen": -0.9176217913627625,
+      "logits/rejected": -0.9330360889434814,
+      "logps/chosen": -10.697993278503418,
+      "logps/rejected": -42.10285568237305,
+      "loss": 1.1247225999832153,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.7696238160133362,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.17038169503211975,
+      "rewards/margins": 3.24851393699646,
+      "rewards/rejected": -3.078131675720215,
+      "step": 561,
+      "train_speed(iter/s)": 0.005443
+    },
+    {
+      "epoch": 0.45344292882176446,
+      "grad_norm": 7.65782356262207,
+      "learning_rate": 0.00012340292875630894,
+      "logits/chosen": -0.8997213840484619,
+      "logits/rejected": -0.9024547338485718,
+      "logps/chosen": -11.049768447875977,
+      "logps/rejected": -39.89359664916992,
+      "loss": 0.6355679035186768,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4390224814414978,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.14920279383659363,
+      "rewards/margins": 3.1954450607299805,
+      "rewards/rejected": -3.0462422370910645,
+      "step": 562,
+      "train_speed(iter/s)": 0.005444
+    },
+    {
+      "epoch": 0.4542497667734046,
+      "grad_norm": 35.03257369995117,
+      "learning_rate": 0.00012314334279289106,
+      "logits/chosen": -0.8772480487823486,
+      "logits/rejected": -0.8769461512565613,
+      "logps/chosen": -10.01123046875,
+      "logps/rejected": -36.706871032714844,
+      "loss": 0.7129454612731934,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4833950400352478,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.35808807611465454,
+      "rewards/margins": 2.8761751651763916,
+      "rewards/rejected": -2.5180869102478027,
+      "step": 563,
+      "train_speed(iter/s)": 0.005444
+    },
+    {
+      "epoch": 0.4550566047250448,
+      "grad_norm": 12.682050704956055,
+      "learning_rate": 0.00012288359194769982,
+      "logits/chosen": -0.8561194539070129,
+      "logits/rejected": -0.8639140725135803,
+      "logps/chosen": -8.79015064239502,
+      "logps/rejected": -37.45537567138672,
+      "loss": 0.8168308734893799,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5434002876281738,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.16478748619556427,
+      "rewards/margins": 2.5817179679870605,
+      "rewards/rejected": -2.4169304370880127,
+      "step": 564,
+      "train_speed(iter/s)": 0.005444
+    },
+    {
+      "epoch": 0.4558634426766849,
+      "grad_norm": 13.328980445861816,
+      "learning_rate": 0.00012262367807129666,
+      "logits/chosen": -0.9220016598701477,
+      "logits/rejected": -0.930358350276947,
+      "logps/chosen": -9.807029724121094,
+      "logps/rejected": -38.22248840332031,
+      "loss": 0.8756065368652344,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5860471129417419,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.18110670149326324,
+      "rewards/margins": 2.863619565963745,
+      "rewards/rejected": -2.6825127601623535,
+      "step": 565,
+      "train_speed(iter/s)": 0.005444
+    },
+    {
+      "epoch": 0.45667028062832504,
+      "grad_norm": 11.524764060974121,
+      "learning_rate": 0.00012236360301540468,
+      "logits/chosen": -0.8239192962646484,
+      "logits/rejected": -0.8312022686004639,
+      "logps/chosen": -9.089396476745605,
+      "logps/rejected": -47.21116638183594,
+      "loss": 0.7630635499954224,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5738613605499268,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.00014786049723625183,
+      "rewards/margins": 3.374746799468994,
+      "rewards/rejected": -3.374598503112793,
+      "step": 566,
+      "train_speed(iter/s)": 0.005444
+    },
+    {
+      "epoch": 0.4574771185799652,
+      "grad_norm": 2.014758586883545,
+      "learning_rate": 0.0001221033686328952,
+      "logits/chosen": -0.8044052720069885,
+      "logits/rejected": -0.8143394589424133,
+      "logps/chosen": -9.542470932006836,
+      "logps/rejected": -41.68825912475586,
+      "loss": 0.5237497091293335,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.34950196743011475,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3110518455505371,
+      "rewards/margins": 2.929806709289551,
+      "rewards/rejected": -2.6187548637390137,
+      "step": 567,
+      "train_speed(iter/s)": 0.005445
+    },
+    {
+      "epoch": 0.45828395653160536,
+      "grad_norm": 13.059274673461914,
+      "learning_rate": 0.00012184297677777463,
+      "logits/chosen": -0.8409905433654785,
+      "logits/rejected": -0.8475866317749023,
+      "logps/chosen": -10.23369312286377,
+      "logps/rejected": -44.068084716796875,
+      "loss": 0.49647825956344604,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.363809198141098,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2194375842809677,
+      "rewards/margins": 3.2950479984283447,
+      "rewards/rejected": -3.075610399246216,
+      "step": 568,
+      "train_speed(iter/s)": 0.005444
+    },
+    {
+      "epoch": 0.4590907944832455,
+      "grad_norm": 2.138056993484497,
+      "learning_rate": 0.00012158242930517134,
+      "logits/chosen": -0.8567276000976562,
+      "logits/rejected": -0.86031174659729,
+      "logps/chosen": -7.590709209442139,
+      "logps/rejected": -39.7437858581543,
+      "loss": 0.4562058448791504,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2901318669319153,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.13064348697662354,
+      "rewards/margins": 3.0062763690948486,
+      "rewards/rejected": -2.8756330013275146,
+      "step": 569,
+      "train_speed(iter/s)": 0.005444
+    },
+    {
+      "epoch": 0.4598976324348857,
+      "grad_norm": 11.261893272399902,
+      "learning_rate": 0.00012132172807132232,
+      "logits/chosen": -0.832809567451477,
+      "logits/rejected": -0.8412526845932007,
+      "logps/chosen": -7.241992473602295,
+      "logps/rejected": -37.31771469116211,
+      "loss": 0.6455690264701843,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3702318072319031,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1550164818763733,
+      "rewards/margins": 2.853445291519165,
+      "rewards/rejected": -2.6984288692474365,
+      "step": 570,
+      "train_speed(iter/s)": 0.005445
+    },
+    {
+      "epoch": 0.4607044703865258,
+      "grad_norm": 3.706634044647217,
+      "learning_rate": 0.00012106087493356002,
+      "logits/chosen": -0.8279977440834045,
+      "logits/rejected": -0.8347707390785217,
+      "logps/chosen": -8.675919532775879,
+      "logps/rejected": -47.83354949951172,
+      "loss": 0.5063999891281128,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.283450186252594,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1277650147676468,
+      "rewards/margins": 3.40612530708313,
+      "rewards/rejected": -3.278360366821289,
+      "step": 571,
+      "train_speed(iter/s)": 0.005445
+    },
+    {
+      "epoch": 0.46151130833816595,
+      "grad_norm": 6.952916145324707,
+      "learning_rate": 0.0001207998717502991,
+      "logits/chosen": -0.8791694641113281,
+      "logits/rejected": -0.8805236220359802,
+      "logps/chosen": -9.15687370300293,
+      "logps/rejected": -35.78827667236328,
+      "loss": 0.7411000728607178,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3683326244354248,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.06607329100370407,
+      "rewards/margins": 2.8387465476989746,
+      "rewards/rejected": -2.7726731300354004,
+      "step": 572,
+      "train_speed(iter/s)": 0.005445
+    },
+    {
+      "epoch": 0.4623181462898061,
+      "grad_norm": 5.609523773193359,
+      "learning_rate": 0.00012053872038102327,
+      "logits/chosen": -0.8049792051315308,
+      "logits/rejected": -0.8103649020195007,
+      "logps/chosen": -9.753730773925781,
+      "logps/rejected": -34.45492935180664,
+      "loss": 0.7352737188339233,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3698999285697937,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.2235478013753891,
+      "rewards/margins": 2.4985084533691406,
+      "rewards/rejected": -2.274960517883301,
+      "step": 573,
+      "train_speed(iter/s)": 0.005445
+    },
+    {
+      "epoch": 0.46312498424144627,
+      "grad_norm": 3.0912234783172607,
+      "learning_rate": 0.00012027742268627185,
+      "logits/chosen": -0.837486743927002,
+      "logits/rejected": -0.8439161777496338,
+      "logps/chosen": -9.795650482177734,
+      "logps/rejected": -50.44477844238281,
+      "loss": 0.4414077401161194,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3059607744216919,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.32000604271888733,
+      "rewards/margins": 4.076040744781494,
+      "rewards/rejected": -3.7560343742370605,
+      "step": 574,
+      "train_speed(iter/s)": 0.005445
+    },
+    {
+      "epoch": 0.46393182219308643,
+      "grad_norm": 3.1543638706207275,
+      "learning_rate": 0.0001200159805276268,
+      "logits/chosen": -0.7970680594444275,
+      "logits/rejected": -0.8043116927146912,
+      "logps/chosen": -9.85317611694336,
+      "logps/rejected": -41.96942901611328,
+      "loss": 0.5125235319137573,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.31278592348098755,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3057025671005249,
+      "rewards/margins": 2.943936824798584,
+      "rewards/rejected": -2.6382341384887695,
+      "step": 575,
+      "train_speed(iter/s)": 0.005445
+    },
+    {
+      "epoch": 0.46473866014472653,
+      "grad_norm": 1.7856658697128296,
+      "learning_rate": 0.0001197543957676991,
+      "logits/chosen": -0.7884432077407837,
+      "logits/rejected": -0.8058311939239502,
+      "logps/chosen": -4.567942142486572,
+      "logps/rejected": -44.635986328125,
+      "loss": 0.29785671830177307,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.14283448457717896,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2166459560394287,
+      "rewards/margins": 3.13464617729187,
+      "rewards/rejected": -2.9180002212524414,
+      "step": 576,
+      "train_speed(iter/s)": 0.005445
+    },
+    {
+      "epoch": 0.4655454980963667,
+      "grad_norm": 4.239716529846191,
+      "learning_rate": 0.00011949267027011583,
+      "logits/chosen": -0.8037387728691101,
+      "logits/rejected": -0.812914252281189,
+      "logps/chosen": -10.53479290008545,
+      "logps/rejected": -40.92039489746094,
+      "loss": 0.6195076107978821,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4382752776145935,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3542560636997223,
+      "rewards/margins": 2.9503233432769775,
+      "rewards/rejected": -2.596067428588867,
+      "step": 577,
+      "train_speed(iter/s)": 0.005445
+    },
+    {
+      "epoch": 0.46635233604800685,
+      "grad_norm": 3.537693500518799,
+      "learning_rate": 0.00011923080589950672,
+      "logits/chosen": -0.8336210250854492,
+      "logits/rejected": -0.847730278968811,
+      "logps/chosen": -7.870901107788086,
+      "logps/rejected": -47.519771575927734,
+      "loss": 0.5777885913848877,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.29648178815841675,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.21377240121364594,
+      "rewards/margins": 2.789757490158081,
+      "rewards/rejected": -2.5759849548339844,
+      "step": 578,
+      "train_speed(iter/s)": 0.005445
+    },
+    {
+      "epoch": 0.467159173999647,
+      "grad_norm": 3.2539420127868652,
+      "learning_rate": 0.00011896880452149077,
+      "logits/chosen": -0.8004827499389648,
+      "logits/rejected": -0.8014000058174133,
+      "logps/chosen": -9.597652435302734,
+      "logps/rejected": -33.71527862548828,
+      "loss": 0.5787873864173889,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.37172919511795044,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.4151571989059448,
+      "rewards/margins": 2.7820167541503906,
+      "rewards/rejected": -2.3668594360351562,
+      "step": 579,
+      "train_speed(iter/s)": 0.005445
+    },
+    {
+      "epoch": 0.4679660119512872,
+      "grad_norm": 2.9435672760009766,
+      "learning_rate": 0.00011870666800266322,
+      "logits/chosen": -0.8050804138183594,
+      "logits/rejected": -0.8147591948509216,
+      "logps/chosen": -3.9187402725219727,
+      "logps/rejected": -40.465240478515625,
+      "loss": 0.3844418525695801,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.20655041933059692,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.248398095369339,
+      "rewards/margins": 2.9495856761932373,
+      "rewards/rejected": -2.7011878490448,
+      "step": 580,
+      "train_speed(iter/s)": 0.005445
+    },
+    {
+      "epoch": 0.46877284990292734,
+      "grad_norm": 10.298157691955566,
+      "learning_rate": 0.00011844439821058197,
+      "logits/chosen": -0.7727063894271851,
+      "logits/rejected": -0.7812480926513672,
+      "logps/chosen": -8.926658630371094,
+      "logps/rejected": -36.966217041015625,
+      "loss": 0.658978283405304,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.41324326395988464,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.28273195028305054,
+      "rewards/margins": 2.469106674194336,
+      "rewards/rejected": -2.1863749027252197,
+      "step": 581,
+      "train_speed(iter/s)": 0.005446
+    },
+    {
+      "epoch": 0.46957968785456744,
+      "grad_norm": 3.50510311126709,
+      "learning_rate": 0.00011818199701375454,
+      "logits/chosen": -0.8110121488571167,
+      "logits/rejected": -0.8104444742202759,
+      "logps/chosen": -12.34595775604248,
+      "logps/rejected": -45.01907730102539,
+      "loss": 0.6881570219993591,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4258965253829956,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.1440855711698532,
+      "rewards/margins": 2.760430335998535,
+      "rewards/rejected": -2.9045162200927734,
+      "step": 582,
+      "train_speed(iter/s)": 0.005446
+    },
+    {
+      "epoch": 0.4703865258062076,
+      "grad_norm": 3.7001280784606934,
+      "learning_rate": 0.00011791946628162447,
+      "logits/chosen": -0.8208535313606262,
+      "logits/rejected": -0.8212382197380066,
+      "logps/chosen": -8.028635025024414,
+      "logps/rejected": -33.360164642333984,
+      "loss": 0.5595318078994751,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.33058375120162964,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.27441638708114624,
+      "rewards/margins": 2.571678638458252,
+      "rewards/rejected": -2.29726243019104,
+      "step": 583,
+      "train_speed(iter/s)": 0.005446
+    },
+    {
+      "epoch": 0.47119336375784776,
+      "grad_norm": 8.616028785705566,
+      "learning_rate": 0.00011765680788455831,
+      "logits/chosen": -0.8166062235832214,
+      "logits/rejected": -0.816809892654419,
+      "logps/chosen": -11.542019844055176,
+      "logps/rejected": -35.4975700378418,
+      "loss": 1.278072714805603,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.8869698643684387,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.05604610964655876,
+      "rewards/margins": 2.5545907020568848,
+      "rewards/rejected": -2.6106367111206055,
+      "step": 584,
+      "train_speed(iter/s)": 0.005446
+    },
+    {
+      "epoch": 0.4720002017094879,
+      "grad_norm": 7.32548189163208,
+      "learning_rate": 0.00011739402369383203,
+      "logits/chosen": -0.8029976487159729,
+      "logits/rejected": -0.8099737763404846,
+      "logps/chosen": -6.423218250274658,
+      "logps/rejected": -35.83351516723633,
+      "loss": 0.9214006066322327,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5963305234909058,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1106901690363884,
+      "rewards/margins": 2.674905776977539,
+      "rewards/rejected": -2.564215660095215,
+      "step": 585,
+      "train_speed(iter/s)": 0.005446
+    },
+    {
+      "epoch": 0.4728070396611281,
+      "grad_norm": 12.532291412353516,
+      "learning_rate": 0.00011713111558161791,
+      "logits/chosen": -0.856770932674408,
+      "logits/rejected": -0.8620243668556213,
+      "logps/chosen": -7.437289237976074,
+      "logps/rejected": -31.561695098876953,
+      "loss": 0.7550098896026611,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.47793805599212646,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.24787144362926483,
+      "rewards/margins": 2.3619582653045654,
+      "rewards/rejected": -2.114086627960205,
+      "step": 586,
+      "train_speed(iter/s)": 0.005446
+    },
+    {
+      "epoch": 0.4736138776127682,
+      "grad_norm": 11.558775901794434,
+      "learning_rate": 0.00011686808542097096,
+      "logits/chosen": -0.8273903727531433,
+      "logits/rejected": -0.831815242767334,
+      "logps/chosen": -12.218116760253906,
+      "logps/rejected": -33.31938171386719,
+      "loss": 0.6346630454063416,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.4017612338066101,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2919488251209259,
+      "rewards/margins": 2.352766990661621,
+      "rewards/rejected": -2.0608184337615967,
+      "step": 587,
+      "train_speed(iter/s)": 0.005446
+    },
+    {
+      "epoch": 0.47442071556440835,
+      "grad_norm": 6.823660373687744,
+      "learning_rate": 0.00011660493508581582,
+      "logits/chosen": -0.8630138635635376,
+      "logits/rejected": -0.8689411878585815,
+      "logps/chosen": -8.854455947875977,
+      "logps/rejected": -30.525468826293945,
+      "loss": 0.6868352890014648,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.44286802411079407,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.23568075895309448,
+      "rewards/margins": 2.2586841583251953,
+      "rewards/rejected": -2.023003578186035,
+      "step": 588,
+      "train_speed(iter/s)": 0.005446
+    },
+    {
+      "epoch": 0.4752275535160485,
+      "grad_norm": 7.324416637420654,
+      "learning_rate": 0.00011634166645093325,
+      "logits/chosen": -0.9088298678398132,
+      "logits/rejected": -0.9183266162872314,
+      "logps/chosen": -7.649410724639893,
+      "logps/rejected": -33.78176498413086,
+      "loss": 0.5547029972076416,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2729610502719879,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.39127102494239807,
+      "rewards/margins": 2.385117292404175,
+      "rewards/rejected": -1.9938466548919678,
+      "step": 589,
+      "train_speed(iter/s)": 0.005447
+    },
+    {
+      "epoch": 0.47603439146768867,
+      "grad_norm": 3.2639598846435547,
+      "learning_rate": 0.00011607828139194683,
+      "logits/chosen": -0.8846572041511536,
+      "logits/rejected": -0.8930388689041138,
+      "logps/chosen": -10.60714340209961,
+      "logps/rejected": -44.169132232666016,
+      "loss": 0.6324015259742737,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.41405993700027466,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.2574876546859741,
+      "rewards/margins": 2.925980567932129,
+      "rewards/rejected": -2.6684930324554443,
+      "step": 590,
+      "train_speed(iter/s)": 0.005447
+    },
+    {
+      "epoch": 0.4768412294193288,
+      "grad_norm": 3.027688980102539,
+      "learning_rate": 0.00011581478178530958,
+      "logits/chosen": -0.8636766076087952,
+      "logits/rejected": -0.8625022172927856,
+      "logps/chosen": -6.506802558898926,
+      "logps/rejected": -32.95680618286133,
+      "loss": 0.5603009462356567,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.26980704069137573,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.25332656502723694,
+      "rewards/margins": 2.5316824913024902,
+      "rewards/rejected": -2.278355836868286,
+      "step": 591,
+      "train_speed(iter/s)": 0.005447
+    },
+    {
+      "epoch": 0.477648067370969,
+      "grad_norm": 2.2653770446777344,
+      "learning_rate": 0.00011555116950829059,
+      "logits/chosen": -0.8657379150390625,
+      "logits/rejected": -0.8774781227111816,
+      "logps/chosen": -10.084299087524414,
+      "logps/rejected": -44.07289123535156,
+      "loss": 0.517981767654419,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.26441580057144165,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.21334537863731384,
+      "rewards/margins": 3.083986282348633,
+      "rewards/rejected": -2.870640993118286,
+      "step": 592,
+      "train_speed(iter/s)": 0.005447
+    },
+    {
+      "epoch": 0.4784549053226091,
+      "grad_norm": 4.369250297546387,
+      "learning_rate": 0.00011528744643896172,
+      "logits/chosen": -0.927314281463623,
+      "logits/rejected": -0.9315405488014221,
+      "logps/chosen": -12.392212867736816,
+      "logps/rejected": -32.89162063598633,
+      "loss": 0.8612978458404541,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5310938954353333,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.00041678547859191895,
+      "rewards/margins": 2.2074103355407715,
+      "rewards/rejected": -2.207827091217041,
+      "step": 593,
+      "train_speed(iter/s)": 0.005447
+    },
+    {
+      "epoch": 0.47926174327424925,
+      "grad_norm": 2.4553322792053223,
+      "learning_rate": 0.00011502361445618404,
+      "logits/chosen": -0.8954474329948425,
+      "logits/rejected": -0.8992701172828674,
+      "logps/chosen": -8.206144332885742,
+      "logps/rejected": -31.729503631591797,
+      "loss": 0.5183227062225342,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.3380415439605713,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.39920157194137573,
+      "rewards/margins": 2.6823015213012695,
+      "rewards/rejected": -2.283099889755249,
+      "step": 594,
+      "train_speed(iter/s)": 0.005447
+    },
+    {
+      "epoch": 0.4800685812258894,
+      "grad_norm": 5.501280307769775,
+      "learning_rate": 0.0001147596754395947,
+      "logits/chosen": -0.9040945768356323,
+      "logits/rejected": -0.9091372489929199,
+      "logps/chosen": -7.491284370422363,
+      "logps/rejected": -29.543960571289062,
+      "loss": 0.8046502470970154,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.384172260761261,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.06815558671951294,
+      "rewards/margins": 2.0989913940429688,
+      "rewards/rejected": -2.0308361053466797,
+      "step": 595,
+      "train_speed(iter/s)": 0.005447
+    },
+    {
+      "epoch": 0.4808754191775296,
+      "grad_norm": 1.8338180780410767,
+      "learning_rate": 0.00011449563126959323,
+      "logits/chosen": -0.8833913207054138,
+      "logits/rejected": -0.8990954756736755,
+      "logps/chosen": -6.696419715881348,
+      "logps/rejected": -45.6534423828125,
+      "loss": 0.38033589720726013,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2674116790294647,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5114734768867493,
+      "rewards/margins": 3.3956971168518066,
+      "rewards/rejected": -2.884223699569702,
+      "step": 596,
+      "train_speed(iter/s)": 0.005447
+    },
+    {
+      "epoch": 0.48168225712916973,
+      "grad_norm": 11.115188598632812,
+      "learning_rate": 0.00011423148382732853,
+      "logits/chosen": -0.9077367782592773,
+      "logits/rejected": -0.9106470346450806,
+      "logps/chosen": -9.558808326721191,
+      "logps/rejected": -36.79584884643555,
+      "loss": 0.8870832324028015,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.42637699842453003,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.12174645066261292,
+      "rewards/margins": 2.7100558280944824,
+      "rewards/rejected": -2.8318021297454834,
+      "step": 597,
+      "train_speed(iter/s)": 0.005447
+    },
+    {
+      "epoch": 0.48248909508080984,
+      "grad_norm": 1.7906243801116943,
+      "learning_rate": 0.00011396723499468502,
+      "logits/chosen": -0.8848870396614075,
+      "logits/rejected": -0.8983874917030334,
+      "logps/chosen": -5.4425048828125,
+      "logps/rejected": -44.632118225097656,
+      "loss": 0.37124451994895935,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.20685873925685883,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.1721365600824356,
+      "rewards/margins": 2.9007062911987305,
+      "rewards/rejected": -2.728569746017456,
+      "step": 598,
+      "train_speed(iter/s)": 0.005447
+    },
+    {
+      "epoch": 0.48329593303245,
+      "grad_norm": 5.615041732788086,
+      "learning_rate": 0.00011370288665426965,
+      "logits/chosen": -0.8737161159515381,
+      "logits/rejected": -0.8760817050933838,
+      "logps/chosen": -19.09202766418457,
+      "logps/rejected": -35.31637954711914,
+      "loss": 0.9512906670570374,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.5964938402175903,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.0961245745420456,
+      "rewards/margins": 2.3884572982788086,
+      "rewards/rejected": -2.292332649230957,
+      "step": 599,
+      "train_speed(iter/s)": 0.005447
+    },
+    {
+      "epoch": 0.48410277098409016,
+      "grad_norm": 2.1948606967926025,
+      "learning_rate": 0.00011343844068939815,
+      "logits/chosen": -0.8662118911743164,
+      "logits/rejected": -0.8754556179046631,
+      "logps/chosen": -7.872878074645996,
+      "logps/rejected": -43.61811065673828,
+      "loss": 0.4264710247516632,
+      "memory(GiB)": 46.83,
+      "nll_loss": 0.2550013065338135,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.17934425175189972,
+      "rewards/margins": 3.2902426719665527,
+      "rewards/rejected": -3.110898494720459,
+      "step": 600,
+      "train_speed(iter/s)": 0.005448
+    },
+    {
+      "epoch": 0.48410277098409016,
+      "eval_logits/chosen": -0.8526069521903992,
+      "eval_logits/rejected": -0.8563287854194641,
+      "eval_logps/chosen": -8.513988494873047,
+      "eval_logps/rejected": -39.23109817504883,
+      "eval_loss": 0.5813677310943604,
+      "eval_nll_loss": 0.36714690923690796,
+      "eval_rewards/accuracies": 0.9075000286102295,
+      "eval_rewards/chosen": 0.1680423766374588,
+      "eval_rewards/margins": 2.8004705905914307,
+      "eval_rewards/rejected": -2.6324281692504883,
+      "eval_runtime": 933.4995,
+      "eval_samples_per_second": 0.428,
+      "eval_steps_per_second": 0.428,
+      "step": 600
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 1239,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.3899480603570995e+18,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}