diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,5434 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.451319381255687,
+  "eval_steps": 500,
+  "global_step": 300,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004852896572641796,
+      "grad_norm": 3.7387311458587646,
+      "learning_rate": 6.451612903225806e-06,
+      "logits/chosen": -0.5510790944099426,
+      "logits/rejected": -0.5881398916244507,
+      "logps/chosen": -12.010782241821289,
+      "logps/rejected": -14.501535415649414,
+      "loss": 1.3914958238601685,
+      "memory(GiB)": 30.65,
+      "nll_loss": 0.6983487010002136,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1,
+      "train_speed(iter/s)": 0.010451
+    },
+    {
+      "epoch": 0.009705793145283591,
+      "grad_norm": 5.074642658233643,
+      "learning_rate": 1.2903225806451613e-05,
+      "logits/chosen": -0.523028552532196,
+      "logits/rejected": -0.48799216747283936,
+      "logps/chosen": -15.598363876342773,
+      "logps/rejected": -12.491965293884277,
+      "loss": 1.3891178369522095,
+      "memory(GiB)": 33.32,
+      "nll_loss": 0.6959705948829651,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 2,
+      "train_speed(iter/s)": 0.010818
+    },
+    {
+      "epoch": 0.014558689717925387,
+      "grad_norm": 4.8900933265686035,
+      "learning_rate": 1.935483870967742e-05,
+      "logits/chosen": -0.6194146871566772,
+      "logits/rejected": -0.6501045227050781,
+      "logps/chosen": -4.106720924377441,
+      "logps/rejected": -9.803337097167969,
+      "loss": 1.0989456176757812,
+      "memory(GiB)": 33.32,
+      "nll_loss": 0.4074462950229645,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0007492969743907452,
+      "rewards/margins": 0.003412690246477723,
+      "rewards/rejected": -0.0026633935049176216,
+      "step": 3,
+      "train_speed(iter/s)": 0.010979
+    },
+    {
+      "epoch": 0.019411586290567183,
+      "grad_norm": 4.551450729370117,
+      "learning_rate": 2.5806451612903226e-05,
+      "logits/chosen": -0.4892222285270691,
+      "logits/rejected": -0.6288261413574219,
+      "logps/chosen": -11.312190055847168,
+      "logps/rejected": -19.471235275268555,
+      "loss": 1.4801559448242188,
+      "memory(GiB)": 36.76,
+      "nll_loss": 0.788896381855011,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.009614704176783562,
+      "rewards/margins": 0.003918787930160761,
+      "rewards/rejected": 0.00569591810926795,
+      "step": 4,
+      "train_speed(iter/s)": 0.011017
+    },
+    {
+      "epoch": 0.024264482863208977,
+      "grad_norm": 5.593626499176025,
+      "learning_rate": 3.2258064516129034e-05,
+      "logits/chosen": -0.5494186282157898,
+      "logits/rejected": -0.6138424277305603,
+      "logps/chosen": -9.588997840881348,
+      "logps/rejected": -13.734201431274414,
+      "loss": 1.360138177871704,
+      "memory(GiB)": 36.76,
+      "nll_loss": 0.6719292998313904,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.006837800145149231,
+      "rewards/margins": 0.01003339048475027,
+      "rewards/rejected": -0.003195591736584902,
+      "step": 5,
+      "train_speed(iter/s)": 0.011063
+    },
+    {
+      "epoch": 0.029117379435850774,
+      "grad_norm": 4.7115631103515625,
+      "learning_rate": 3.870967741935484e-05,
+      "logits/chosen": -0.5902564525604248,
+      "logits/rejected": -0.6040255427360535,
+      "logps/chosen": -10.28292465209961,
+      "logps/rejected": -10.143241882324219,
+      "loss": 1.4910708665847778,
+      "memory(GiB)": 36.76,
+      "nll_loss": 0.7961816787719727,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.010182744823396206,
+      "rewards/margins": -0.0029990212060511112,
+      "rewards/rejected": 0.013181766495108604,
+      "step": 6,
+      "train_speed(iter/s)": 0.011101
+    },
+    {
+      "epoch": 0.03397027600849257,
+      "grad_norm": 3.242079257965088,
+      "learning_rate": 4.516129032258064e-05,
+      "logits/chosen": -0.5178452730178833,
+      "logits/rejected": -0.5171934366226196,
+      "logps/chosen": -11.35531997680664,
+      "logps/rejected": -13.940824508666992,
+      "loss": 1.256138563156128,
+      "memory(GiB)": 36.76,
+      "nll_loss": 0.5596349239349365,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.027008483186364174,
+      "rewards/margins": -0.005357709713280201,
+      "rewards/rejected": 0.03236619383096695,
+      "step": 7,
+      "train_speed(iter/s)": 0.01111
+    },
+    {
+      "epoch": 0.038823172581134366,
+      "grad_norm": 4.068852424621582,
+      "learning_rate": 5.161290322580645e-05,
+      "logits/chosen": -0.5522034168243408,
+      "logits/rejected": -0.5255574584007263,
+      "logps/chosen": -12.265619277954102,
+      "logps/rejected": -11.32555866241455,
+      "loss": 1.3358875513076782,
+      "memory(GiB)": 36.76,
+      "nll_loss": 0.6352814435958862,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": 0.021265771239995956,
+      "rewards/margins": -0.011079448275268078,
+      "rewards/rejected": 0.03234522044658661,
+      "step": 8,
+      "train_speed(iter/s)": 0.011128
+    },
+    {
+      "epoch": 0.04367606915377616,
+      "grad_norm": 3.7364556789398193,
+      "learning_rate": 5.8064516129032266e-05,
+      "logits/chosen": -0.5157386064529419,
+      "logits/rejected": -0.5693979263305664,
+      "logps/chosen": -8.727921485900879,
+      "logps/rejected": -15.667279243469238,
+      "loss": 1.2342674732208252,
+      "memory(GiB)": 36.76,
+      "nll_loss": 0.565812349319458,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.06674117594957352,
+      "rewards/margins": 0.05803339183330536,
+      "rewards/rejected": 0.008707791566848755,
+      "step": 9,
+      "train_speed(iter/s)": 0.01114
+    },
+    {
+      "epoch": 0.048528965726417954,
+      "grad_norm": 5.376058578491211,
+      "learning_rate": 6.451612903225807e-05,
+      "logits/chosen": -0.5336275696754456,
+      "logits/rejected": -0.5664803385734558,
+      "logps/chosen": -10.200874328613281,
+      "logps/rejected": -14.752674102783203,
+      "loss": 1.3360637426376343,
+      "memory(GiB)": 36.76,
+      "nll_loss": 0.6595087647438049,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.05797234922647476,
+      "rewards/margins": 0.05362413078546524,
+      "rewards/rejected": 0.00434822216629982,
+      "step": 10,
+      "train_speed(iter/s)": 0.011149
+    },
+    {
+      "epoch": 0.05338186229905975,
+      "grad_norm": 3.41105318069458,
+      "learning_rate": 7.096774193548388e-05,
+      "logits/chosen": -0.5836127996444702,
+      "logits/rejected": -0.5781168341636658,
+      "logps/chosen": -16.744802474975586,
+      "logps/rejected": -10.854264259338379,
+      "loss": 1.4587984085083008,
+      "memory(GiB)": 36.76,
+      "nll_loss": 0.8293395042419434,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.10905362665653229,
+      "rewards/margins": 0.15107941627502441,
+      "rewards/rejected": -0.04202580079436302,
+      "step": 11,
+      "train_speed(iter/s)": 0.011156
+    },
+    {
+      "epoch": 0.05823475887170155,
+      "grad_norm": 3.8034191131591797,
+      "learning_rate": 7.741935483870968e-05,
+      "logits/chosen": -0.5040208101272583,
+      "logits/rejected": -0.5330135822296143,
+      "logps/chosen": -9.613079071044922,
+      "logps/rejected": -11.064685821533203,
+      "loss": 1.267149806022644,
+      "memory(GiB)": 36.76,
+      "nll_loss": 0.6177489161491394,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.02217767760157585,
+      "rewards/margins": 0.10778125375509262,
+      "rewards/rejected": -0.08560357987880707,
+      "step": 12,
+      "train_speed(iter/s)": 0.011164
+    },
+    {
+      "epoch": 0.06308765544434335,
+      "grad_norm": 3.453901767730713,
+      "learning_rate": 8.387096774193549e-05,
+      "logits/chosen": -0.5467038154602051,
+      "logits/rejected": -0.577677845954895,
+      "logps/chosen": -8.353002548217773,
+      "logps/rejected": -13.458589553833008,
+      "loss": 1.233524203300476,
+      "memory(GiB)": 36.76,
+      "nll_loss": 0.5735047459602356,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0792345255613327,
+      "rewards/margins": 0.10825661569833755,
+      "rewards/rejected": -0.02902209758758545,
+      "step": 13,
+      "train_speed(iter/s)": 0.011171
+    },
+    {
+      "epoch": 0.06794055201698514,
+      "grad_norm": 3.1424827575683594,
+      "learning_rate": 9.032258064516129e-05,
+      "logits/chosen": -0.5190276503562927,
+      "logits/rejected": -0.5021806359291077,
+      "logps/chosen": -11.147318840026855,
+      "logps/rejected": -11.436860084533691,
+      "loss": 1.254595398902893,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5940842628479004,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 0.12143825739622116,
+      "rewards/margins": 0.11505899578332901,
+      "rewards/rejected": 0.006379259284585714,
+      "step": 14,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.07279344858962693,
+      "grad_norm": 3.430884599685669,
+      "learning_rate": 9.677419354838711e-05,
+      "logits/chosen": -0.4556944668292999,
+      "logits/rejected": -0.4863564968109131,
+      "logps/chosen": -13.696606636047363,
+      "logps/rejected": -16.93030548095703,
+      "loss": 1.4541280269622803,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.7789333462715149,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.06093275174498558,
+      "rewards/margins": 0.06727998703718185,
+      "rewards/rejected": -0.006347230635583401,
+      "step": 15,
+      "train_speed(iter/s)": 0.011173
+    },
+    {
+      "epoch": 0.07764634516226873,
+      "grad_norm": 5.697832107543945,
+      "learning_rate": 0.0001032258064516129,
+      "logits/chosen": -0.5404947400093079,
+      "logits/rejected": -0.5623520016670227,
+      "logps/chosen": -8.202204704284668,
+      "logps/rejected": -12.407597541809082,
+      "loss": 1.244711995124817,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.601537823677063,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.14254164695739746,
+      "rewards/margins": 0.16008448600769043,
+      "rewards/rejected": -0.017542842775583267,
+      "step": 16,
+      "train_speed(iter/s)": 0.011181
+    },
+    {
+      "epoch": 0.08249924173491052,
+      "grad_norm": 3.0015194416046143,
+      "learning_rate": 0.00010967741935483871,
+      "logits/chosen": -0.5660006999969482,
+      "logits/rejected": -0.5559377074241638,
+      "logps/chosen": -10.180459022521973,
+      "logps/rejected": -10.599821090698242,
+      "loss": 1.2977104187011719,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6220536231994629,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.12847113609313965,
+      "rewards/margins": 0.0701126754283905,
+      "rewards/rejected": 0.058358460664749146,
+      "step": 17,
+      "train_speed(iter/s)": 0.011186
+    },
+    {
+      "epoch": 0.08735213830755233,
+      "grad_norm": 5.351679801940918,
+      "learning_rate": 0.00011612903225806453,
+      "logits/chosen": -0.4931953251361847,
+      "logits/rejected": -0.5151236653327942,
+      "logps/chosen": -9.91970157623291,
+      "logps/rejected": -17.587669372558594,
+      "loss": 1.2324532270431519,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6188042163848877,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.10037277638912201,
+      "rewards/margins": 0.1970798224210739,
+      "rewards/rejected": -0.0967070460319519,
+      "step": 18,
+      "train_speed(iter/s)": 0.011184
+    },
+    {
+      "epoch": 0.09220503488019412,
+      "grad_norm": 2.8047103881835938,
+      "learning_rate": 0.00012258064516129034,
+      "logits/chosen": -0.5366685390472412,
+      "logits/rejected": -0.5188369154930115,
+      "logps/chosen": -11.985616683959961,
+      "logps/rejected": -9.721867561340332,
+      "loss": 1.243293285369873,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5873799920082092,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.1405532956123352,
+      "rewards/margins": 0.13308030366897583,
+      "rewards/rejected": 0.007473001256585121,
+      "step": 19,
+      "train_speed(iter/s)": 0.011187
+    },
+    {
+      "epoch": 0.09705793145283591,
+      "grad_norm": 3.8547439575195312,
+      "learning_rate": 0.00012903225806451613,
+      "logits/chosen": -0.4702628254890442,
+      "logits/rejected": -0.45280686020851135,
+      "logps/chosen": -11.422282218933105,
+      "logps/rejected": -13.049072265625,
+      "loss": 1.1890830993652344,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.49578943848609924,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.15254227817058563,
+      "rewards/margins": 0.06372587382793427,
+      "rewards/rejected": 0.08881638944149017,
+      "step": 20,
+      "train_speed(iter/s)": 0.011186
+    },
+    {
+      "epoch": 0.10191082802547771,
+      "grad_norm": 3.9698216915130615,
+      "learning_rate": 0.00013548387096774193,
+      "logits/chosen": -0.46789583563804626,
+      "logits/rejected": -0.511239230632782,
+      "logps/chosen": -6.999385833740234,
+      "logps/rejected": -15.381391525268555,
+      "loss": 1.1891487836837769,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.4858560860157013,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.0561373308300972,
+      "rewards/margins": 0.021670352667570114,
+      "rewards/rejected": 0.034466978162527084,
+      "step": 21,
+      "train_speed(iter/s)": 0.01119
+    },
+    {
+      "epoch": 0.1067637245981195,
+      "grad_norm": 2.810508966445923,
+      "learning_rate": 0.00014193548387096775,
+      "logits/chosen": -0.43238648772239685,
+      "logits/rejected": -0.5333250164985657,
+      "logps/chosen": -10.029150009155273,
+      "logps/rejected": -19.058868408203125,
+      "loss": 1.1820727586746216,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5326224565505981,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.09418301284313202,
+      "rewards/margins": 0.12642550468444824,
+      "rewards/rejected": -0.03224247321486473,
+      "step": 22,
+      "train_speed(iter/s)": 0.011191
+    },
+    {
+      "epoch": 0.1116166211707613,
+      "grad_norm": 5.709966659545898,
+      "learning_rate": 0.00014838709677419355,
+      "logits/chosen": -0.42264610528945923,
+      "logits/rejected": -0.45910894870758057,
+      "logps/chosen": -16.804805755615234,
+      "logps/rejected": -18.372587203979492,
+      "loss": 1.3850518465042114,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.8230935335159302,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.05581340938806534,
+      "rewards/margins": 0.3414239287376404,
+      "rewards/rejected": -0.285610556602478,
+      "step": 23,
+      "train_speed(iter/s)": 0.01119
+    },
+    {
+      "epoch": 0.1164695177434031,
+      "grad_norm": 8.213228225708008,
+      "learning_rate": 0.00015483870967741937,
+      "logits/chosen": -0.43471360206604004,
+      "logits/rejected": -0.4376092553138733,
+      "logps/chosen": -10.293994903564453,
+      "logps/rejected": -12.654733657836914,
+      "loss": 1.2845356464385986,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6288778781890869,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.019672194495797157,
+      "rewards/margins": 0.11052073538303375,
+      "rewards/rejected": -0.09084855020046234,
+      "step": 24,
+      "train_speed(iter/s)": 0.011191
+    },
+    {
+      "epoch": 0.12132241431604489,
+      "grad_norm": 2.5066726207733154,
+      "learning_rate": 0.00016129032258064516,
+      "logits/chosen": -0.45425349473953247,
+      "logits/rejected": -0.46823209524154663,
+      "logps/chosen": -5.9361162185668945,
+      "logps/rejected": -12.616898536682129,
+      "loss": 0.9625968933105469,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.3818866014480591,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.14780059456825256,
+      "rewards/margins": 0.3101717531681061,
+      "rewards/rejected": -0.16237111389636993,
+      "step": 25,
+      "train_speed(iter/s)": 0.011195
+    },
+    {
+      "epoch": 0.1261753108886867,
+      "grad_norm": 3.1320712566375732,
+      "learning_rate": 0.00016774193548387098,
+      "logits/chosen": -0.47818127274513245,
+      "logits/rejected": -0.43204230070114136,
+      "logps/chosen": -13.965362548828125,
+      "logps/rejected": -15.635419845581055,
+      "loss": 1.1598286628723145,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6072496175765991,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.06804848462343216,
+      "rewards/margins": 0.44113460183143616,
+      "rewards/rejected": -0.3730861246585846,
+      "step": 26,
+      "train_speed(iter/s)": 0.011195
+    },
+    {
+      "epoch": 0.13102820746132848,
+      "grad_norm": 3.8042213916778564,
+      "learning_rate": 0.00017419354838709678,
+      "logits/chosen": -0.449899286031723,
+      "logits/rejected": -0.4721740484237671,
+      "logps/chosen": -9.528651237487793,
+      "logps/rejected": -12.257569313049316,
+      "loss": 1.2201637029647827,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6225947737693787,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.07078885287046432,
+      "rewards/margins": 0.28294628858566284,
+      "rewards/rejected": -0.21215742826461792,
+      "step": 27,
+      "train_speed(iter/s)": 0.011198
+    },
+    {
+      "epoch": 0.13588110403397027,
+      "grad_norm": 2.5645625591278076,
+      "learning_rate": 0.00018064516129032257,
+      "logits/chosen": -0.3968259394168854,
+      "logits/rejected": -0.41532400250434875,
+      "logps/chosen": -7.651938438415527,
+      "logps/rejected": -15.575130462646484,
+      "loss": 0.9698889851570129,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.4171852469444275,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.1875855028629303,
+      "rewards/margins": 0.46900543570518494,
+      "rewards/rejected": -0.28141990303993225,
+      "step": 28,
+      "train_speed(iter/s)": 0.011199
+    },
+    {
+      "epoch": 0.14073400060661206,
+      "grad_norm": 3.2087931632995605,
+      "learning_rate": 0.0001870967741935484,
+      "logits/chosen": -0.38016778230667114,
+      "logits/rejected": -0.45883065462112427,
+      "logps/chosen": -9.820419311523438,
+      "logps/rejected": -17.21317481994629,
+      "loss": 1.156844973564148,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5829570293426514,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.21587063372135162,
+      "rewards/margins": 0.3926744759082794,
+      "rewards/rejected": -0.1768038272857666,
+      "step": 29,
+      "train_speed(iter/s)": 0.0112
+    },
+    {
+      "epoch": 0.14558689717925385,
+      "grad_norm": 3.5056474208831787,
+      "learning_rate": 0.00019354838709677422,
+      "logits/chosen": -0.43780362606048584,
+      "logits/rejected": -0.46428602933883667,
+      "logps/chosen": -12.7692289352417,
+      "logps/rejected": -16.947650909423828,
+      "loss": 1.2092680931091309,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.7365323305130005,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.19128330051898956,
+      "rewards/margins": 0.7160227298736572,
+      "rewards/rejected": -0.5247394442558289,
+      "step": 30,
+      "train_speed(iter/s)": 0.011202
+    },
+    {
+      "epoch": 0.15043979375189567,
+      "grad_norm": 7.983421802520752,
+      "learning_rate": 0.0002,
+      "logits/chosen": -0.3911149799823761,
+      "logits/rejected": -0.4533422887325287,
+      "logps/chosen": -11.242105484008789,
+      "logps/rejected": -16.97354507446289,
+      "loss": 1.3208513259887695,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.7320083975791931,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.08527207374572754,
+      "rewards/margins": 0.4059477746486664,
+      "rewards/rejected": -0.4912198483943939,
+      "step": 31,
+      "train_speed(iter/s)": 0.011204
+    },
+    {
+      "epoch": 0.15529269032453746,
+      "grad_norm": 7.263556957244873,
+      "learning_rate": 0.0001999985678367986,
+      "logits/chosen": -0.4142841398715973,
+      "logits/rejected": -0.49599555134773254,
+      "logps/chosen": -8.028633117675781,
+      "logps/rejected": -19.247087478637695,
+      "loss": 1.1268211603164673,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5463874936103821,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.16298356652259827,
+      "rewards/margins": 0.4510626792907715,
+      "rewards/rejected": -0.6140461564064026,
+      "step": 32,
+      "train_speed(iter/s)": 0.011206
+    },
+    {
+      "epoch": 0.16014558689717925,
+      "grad_norm": 7.569846153259277,
+      "learning_rate": 0.0001999942713882162,
+      "logits/chosen": -0.4265003502368927,
+      "logits/rejected": -0.42608755826950073,
+      "logps/chosen": -9.83013916015625,
+      "logps/rejected": -17.054443359375,
+      "loss": 1.2255939245224,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6033804416656494,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.24369043111801147,
+      "rewards/margins": 0.24309512972831726,
+      "rewards/rejected": -0.48678556084632874,
+      "step": 33,
+      "train_speed(iter/s)": 0.011209
+    },
+    {
+      "epoch": 0.16499848346982104,
+      "grad_norm": 3.240605592727661,
+      "learning_rate": 0.00019998711077731716,
+      "logits/chosen": -0.4495273232460022,
+      "logits/rejected": -0.3768068850040436,
+      "logps/chosen": -12.2802095413208,
+      "logps/rejected": -12.816421508789062,
+      "loss": 1.085668683052063,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5540680289268494,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.0720956027507782,
+      "rewards/margins": 0.47421759366989136,
+      "rewards/rejected": -0.40212199091911316,
+      "step": 34,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.16985138004246284,
+      "grad_norm": 2.972562551498413,
+      "learning_rate": 0.00019997708620920465,
+      "logits/chosen": -0.4633820354938507,
+      "logits/rejected": -0.5037291646003723,
+      "logps/chosen": -8.281206130981445,
+      "logps/rejected": -17.908266067504883,
+      "loss": 1.0561590194702148,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6168872714042664,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.13331066071987152,
+      "rewards/margins": 0.9570158123970032,
+      "rewards/rejected": -0.8237051963806152,
+      "step": 35,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.17470427661510465,
+      "grad_norm": 5.980532646179199,
+      "learning_rate": 0.00019996419797101514,
+      "logits/chosen": -0.39558717608451843,
+      "logits/rejected": -0.431841105222702,
+      "logps/chosen": -10.584002494812012,
+      "logps/rejected": -19.50037956237793,
+      "loss": 1.2352135181427002,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6387596130371094,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.029237767681479454,
+      "rewards/margins": 0.46423807740211487,
+      "rewards/rejected": -0.43500036001205444,
+      "step": 36,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.17955717318774644,
+      "grad_norm": 5.786473751068115,
+      "learning_rate": 0.00019994844643190975,
+      "logits/chosen": -0.3947299122810364,
+      "logits/rejected": -0.5335777997970581,
+      "logps/chosen": -10.800368309020996,
+      "logps/rejected": -19.217914581298828,
+      "loss": 1.4149001836776733,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.8904857635498047,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.04618726670742035,
+      "rewards/margins": 0.49434420466423035,
+      "rewards/rejected": -0.5405314564704895,
+      "step": 37,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.18441006976038823,
+      "grad_norm": 2.6556568145751953,
+      "learning_rate": 0.00019992983204306403,
+      "logits/chosen": -0.39078089594841003,
+      "logits/rejected": -0.39872410893440247,
+      "logps/chosen": -12.065397262573242,
+      "logps/rejected": -19.61814308166504,
+      "loss": 1.1651582717895508,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5659552812576294,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.06024760380387306,
+      "rewards/margins": 0.4251976013183594,
+      "rewards/rejected": -0.36494994163513184,
+      "step": 38,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.18926296633303002,
+      "grad_norm": 4.3966779708862305,
+      "learning_rate": 0.0001999083553376548,
+      "logits/chosen": -0.4178510308265686,
+      "logits/rejected": -0.42042505741119385,
+      "logps/chosen": -10.572690963745117,
+      "logps/rejected": -17.888080596923828,
+      "loss": 1.1389288902282715,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6220788955688477,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.14774595201015472,
+      "rewards/margins": 0.5430118441581726,
+      "rewards/rejected": -0.3952659070491791,
+      "step": 39,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.19411586290567182,
+      "grad_norm": 2.347978115081787,
+      "learning_rate": 0.00019988401693084502,
+      "logits/chosen": -0.4415282905101776,
+      "logits/rejected": -0.4420144259929657,
+      "logps/chosen": -10.284622192382812,
+      "logps/rejected": -15.651193618774414,
+      "loss": 1.131446123123169,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5388365387916565,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.08026871085166931,
+      "rewards/margins": 0.271782785654068,
+      "rewards/rejected": -0.19151407480239868,
+      "step": 40,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.1989687594783136,
+      "grad_norm": 3.0811686515808105,
+      "learning_rate": 0.0001998568175197661,
+      "logits/chosen": -0.48944562673568726,
+      "logits/rejected": -0.48836228251457214,
+      "logps/chosen": -10.908992767333984,
+      "logps/rejected": -16.806554794311523,
+      "loss": 1.1378778219223022,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5336601734161377,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.026157543063163757,
+      "rewards/margins": 0.3137216567993164,
+      "rewards/rejected": -0.33987918496131897,
+      "step": 41,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.20382165605095542,
+      "grad_norm": 3.496178150177002,
+      "learning_rate": 0.00019982675788349794,
+      "logits/chosen": -0.4175988435745239,
+      "logits/rejected": -0.49307048320770264,
+      "logps/chosen": -8.874513626098633,
+      "logps/rejected": -16.90668296813965,
+      "loss": 1.237853765487671,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6065552830696106,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.028882348909974098,
+      "rewards/margins": 0.22182509303092957,
+      "rewards/rejected": -0.19294273853302002,
+      "step": 42,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.20867455262359721,
+      "grad_norm": 2.585383892059326,
+      "learning_rate": 0.00019979383888304666,
+      "logits/chosen": -0.40949565172195435,
+      "logits/rejected": -0.3888520896434784,
+      "logps/chosen": -12.942455291748047,
+      "logps/rejected": -16.93043327331543,
+      "loss": 1.1456714868545532,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5834133625030518,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.029632963240146637,
+      "rewards/margins": 0.47202715277671814,
+      "rewards/rejected": -0.4423941969871521,
+      "step": 43,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.213527449196239,
+      "grad_norm": 2.6477532386779785,
+      "learning_rate": 0.0001997580614613199,
+      "logits/chosen": -0.4283292591571808,
+      "logits/rejected": -0.417165070772171,
+      "logps/chosen": -11.886260986328125,
+      "logps/rejected": -22.782238006591797,
+      "loss": 1.126604676246643,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6129148602485657,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.020920906215906143,
+      "rewards/margins": 0.8850383162498474,
+      "rewards/rejected": -0.9059593677520752,
+      "step": 44,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.2183803457688808,
+      "grad_norm": 2.563398599624634,
+      "learning_rate": 0.00019971942664309968,
+      "logits/chosen": -0.4108792245388031,
+      "logits/rejected": -0.4292621612548828,
+      "logps/chosen": -10.143468856811523,
+      "logps/rejected": -21.13189125061035,
+      "loss": 0.957160472869873,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.4789010286331177,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.04898708686232567,
+      "rewards/margins": 0.9526273012161255,
+      "rewards/rejected": -0.9036401510238647,
+      "step": 45,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.2232332423415226,
+      "grad_norm": 5.1497344970703125,
+      "learning_rate": 0.00019967793553501346,
+      "logits/chosen": -0.3103107213973999,
+      "logits/rejected": -0.3952590227127075,
+      "logps/chosen": -12.430742263793945,
+      "logps/rejected": -20.878339767456055,
+      "loss": 1.2279822826385498,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6726821660995483,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.020153822377324104,
+      "rewards/margins": 0.7637170553207397,
+      "rewards/rejected": -0.7435632348060608,
+      "step": 46,
+      "train_speed(iter/s)": 0.011209
+    },
+    {
+      "epoch": 0.2280861389141644,
+      "grad_norm": 3.151014566421509,
+      "learning_rate": 0.00019963358932550192,
+      "logits/chosen": -0.3890960216522217,
+      "logits/rejected": -0.4021611511707306,
+      "logps/chosen": -10.522011756896973,
+      "logps/rejected": -18.351852416992188,
+      "loss": 1.1763105392456055,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5507612228393555,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.15107618272304535,
+      "rewards/margins": 0.30810290575027466,
+      "rewards/rejected": -0.4591791033744812,
+      "step": 47,
+      "train_speed(iter/s)": 0.011208
+    },
+    {
+      "epoch": 0.2329390354868062,
+      "grad_norm": 4.961374759674072,
+      "learning_rate": 0.00019958638928478523,
+      "logits/chosen": -0.31074726581573486,
+      "logits/rejected": -0.3801327645778656,
+      "logps/chosen": -9.19954776763916,
+      "logps/rejected": -21.655811309814453,
+      "loss": 0.9774930477142334,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5149185657501221,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.030308127403259277,
+      "rewards/margins": 0.8868625164031982,
+      "rewards/rejected": -0.8565543293952942,
+      "step": 48,
+      "train_speed(iter/s)": 0.011208
+    },
+    {
+      "epoch": 0.23779193205944799,
+      "grad_norm": 4.1918511390686035,
+      "learning_rate": 0.00019953633676482669,
+      "logits/chosen": -0.3845188617706299,
+      "logits/rejected": -0.4037459194660187,
+      "logps/chosen": -14.745367050170898,
+      "logps/rejected": -18.788145065307617,
+      "loss": 1.2587707042694092,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.7012693285942078,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.11992412805557251,
+      "rewards/margins": 0.648144543170929,
+      "rewards/rejected": -0.5282202959060669,
+      "step": 49,
+      "train_speed(iter/s)": 0.011207
+    },
+    {
+      "epoch": 0.24264482863208978,
+      "grad_norm": 3.7718863487243652,
+      "learning_rate": 0.00019948343319929374,
+      "logits/chosen": -0.3694141209125519,
+      "logits/rejected": -0.474610835313797,
+      "logps/chosen": -10.025066375732422,
+      "logps/rejected": -20.226930618286133,
+      "loss": 1.0972646474838257,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5580352544784546,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.015783965587615967,
+      "rewards/margins": 0.48214325308799744,
+      "rewards/rejected": -0.4979272484779358,
+      "step": 50,
+      "train_speed(iter/s)": 0.011207
+    },
+    {
+      "epoch": 0.24749772520473157,
+      "grad_norm": 3.204578161239624,
+      "learning_rate": 0.00019942768010351727,
+      "logits/chosen": -0.5031870007514954,
+      "logits/rejected": -0.46329694986343384,
+      "logps/chosen": -13.154851913452148,
+      "logps/rejected": -16.299560546875,
+      "loss": 1.1670925617218018,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.648939311504364,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.07203729450702667,
+      "rewards/margins": 0.5248441696166992,
+      "rewards/rejected": -0.5968814492225647,
+      "step": 51,
+      "train_speed(iter/s)": 0.011208
+    },
+    {
+      "epoch": 0.2523506217773734,
+      "grad_norm": 3.282288074493408,
+      "learning_rate": 0.00019936907907444788,
+      "logits/chosen": -0.40597957372665405,
+      "logits/rejected": -0.4711161255836487,
+      "logps/chosen": -10.530755996704102,
+      "logps/rejected": -18.246557235717773,
+      "loss": 1.1418707370758057,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6192983984947205,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.012871884740889072,
+      "rewards/margins": 0.4392779469490051,
+      "rewards/rejected": -0.4264060854911804,
+      "step": 52,
+      "train_speed(iter/s)": 0.011209
+    },
+    {
+      "epoch": 0.25720351835001515,
+      "grad_norm": 3.151282787322998,
+      "learning_rate": 0.00019930763179061036,
+      "logits/chosen": -0.4330732226371765,
+      "logits/rejected": -0.456554651260376,
+      "logps/chosen": -8.236556053161621,
+      "logps/rejected": -15.27733039855957,
+      "loss": 1.168614149093628,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6144590973854065,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.12388759851455688,
+      "rewards/margins": 0.6339080333709717,
+      "rewards/rejected": -0.5100203156471252,
+      "step": 53,
+      "train_speed(iter/s)": 0.011209
+    },
+    {
+      "epoch": 0.26205641492265697,
+      "grad_norm": 3.4259731769561768,
+      "learning_rate": 0.00019924334001205542,
+      "logits/chosen": -0.4073447287082672,
+      "logits/rejected": -0.42736995220184326,
+      "logps/chosen": -11.766214370727539,
+      "logps/rejected": -20.65784454345703,
+      "loss": 1.234757423400879,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6889488697052002,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.08055731654167175,
+      "rewards/margins": 0.7608071565628052,
+      "rewards/rejected": -0.680249810218811,
+      "step": 54,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.26690931149529873,
+      "grad_norm": 2.5751075744628906,
+      "learning_rate": 0.00019917620558030947,
+      "logits/chosen": -0.45161640644073486,
+      "logits/rejected": -0.40232208371162415,
+      "logps/chosen": -10.377983093261719,
+      "logps/rejected": -15.469682693481445,
+      "loss": 1.1288338899612427,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6078510284423828,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.13011425733566284,
+      "rewards/margins": 0.7133916020393372,
+      "rewards/rejected": -0.5832774043083191,
+      "step": 55,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.27176220806794055,
+      "grad_norm": 1.9270464181900024,
+      "learning_rate": 0.0001991062304183218,
+      "logits/chosen": -0.3679361641407013,
+      "logits/rejected": -0.4820949137210846,
+      "logps/chosen": -6.5226335525512695,
+      "logps/rejected": -21.38983917236328,
+      "loss": 0.9311513900756836,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.48036620020866394,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1770131140947342,
+      "rewards/margins": 0.9639515280723572,
+      "rewards/rejected": -0.7869384288787842,
+      "step": 56,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.27661510464058237,
+      "grad_norm": 2.343013048171997,
+      "learning_rate": 0.0001990334165304094,
+      "logits/chosen": -0.3758355975151062,
+      "logits/rejected": -0.446993350982666,
+      "logps/chosen": -10.01812744140625,
+      "logps/rejected": -22.936779022216797,
+      "loss": 1.017271637916565,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5227488875389099,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.20892874896526337,
+      "rewards/margins": 0.9375370144844055,
+      "rewards/rejected": -0.728608250617981,
+      "step": 57,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.28146800121322413,
+      "grad_norm": 3.4284698963165283,
+      "learning_rate": 0.00019895776600219973,
+      "logits/chosen": -0.3820294141769409,
+      "logits/rejected": -0.4261220097541809,
+      "logps/chosen": -12.147420883178711,
+      "logps/rejected": -21.496156692504883,
+      "loss": 1.173353910446167,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.579372763633728,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.036380209028720856,
+      "rewards/margins": 0.430204302072525,
+      "rewards/rejected": -0.46658453345298767,
+      "step": 58,
+      "train_speed(iter/s)": 0.011209
+    },
+    {
+      "epoch": 0.28632089778586595,
+      "grad_norm": 3.7487685680389404,
+      "learning_rate": 0.0001988792810005708,
+      "logits/chosen": -0.3050299882888794,
+      "logits/rejected": -0.40809452533721924,
+      "logps/chosen": -10.987959861755371,
+      "logps/rejected": -24.92184066772461,
+      "loss": 1.2469043731689453,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6960954070091248,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.009784307330846786,
+      "rewards/margins": 0.4835450351238251,
+      "rewards/rejected": -0.473760724067688,
+      "step": 59,
+      "train_speed(iter/s)": 0.011208
+    },
+    {
+      "epoch": 0.2911737943585077,
+      "grad_norm": 3.093308448791504,
+      "learning_rate": 0.00019879796377358927,
+      "logits/chosen": -0.37827837467193604,
+      "logits/rejected": -0.4184655547142029,
+      "logps/chosen": -7.49070930480957,
+      "logps/rejected": -17.80307388305664,
+      "loss": 1.0403252840042114,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.49073174595832825,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.00874985009431839,
+      "rewards/margins": 0.46609920263290405,
+      "rewards/rejected": -0.47484904527664185,
+      "step": 60,
+      "train_speed(iter/s)": 0.011209
+    },
+    {
+      "epoch": 0.2960266909311495,
+      "grad_norm": 3.3970422744750977,
+      "learning_rate": 0.00019871381665044594,
+      "logits/chosen": -0.3776484727859497,
+      "logits/rejected": -0.3863822817802429,
+      "logps/chosen": -9.921972274780273,
+      "logps/rejected": -15.067032814025879,
+      "loss": 1.250232219696045,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.7360854744911194,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.09412748366594315,
+      "rewards/margins": 0.5320085287094116,
+      "rewards/rejected": -0.4378809928894043,
+      "step": 61,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.30087958750379135,
+      "grad_norm": 2.651646137237549,
+      "learning_rate": 0.00019862684204138907,
+      "logits/chosen": -0.3813178539276123,
+      "logits/rejected": -0.3218821585178375,
+      "logps/chosen": -12.840286254882812,
+      "logps/rejected": -17.035186767578125,
+      "loss": 1.1197326183319092,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6345531940460205,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.10404972732067108,
+      "rewards/margins": 0.6996921896934509,
+      "rewards/rejected": -0.5956425070762634,
+      "step": 62,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.3057324840764331,
+      "grad_norm": 2.7409791946411133,
+      "learning_rate": 0.00019853704243765532,
+      "logits/chosen": -0.21578770875930786,
+      "logits/rejected": -0.40845662355422974,
+      "logps/chosen": -9.395906448364258,
+      "logps/rejected": -28.66230583190918,
+      "loss": 1.0102654695510864,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6361392736434937,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.23129011690616608,
+      "rewards/margins": 1.06308913230896,
+      "rewards/rejected": -0.8317990303039551,
+      "step": 63,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.3105853806490749,
+      "grad_norm": 2.692429304122925,
+      "learning_rate": 0.00019844442041139847,
+      "logits/chosen": -0.35631364583969116,
+      "logits/rejected": -0.3108435571193695,
+      "logps/chosen": -11.395570755004883,
+      "logps/rejected": -16.973485946655273,
+      "loss": 0.9985603094100952,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5419827699661255,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.07737573981285095,
+      "rewards/margins": 0.7498005628585815,
+      "rewards/rejected": -0.672424852848053,
+      "step": 64,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.3154382772217167,
+      "grad_norm": 5.938281536102295,
+      "learning_rate": 0.00019834897861561573,
+      "logits/chosen": -0.3479470908641815,
+      "logits/rejected": -0.3646903932094574,
+      "logps/chosen": -8.458881378173828,
+      "logps/rejected": -18.25616455078125,
+      "loss": 1.0791547298431396,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5545474290847778,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.05356256291270256,
+      "rewards/margins": 0.5557125210762024,
+      "rewards/rejected": -0.6092751026153564,
+      "step": 65,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.3202911737943585,
+      "grad_norm": 2.738814353942871,
+      "learning_rate": 0.00019825071978407156,
+      "logits/chosen": -0.29352137446403503,
+      "logits/rejected": -0.2870010733604431,
+      "logps/chosen": -12.466264724731445,
+      "logps/rejected": -22.961450576782227,
+      "loss": 0.9836492538452148,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5442343354225159,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.123663991689682,
+      "rewards/margins": 0.9405882954597473,
+      "rewards/rejected": -0.8169243335723877,
+      "step": 66,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.3251440703670003,
+      "grad_norm": 3.51104998588562,
+      "learning_rate": 0.00019814964673121966,
+      "logits/chosen": -0.3337523937225342,
+      "logits/rejected": -0.3147628903388977,
+      "logps/chosen": -11.947603225708008,
+      "logps/rejected": -17.78832244873047,
+      "loss": 1.2882845401763916,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.8149017095565796,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.04635877162218094,
+      "rewards/margins": 0.7259438037872314,
+      "rewards/rejected": -0.7723026275634766,
+      "step": 67,
+      "train_speed(iter/s)": 0.011213
+    },
+    {
+      "epoch": 0.3299969669396421,
+      "grad_norm": 3.068816661834717,
+      "learning_rate": 0.00019804576235212218,
+      "logits/chosen": -0.2215232253074646,
+      "logits/rejected": -0.2657161355018616,
+      "logps/chosen": -11.033540725708008,
+      "logps/rejected": -25.50714874267578,
+      "loss": 0.8847059607505798,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5216297507286072,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.09357042610645294,
+      "rewards/margins": 1.1900687217712402,
+      "rewards/rejected": -1.0964982509613037,
+      "step": 68,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.3348498635122839,
+      "grad_norm": 2.5830941200256348,
+      "learning_rate": 0.00019793906962236682,
+      "logits/chosen": -0.24091127514839172,
+      "logits/rejected": -0.29128578305244446,
+      "logps/chosen": -10.63074016571045,
+      "logps/rejected": -26.628215789794922,
+      "loss": 1.0002039670944214,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5858277082443237,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.06674355268478394,
+      "rewards/margins": 1.1017783880233765,
+      "rewards/rejected": -1.0350347757339478,
+      "step": 69,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.33970276008492567,
+      "grad_norm": 2.7311224937438965,
+      "learning_rate": 0.00019782957159798157,
+      "logits/chosen": -0.30429190397262573,
+      "logits/rejected": -0.2560291290283203,
+      "logps/chosen": -12.162567138671875,
+      "logps/rejected": -17.434350967407227,
+      "loss": 1.063048005104065,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6400512456893921,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.18220071494579315,
+      "rewards/margins": 1.0044963359832764,
+      "rewards/rejected": -0.8222954869270325,
+      "step": 70,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.3445556566575675,
+      "grad_norm": 3.414921760559082,
+      "learning_rate": 0.0001977172714153473,
+      "logits/chosen": -0.28515180945396423,
+      "logits/rejected": -0.3108617961406708,
+      "logps/chosen": -10.81340503692627,
+      "logps/rejected": -26.520463943481445,
+      "loss": 1.1172614097595215,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.685325026512146,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1235567182302475,
+      "rewards/margins": 1.0347576141357422,
+      "rewards/rejected": -0.9112008810043335,
+      "step": 71,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.3494085532302093,
+      "grad_norm": 4.576763153076172,
+      "learning_rate": 0.00019760217229110777,
+      "logits/chosen": -0.36723142862319946,
+      "logits/rejected": -0.3557523488998413,
+      "logps/chosen": -12.69983196258545,
+      "logps/rejected": -20.64535140991211,
+      "loss": 1.1201751232147217,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.692030668258667,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.049573928117752075,
+      "rewards/margins": 1.0138797760009766,
+      "rewards/rejected": -0.9643057584762573,
+      "step": 72,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.35426144980285107,
+      "grad_norm": 4.538747310638428,
+      "learning_rate": 0.00019748427752207755,
+      "logits/chosen": -0.34310680627822876,
+      "logits/rejected": -0.366294801235199,
+      "logps/chosen": -12.221488952636719,
+      "logps/rejected": -21.165037155151367,
+      "loss": 1.2563344240188599,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.7162244915962219,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.12930546700954437,
+      "rewards/margins": 0.6588571667671204,
+      "rewards/rejected": -0.7881627082824707,
+      "step": 73,
+      "train_speed(iter/s)": 0.011213
+    },
+    {
+      "epoch": 0.3591143463754929,
+      "grad_norm": 2.0262386798858643,
+      "learning_rate": 0.00019736359048514766,
+      "logits/chosen": -0.334412157535553,
+      "logits/rejected": -0.37951967120170593,
+      "logps/chosen": -14.834428787231445,
+      "logps/rejected": -21.54924964904785,
+      "loss": 1.0132083892822266,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6331067085266113,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1725282073020935,
+      "rewards/margins": 1.028924822807312,
+      "rewards/rejected": -0.8563966751098633,
+      "step": 74,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.36396724294813465,
+      "grad_norm": 3.2811484336853027,
+      "learning_rate": 0.00019724011463718885,
+      "logits/chosen": -0.360372394323349,
+      "logits/rejected": -0.35564228892326355,
+      "logps/chosen": -11.420612335205078,
+      "logps/rejected": -19.772428512573242,
+      "loss": 1.2705371379852295,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6594747304916382,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.09604446589946747,
+      "rewards/margins": 0.5641387701034546,
+      "rewards/rejected": -0.660183310508728,
+      "step": 75,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.36882013952077647,
+      "grad_norm": 3.251281976699829,
+      "learning_rate": 0.0001971138535149523,
+      "logits/chosen": -0.3563613295555115,
+      "logits/rejected": -0.38878530263900757,
+      "logps/chosen": -12.576432228088379,
+      "logps/rejected": -21.920602798461914,
+      "loss": 0.9680290222167969,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5066214799880981,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.009179912507534027,
+      "rewards/margins": 0.7688724994659424,
+      "rewards/rejected": -0.7780523896217346,
+      "step": 76,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.37367303609341823,
+      "grad_norm": 2.962810516357422,
+      "learning_rate": 0.00019698481073496878,
+      "logits/chosen": -0.3119925856590271,
+      "logits/rejected": -0.4056414067745209,
+      "logps/chosen": -9.659420013427734,
+      "logps/rejected": -23.653026580810547,
+      "loss": 0.9925091862678528,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5779968500137329,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.030417252331972122,
+      "rewards/margins": 0.987002968788147,
+      "rewards/rejected": -0.956585705280304,
+      "step": 77,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.37852593266606005,
+      "grad_norm": 10.599248886108398,
+      "learning_rate": 0.00019685298999344458,
+      "logits/chosen": -0.35163983702659607,
+      "logits/rejected": -0.3840740919113159,
+      "logps/chosen": -8.082690238952637,
+      "logps/rejected": -25.49774169921875,
+      "loss": 1.0417295694351196,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.7148680686950684,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.03415816277265549,
+      "rewards/margins": 1.2999666929244995,
+      "rewards/rejected": -1.3341249227523804,
+      "step": 78,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.38337882923870187,
+      "grad_norm": 3.775139331817627,
+      "learning_rate": 0.00019671839506615614,
+      "logits/chosen": -0.3263033926486969,
+      "logits/rejected": -0.336015522480011,
+      "logps/chosen": -12.873096466064453,
+      "logps/rejected": -25.73821258544922,
+      "loss": 1.011587142944336,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5964220762252808,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.15943416953086853,
+      "rewards/margins": 1.001997947692871,
+      "rewards/rejected": -0.8425638675689697,
+      "step": 79,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.38823172581134363,
+      "grad_norm": 2.6962356567382812,
+      "learning_rate": 0.00019658102980834146,
+      "logits/chosen": -0.42636168003082275,
+      "logits/rejected": -0.40870201587677,
+      "logps/chosen": -9.675471305847168,
+      "logps/rejected": -17.18560791015625,
+      "loss": 1.0823123455047607,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6633837819099426,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.09595775604248047,
+      "rewards/margins": 0.8637285232543945,
+      "rewards/rejected": -0.7677708268165588,
+      "step": 80,
+      "train_speed(iter/s)": 0.011213
+    },
+    {
+      "epoch": 0.39308462238398545,
+      "grad_norm": 3.252925395965576,
+      "learning_rate": 0.0001964408981545898,
+      "logits/chosen": -0.3987385034561157,
+      "logits/rejected": -0.3940485715866089,
+      "logps/chosen": -11.167372703552246,
+      "logps/rejected": -19.442848205566406,
+      "loss": 1.2748379707336426,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.8224515914916992,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.004448680207133293,
+      "rewards/margins": 0.9587394595146179,
+      "rewards/rejected": -0.9631881713867188,
+      "step": 81,
+      "train_speed(iter/s)": 0.011213
+    },
+    {
+      "epoch": 0.3979375189566272,
+      "grad_norm": 2.8878297805786133,
+      "learning_rate": 0.00019629800411872928,
+      "logits/chosen": -0.35343819856643677,
+      "logits/rejected": -0.4096568822860718,
+      "logps/chosen": -8.287420272827148,
+      "logps/rejected": -20.879899978637695,
+      "loss": 0.9840810894966125,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5656735897064209,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.13655401766300201,
+      "rewards/margins": 0.9052618145942688,
+      "rewards/rejected": -0.768707811832428,
+      "step": 82,
+      "train_speed(iter/s)": 0.011214
+    },
+    {
+      "epoch": 0.40279041552926903,
+      "grad_norm": 2.34645676612854,
+      "learning_rate": 0.00019615235179371135,
+      "logits/chosen": -0.31612086296081543,
+      "logits/rejected": -0.39879709482192993,
+      "logps/chosen": -9.401403427124023,
+      "logps/rejected": -27.325342178344727,
+      "loss": 0.888444721698761,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.569918692111969,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.23369260132312775,
+      "rewards/margins": 1.3009941577911377,
+      "rewards/rejected": -1.0673015117645264,
+      "step": 83,
+      "train_speed(iter/s)": 0.011214
+    },
+    {
+      "epoch": 0.40764331210191085,
+      "grad_norm": 2.7571706771850586,
+      "learning_rate": 0.00019600394535149404,
+      "logits/chosen": -0.34237152338027954,
+      "logits/rejected": -0.38029745221138,
+      "logps/chosen": -9.792423248291016,
+      "logps/rejected": -21.61672592163086,
+      "loss": 1.0087002515792847,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5312238335609436,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.07024291902780533,
+      "rewards/margins": 0.8068092465400696,
+      "rewards/rejected": -0.7365662455558777,
+      "step": 84,
+      "train_speed(iter/s)": 0.011214
+    },
+    {
+      "epoch": 0.4124962086745526,
+      "grad_norm": 2.0822556018829346,
+      "learning_rate": 0.0001958527890429223,
+      "logits/chosen": -0.3475739657878876,
+      "logits/rejected": -0.31929460167884827,
+      "logps/chosen": -13.526400566101074,
+      "logps/rejected": -25.060049057006836,
+      "loss": 0.8273504376411438,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5433682203292847,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2506234347820282,
+      "rewards/margins": 1.4649745225906372,
+      "rewards/rejected": -1.2143512964248657,
+      "step": 85,
+      "train_speed(iter/s)": 0.011214
+    },
+    {
+      "epoch": 0.41734910524719443,
+      "grad_norm": 2.4087798595428467,
+      "learning_rate": 0.0001956988871976062,
+      "logits/chosen": -0.27175548672676086,
+      "logits/rejected": -0.3286592662334442,
+      "logps/chosen": -7.984870910644531,
+      "logps/rejected": -24.256772994995117,
+      "loss": 0.7937061786651611,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.42936819791793823,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.059691205620765686,
+      "rewards/margins": 1.0793851613998413,
+      "rewards/rejected": -1.0196938514709473,
+      "step": 86,
+      "train_speed(iter/s)": 0.011214
+    },
+    {
+      "epoch": 0.4222020018198362,
+      "grad_norm": 2.174376964569092,
+      "learning_rate": 0.00019554224422379686,
+      "logits/chosen": -0.23324547708034515,
+      "logits/rejected": -0.30267754197120667,
+      "logps/chosen": -11.11595344543457,
+      "logps/rejected": -27.55086326599121,
+      "loss": 0.799884021282196,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.4499448537826538,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.12257401645183563,
+      "rewards/margins": 1.2369678020477295,
+      "rewards/rejected": -1.114393711090088,
+      "step": 87,
+      "train_speed(iter/s)": 0.011214
+    },
+    {
+      "epoch": 0.427054898392478,
+      "grad_norm": 4.111970901489258,
+      "learning_rate": 0.00019538286460826041,
+      "logits/chosen": -0.2658248543739319,
+      "logits/rejected": -0.2973652482032776,
+      "logps/chosen": -8.75698471069336,
+      "logps/rejected": -22.72409439086914,
+      "loss": 0.8912484645843506,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.48959213495254517,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.09087920188903809,
+      "rewards/margins": 1.290701150894165,
+      "rewards/rejected": -1.199821949005127,
+      "step": 88,
+      "train_speed(iter/s)": 0.011214
+    },
+    {
+      "epoch": 0.43190779496511983,
+      "grad_norm": 3.519275665283203,
+      "learning_rate": 0.0001952207529161492,
+      "logits/chosen": -0.3236900866031647,
+      "logits/rejected": -0.29652339220046997,
+      "logps/chosen": -15.083149909973145,
+      "logps/rejected": -29.6583251953125,
+      "loss": 1.028883695602417,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.7164204120635986,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.056427597999572754,
+      "rewards/margins": 1.5590038299560547,
+      "rewards/rejected": -1.5025763511657715,
+      "step": 89,
+      "train_speed(iter/s)": 0.011214
+    },
+    {
+      "epoch": 0.4367606915377616,
+      "grad_norm": 5.104607105255127,
+      "learning_rate": 0.00019505591379087128,
+      "logits/chosen": -0.30673032999038696,
+      "logits/rejected": -0.284626841545105,
+      "logps/chosen": -18.149417877197266,
+      "logps/rejected": -27.66114616394043,
+      "loss": 1.1724226474761963,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.8443087935447693,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.12549515068531036,
+      "rewards/margins": 1.6692070960998535,
+      "rewards/rejected": -1.5437119007110596,
+      "step": 90,
+      "train_speed(iter/s)": 0.011214
+    },
+    {
+      "epoch": 0.4416135881104034,
+      "grad_norm": 5.404626846313477,
+      "learning_rate": 0.00019488835195395718,
+      "logits/chosen": -0.25754019618034363,
+      "logits/rejected": -0.25054478645324707,
+      "logps/chosen": -11.13974380493164,
+      "logps/rejected": -29.229190826416016,
+      "loss": 0.9712298512458801,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.614799976348877,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.06202991306781769,
+      "rewards/margins": 1.4658480882644653,
+      "rewards/rejected": -1.4038183689117432,
+      "step": 91,
+      "train_speed(iter/s)": 0.011214
+    },
+    {
+      "epoch": 0.4464664846830452,
+      "grad_norm": 4.149505138397217,
+      "learning_rate": 0.00019471807220492487,
+      "logits/chosen": -0.3054165542125702,
+      "logits/rejected": -0.3217582106590271,
+      "logps/chosen": -7.596218585968018,
+      "logps/rejected": -24.510986328125,
+      "loss": 0.8925672173500061,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5096707344055176,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.05373791605234146,
+      "rewards/margins": 1.6141862869262695,
+      "rewards/rejected": -1.560448408126831,
+      "step": 92,
+      "train_speed(iter/s)": 0.011214
+    },
+    {
+      "epoch": 0.451319381255687,
+      "grad_norm": 2.699763536453247,
+      "learning_rate": 0.00019454507942114218,
+      "logits/chosen": -0.4031309485435486,
+      "logits/rejected": -0.3797576129436493,
+      "logps/chosen": -11.298748970031738,
+      "logps/rejected": -21.17618751525879,
+      "loss": 0.9288226366043091,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5129250884056091,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.09262087196111679,
+      "rewards/margins": 1.186156988143921,
+      "rewards/rejected": -1.0935360193252563,
+      "step": 93,
+      "train_speed(iter/s)": 0.011215
+    },
+    {
+      "epoch": 0.4561722778283288,
+      "grad_norm": 2.987886905670166,
+      "learning_rate": 0.00019436937855768705,
+      "logits/chosen": -0.3467521667480469,
+      "logits/rejected": -0.38047531247138977,
+      "logps/chosen": -8.127204895019531,
+      "logps/rejected": -27.24859619140625,
+      "loss": 0.9090497493743896,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5626466274261475,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.01449224166572094,
+      "rewards/margins": 1.3198893070220947,
+      "rewards/rejected": -1.3053967952728271,
+      "step": 94,
+      "train_speed(iter/s)": 0.011216
+    },
+    {
+      "epoch": 0.46102517440097057,
+      "grad_norm": 6.682155609130859,
+      "learning_rate": 0.0001941909746472057,
+      "logits/chosen": -0.36795341968536377,
+      "logits/rejected": -0.33038249611854553,
+      "logps/chosen": -14.30661392211914,
+      "logps/rejected": -27.33529281616211,
+      "loss": 1.2022923231124878,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6685870289802551,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.14697454869747162,
+      "rewards/margins": 0.7069932818412781,
+      "rewards/rejected": -0.8539677858352661,
+      "step": 95,
+      "train_speed(iter/s)": 0.011215
+    },
+    {
+      "epoch": 0.4658780709736124,
+      "grad_norm": 6.086058616638184,
+      "learning_rate": 0.00019400987279976847,
+      "logits/chosen": -0.4552175998687744,
+      "logits/rejected": -0.3753393590450287,
+      "logps/chosen": -12.3525390625,
+      "logps/rejected": -24.96709632873535,
+      "loss": 1.1781623363494873,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.8305119276046753,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.04669938236474991,
+      "rewards/margins": 1.6278446912765503,
+      "rewards/rejected": -1.674544095993042,
+      "step": 96,
+      "train_speed(iter/s)": 0.011216
+    },
+    {
+      "epoch": 0.47073096754625415,
+      "grad_norm": 2.856499671936035,
+      "learning_rate": 0.00019382607820272343,
+      "logits/chosen": -0.34398210048675537,
+      "logits/rejected": -0.3632335662841797,
+      "logps/chosen": -13.498963356018066,
+      "logps/rejected": -23.799148559570312,
+      "loss": 0.9125381708145142,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6083617806434631,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.16915112733840942,
+      "rewards/margins": 1.5896618366241455,
+      "rewards/rejected": -1.4205105304718018,
+      "step": 97,
+      "train_speed(iter/s)": 0.011216
+    },
+    {
+      "epoch": 0.47558386411889597,
+      "grad_norm": 2.2536661624908447,
+      "learning_rate": 0.00019363959612054768,
+      "logits/chosen": -0.3522893190383911,
+      "logits/rejected": -0.3821050822734833,
+      "logps/chosen": -7.958831310272217,
+      "logps/rejected": -22.634958267211914,
+      "loss": 0.9238535165786743,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5074257254600525,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.05879054591059685,
+      "rewards/margins": 1.1695164442062378,
+      "rewards/rejected": -1.1107258796691895,
+      "step": 98,
+      "train_speed(iter/s)": 0.011217
+    },
+    {
+      "epoch": 0.48043676069153773,
+      "grad_norm": 4.499397277832031,
+      "learning_rate": 0.00019345043189469675,
+      "logits/chosen": -0.3616553843021393,
+      "logits/rejected": -0.3694867789745331,
+      "logps/chosen": -9.15666389465332,
+      "logps/rejected": -22.706497192382812,
+      "loss": 0.8626678586006165,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.4881688058376312,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.22088897228240967,
+      "rewards/margins": 1.2141640186309814,
+      "rewards/rejected": -0.993274986743927,
+      "step": 99,
+      "train_speed(iter/s)": 0.011217
+    },
+    {
+      "epoch": 0.48528965726417955,
+      "grad_norm": 2.5381503105163574,
+      "learning_rate": 0.00019325859094345148,
+      "logits/chosen": -0.3770003914833069,
+      "logits/rejected": -0.4174918234348297,
+      "logps/chosen": -10.83204460144043,
+      "logps/rejected": -20.01091194152832,
+      "loss": 1.1128623485565186,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6556410193443298,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.08420430123806,
+      "rewards/margins": 0.7714729905128479,
+      "rewards/rejected": -0.6872686743736267,
+      "step": 100,
+      "train_speed(iter/s)": 0.011217
+    },
+    {
+      "epoch": 0.49014255383682137,
+      "grad_norm": 3.571274995803833,
+      "learning_rate": 0.000193064078761763,
+      "logits/chosen": -0.3813248872756958,
+      "logits/rejected": -0.4329451024532318,
+      "logps/chosen": -11.104312896728516,
+      "logps/rejected": -24.20327377319336,
+      "loss": 1.1288418769836426,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.7659333944320679,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.13299986720085144,
+      "rewards/margins": 1.1876386404037476,
+      "rewards/rejected": -1.0546387434005737,
+      "step": 101,
+      "train_speed(iter/s)": 0.011197
+    },
+    {
+      "epoch": 0.49499545040946313,
+      "grad_norm": 3.4420199394226074,
+      "learning_rate": 0.00019286690092109495,
+      "logits/chosen": -0.3884170353412628,
+      "logits/rejected": -0.4291474521160126,
+      "logps/chosen": -8.859447479248047,
+      "logps/rejected": -19.800003051757812,
+      "loss": 1.0438815355300903,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6051428318023682,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.10340974479913712,
+      "rewards/margins": 0.9933844804763794,
+      "rewards/rejected": -0.8899747729301453,
+      "step": 102,
+      "train_speed(iter/s)": 0.011197
+    },
+    {
+      "epoch": 0.49984834698210495,
+      "grad_norm": 3.0272207260131836,
+      "learning_rate": 0.00019266706306926437,
+      "logits/chosen": -0.28712770342826843,
+      "logits/rejected": -0.482435941696167,
+      "logps/chosen": -8.935839653015137,
+      "logps/rejected": -26.6412353515625,
+      "loss": 1.1441514492034912,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6857534646987915,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.1457400619983673,
+      "rewards/margins": 1.0326664447784424,
+      "rewards/rejected": -0.8869264125823975,
+      "step": 103,
+      "train_speed(iter/s)": 0.011197
+    },
+    {
+      "epoch": 0.5047012435547468,
+      "grad_norm": 2.2835068702697754,
+      "learning_rate": 0.00019246457093027955,
+      "logits/chosen": -0.3885180354118347,
+      "logits/rejected": -0.4434734582901001,
+      "logps/chosen": -7.748335838317871,
+      "logps/rejected": -27.667823791503906,
+      "loss": 0.7652894258499146,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.3754329979419708,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.06499351561069489,
+      "rewards/margins": 1.0574610233306885,
+      "rewards/rejected": -0.9924674034118652,
+      "step": 104,
+      "train_speed(iter/s)": 0.011197
+    },
+    {
+      "epoch": 0.5095541401273885,
+      "grad_norm": 3.3575081825256348,
+      "learning_rate": 0.0001922594303041764,
+      "logits/chosen": -0.314767062664032,
+      "logits/rejected": -0.418552041053772,
+      "logps/chosen": -9.141714096069336,
+      "logps/rejected": -26.07691192626953,
+      "loss": 0.8440835475921631,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.53468257188797,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.18071942031383514,
+      "rewards/margins": 1.4176182746887207,
+      "rewards/rejected": -1.2368988990783691,
+      "step": 105,
+      "train_speed(iter/s)": 0.011197
+    },
+    {
+      "epoch": 0.5144070367000303,
+      "grad_norm": 3.938190460205078,
+      "learning_rate": 0.0001920516470668519,
+      "logits/chosen": -0.38045060634613037,
+      "logits/rejected": -0.46996402740478516,
+      "logps/chosen": -9.252923965454102,
+      "logps/rejected": -25.073986053466797,
+      "loss": 0.8858380913734436,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5138120651245117,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.24657589197158813,
+      "rewards/margins": 1.3090505599975586,
+      "rewards/rejected": -1.0624747276306152,
+      "step": 106,
+      "train_speed(iter/s)": 0.011198
+    },
+    {
+      "epoch": 0.5192599332726722,
+      "grad_norm": 3.807465076446533,
+      "learning_rate": 0.00019184122716989625,
+      "logits/chosen": -0.38855308294296265,
+      "logits/rejected": -0.384641170501709,
+      "logps/chosen": -10.419635772705078,
+      "logps/rejected": -23.18486213684082,
+      "loss": 1.1566160917282104,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6819642782211304,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.015541527420282364,
+      "rewards/margins": 1.0274978876113892,
+      "rewards/rejected": -1.0430394411087036,
+      "step": 107,
+      "train_speed(iter/s)": 0.011198
+    },
+    {
+      "epoch": 0.5241128298453139,
+      "grad_norm": 3.046457529067993,
+      "learning_rate": 0.0001916281766404221,
+      "logits/chosen": -0.41123640537261963,
+      "logits/rejected": -0.4135390818119049,
+      "logps/chosen": -10.575555801391602,
+      "logps/rejected": -25.36039924621582,
+      "loss": 0.9802708625793457,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5873898863792419,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.016565335914492607,
+      "rewards/margins": 1.2314006090164185,
+      "rewards/rejected": -1.2479660511016846,
+      "step": 108,
+      "train_speed(iter/s)": 0.011199
+    },
+    {
+      "epoch": 0.5289657264179557,
+      "grad_norm": 5.713778495788574,
+      "learning_rate": 0.00019141250158089202,
+      "logits/chosen": -0.36335325241088867,
+      "logits/rejected": -0.42550188302993774,
+      "logps/chosen": -11.970105171203613,
+      "logps/rejected": -30.854656219482422,
+      "loss": 1.0406579971313477,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.7013589143753052,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.020902972668409348,
+      "rewards/margins": 1.5762500762939453,
+      "rewards/rejected": -1.5553470849990845,
+      "step": 109,
+      "train_speed(iter/s)": 0.011199
+    },
+    {
+      "epoch": 0.5338186229905975,
+      "grad_norm": 3.487213373184204,
+      "learning_rate": 0.0001911942081689437,
+      "logits/chosen": -0.33955633640289307,
+      "logits/rejected": -0.4554653465747833,
+      "logps/chosen": -13.618782997131348,
+      "logps/rejected": -27.65629005432129,
+      "loss": 1.2663241624832153,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.8004875183105469,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.034927308559417725,
+      "rewards/margins": 1.3416428565979004,
+      "rewards/rejected": -1.306715488433838,
+      "step": 110,
+      "train_speed(iter/s)": 0.011199
+    },
+    {
+      "epoch": 0.5386715195632393,
+      "grad_norm": 5.0910491943359375,
+      "learning_rate": 0.00019097330265721295,
+      "logits/chosen": -0.2648548185825348,
+      "logits/rejected": -0.3673378527164459,
+      "logps/chosen": -12.02265453338623,
+      "logps/rejected": -30.836153030395508,
+      "loss": 1.0310111045837402,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5808767676353455,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.09649889171123505,
+      "rewards/margins": 1.15187406539917,
+      "rewards/rejected": -1.248373031616211,
+      "step": 111,
+      "train_speed(iter/s)": 0.011199
+    },
+    {
+      "epoch": 0.5435244161358811,
+      "grad_norm": 2.8877944946289062,
+      "learning_rate": 0.00019074979137315468,
+      "logits/chosen": -0.4370982348918915,
+      "logits/rejected": -0.41886797547340393,
+      "logps/chosen": -8.66671371459961,
+      "logps/rejected": -21.121248245239258,
+      "loss": 0.915783703327179,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5682031512260437,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.0029869861900806427,
+      "rewards/margins": 1.161545753479004,
+      "rewards/rejected": -1.1645326614379883,
+      "step": 112,
+      "train_speed(iter/s)": 0.0112
+    },
+    {
+      "epoch": 0.5483773127085229,
+      "grad_norm": 6.939223289489746,
+      "learning_rate": 0.0001905236807188616,
+      "logits/chosen": -0.32761722803115845,
+      "logits/rejected": -0.3690508306026459,
+      "logps/chosen": -9.920032501220703,
+      "logps/rejected": -33.124969482421875,
+      "loss": 0.9988139867782593,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6303930282592773,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.05219712853431702,
+      "rewards/margins": 1.4993199110031128,
+      "rewards/rejected": -1.5515170097351074,
+      "step": 113,
+      "train_speed(iter/s)": 0.0112
+    },
+    {
+      "epoch": 0.5532302092811647,
+      "grad_norm": 3.1117734909057617,
+      "learning_rate": 0.00019029497717088086,
+      "logits/chosen": -0.3898760974407196,
+      "logits/rejected": -0.42370229959487915,
+      "logps/chosen": -8.5205078125,
+      "logps/rejected": -28.223445892333984,
+      "loss": 0.9178063869476318,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5913013219833374,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.11722102761268616,
+      "rewards/margins": 1.5129557847976685,
+      "rewards/rejected": -1.3957347869873047,
+      "step": 114,
+      "train_speed(iter/s)": 0.0112
+    },
+    {
+      "epoch": 0.5580831058538065,
+      "grad_norm": 9.934938430786133,
+      "learning_rate": 0.00019006368728002862,
+      "logits/chosen": -0.3294893503189087,
+      "logits/rejected": -0.4310123026371002,
+      "logps/chosen": -7.87459135055542,
+      "logps/rejected": -26.407917022705078,
+      "loss": 1.0052833557128906,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6052643656730652,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.08163556456565857,
+      "rewards/margins": 1.377348780632019,
+      "rewards/rejected": -1.295713186264038,
+      "step": 115,
+      "train_speed(iter/s)": 0.011201
+    },
+    {
+      "epoch": 0.5629360024264483,
+      "grad_norm": 2.110900640487671,
+      "learning_rate": 0.00018982981767120231,
+      "logits/chosen": -0.2874082624912262,
+      "logits/rejected": -0.27834200859069824,
+      "logps/chosen": -11.620476722717285,
+      "logps/rejected": -28.311389923095703,
+      "loss": 0.8064985871315002,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.4605460464954376,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.03009045496582985,
+      "rewards/margins": 1.43373441696167,
+      "rewards/rejected": -1.4036438465118408,
+      "step": 116,
+      "train_speed(iter/s)": 0.0112
+    },
+    {
+      "epoch": 0.5677888989990901,
+      "grad_norm": 4.351490497589111,
+      "learning_rate": 0.0001895933750431908,
+      "logits/chosen": -0.31533291935920715,
+      "logits/rejected": -0.33318501710891724,
+      "logps/chosen": -9.928375244140625,
+      "logps/rejected": -26.295669555664062,
+      "loss": 1.0582681894302368,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6660478711128235,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.0869743824005127,
+      "rewards/margins": 1.2630525827407837,
+      "rewards/rejected": -1.3500269651412964,
+      "step": 117,
+      "train_speed(iter/s)": 0.011201
+    },
+    {
+      "epoch": 0.5726417955717319,
+      "grad_norm": 2.8951752185821533,
+      "learning_rate": 0.0001893543661684828,
+      "logits/chosen": -0.33839067816734314,
+      "logits/rejected": -0.3445620536804199,
+      "logps/chosen": -8.102829933166504,
+      "logps/rejected": -22.460590362548828,
+      "loss": 0.7739503979682922,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.43036913871765137,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1979444921016693,
+      "rewards/margins": 1.243354082107544,
+      "rewards/rejected": -1.0454095602035522,
+      "step": 118,
+      "train_speed(iter/s)": 0.011201
+    },
+    {
+      "epoch": 0.5774946921443737,
+      "grad_norm": 7.867568492889404,
+      "learning_rate": 0.00018911279789307254,
+      "logits/chosen": -0.2516986131668091,
+      "logits/rejected": -0.30931952595710754,
+      "logps/chosen": -8.088445663452148,
+      "logps/rejected": -28.628847122192383,
+      "loss": 0.8760090470314026,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6033605933189392,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.18770426511764526,
+      "rewards/margins": 1.7780780792236328,
+      "rewards/rejected": -1.5903738737106323,
+      "step": 119,
+      "train_speed(iter/s)": 0.011201
+    },
+    {
+      "epoch": 0.5823475887170154,
+      "grad_norm": 9.06499195098877,
+      "learning_rate": 0.00018886867713626394,
+      "logits/chosen": -0.31181472539901733,
+      "logits/rejected": -0.35031330585479736,
+      "logps/chosen": -6.005623817443848,
+      "logps/rejected": -24.879501342773438,
+      "loss": 0.7998319864273071,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5190199613571167,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.15923379361629486,
+      "rewards/margins": 1.547129511833191,
+      "rewards/rejected": -1.3878957033157349,
+      "step": 120,
+      "train_speed(iter/s)": 0.011202
+    },
+    {
+      "epoch": 0.5872004852896573,
+      "grad_norm": 2.1969258785247803,
+      "learning_rate": 0.00018862201089047228,
+      "logits/chosen": -0.36267367005348206,
+      "logits/rejected": -0.35898861289024353,
+      "logps/chosen": -9.9768648147583,
+      "logps/rejected": -31.001258850097656,
+      "loss": 0.8342351913452148,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6177542805671692,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3305099904537201,
+      "rewards/margins": 2.1078670024871826,
+      "rewards/rejected": -1.7773568630218506,
+      "step": 121,
+      "train_speed(iter/s)": 0.011202
+    },
+    {
+      "epoch": 0.592053381862299,
+      "grad_norm": 4.638126850128174,
+      "learning_rate": 0.000188372806221024,
+      "logits/chosen": -0.2871503531932831,
+      "logits/rejected": -0.3778407573699951,
+      "logps/chosen": -11.12118911743164,
+      "logps/rejected": -28.530237197875977,
+      "loss": 1.1293538808822632,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.7672280669212341,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.024127217009663582,
+      "rewards/margins": 1.5403187274932861,
+      "rewards/rejected": -1.5644460916519165,
+      "step": 122,
+      "train_speed(iter/s)": 0.011203
+    },
+    {
+      "epoch": 0.5969062784349408,
+      "grad_norm": 2.8399314880371094,
+      "learning_rate": 0.0001881210702659542,
+      "logits/chosen": -0.2696555554866791,
+      "logits/rejected": -0.2814023196697235,
+      "logps/chosen": -12.141545295715332,
+      "logps/rejected": -28.03343391418457,
+      "loss": 0.8627264499664307,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5336970090866089,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.07849591970443726,
+      "rewards/margins": 1.3175827264785767,
+      "rewards/rejected": -1.2390867471694946,
+      "step": 123,
+      "train_speed(iter/s)": 0.011203
+    },
+    {
+      "epoch": 0.6017591750075827,
+      "grad_norm": 2.7301621437072754,
+      "learning_rate": 0.00018786681023580235,
+      "logits/chosen": -0.26133084297180176,
+      "logits/rejected": -0.2869059145450592,
+      "logps/chosen": -12.609733581542969,
+      "logps/rejected": -32.32643508911133,
+      "loss": 0.7636784315109253,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.4288833439350128,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.08298669010400772,
+      "rewards/margins": 1.697174072265625,
+      "rewards/rejected": -1.6141873598098755,
+      "step": 124,
+      "train_speed(iter/s)": 0.011202
+    },
+    {
+      "epoch": 0.6066120715802245,
+      "grad_norm": 4.0161309242248535,
+      "learning_rate": 0.0001876100334134056,
+      "logits/chosen": -0.28447672724723816,
+      "logits/rejected": -0.3584819436073303,
+      "logps/chosen": -17.320146560668945,
+      "logps/rejected": -27.78953742980957,
+      "loss": 1.2440454959869385,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.867017924785614,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.12772110104560852,
+      "rewards/margins": 1.4953205585479736,
+      "rewards/rejected": -1.3675994873046875,
+      "step": 125,
+      "train_speed(iter/s)": 0.011202
+    },
+    {
+      "epoch": 0.6114649681528662,
+      "grad_norm": 4.7182159423828125,
+      "learning_rate": 0.00018735074715369026,
+      "logits/chosen": -0.3351546823978424,
+      "logits/rejected": -0.34775683283805847,
+      "logps/chosen": -12.783681869506836,
+      "logps/rejected": -28.113178253173828,
+      "loss": 1.1283553838729858,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.7738282084465027,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.028632041066884995,
+      "rewards/margins": 1.5667368173599243,
+      "rewards/rejected": -1.5381048917770386,
+      "step": 126,
+      "train_speed(iter/s)": 0.011203
+    },
+    {
+      "epoch": 0.6163178647255081,
+      "grad_norm": 3.1524689197540283,
+      "learning_rate": 0.00018708895888346118,
+      "logits/chosen": -0.33259081840515137,
+      "logits/rejected": -0.38818880915641785,
+      "logps/chosen": -8.918370246887207,
+      "logps/rejected": -27.71927261352539,
+      "loss": 0.9142798185348511,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6048630475997925,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16594702005386353,
+      "rewards/margins": 1.7087321281433105,
+      "rewards/rejected": -1.5427850484848022,
+      "step": 127,
+      "train_speed(iter/s)": 0.011203
+    },
+    {
+      "epoch": 0.6211707612981499,
+      "grad_norm": 2.636528491973877,
+      "learning_rate": 0.00018682467610118883,
+      "logits/chosen": -0.2616899013519287,
+      "logits/rejected": -0.31121164560317993,
+      "logps/chosen": -7.396364212036133,
+      "logps/rejected": -26.648653030395508,
+      "loss": 0.685588538646698,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.40424948930740356,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.18888305127620697,
+      "rewards/margins": 1.6681468486785889,
+      "rewards/rejected": -1.4792636632919312,
+      "step": 128,
+      "train_speed(iter/s)": 0.011203
+    },
+    {
+      "epoch": 0.6260236578707916,
+      "grad_norm": 5.758305072784424,
+      "learning_rate": 0.00018655790637679478,
+      "logits/chosen": -0.3434281349182129,
+      "logits/rejected": -0.36044564843177795,
+      "logps/chosen": -8.97478199005127,
+      "logps/rejected": -31.279216766357422,
+      "loss": 0.9071744680404663,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5397835373878479,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.0950365662574768,
+      "rewards/margins": 1.9973032474517822,
+      "rewards/rejected": -1.9022667407989502,
+      "step": 129,
+      "train_speed(iter/s)": 0.011203
+    },
+    {
+      "epoch": 0.6308765544434334,
+      "grad_norm": 5.446237087249756,
+      "learning_rate": 0.00018628865735143463,
+      "logits/chosen": -0.40321969985961914,
+      "logits/rejected": -0.44060012698173523,
+      "logps/chosen": -7.757773399353027,
+      "logps/rejected": -26.798311233520508,
+      "loss": 0.9266724586486816,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5693855881690979,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.0821087509393692,
+      "rewards/margins": 1.3789818286895752,
+      "rewards/rejected": -1.4610905647277832,
+      "step": 130,
+      "train_speed(iter/s)": 0.011204
+    },
+    {
+      "epoch": 0.6357294510160753,
+      "grad_norm": 3.8205740451812744,
+      "learning_rate": 0.00018601693673727933,
+      "logits/chosen": -0.40578073263168335,
+      "logits/rejected": -0.45195069909095764,
+      "logps/chosen": -12.803248405456543,
+      "logps/rejected": -31.195695877075195,
+      "loss": 0.9055405855178833,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6313671469688416,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.27764347195625305,
+      "rewards/margins": 1.7698627710342407,
+      "rewards/rejected": -1.492219090461731,
+      "step": 131,
+      "train_speed(iter/s)": 0.011204
+    },
+    {
+      "epoch": 0.640582347588717,
+      "grad_norm": 4.257453918457031,
+      "learning_rate": 0.00018574275231729418,
+      "logits/chosen": -0.3832516670227051,
+      "logits/rejected": -0.40236127376556396,
+      "logps/chosen": -10.488710403442383,
+      "logps/rejected": -29.920001983642578,
+      "loss": 0.9821866154670715,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6411001682281494,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.07582154870033264,
+      "rewards/margins": 1.663382649421692,
+      "rewards/rejected": -1.7392042875289917,
+      "step": 132,
+      "train_speed(iter/s)": 0.011204
+    },
+    {
+      "epoch": 0.6454352441613588,
+      "grad_norm": 5.575984954833984,
+      "learning_rate": 0.00018546611194501594,
+      "logits/chosen": -0.3834778666496277,
+      "logits/rejected": -0.4196530878543854,
+      "logps/chosen": -8.930424690246582,
+      "logps/rejected": -34.139522552490234,
+      "loss": 0.9412792921066284,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5637305974960327,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.2074286788702011,
+      "rewards/margins": 1.6988741159439087,
+      "rewards/rejected": -1.9063026905059814,
+      "step": 133,
+      "train_speed(iter/s)": 0.011204
+    },
+    {
+      "epoch": 0.6502881407340007,
+      "grad_norm": 2.6553735733032227,
+      "learning_rate": 0.00018518702354432774,
+      "logits/chosen": -0.302835613489151,
+      "logits/rejected": -0.4271845519542694,
+      "logps/chosen": -6.828452110290527,
+      "logps/rejected": -32.71015167236328,
+      "loss": 0.8524077534675598,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5101207494735718,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.007050292566418648,
+      "rewards/margins": 1.6347914934158325,
+      "rewards/rejected": -1.6277412176132202,
+      "step": 134,
+      "train_speed(iter/s)": 0.011204
+    },
+    {
+      "epoch": 0.6551410373066424,
+      "grad_norm": 2.3347177505493164,
+      "learning_rate": 0.0001849054951092324,
+      "logits/chosen": -0.4007658362388611,
+      "logits/rejected": -0.4008050560951233,
+      "logps/chosen": -10.33335018157959,
+      "logps/rejected": -23.038896560668945,
+      "loss": 0.8887284398078918,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5501910448074341,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2812037765979767,
+      "rewards/margins": 1.3908374309539795,
+      "rewards/rejected": -1.1096336841583252,
+      "step": 135,
+      "train_speed(iter/s)": 0.011205
+    },
+    {
+      "epoch": 0.6599939338792842,
+      "grad_norm": 2.9783730506896973,
+      "learning_rate": 0.00018462153470362322,
+      "logits/chosen": -0.3893599808216095,
+      "logits/rejected": -0.44760093092918396,
+      "logps/chosen": -7.99445915222168,
+      "logps/rejected": -26.292232513427734,
+      "loss": 0.9434449672698975,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5910047292709351,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.11856328696012497,
+      "rewards/margins": 1.402287244796753,
+      "rewards/rejected": -1.2837239503860474,
+      "step": 136,
+      "train_speed(iter/s)": 0.011205
+    },
+    {
+      "epoch": 0.6648468304519259,
+      "grad_norm": 2.7139804363250732,
+      "learning_rate": 0.00018433515046105306,
+      "logits/chosen": -0.4038199782371521,
+      "logits/rejected": -0.3786419630050659,
+      "logps/chosen": -11.153128623962402,
+      "logps/rejected": -27.12870216369629,
+      "loss": 0.8474470376968384,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5868622660636902,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.18890687823295593,
+      "rewards/margins": 1.7894312143325806,
+      "rewards/rejected": -1.6005244255065918,
+      "step": 137,
+      "train_speed(iter/s)": 0.011205
+    },
+    {
+      "epoch": 0.6696997270245678,
+      "grad_norm": 2.877267599105835,
+      "learning_rate": 0.0001840463505845014,
+      "logits/chosen": -0.30394551157951355,
+      "logits/rejected": -0.42835158109664917,
+      "logps/chosen": -7.929500102996826,
+      "logps/rejected": -30.011865615844727,
+      "loss": 0.806150496006012,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.49172037839889526,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.15592113137245178,
+      "rewards/margins": 1.6279278993606567,
+      "rewards/rejected": -1.4720066785812378,
+      "step": 138,
+      "train_speed(iter/s)": 0.011205
+    },
+    {
+      "epoch": 0.6745526235972096,
+      "grad_norm": 2.5812747478485107,
+      "learning_rate": 0.0001837551433461393,
+      "logits/chosen": -0.3570711016654968,
+      "logits/rejected": -0.40324440598487854,
+      "logps/chosen": -9.332073211669922,
+      "logps/rejected": -23.627119064331055,
+      "loss": 0.7940301299095154,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.480025053024292,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.24301281571388245,
+      "rewards/margins": 1.4770238399505615,
+      "rewards/rejected": -1.234011173248291,
+      "step": 139,
+      "train_speed(iter/s)": 0.011206
+    },
+    {
+      "epoch": 0.6794055201698513,
+      "grad_norm": 9.010900497436523,
+      "learning_rate": 0.00018346153708709268,
+      "logits/chosen": -0.36091697216033936,
+      "logits/rejected": -0.36146706342697144,
+      "logps/chosen": -11.872391700744629,
+      "logps/rejected": -22.261938095092773,
+      "loss": 1.231181263923645,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.7421593070030212,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.055155493319034576,
+      "rewards/margins": 0.9281628131866455,
+      "rewards/rejected": -0.8730072975158691,
+      "step": 140,
+      "train_speed(iter/s)": 0.011205
+    },
+    {
+      "epoch": 0.6842584167424932,
+      "grad_norm": 2.032696485519409,
+      "learning_rate": 0.00018316554021720306,
+      "logits/chosen": -0.32833001017570496,
+      "logits/rejected": -0.44503307342529297,
+      "logps/chosen": -7.086223602294922,
+      "logps/rejected": -27.835424423217773,
+      "loss": 0.8454415202140808,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.49735450744628906,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.16136682033538818,
+      "rewards/margins": 1.4422115087509155,
+      "rewards/rejected": -1.2808446884155273,
+      "step": 141,
+      "train_speed(iter/s)": 0.011206
+    },
+    {
+      "epoch": 0.689111313315135,
+      "grad_norm": 8.792712211608887,
+      "learning_rate": 0.00018286716121478693,
+      "logits/chosen": -0.32804882526397705,
+      "logits/rejected": -0.4235531687736511,
+      "logps/chosen": -7.5324883460998535,
+      "logps/rejected": -31.770389556884766,
+      "loss": 0.6662007570266724,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.43374398350715637,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.18283605575561523,
+      "rewards/margins": 2.050058364868164,
+      "rewards/rejected": -1.8672223091125488,
+      "step": 142,
+      "train_speed(iter/s)": 0.011206
+    },
+    {
+      "epoch": 0.6939642098877767,
+      "grad_norm": 3.7807235717773438,
+      "learning_rate": 0.00018256640862639287,
+      "logits/chosen": -0.31429523229599,
+      "logits/rejected": -0.42787081003189087,
+      "logps/chosen": -7.7822160720825195,
+      "logps/rejected": -30.884796142578125,
+      "loss": 0.823533296585083,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.556084394454956,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.06609532237052917,
+      "rewards/margins": 1.6945431232452393,
+      "rewards/rejected": -1.6284478902816772,
+      "step": 143,
+      "train_speed(iter/s)": 0.011206
+    },
+    {
+      "epoch": 0.6988171064604186,
+      "grad_norm": 2.3116719722747803,
+      "learning_rate": 0.00018226329106655668,
+      "logits/chosen": -0.4266669750213623,
+      "logits/rejected": -0.4430805444717407,
+      "logps/chosen": -9.213752746582031,
+      "logps/rejected": -29.852783203125,
+      "loss": 0.671653687953949,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.434116929769516,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08867526799440384,
+      "rewards/margins": 1.9470399618148804,
+      "rewards/rejected": -1.8583647012710571,
+      "step": 144,
+      "train_speed(iter/s)": 0.011206
+    },
+    {
+      "epoch": 0.7036700030330604,
+      "grad_norm": 3.421821355819702,
+      "learning_rate": 0.00018195781721755465,
+      "logits/chosen": -0.44033709168434143,
+      "logits/rejected": -0.4357948899269104,
+      "logps/chosen": -9.914063453674316,
+      "logps/rejected": -33.468997955322266,
+      "loss": 0.911195695400238,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5890682935714722,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.049831733107566833,
+      "rewards/margins": 2.241945505142212,
+      "rewards/rejected": -2.1921138763427734,
+      "step": 145,
+      "train_speed(iter/s)": 0.011207
+    },
+    {
+      "epoch": 0.7085228996057021,
+      "grad_norm": 2.5243051052093506,
+      "learning_rate": 0.00018164999582915488,
+      "logits/chosen": -0.3629850447177887,
+      "logits/rejected": -0.4418475925922394,
+      "logps/chosen": -12.760583877563477,
+      "logps/rejected": -46.22138595581055,
+      "loss": 0.8992067575454712,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.685957133769989,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3213002681732178,
+      "rewards/margins": 3.270735502243042,
+      "rewards/rejected": -2.9494354724884033,
+      "step": 146,
+      "train_speed(iter/s)": 0.011206
+    },
+    {
+      "epoch": 0.7133757961783439,
+      "grad_norm": 4.871396064758301,
+      "learning_rate": 0.00018133983571836667,
+      "logits/chosen": -0.33027127385139465,
+      "logits/rejected": -0.4520616829395294,
+      "logps/chosen": -10.541389465332031,
+      "logps/rejected": -41.49027633666992,
+      "loss": 1.0133702754974365,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6680970788002014,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.13710376620292664,
+      "rewards/margins": 2.4736220836639404,
+      "rewards/rejected": -2.3365182876586914,
+      "step": 147,
+      "train_speed(iter/s)": 0.011206
+    },
+    {
+      "epoch": 0.7182286927509858,
+      "grad_norm": 3.6936795711517334,
+      "learning_rate": 0.000181027345769188,
+      "logits/chosen": -0.41163426637649536,
+      "logits/rejected": -0.40933167934417725,
+      "logps/chosen": -9.165144920349121,
+      "logps/rejected": -30.310758590698242,
+      "loss": 0.7363777160644531,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.42432036995887756,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.02400527521967888,
+      "rewards/margins": 1.7538728713989258,
+      "rewards/rejected": -1.7778780460357666,
+      "step": 148,
+      "train_speed(iter/s)": 0.011206
+    },
+    {
+      "epoch": 0.7230815893236275,
+      "grad_norm": 6.49341344833374,
+      "learning_rate": 0.00018071253493235094,
+      "logits/chosen": -0.4786967933177948,
+      "logits/rejected": -0.4735773801803589,
+      "logps/chosen": -12.567697525024414,
+      "logps/rejected": -31.06718635559082,
+      "loss": 1.0480177402496338,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.7017223834991455,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.04890735074877739,
+      "rewards/margins": 1.8125377893447876,
+      "rewards/rejected": -1.8614450693130493,
+      "step": 149,
+      "train_speed(iter/s)": 0.011206
+    },
+    {
+      "epoch": 0.7279344858962693,
+      "grad_norm": 11.014755249023438,
+      "learning_rate": 0.0001803954122250654,
+      "logits/chosen": -0.493363618850708,
+      "logits/rejected": -0.560202419757843,
+      "logps/chosen": -11.512059211730957,
+      "logps/rejected": -37.07139587402344,
+      "loss": 1.237532138824463,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.7943273782730103,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.1521718055009842,
+      "rewards/margins": 2.0435235500335693,
+      "rewards/rejected": -2.195695161819458,
+      "step": 150,
+      "train_speed(iter/s)": 0.011207
+    },
+    {
+      "epoch": 0.7327873824689112,
+      "grad_norm": 4.69052791595459,
+      "learning_rate": 0.00018007598673076093,
+      "logits/chosen": -0.4235774278640747,
+      "logits/rejected": -0.4463633596897125,
+      "logps/chosen": -11.031900405883789,
+      "logps/rejected": -27.883403778076172,
+      "loss": 1.107750415802002,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.7605136632919312,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.05042722821235657,
+      "rewards/margins": 1.6558500528335571,
+      "rewards/rejected": -1.7062770128250122,
+      "step": 151,
+      "train_speed(iter/s)": 0.011207
+    },
+    {
+      "epoch": 0.7376402790415529,
+      "grad_norm": 2.4655253887176514,
+      "learning_rate": 0.00017975426759882614,
+      "logits/chosen": -0.44894903898239136,
+      "logits/rejected": -0.4738643765449524,
+      "logps/chosen": -9.968032836914062,
+      "logps/rejected": -29.113271713256836,
+      "loss": 0.7568652629852295,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5070943236351013,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.34214267134666443,
+      "rewards/margins": 1.9045073986053467,
+      "rewards/rejected": -1.5623646974563599,
+      "step": 152,
+      "train_speed(iter/s)": 0.011208
+    },
+    {
+      "epoch": 0.7424931756141947,
+      "grad_norm": 3.0639827251434326,
+      "learning_rate": 0.00017943026404434719,
+      "logits/chosen": -0.45172956585884094,
+      "logits/rejected": -0.5009413957595825,
+      "logps/chosen": -9.160325050354004,
+      "logps/rejected": -31.873594284057617,
+      "loss": 0.9305005669593811,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.6987206935882568,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.11259810626506805,
+      "rewards/margins": 1.9045584201812744,
+      "rewards/rejected": -1.7919602394104004,
+      "step": 153,
+      "train_speed(iter/s)": 0.011208
+    },
+    {
+      "epoch": 0.7473460721868365,
+      "grad_norm": 3.0101685523986816,
+      "learning_rate": 0.00017910398534784337,
+      "logits/chosen": -0.3823431730270386,
+      "logits/rejected": -0.47442248463630676,
+      "logps/chosen": -10.423698425292969,
+      "logps/rejected": -23.59076690673828,
+      "loss": 1.046834945678711,
+      "memory(GiB)": 40.31,
+      "nll_loss": 0.5919281244277954,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2845015525817871,
+      "rewards/margins": 1.1300426721572876,
+      "rewards/rejected": -0.8455410599708557,
+      "step": 154,
+      "train_speed(iter/s)": 0.011208
+    },
+    {
+      "epoch": 0.7521989687594783,
+      "grad_norm": 6.684966087341309,
+      "learning_rate": 0.00017877544085500156,
+      "logits/chosen": -0.32389581203460693,
+      "logits/rejected": -0.4660918116569519,
+      "logps/chosen": -5.926966190338135,
+      "logps/rejected": -32.869140625,
+      "loss": 0.8766897916793823,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5428420305252075,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.006232857704162598,
+      "rewards/margins": 1.7234890460968018,
+      "rewards/rejected": -1.7172563076019287,
+      "step": 155,
+      "train_speed(iter/s)": 0.011208
+    },
+    {
+      "epoch": 0.7570518653321201,
+      "grad_norm": 3.565051317214966,
+      "learning_rate": 0.00017844463997640842,
+      "logits/chosen": -0.40359729528427124,
+      "logits/rejected": -0.44952863454818726,
+      "logps/chosen": -11.961484909057617,
+      "logps/rejected": -28.9517879486084,
+      "loss": 1.0656830072402954,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.6759463548660278,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.06430176645517349,
+      "rewards/margins": 1.5096461772918701,
+      "rewards/rejected": -1.44534432888031,
+      "step": 156,
+      "train_speed(iter/s)": 0.011208
+    },
+    {
+      "epoch": 0.7619047619047619,
+      "grad_norm": 2.5383567810058594,
+      "learning_rate": 0.00017811159218728082,
+      "logits/chosen": -0.39084097743034363,
+      "logits/rejected": -0.4242747128009796,
+      "logps/chosen": -8.804425239562988,
+      "logps/rejected": -22.249990463256836,
+      "loss": 0.9147839546203613,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.590728759765625,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2870722711086273,
+      "rewards/margins": 1.4324426651000977,
+      "rewards/rejected": -1.1453704833984375,
+      "step": 157,
+      "train_speed(iter/s)": 0.011208
+    },
+    {
+      "epoch": 0.7667576584774037,
+      "grad_norm": 2.662156343460083,
+      "learning_rate": 0.00017777630702719453,
+      "logits/chosen": -0.32536131143569946,
+      "logits/rejected": -0.397830992937088,
+      "logps/chosen": -9.738006591796875,
+      "logps/rejected": -30.069774627685547,
+      "loss": 0.8968651294708252,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.527851402759552,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.08923595398664474,
+      "rewards/margins": 1.6620792150497437,
+      "rewards/rejected": -1.5728431940078735,
+      "step": 158,
+      "train_speed(iter/s)": 0.011208
+    },
+    {
+      "epoch": 0.7716105550500455,
+      "grad_norm": 3.2361485958099365,
+      "learning_rate": 0.00017743879409981095,
+      "logits/chosen": -0.3466249704360962,
+      "logits/rejected": -0.41640138626098633,
+      "logps/chosen": -11.293127059936523,
+      "logps/rejected": -30.880006790161133,
+      "loss": 0.9047757387161255,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5920196175575256,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.030911901965737343,
+      "rewards/margins": 1.6093332767486572,
+      "rewards/rejected": -1.5784213542938232,
+      "step": 159,
+      "train_speed(iter/s)": 0.011208
+    },
+    {
+      "epoch": 0.7764634516226873,
+      "grad_norm": 2.2754669189453125,
+      "learning_rate": 0.00017709906307260194,
+      "logits/chosen": -0.3480105400085449,
+      "logits/rejected": -0.4323396682739258,
+      "logps/chosen": -5.929109573364258,
+      "logps/rejected": -34.05678939819336,
+      "loss": 0.7185795903205872,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4936397075653076,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.06835521012544632,
+      "rewards/margins": 2.1114068031311035,
+      "rewards/rejected": -2.0430517196655273,
+      "step": 160,
+      "train_speed(iter/s)": 0.011209
+    },
+    {
+      "epoch": 0.7813163481953291,
+      "grad_norm": 2.226016044616699,
+      "learning_rate": 0.000176757123676573,
+      "logits/chosen": -0.39186131954193115,
+      "logits/rejected": -0.3231448531150818,
+      "logps/chosen": -12.739521026611328,
+      "logps/rejected": -28.13015365600586,
+      "loss": 0.8966748714447021,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5328811407089233,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.17805510759353638,
+      "rewards/margins": 1.8448935747146606,
+      "rewards/rejected": -1.6668384075164795,
+      "step": 161,
+      "train_speed(iter/s)": 0.011208
+    },
+    {
+      "epoch": 0.7861692447679709,
+      "grad_norm": 2.2700612545013428,
+      "learning_rate": 0.00017641298570598459,
+      "logits/chosen": -0.36923766136169434,
+      "logits/rejected": -0.41111743450164795,
+      "logps/chosen": -6.442410469055176,
+      "logps/rejected": -29.95729637145996,
+      "loss": 0.7110913991928101,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4247634708881378,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.08174540847539902,
+      "rewards/margins": 1.7869547605514526,
+      "rewards/rejected": -1.7052093744277954,
+      "step": 162,
+      "train_speed(iter/s)": 0.011208
+    },
+    {
+      "epoch": 0.7910221413406127,
+      "grad_norm": 5.662917613983154,
+      "learning_rate": 0.0001760666590180714,
+      "logits/chosen": -0.2990395426750183,
+      "logits/rejected": -0.4443129301071167,
+      "logps/chosen": -11.827997207641602,
+      "logps/rejected": -39.40394592285156,
+      "loss": 0.9818216562271118,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.6332793831825256,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.07979752123355865,
+      "rewards/margins": 2.349473237991333,
+      "rewards/rejected": -2.4292705059051514,
+      "step": 163,
+      "train_speed(iter/s)": 0.011208
+    },
+    {
+      "epoch": 0.7958750379132544,
+      "grad_norm": 2.659315586090088,
+      "learning_rate": 0.0001757181535327602,
+      "logits/chosen": -0.41728484630584717,
+      "logits/rejected": -0.44338852167129517,
+      "logps/chosen": -7.938527584075928,
+      "logps/rejected": -30.74227523803711,
+      "loss": 0.8022794723510742,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4814991056919098,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.056870464235544205,
+      "rewards/margins": 1.9418084621429443,
+      "rewards/rejected": -1.8849378824234009,
+      "step": 164,
+      "train_speed(iter/s)": 0.011208
+    },
+    {
+      "epoch": 0.8007279344858963,
+      "grad_norm": 2.9071202278137207,
+      "learning_rate": 0.00017536747923238566,
+      "logits/chosen": -0.3409942090511322,
+      "logits/rejected": -0.457685649394989,
+      "logps/chosen": -7.500731468200684,
+      "logps/rejected": -44.705909729003906,
+      "loss": 0.6390520930290222,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.38755032420158386,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.19773569703102112,
+      "rewards/margins": 3.0309081077575684,
+      "rewards/rejected": -2.83317232131958,
+      "step": 165,
+      "train_speed(iter/s)": 0.011208
+    },
+    {
+      "epoch": 0.8055808310585381,
+      "grad_norm": 3.0054497718811035,
+      "learning_rate": 0.00017501464616140436,
+      "logits/chosen": -0.41681885719299316,
+      "logits/rejected": -0.45619145035743713,
+      "logps/chosen": -8.748014450073242,
+      "logps/rejected": -35.70191955566406,
+      "loss": 0.7890005111694336,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5446341633796692,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.16440145671367645,
+      "rewards/margins": 2.74184250831604,
+      "rewards/rejected": -2.5774412155151367,
+      "step": 166,
+      "train_speed(iter/s)": 0.011209
+    },
+    {
+      "epoch": 0.8104337276311798,
+      "grad_norm": 3.300097942352295,
+      "learning_rate": 0.000174659664426107,
+      "logits/chosen": -0.4365924298763275,
+      "logits/rejected": -0.45622488856315613,
+      "logps/chosen": -11.21609115600586,
+      "logps/rejected": -34.401222229003906,
+      "loss": 0.9579750299453735,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.7187469601631165,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.14886964857578278,
+      "rewards/margins": 2.472686529159546,
+      "rewards/rejected": -2.323817253112793,
+      "step": 167,
+      "train_speed(iter/s)": 0.011209
+    },
+    {
+      "epoch": 0.8152866242038217,
+      "grad_norm": 2.5296413898468018,
+      "learning_rate": 0.00017430254419432931,
+      "logits/chosen": -0.41913431882858276,
+      "logits/rejected": -0.41736355423927307,
+      "logps/chosen": -13.593805313110352,
+      "logps/rejected": -33.63285827636719,
+      "loss": 0.6976242065429688,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.46450674533843994,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2216261476278305,
+      "rewards/margins": 2.4022936820983887,
+      "rewards/rejected": -2.1806676387786865,
+      "step": 168,
+      "train_speed(iter/s)": 0.011209
+    },
+    {
+      "epoch": 0.8201395207764635,
+      "grad_norm": 21.119770050048828,
+      "learning_rate": 0.00017394329569516028,
+      "logits/chosen": -0.3196732997894287,
+      "logits/rejected": -0.47898930311203003,
+      "logps/chosen": -11.645589828491211,
+      "logps/rejected": -45.33073425292969,
+      "loss": 1.067216157913208,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.8613059520721436,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.024522848427295685,
+      "rewards/margins": 2.9725117683410645,
+      "rewards/rejected": -2.947988748550415,
+      "step": 169,
+      "train_speed(iter/s)": 0.011209
+    },
+    {
+      "epoch": 0.8249924173491052,
+      "grad_norm": 2.541898012161255,
+      "learning_rate": 0.00017358192921864953,
+      "logits/chosen": -0.40079769492149353,
+      "logits/rejected": -0.4508993923664093,
+      "logps/chosen": -10.424609184265137,
+      "logps/rejected": -34.24469757080078,
+      "loss": 0.7411442399024963,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4615729749202728,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.05271195247769356,
+      "rewards/margins": 2.12577223777771,
+      "rewards/rejected": -2.0730605125427246,
+      "step": 170,
+      "train_speed(iter/s)": 0.011209
+    },
+    {
+      "epoch": 0.8298453139217471,
+      "grad_norm": 4.101256370544434,
+      "learning_rate": 0.00017321845511551244,
+      "logits/chosen": -0.38883864879608154,
+      "logits/rejected": -0.46420595049858093,
+      "logps/chosen": -9.728652954101562,
+      "logps/rejected": -49.25832748413086,
+      "loss": 0.7956501245498657,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.6269941926002502,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.07354284077882767,
+      "rewards/margins": 3.7279632091522217,
+      "rewards/rejected": -3.6544198989868164,
+      "step": 171,
+      "train_speed(iter/s)": 0.011209
+    },
+    {
+      "epoch": 0.8346982104943889,
+      "grad_norm": 6.031051158905029,
+      "learning_rate": 0.00017285288379683376,
+      "logits/chosen": -0.4501774311065674,
+      "logits/rejected": -0.49218156933784485,
+      "logps/chosen": -12.754474639892578,
+      "logps/rejected": -35.773963928222656,
+      "loss": 0.9267921447753906,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.6455193758010864,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.23461803793907166,
+      "rewards/margins": 2.5618717670440674,
+      "rewards/rejected": -2.327253580093384,
+      "step": 172,
+      "train_speed(iter/s)": 0.011209
+    },
+    {
+      "epoch": 0.8395511070670306,
+      "grad_norm": 2.7078492641448975,
+      "learning_rate": 0.00017248522573376932,
+      "logits/chosen": -0.5002922415733337,
+      "logits/rejected": -0.4927571713924408,
+      "logps/chosen": -12.274182319641113,
+      "logps/rejected": -38.476226806640625,
+      "loss": 0.8795382976531982,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.679415225982666,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.20902375876903534,
+      "rewards/margins": 2.886625289916992,
+      "rewards/rejected": -2.6776018142700195,
+      "step": 173,
+      "train_speed(iter/s)": 0.011209
+    },
+    {
+      "epoch": 0.8444040036396724,
+      "grad_norm": 3.0864455699920654,
+      "learning_rate": 0.00017211549145724602,
+      "logits/chosen": -0.44441068172454834,
+      "logits/rejected": -0.5031411647796631,
+      "logps/chosen": -6.964417934417725,
+      "logps/rejected": -47.59633255004883,
+      "loss": 0.5743454694747925,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.39823874831199646,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1409493237733841,
+      "rewards/margins": 3.664174795150757,
+      "rewards/rejected": -3.5232253074645996,
+      "step": 174,
+      "train_speed(iter/s)": 0.011209
+    },
+    {
+      "epoch": 0.8492569002123143,
+      "grad_norm": 13.482826232910156,
+      "learning_rate": 0.00017174369155766037,
+      "logits/chosen": -0.44830185174942017,
+      "logits/rejected": -0.4931170344352722,
+      "logps/chosen": -11.00506591796875,
+      "logps/rejected": -38.13721466064453,
+      "loss": 1.102919578552246,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.8681702017784119,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.020436841994524002,
+      "rewards/margins": 2.716992139816284,
+      "rewards/rejected": -2.6965548992156982,
+      "step": 175,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.854109796784956,
+      "grad_norm": 2.788137435913086,
+      "learning_rate": 0.0001713698366845751,
+      "logits/chosen": -0.41955456137657166,
+      "logits/rejected": -0.4849671423435211,
+      "logps/chosen": -10.713014602661133,
+      "logps/rejected": -38.87740707397461,
+      "loss": 0.8198617696762085,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5618011951446533,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.052750274538993835,
+      "rewards/margins": 2.4263105392456055,
+      "rewards/rejected": -2.3735604286193848,
+      "step": 176,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.8589626933575978,
+      "grad_norm": 2.8578059673309326,
+      "learning_rate": 0.00017099393754641402,
+      "logits/chosen": -0.4903329014778137,
+      "logits/rejected": -0.44835591316223145,
+      "logps/chosen": -12.698493957519531,
+      "logps/rejected": -31.339452743530273,
+      "loss": 0.8736453056335449,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5660265684127808,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.12503781914710999,
+      "rewards/margins": 2.115137815475464,
+      "rewards/rejected": -1.9901000261306763,
+      "step": 177,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.8638155899302397,
+      "grad_norm": 14.433143615722656,
+      "learning_rate": 0.0001706160049101554,
+      "logits/chosen": -0.39836981892585754,
+      "logits/rejected": -0.4502709209918976,
+      "logps/chosen": -12.003418922424316,
+      "logps/rejected": -33.39105224609375,
+      "loss": 0.9455697536468506,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.6791843771934509,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2257295846939087,
+      "rewards/margins": 2.307054042816162,
+      "rewards/rejected": -2.081324577331543,
+      "step": 178,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.8686684865028814,
+      "grad_norm": 4.972922325134277,
+      "learning_rate": 0.00017023604960102355,
+      "logits/chosen": -0.48707541823387146,
+      "logits/rejected": -0.43483617901802063,
+      "logps/chosen": -15.320083618164062,
+      "logps/rejected": -26.741153717041016,
+      "loss": 1.0946437120437622,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.7049025893211365,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.14932136237621307,
+      "rewards/margins": 1.6685651540756226,
+      "rewards/rejected": -1.519243836402893,
+      "step": 179,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.8735213830755232,
+      "grad_norm": 7.616319179534912,
+      "learning_rate": 0.00016985408250217864,
+      "logits/chosen": -0.48446008563041687,
+      "logits/rejected": -0.5007644891738892,
+      "logps/chosen": -9.878619194030762,
+      "logps/rejected": -33.84723663330078,
+      "loss": 0.91806560754776,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.588477373123169,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.13273106515407562,
+      "rewards/margins": 2.1440441608428955,
+      "rewards/rejected": -2.0113131999969482,
+      "step": 180,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.878374279648165,
+      "grad_norm": 5.360206604003906,
+      "learning_rate": 0.00016947011455440523,
+      "logits/chosen": -0.45152074098587036,
+      "logits/rejected": -0.4405006766319275,
+      "logps/chosen": -12.04659652709961,
+      "logps/rejected": -29.337995529174805,
+      "loss": 0.9476042985916138,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5956300497055054,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.007146604359149933,
+      "rewards/margins": 1.6998629570007324,
+      "rewards/rejected": -1.6927162408828735,
+      "step": 181,
+      "train_speed(iter/s)": 0.01121
+    },
+    {
+      "epoch": 0.8832271762208068,
+      "grad_norm": 2.752962589263916,
+      "learning_rate": 0.00016908415675579854,
+      "logits/chosen": -0.400238573551178,
+      "logits/rejected": -0.5106572508811951,
+      "logps/chosen": -5.792488098144531,
+      "logps/rejected": -31.905841827392578,
+      "loss": 0.7257025241851807,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4335445463657379,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.0758802518248558,
+      "rewards/margins": 1.8862504959106445,
+      "rewards/rejected": -1.8103703260421753,
+      "step": 182,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.8880800727934486,
+      "grad_norm": 2.3120791912078857,
+      "learning_rate": 0.00016869622016144973,
+      "logits/chosen": -0.5098201632499695,
+      "logits/rejected": -0.4519084095954895,
+      "logps/chosen": -8.241873741149902,
+      "logps/rejected": -25.190824508666992,
+      "loss": 0.7128751277923584,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.44536417722702026,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2868940234184265,
+      "rewards/margins": 1.7276583909988403,
+      "rewards/rejected": -1.4407645463943481,
+      "step": 183,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.8929329693660903,
+      "grad_norm": 2.231217861175537,
+      "learning_rate": 0.0001683063158831291,
+      "logits/chosen": -0.4472496509552002,
+      "logits/rejected": -0.4929869771003723,
+      "logps/chosen": -7.548064231872559,
+      "logps/rejected": -31.63290023803711,
+      "loss": 0.7525639533996582,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4533500671386719,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2907182276248932,
+      "rewards/margins": 2.194798231124878,
+      "rewards/rejected": -1.9040801525115967,
+      "step": 184,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.8977858659387322,
+      "grad_norm": 2.4883766174316406,
+      "learning_rate": 0.00016791445508896783,
+      "logits/chosen": -0.36559921503067017,
+      "logits/rejected": -0.42163488268852234,
+      "logps/chosen": -11.388641357421875,
+      "logps/rejected": -32.0931510925293,
+      "loss": 0.7775099873542786,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5461112856864929,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.22250865399837494,
+      "rewards/margins": 1.9723868370056152,
+      "rewards/rejected": -1.749878168106079,
+      "step": 185,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.902638762511374,
+      "grad_norm": 3.259838819503784,
+      "learning_rate": 0.00016752064900313815,
+      "logits/chosen": -0.5012748837471008,
+      "logits/rejected": -0.4815691113471985,
+      "logps/chosen": -8.668580055236816,
+      "logps/rejected": -27.685821533203125,
+      "loss": 0.7447406649589539,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.46015244722366333,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.07047421485185623,
+      "rewards/margins": 1.7800168991088867,
+      "rewards/rejected": -1.7095428705215454,
+      "step": 186,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.9074916590840157,
+      "grad_norm": 4.5533576011657715,
+      "learning_rate": 0.0001671249089055317,
+      "logits/chosen": -0.5065349340438843,
+      "logits/rejected": -0.4818389117717743,
+      "logps/chosen": -9.181870460510254,
+      "logps/rejected": -33.568580627441406,
+      "loss": 0.7996796369552612,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5168719291687012,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.10808134078979492,
+      "rewards/margins": 2.115391731262207,
+      "rewards/rejected": -2.007310628890991,
+      "step": 187,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.9123445556566576,
+      "grad_norm": 4.431750297546387,
+      "learning_rate": 0.0001667272461314366,
+      "logits/chosen": -0.48659980297088623,
+      "logits/rejected": -0.487091988325119,
+      "logps/chosen": -10.220385551452637,
+      "logps/rejected": -35.60252380371094,
+      "loss": 0.9328407049179077,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.6412612199783325,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.01909228041768074,
+      "rewards/margins": 2.4949698448181152,
+      "rewards/rejected": -2.475877285003662,
+      "step": 188,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.9171974522292994,
+      "grad_norm": 8.757882118225098,
+      "learning_rate": 0.00016632767207121267,
+      "logits/chosen": -0.33902058005332947,
+      "logits/rejected": -0.4704265594482422,
+      "logps/chosen": -10.807303428649902,
+      "logps/rejected": -48.82100296020508,
+      "loss": 1.2881890535354614,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.8564696907997131,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.06860647350549698,
+      "rewards/margins": 2.6622214317321777,
+      "rewards/rejected": -2.7308273315429688,
+      "step": 189,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.9220503488019411,
+      "grad_norm": 9.251166343688965,
+      "learning_rate": 0.00016592619816996527,
+      "logits/chosen": -0.47989535331726074,
+      "logits/rejected": -0.5115556716918945,
+      "logps/chosen": -15.828009605407715,
+      "logps/rejected": -35.08710861206055,
+      "loss": 0.9385941624641418,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.632335901260376,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.0631108283996582,
+      "rewards/margins": 1.8624895811080933,
+      "rewards/rejected": -1.799378752708435,
+      "step": 190,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.9269032453745829,
+      "grad_norm": 4.995007038116455,
+      "learning_rate": 0.0001655228359272173,
+      "logits/chosen": -0.45928898453712463,
+      "logits/rejected": -0.42803335189819336,
+      "logps/chosen": -11.98594856262207,
+      "logps/rejected": -27.537172317504883,
+      "loss": 1.0162464380264282,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.6742801070213318,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.0864061713218689,
+      "rewards/margins": 1.6110825538635254,
+      "rewards/rejected": -1.5246765613555908,
+      "step": 191,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.9317561419472248,
+      "grad_norm": 8.054399490356445,
+      "learning_rate": 0.00016511759689657999,
+      "logits/chosen": -0.3773999512195587,
+      "logits/rejected": -0.42789533734321594,
+      "logps/chosen": -12.406736373901367,
+      "logps/rejected": -39.963958740234375,
+      "loss": 0.7866171598434448,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5565248131752014,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.005689222365617752,
+      "rewards/margins": 2.343252658843994,
+      "rewards/rejected": -2.3489420413970947,
+      "step": 192,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.9366090385198665,
+      "grad_norm": 3.3172736167907715,
+      "learning_rate": 0.00016471049268542188,
+      "logits/chosen": -0.46051445603370667,
+      "logits/rejected": -0.41965189576148987,
+      "logps/chosen": -10.034582138061523,
+      "logps/rejected": -36.16367721557617,
+      "loss": 0.7486264109611511,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4820393919944763,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.0357435941696167,
+      "rewards/margins": 2.641000747680664,
+      "rewards/rejected": -2.605257034301758,
+      "step": 193,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.9414619350925083,
+      "grad_norm": 2.3992457389831543,
+      "learning_rate": 0.00016430153495453642,
+      "logits/chosen": -0.5328064560890198,
+      "logits/rejected": -0.47638142108917236,
+      "logps/chosen": -11.91832160949707,
+      "logps/rejected": -36.306785583496094,
+      "loss": 0.8761617541313171,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.6264892220497131,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1922227442264557,
+      "rewards/margins": 2.7257025241851807,
+      "rewards/rejected": -2.533479690551758,
+      "step": 194,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.9463148316651502,
+      "grad_norm": 4.527428150177002,
+      "learning_rate": 0.00016389073541780782,
+      "logits/chosen": -0.46726107597351074,
+      "logits/rejected": -0.47500157356262207,
+      "logps/chosen": -8.38770580291748,
+      "logps/rejected": -29.87854766845703,
+      "loss": 0.7214972376823425,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4556006193161011,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.08811609447002411,
+      "rewards/margins": 1.9756295680999756,
+      "rewards/rejected": -1.8875133991241455,
+      "step": 195,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.9511677282377919,
+      "grad_norm": 3.743474006652832,
+      "learning_rate": 0.00016347810584187568,
+      "logits/chosen": -0.5369599461555481,
+      "logits/rejected": -0.5066350102424622,
+      "logps/chosen": -8.895236015319824,
+      "logps/rejected": -26.281404495239258,
+      "loss": 0.8112626075744629,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5115976333618164,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.04646964743733406,
+      "rewards/margins": 1.599694848060608,
+      "rewards/rejected": -1.5532252788543701,
+      "step": 196,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.9560206248104337,
+      "grad_norm": 2.772148370742798,
+      "learning_rate": 0.00016306365804579794,
+      "logits/chosen": -0.36604201793670654,
+      "logits/rejected": -0.4730686545372009,
+      "logps/chosen": -9.536896705627441,
+      "logps/rejected": -40.390995025634766,
+      "loss": 0.6381856799125671,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4115173816680908,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2083117514848709,
+      "rewards/margins": 2.512366771697998,
+      "rewards/rejected": -2.3040552139282227,
+      "step": 197,
+      "train_speed(iter/s)": 0.011211
+    },
+    {
+      "epoch": 0.9608735213830755,
+      "grad_norm": 3.5839767456054688,
+      "learning_rate": 0.0001626474039007122,
+      "logits/chosen": -0.4816967844963074,
+      "logits/rejected": -0.47818446159362793,
+      "logps/chosen": -7.637480735778809,
+      "logps/rejected": -29.050764083862305,
+      "loss": 0.6914876103401184,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3912826478481293,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.2960473895072937,
+      "rewards/margins": 2.151468515396118,
+      "rewards/rejected": -1.8554210662841797,
+      "step": 198,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.9657264179557173,
+      "grad_norm": 2.633289098739624,
+      "learning_rate": 0.00016222935532949587,
+      "logits/chosen": -0.4851294755935669,
+      "logits/rejected": -0.4861046373844147,
+      "logps/chosen": -10.165040969848633,
+      "logps/rejected": -34.33792495727539,
+      "loss": 0.7900617718696594,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5045729279518127,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.33929818868637085,
+      "rewards/margins": 2.4269657135009766,
+      "rewards/rejected": -2.08766770362854,
+      "step": 199,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.9705793145283591,
+      "grad_norm": 4.324284553527832,
+      "learning_rate": 0.0001618095243064245,
+      "logits/chosen": -0.486560583114624,
+      "logits/rejected": -0.46493133902549744,
+      "logps/chosen": -7.55566930770874,
+      "logps/rejected": -32.536399841308594,
+      "loss": 0.7669270038604736,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.45861828327178955,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.0628332868218422,
+      "rewards/margins": 2.340352773666382,
+      "rewards/rejected": -2.277519464492798,
+      "step": 200,
+      "train_speed(iter/s)": 0.011212
+    },
+    {
+      "epoch": 0.9754322111010009,
+      "grad_norm": 1.9603885412216187,
+      "learning_rate": 0.000161387922856829,
+      "logits/chosen": -0.5021265149116516,
+      "logits/rejected": -0.5176340937614441,
+      "logps/chosen": -9.77086067199707,
+      "logps/rejected": -40.27810287475586,
+      "loss": 0.7130787968635559,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4609830379486084,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.18531039357185364,
+      "rewards/margins": 2.942089080810547,
+      "rewards/rejected": -2.7567787170410156,
+      "step": 201,
+      "train_speed(iter/s)": 0.011203
+    },
+    {
+      "epoch": 0.9802851076736427,
+      "grad_norm": 2.769995927810669,
+      "learning_rate": 0.00016096456305675095,
+      "logits/chosen": -0.44220247864723206,
+      "logits/rejected": -0.5238637328147888,
+      "logps/chosen": -7.195171356201172,
+      "logps/rejected": -44.38209533691406,
+      "loss": 0.6904028058052063,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5173193216323853,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3396221101284027,
+      "rewards/margins": 3.3168182373046875,
+      "rewards/rejected": -2.977196216583252,
+      "step": 202,
+      "train_speed(iter/s)": 0.011203
+    },
+    {
+      "epoch": 0.9851380042462845,
+      "grad_norm": 3.073450803756714,
+      "learning_rate": 0.00016053945703259692,
+      "logits/chosen": -0.47079914808273315,
+      "logits/rejected": -0.4507486820220947,
+      "logps/chosen": -9.377758979797363,
+      "logps/rejected": -35.4901008605957,
+      "loss": 0.8126031756401062,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5169999599456787,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.07428787648677826,
+      "rewards/margins": 2.5271918773651123,
+      "rewards/rejected": -2.452903985977173,
+      "step": 203,
+      "train_speed(iter/s)": 0.011203
+    },
+    {
+      "epoch": 0.9899909008189263,
+      "grad_norm": 3.061457872390747,
+      "learning_rate": 0.00016011261696079098,
+      "logits/chosen": -0.43547484278678894,
+      "logits/rejected": -0.5544446110725403,
+      "logps/chosen": -11.976186752319336,
+      "logps/rejected": -53.08049392700195,
+      "loss": 0.8855732679367065,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.6567856073379517,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1509181559085846,
+      "rewards/margins": 3.721925973892212,
+      "rewards/rejected": -3.5710079669952393,
+      "step": 204,
+      "train_speed(iter/s)": 0.011203
+    },
+    {
+      "epoch": 0.9948437973915681,
+      "grad_norm": 2.193002223968506,
+      "learning_rate": 0.00015968405506742599,
+      "logits/chosen": -0.5186320543289185,
+      "logits/rejected": -0.5116863250732422,
+      "logps/chosen": -9.911492347717285,
+      "logps/rejected": -34.19422912597656,
+      "loss": 0.9093415141105652,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.6466048359870911,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2533470094203949,
+      "rewards/margins": 2.494154214859009,
+      "rewards/rejected": -2.240807294845581,
+      "step": 205,
+      "train_speed(iter/s)": 0.011204
+    },
+    {
+      "epoch": 0.9996966939642099,
+      "grad_norm": 2.4486677646636963,
+      "learning_rate": 0.00015925378362791347,
+      "logits/chosen": -0.49749520421028137,
+      "logits/rejected": -0.5242308378219604,
+      "logps/chosen": -7.672642707824707,
+      "logps/rejected": -41.62055206298828,
+      "loss": 0.6428540349006653,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3894902467727661,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.07286091148853302,
+      "rewards/margins": 3.0560548305511475,
+      "rewards/rejected": -2.983193874359131,
+      "step": 206,
+      "train_speed(iter/s)": 0.011204
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 2.4486677646636963,
+      "learning_rate": 0.0001588218149666318,
+      "logits/chosen": -0.0672072023153305,
+      "logits/rejected": -0.3169337511062622,
+      "logps/chosen": -3.5387771129608154,
+      "logps/rejected": -19.517276763916016,
+      "loss": 0.10221779346466064,
+      "memory(GiB)": 43.93,
+      "nll_loss": 1.1795923709869385,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.21849411725997925,
+      "rewards/margins": 0.8076673746109009,
+      "rewards/rejected": -0.5891733169555664,
+      "step": 207,
+      "train_speed(iter/s)": 0.011255
+    },
+    {
+      "epoch": 1.0048528965726418,
+      "grad_norm": 1.7180174589157104,
+      "learning_rate": 0.00015838816145657343,
+      "logits/chosen": -0.41013669967651367,
+      "logits/rejected": -0.5080668330192566,
+      "logps/chosen": -8.022540092468262,
+      "logps/rejected": -41.35959243774414,
+      "loss": 0.5290953516960144,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.42788973450660706,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.43706047534942627,
+      "rewards/margins": 3.0258355140686035,
+      "rewards/rejected": -2.588775157928467,
+      "step": 208,
+      "train_speed(iter/s)": 0.011253
+    },
+    {
+      "epoch": 1.0097057931452835,
+      "grad_norm": 1.6782983541488647,
+      "learning_rate": 0.00015795283551899036,
+      "logits/chosen": -0.4570477306842804,
+      "logits/rejected": -0.4886360168457031,
+      "logps/chosen": -9.411005973815918,
+      "logps/rejected": -43.55297088623047,
+      "loss": 0.5858383774757385,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.44828641414642334,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3919137418270111,
+      "rewards/margins": 3.3793742656707764,
+      "rewards/rejected": -2.9874606132507324,
+      "step": 209,
+      "train_speed(iter/s)": 0.011253
+    },
+    {
+      "epoch": 1.0145586897179253,
+      "grad_norm": 1.8372869491577148,
+      "learning_rate": 0.00015751584962303832,
+      "logits/chosen": -0.45137134194374084,
+      "logits/rejected": -0.49931854009628296,
+      "logps/chosen": -5.573295593261719,
+      "logps/rejected": -44.04582214355469,
+      "loss": 0.47281989455223083,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3471827208995819,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5734691023826599,
+      "rewards/margins": 3.8452658653259277,
+      "rewards/rejected": -3.271796464920044,
+      "step": 210,
+      "train_speed(iter/s)": 0.011253
+    },
+    {
+      "epoch": 1.0194115862905673,
+      "grad_norm": 2.122117280960083,
+      "learning_rate": 0.0001570772162854197,
+      "logits/chosen": -0.4932425022125244,
+      "logits/rejected": -0.5862610936164856,
+      "logps/chosen": -5.677023410797119,
+      "logps/rejected": -67.28507995605469,
+      "loss": 0.5105875134468079,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4059942364692688,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.20374003052711487,
+      "rewards/margins": 5.488041877746582,
+      "rewards/rejected": -5.284302234649658,
+      "step": 211,
+      "train_speed(iter/s)": 0.011253
+    },
+    {
+      "epoch": 1.024264482863209,
+      "grad_norm": 2.133958339691162,
+      "learning_rate": 0.00015663694807002506,
+      "logits/chosen": -0.5366697311401367,
+      "logits/rejected": -0.627138078212738,
+      "logps/chosen": -9.758528709411621,
+      "logps/rejected": -38.330787658691406,
+      "loss": 0.5926258563995361,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4576071500778198,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.45621615648269653,
+      "rewards/margins": 2.72711181640625,
+      "rewards/rejected": -2.2708957195281982,
+      "step": 212,
+      "train_speed(iter/s)": 0.011253
+    },
+    {
+      "epoch": 1.0291173794358508,
+      "grad_norm": 5.999765872955322,
+      "learning_rate": 0.00015619505758757312,
+      "logits/chosen": -0.5824704766273499,
+      "logits/rejected": -0.5835531949996948,
+      "logps/chosen": -7.275055885314941,
+      "logps/rejected": -45.14403533935547,
+      "loss": 0.6712980270385742,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5353971719741821,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.28136032819747925,
+      "rewards/margins": 3.9529309272766113,
+      "rewards/rejected": -3.6715707778930664,
+      "step": 213,
+      "train_speed(iter/s)": 0.011253
+    },
+    {
+      "epoch": 1.0339702760084926,
+      "grad_norm": 3.805849313735962,
+      "learning_rate": 0.0001557515574952496,
+      "logits/chosen": -0.5069078207015991,
+      "logits/rejected": -0.55241459608078,
+      "logps/chosen": -7.691582202911377,
+      "logps/rejected": -50.89150619506836,
+      "loss": 0.6278761029243469,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5273099541664124,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.492880642414093,
+      "rewards/margins": 4.282929420471191,
+      "rewards/rejected": -3.790048122406006,
+      "step": 214,
+      "train_speed(iter/s)": 0.011253
+    },
+    {
+      "epoch": 1.0388231725811343,
+      "grad_norm": 10.641289710998535,
+      "learning_rate": 0.00015530646049634474,
+      "logits/chosen": -0.532200813293457,
+      "logits/rejected": -0.6343463659286499,
+      "logps/chosen": -10.347488403320312,
+      "logps/rejected": -58.461692810058594,
+      "loss": 0.7014778256416321,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5599174499511719,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.08643528819084167,
+      "rewards/margins": 4.3690185546875,
+      "rewards/rejected": -4.282583713531494,
+      "step": 215,
+      "train_speed(iter/s)": 0.011253
+    },
+    {
+      "epoch": 1.043676069153776,
+      "grad_norm": 4.997859954833984,
+      "learning_rate": 0.00015485977933988947,
+      "logits/chosen": -0.5499998331069946,
+      "logits/rejected": -0.6231508255004883,
+      "logps/chosen": -8.2232027053833,
+      "logps/rejected": -70.93086242675781,
+      "loss": 0.5264535546302795,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.41253799200057983,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3687896430492401,
+      "rewards/margins": 5.875752925872803,
+      "rewards/rejected": -5.50696325302124,
+      "step": 216,
+      "train_speed(iter/s)": 0.011253
+    },
+    {
+      "epoch": 1.0485289657264178,
+      "grad_norm": 1.6605055332183838,
+      "learning_rate": 0.00015441152682029,
+      "logits/chosen": -0.5347371101379395,
+      "logits/rejected": -0.6293535232543945,
+      "logps/chosen": -6.7471184730529785,
+      "logps/rejected": -57.79261016845703,
+      "loss": 0.4279806613922119,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3485845923423767,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4243243932723999,
+      "rewards/margins": 4.480070114135742,
+      "rewards/rejected": -4.055746078491211,
+      "step": 217,
+      "train_speed(iter/s)": 0.011253
+    },
+    {
+      "epoch": 1.0533818622990598,
+      "grad_norm": 1.3759158849716187,
+      "learning_rate": 0.00015396171577696167,
+      "logits/chosen": -0.5051631927490234,
+      "logits/rejected": -0.6489286422729492,
+      "logps/chosen": -6.5541839599609375,
+      "logps/rejected": -61.18578338623047,
+      "loss": 0.40871456265449524,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.34622615575790405,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.8713262677192688,
+      "rewards/margins": 5.349825859069824,
+      "rewards/rejected": -4.4785003662109375,
+      "step": 218,
+      "train_speed(iter/s)": 0.011252
+    },
+    {
+      "epoch": 1.0582347588717016,
+      "grad_norm": 1.5643813610076904,
+      "learning_rate": 0.00015351035909396088,
+      "logits/chosen": -0.5682575702667236,
+      "logits/rejected": -0.5700003504753113,
+      "logps/chosen": -7.853461265563965,
+      "logps/rejected": -37.5399284362793,
+      "loss": 0.5135300755500793,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.39472249150276184,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.44226574897766113,
+      "rewards/margins": 2.9900660514831543,
+      "rewards/rejected": -2.5478007793426514,
+      "step": 219,
+      "train_speed(iter/s)": 0.011252
+    },
+    {
+      "epoch": 1.0630876554443434,
+      "grad_norm": 1.9472591876983643,
+      "learning_rate": 0.0001530574696996164,
+      "logits/chosen": -0.6036070585250854,
+      "logits/rejected": -0.6442514061927795,
+      "logps/chosen": -7.859013557434082,
+      "logps/rejected": -51.61414337158203,
+      "loss": 0.47415411472320557,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3808107376098633,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4538901150226593,
+      "rewards/margins": 4.290732383728027,
+      "rewards/rejected": -3.8368425369262695,
+      "step": 220,
+      "train_speed(iter/s)": 0.011252
+    },
+    {
+      "epoch": 1.0679405520169851,
+      "grad_norm": 7.173949241638184,
+      "learning_rate": 0.00015260306056615858,
+      "logits/chosen": -0.5031501650810242,
+      "logits/rejected": -0.5708950757980347,
+      "logps/chosen": -8.430438041687012,
+      "logps/rejected": -49.38084030151367,
+      "loss": 0.6345080733299255,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4748482406139374,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5568549633026123,
+      "rewards/margins": 3.7820630073547363,
+      "rewards/rejected": -3.225208044052124,
+      "step": 221,
+      "train_speed(iter/s)": 0.011252
+    },
+    {
+      "epoch": 1.0727934485896269,
+      "grad_norm": 2.499420404434204,
+      "learning_rate": 0.0001521471447093483,
+      "logits/chosen": -0.5780302286148071,
+      "logits/rejected": -0.652087390422821,
+      "logps/chosen": -6.439700126647949,
+      "logps/rejected": -47.79240036010742,
+      "loss": 0.6468124389648438,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5188155174255371,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4574861526489258,
+      "rewards/margins": 4.022400856018066,
+      "rewards/rejected": -3.5649144649505615,
+      "step": 222,
+      "train_speed(iter/s)": 0.011251
+    },
+    {
+      "epoch": 1.0776463451622686,
+      "grad_norm": 2.868643045425415,
+      "learning_rate": 0.0001516897351881038,
+      "logits/chosen": -0.48600101470947266,
+      "logits/rejected": -0.7179228067398071,
+      "logps/chosen": -7.020376682281494,
+      "logps/rejected": -63.106910705566406,
+      "loss": 0.5186583995819092,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.45133551955223083,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4073833227157593,
+      "rewards/margins": 5.3973588943481445,
+      "rewards/rejected": -4.989975929260254,
+      "step": 223,
+      "train_speed(iter/s)": 0.011251
+    },
+    {
+      "epoch": 1.0824992417349106,
+      "grad_norm": 2.9537835121154785,
+      "learning_rate": 0.00015123084510412676,
+      "logits/chosen": -0.6880484223365784,
+      "logits/rejected": -0.6554800868034363,
+      "logps/chosen": -12.542672157287598,
+      "logps/rejected": -41.370513916015625,
+      "loss": 0.7187406420707703,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.6188425421714783,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.6864035129547119,
+      "rewards/margins": 3.933318614959717,
+      "rewards/rejected": -3.246915102005005,
+      "step": 224,
+      "train_speed(iter/s)": 0.011251
+    },
+    {
+      "epoch": 1.0873521383075524,
+      "grad_norm": 2.0743846893310547,
+      "learning_rate": 0.00015077048760152702,
+      "logits/chosen": -0.7064493894577026,
+      "logits/rejected": -0.5577601194381714,
+      "logps/chosen": -8.18484878540039,
+      "logps/rejected": -44.98584747314453,
+      "loss": 0.3742949962615967,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3041650652885437,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.6248530745506287,
+      "rewards/margins": 4.210747718811035,
+      "rewards/rejected": -3.58589506149292,
+      "step": 225,
+      "train_speed(iter/s)": 0.011251
+    },
+    {
+      "epoch": 1.0922050348801942,
+      "grad_norm": 3.051504373550415,
+      "learning_rate": 0.00015030867586644605,
+      "logits/chosen": -0.4182083010673523,
+      "logits/rejected": -0.5224165916442871,
+      "logps/chosen": -8.414253234863281,
+      "logps/rejected": -50.0076789855957,
+      "loss": 0.5939789414405823,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.43186870217323303,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2714881896972656,
+      "rewards/margins": 3.5979065895080566,
+      "rewards/rejected": -3.32641863822937,
+      "step": 226,
+      "train_speed(iter/s)": 0.011251
+    },
+    {
+      "epoch": 1.097057931452836,
+      "grad_norm": 19.128318786621094,
+      "learning_rate": 0.00014984542312667935,
+      "logits/chosen": -0.5070249438285828,
+      "logits/rejected": -0.6899809837341309,
+      "logps/chosen": -5.100536823272705,
+      "logps/rejected": -58.729217529296875,
+      "loss": 0.5768824219703674,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.45623981952667236,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.29099348187446594,
+      "rewards/margins": 4.82575798034668,
+      "rewards/rejected": -4.534764766693115,
+      "step": 227,
+      "train_speed(iter/s)": 0.011251
+    },
+    {
+      "epoch": 1.1019108280254777,
+      "grad_norm": 15.456758499145508,
+      "learning_rate": 0.00014938074265129743,
+      "logits/chosen": -0.6614000797271729,
+      "logits/rejected": -0.583651065826416,
+      "logps/chosen": -10.315933227539062,
+      "logps/rejected": -44.13168716430664,
+      "loss": 0.8722174763679504,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.6912877559661865,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4925556778907776,
+      "rewards/margins": 3.7772724628448486,
+      "rewards/rejected": -3.2847163677215576,
+      "step": 228,
+      "train_speed(iter/s)": 0.011251
+    },
+    {
+      "epoch": 1.1067637245981194,
+      "grad_norm": 4.41496467590332,
+      "learning_rate": 0.00014891464775026585,
+      "logits/chosen": -0.6627179980278015,
+      "logits/rejected": -0.5836312770843506,
+      "logps/chosen": -7.832060813903809,
+      "logps/rejected": -51.398380279541016,
+      "loss": 0.42356374859809875,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.33340057730674744,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.33356133103370667,
+      "rewards/margins": 4.259485721588135,
+      "rewards/rejected": -3.92592453956604,
+      "step": 229,
+      "train_speed(iter/s)": 0.011251
+    },
+    {
+      "epoch": 1.1116166211707612,
+      "grad_norm": 2.4247095584869385,
+      "learning_rate": 0.0001484471517740639,
+      "logits/chosen": -0.6520415544509888,
+      "logits/rejected": -0.6481152772903442,
+      "logps/chosen": -7.814848899841309,
+      "logps/rejected": -48.36859130859375,
+      "loss": 0.5468817949295044,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.44277364015579224,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.29666104912757874,
+      "rewards/margins": 4.1083831787109375,
+      "rewards/rejected": -3.8117218017578125,
+      "step": 230,
+      "train_speed(iter/s)": 0.011251
+    },
+    {
+      "epoch": 1.1164695177434032,
+      "grad_norm": 1.3357291221618652,
+      "learning_rate": 0.00014797826811330228,
+      "logits/chosen": -0.5879421830177307,
+      "logits/rejected": -0.5645837187767029,
+      "logps/chosen": -6.680647373199463,
+      "logps/rejected": -54.134193420410156,
+      "loss": 0.38013574481010437,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.33245307207107544,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5417324900627136,
+      "rewards/margins": 4.868451118469238,
+      "rewards/rejected": -4.326718807220459,
+      "step": 231,
+      "train_speed(iter/s)": 0.011251
+    },
+    {
+      "epoch": 1.121322414316045,
+      "grad_norm": 1.5943940877914429,
+      "learning_rate": 0.00014750801019833949,
+      "logits/chosen": -0.568419873714447,
+      "logits/rejected": -0.6231982111930847,
+      "logps/chosen": -5.1526055335998535,
+      "logps/rejected": -46.67970657348633,
+      "loss": 0.43953895568847656,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.35486042499542236,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4100644588470459,
+      "rewards/margins": 3.7241714000701904,
+      "rewards/rejected": -3.3141071796417236,
+      "step": 232,
+      "train_speed(iter/s)": 0.011251
+    },
+    {
+      "epoch": 1.1261753108886867,
+      "grad_norm": 2.373943328857422,
+      "learning_rate": 0.00014703639149889712,
+      "logits/chosen": -0.5270050764083862,
+      "logits/rejected": -0.6688944101333618,
+      "logps/chosen": -6.8727521896362305,
+      "logps/rejected": -54.39302444458008,
+      "loss": 0.5719085931777954,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4776768386363983,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5398850440979004,
+      "rewards/margins": 4.560108184814453,
+      "rewards/rejected": -4.0202226638793945,
+      "step": 233,
+      "train_speed(iter/s)": 0.011251
+    },
+    {
+      "epoch": 1.1310282074613285,
+      "grad_norm": 1.4414172172546387,
+      "learning_rate": 0.00014656342552367416,
+      "logits/chosen": -0.5357221961021423,
+      "logits/rejected": -0.5533539652824402,
+      "logps/chosen": -6.900845527648926,
+      "logps/rejected": -45.86457443237305,
+      "loss": 0.4437209367752075,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3537225127220154,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.38456112146377563,
+      "rewards/margins": 3.7542195320129395,
+      "rewards/rejected": -3.3696582317352295,
+      "step": 234,
+      "train_speed(iter/s)": 0.011251
+    },
+    {
+      "epoch": 1.1358811040339702,
+      "grad_norm": 1.4943245649337769,
+      "learning_rate": 0.00014608912581995983,
+      "logits/chosen": -0.5588849186897278,
+      "logits/rejected": -0.6436609029769897,
+      "logps/chosen": -5.6003923416137695,
+      "logps/rejected": -46.67798614501953,
+      "loss": 0.4039139151573181,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.33202505111694336,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7212438583374023,
+      "rewards/margins": 3.975301742553711,
+      "rewards/rejected": -3.2540581226348877,
+      "step": 235,
+      "train_speed(iter/s)": 0.011251
+    },
+    {
+      "epoch": 1.140734000606612,
+      "grad_norm": 4.428430557250977,
+      "learning_rate": 0.00014561350597324578,
+      "logits/chosen": -0.6040487289428711,
+      "logits/rejected": -0.5729888677597046,
+      "logps/chosen": -10.751192092895508,
+      "logps/rejected": -46.61857223510742,
+      "loss": 0.6180225014686584,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5182946920394897,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.482886403799057,
+      "rewards/margins": 3.716885566711426,
+      "rewards/rejected": -3.233999490737915,
+      "step": 236,
+      "train_speed(iter/s)": 0.011251
+    },
+    {
+      "epoch": 1.1455868971792538,
+      "grad_norm": 1.5909223556518555,
+      "learning_rate": 0.00014513657960683692,
+      "logits/chosen": -0.6496992707252502,
+      "logits/rejected": -0.6147754192352295,
+      "logps/chosen": -9.563628196716309,
+      "logps/rejected": -53.87767028808594,
+      "loss": 0.5457061529159546,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5064206719398499,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5197762250900269,
+      "rewards/margins": 4.888551235198975,
+      "rewards/rejected": -4.368774890899658,
+      "step": 237,
+      "train_speed(iter/s)": 0.011251
+    },
+    {
+      "epoch": 1.1504397937518958,
+      "grad_norm": 2.698936700820923,
+      "learning_rate": 0.00014465836038146107,
+      "logits/chosen": -0.5431322455406189,
+      "logits/rejected": -0.6387051343917847,
+      "logps/chosen": -7.608684539794922,
+      "logps/rejected": -50.657470703125,
+      "loss": 0.5676233172416687,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4664645493030548,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.157401442527771,
+      "rewards/margins": 4.005822658538818,
+      "rewards/rejected": -3.848421096801758,
+      "step": 238,
+      "train_speed(iter/s)": 0.011251
+    },
+    {
+      "epoch": 1.1552926903245375,
+      "grad_norm": 2.1706926822662354,
+      "learning_rate": 0.00014417886199487775,
+      "logits/chosen": -0.5933765172958374,
+      "logits/rejected": -0.5305951237678528,
+      "logps/chosen": -9.74417495727539,
+      "logps/rejected": -41.780517578125,
+      "loss": 0.4316883087158203,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.33147385716438293,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.6860098838806152,
+      "rewards/margins": 3.783914566040039,
+      "rewards/rejected": -3.0979042053222656,
+      "step": 239,
+      "train_speed(iter/s)": 0.01125
+    },
+    {
+      "epoch": 1.1601455868971793,
+      "grad_norm": 2.7774465084075928,
+      "learning_rate": 0.00014369809818148586,
+      "logits/chosen": -0.6653566360473633,
+      "logits/rejected": -0.645525336265564,
+      "logps/chosen": -7.276230812072754,
+      "logps/rejected": -42.255470275878906,
+      "loss": 0.5839024186134338,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.45142966508865356,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3745161294937134,
+      "rewards/margins": 3.7749557495117188,
+      "rewards/rejected": -3.400439977645874,
+      "step": 240,
+      "train_speed(iter/s)": 0.01125
+    },
+    {
+      "epoch": 1.164998483469821,
+      "grad_norm": 2.202160358428955,
+      "learning_rate": 0.00014321608271193018,
+      "logits/chosen": -0.5993486046791077,
+      "logits/rejected": -0.6560922861099243,
+      "logps/chosen": -7.505954742431641,
+      "logps/rejected": -51.6959114074707,
+      "loss": 0.615651547908783,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5654053688049316,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5433130860328674,
+      "rewards/margins": 4.539626598358154,
+      "rewards/rejected": -3.9963133335113525,
+      "step": 241,
+      "train_speed(iter/s)": 0.01125
+    },
+    {
+      "epoch": 1.1698513800424628,
+      "grad_norm": 1.8709537982940674,
+      "learning_rate": 0.00014273282939270714,
+      "logits/chosen": -0.5413939952850342,
+      "logits/rejected": -0.6282732486724854,
+      "logps/chosen": -6.796370029449463,
+      "logps/rejected": -60.840675354003906,
+      "loss": 0.5075444579124451,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.404622882604599,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.33700963854789734,
+      "rewards/margins": 4.8637542724609375,
+      "rewards/rejected": -4.526744365692139,
+      "step": 242,
+      "train_speed(iter/s)": 0.01125
+    },
+    {
+      "epoch": 1.1747042766151046,
+      "grad_norm": 3.5226120948791504,
+      "learning_rate": 0.00014224835206576915,
+      "logits/chosen": -0.6383830308914185,
+      "logits/rejected": -0.6402108073234558,
+      "logps/chosen": -9.347366333007812,
+      "logps/rejected": -52.63505172729492,
+      "loss": 0.524966299533844,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.413347065448761,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4538324773311615,
+      "rewards/margins": 4.46920919418335,
+      "rewards/rejected": -4.015376567840576,
+      "step": 243,
+      "train_speed(iter/s)": 0.01125
+    },
+    {
+      "epoch": 1.1795571731877463,
+      "grad_norm": 2.4066126346588135,
+      "learning_rate": 0.00014176266460812816,
+      "logits/chosen": -0.4784002900123596,
+      "logits/rejected": -0.5839812159538269,
+      "logps/chosen": -5.386353969573975,
+      "logps/rejected": -57.16166687011719,
+      "loss": 0.4779638946056366,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.35747677087783813,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.39205074310302734,
+      "rewards/margins": 4.41473913192749,
+      "rewards/rejected": -4.022688388824463,
+      "step": 244,
+      "train_speed(iter/s)": 0.01125
+    },
+    {
+      "epoch": 1.1844100697603883,
+      "grad_norm": 1.8582743406295776,
+      "learning_rate": 0.0001412757809314583,
+      "logits/chosen": -0.5211676359176636,
+      "logits/rejected": -0.5809144973754883,
+      "logps/chosen": -6.789330005645752,
+      "logps/rejected": -53.503807067871094,
+      "loss": 0.4873069226741791,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.39957380294799805,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3342842757701874,
+      "rewards/margins": 4.531519412994385,
+      "rewards/rejected": -4.197235584259033,
+      "step": 245,
+      "train_speed(iter/s)": 0.01125
+    },
+    {
+      "epoch": 1.18926296633303,
+      "grad_norm": 1.9136053323745728,
+      "learning_rate": 0.00014078771498169727,
+      "logits/chosen": -0.5340145826339722,
+      "logits/rejected": -0.502403974533081,
+      "logps/chosen": -7.623006820678711,
+      "logps/rejected": -48.99563217163086,
+      "loss": 0.5889253616333008,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.44297417998313904,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3839024305343628,
+      "rewards/margins": 4.315854072570801,
+      "rewards/rejected": -3.9319517612457275,
+      "step": 246,
+      "train_speed(iter/s)": 0.01125
+    },
+    {
+      "epoch": 1.1941158629056718,
+      "grad_norm": 5.204230785369873,
+      "learning_rate": 0.0001402984807386469,
+      "logits/chosen": -0.466047078371048,
+      "logits/rejected": -0.6075699329376221,
+      "logps/chosen": -4.622448921203613,
+      "logps/rejected": -61.837646484375,
+      "loss": 0.3498527705669403,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.30633744597435,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3042653203010559,
+      "rewards/margins": 4.933521270751953,
+      "rewards/rejected": -4.629256248474121,
+      "step": 247,
+      "train_speed(iter/s)": 0.01125
+    },
+    {
+      "epoch": 1.1989687594783136,
+      "grad_norm": 2.3503730297088623,
+      "learning_rate": 0.00013980809221557277,
+      "logits/chosen": -0.5060653686523438,
+      "logits/rejected": -0.633462131023407,
+      "logps/chosen": -5.608469009399414,
+      "logps/rejected": -56.556522369384766,
+      "loss": 0.8063190579414368,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.533983051776886,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.13630348443984985,
+      "rewards/margins": 4.395716190338135,
+      "rewards/rejected": -4.259413719177246,
+      "step": 248,
+      "train_speed(iter/s)": 0.01125
+    },
+    {
+      "epoch": 1.2038216560509554,
+      "grad_norm": 5.791271686553955,
+      "learning_rate": 0.0001393165634588029,
+      "logits/chosen": -0.4766685962677002,
+      "logits/rejected": -0.4965994656085968,
+      "logps/chosen": -8.610245704650879,
+      "logps/rejected": -50.26591110229492,
+      "loss": 0.5034311413764954,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4234567880630493,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5894193649291992,
+      "rewards/margins": 3.848754644393921,
+      "rewards/rejected": -3.2593352794647217,
+      "step": 249,
+      "train_speed(iter/s)": 0.01125
+    },
+    {
+      "epoch": 1.2086745526235971,
+      "grad_norm": 3.2736964225769043,
+      "learning_rate": 0.00013882390854732518,
+      "logits/chosen": -0.44910845160484314,
+      "logits/rejected": -0.5383850336074829,
+      "logps/chosen": -6.248256683349609,
+      "logps/rejected": -58.71024703979492,
+      "loss": 0.5866567492485046,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4265459477901459,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2640109658241272,
+      "rewards/margins": 4.627007484436035,
+      "rewards/rejected": -4.362997055053711,
+      "step": 250,
+      "train_speed(iter/s)": 0.01125
+    },
+    {
+      "epoch": 1.213527449196239,
+      "grad_norm": 1.668660044670105,
+      "learning_rate": 0.00013833014159238434,
+      "logits/chosen": -0.5141372084617615,
+      "logits/rejected": -0.4604420065879822,
+      "logps/chosen": -10.546465873718262,
+      "logps/rejected": -49.485198974609375,
+      "loss": 0.5634040236473083,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.43968477845191956,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5591100454330444,
+      "rewards/margins": 4.2043328285217285,
+      "rewards/rejected": -3.6452226638793945,
+      "step": 251,
+      "train_speed(iter/s)": 0.011249
+    },
+    {
+      "epoch": 1.2183803457688809,
+      "grad_norm": 2.68563175201416,
+      "learning_rate": 0.00013783527673707761,
+      "logits/chosen": -0.5613682866096497,
+      "logits/rejected": -0.6005113124847412,
+      "logps/chosen": -7.227791786193848,
+      "logps/rejected": -44.98606491088867,
+      "loss": 0.69660884141922,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.57420814037323,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3206728994846344,
+      "rewards/margins": 3.6510839462280273,
+      "rewards/rejected": -3.3304104804992676,
+      "step": 252,
+      "train_speed(iter/s)": 0.011249
+    },
+    {
+      "epoch": 1.2232332423415226,
+      "grad_norm": 1.2537397146224976,
+      "learning_rate": 0.00013733932815594975,
+      "logits/chosen": -0.5108453631401062,
+      "logits/rejected": -0.5226196050643921,
+      "logps/chosen": -5.904589653015137,
+      "logps/rejected": -42.56575393676758,
+      "loss": 0.5000451803207397,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3508087992668152,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3466809391975403,
+      "rewards/margins": 3.448155403137207,
+      "rewards/rejected": -3.1014742851257324,
+      "step": 253,
+      "train_speed(iter/s)": 0.011249
+    },
+    {
+      "epoch": 1.2280861389141644,
+      "grad_norm": 9.252999305725098,
+      "learning_rate": 0.00013684231005458685,
+      "logits/chosen": -0.4556179940700531,
+      "logits/rejected": -0.5657678842544556,
+      "logps/chosen": -5.150843620300293,
+      "logps/rejected": -50.85475158691406,
+      "loss": 0.48492416739463806,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.39000463485717773,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.28096336126327515,
+      "rewards/margins": 3.930372476577759,
+      "rewards/rejected": -3.6494088172912598,
+      "step": 254,
+      "train_speed(iter/s)": 0.011249
+    },
+    {
+      "epoch": 1.2329390354868062,
+      "grad_norm": 1.8387612104415894,
+      "learning_rate": 0.00013634423666920967,
+      "logits/chosen": -0.5443651080131531,
+      "logits/rejected": -0.5357040166854858,
+      "logps/chosen": -4.140000820159912,
+      "logps/rejected": -41.748435974121094,
+      "loss": 0.42465436458587646,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.30934131145477295,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.44264912605285645,
+      "rewards/margins": 3.596529960632324,
+      "rewards/rejected": -3.1538808345794678,
+      "step": 255,
+      "train_speed(iter/s)": 0.011249
+    },
+    {
+      "epoch": 1.237791932059448,
+      "grad_norm": 2.0355947017669678,
+      "learning_rate": 0.00013584512226626566,
+      "logits/chosen": -0.5308322906494141,
+      "logits/rejected": -0.583949089050293,
+      "logps/chosen": -8.423534393310547,
+      "logps/rejected": -46.4453125,
+      "loss": 0.5745125412940979,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.46543920040130615,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5006444454193115,
+      "rewards/margins": 3.8912205696105957,
+      "rewards/rejected": -3.3905763626098633,
+      "step": 256,
+      "train_speed(iter/s)": 0.011249
+    },
+    {
+      "epoch": 1.2426448286320897,
+      "grad_norm": 13.295863151550293,
+      "learning_rate": 0.0001353449811420205,
+      "logits/chosen": -0.602102518081665,
+      "logits/rejected": -0.62212073802948,
+      "logps/chosen": -9.10683822631836,
+      "logps/rejected": -58.837486267089844,
+      "loss": 0.5338473320007324,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.47453534603118896,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5761138796806335,
+      "rewards/margins": 5.05469274520874,
+      "rewards/rejected": -4.478578567504883,
+      "step": 257,
+      "train_speed(iter/s)": 0.011249
+    },
+    {
+      "epoch": 1.2474977252047315,
+      "grad_norm": 2.8557815551757812,
+      "learning_rate": 0.00013484382762214838,
+      "logits/chosen": -0.5345262289047241,
+      "logits/rejected": -0.6295793056488037,
+      "logps/chosen": -8.022461891174316,
+      "logps/rejected": -51.12040710449219,
+      "loss": 0.9521405696868896,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.7950073480606079,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3921523094177246,
+      "rewards/margins": 4.2459635734558105,
+      "rewards/rejected": -3.853811025619507,
+      "step": 258,
+      "train_speed(iter/s)": 0.011249
+    },
+    {
+      "epoch": 1.2523506217773734,
+      "grad_norm": 1.9786404371261597,
+      "learning_rate": 0.00013434167606132192,
+      "logits/chosen": -0.41818898916244507,
+      "logits/rejected": -0.6048794984817505,
+      "logps/chosen": -4.521004676818848,
+      "logps/rejected": -54.303401947021484,
+      "loss": 0.45169034600257874,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.32108208537101746,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2147521823644638,
+      "rewards/margins": 4.153278350830078,
+      "rewards/rejected": -3.9385263919830322,
+      "step": 259,
+      "train_speed(iter/s)": 0.011249
+    },
+    {
+      "epoch": 1.2572035183500152,
+      "grad_norm": 2.75905179977417,
+      "learning_rate": 0.00013383854084280088,
+      "logits/chosen": -0.505330502986908,
+      "logits/rejected": -0.599029541015625,
+      "logps/chosen": -7.0438127517700195,
+      "logps/rejected": -52.19548416137695,
+      "loss": 0.44328948855400085,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.2728576064109802,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3309274911880493,
+      "rewards/margins": 3.922762393951416,
+      "rewards/rejected": -3.591834783554077,
+      "step": 260,
+      "train_speed(iter/s)": 0.011249
+    },
+    {
+      "epoch": 1.262056414922657,
+      "grad_norm": 2.26991605758667,
+      "learning_rate": 0.00013333443637802018,
+      "logits/chosen": -0.5596145391464233,
+      "logits/rejected": -0.606502890586853,
+      "logps/chosen": -11.379673957824707,
+      "logps/rejected": -57.18223190307617,
+      "loss": 0.6898365020751953,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.577810525894165,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.8344271779060364,
+      "rewards/margins": 5.093077182769775,
+      "rewards/rejected": -4.258649826049805,
+      "step": 261,
+      "train_speed(iter/s)": 0.011249
+    },
+    {
+      "epoch": 1.2669093114952987,
+      "grad_norm": 2.1123597621917725,
+      "learning_rate": 0.00013282937710617704,
+      "logits/chosen": -0.5448533892631531,
+      "logits/rejected": -0.5932585597038269,
+      "logps/chosen": -8.195714950561523,
+      "logps/rejected": -45.97007751464844,
+      "loss": 0.6087915301322937,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.46375781297683716,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3240474462509155,
+      "rewards/margins": 3.550736665725708,
+      "rewards/rejected": -3.226688861846924,
+      "step": 262,
+      "train_speed(iter/s)": 0.011248
+    },
+    {
+      "epoch": 1.2717622080679405,
+      "grad_norm": 2.723811626434326,
+      "learning_rate": 0.0001323233774938176,
+      "logits/chosen": -0.42906200885772705,
+      "logits/rejected": -0.5946815013885498,
+      "logps/chosen": -4.58329439163208,
+      "logps/rejected": -51.42955017089844,
+      "loss": 0.620631754398346,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.44010403752326965,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08876454085111618,
+      "rewards/margins": 3.707002878189087,
+      "rewards/rejected": -3.6182377338409424,
+      "step": 263,
+      "train_speed(iter/s)": 0.011249
+    },
+    {
+      "epoch": 1.2766151046405825,
+      "grad_norm": 3.211887836456299,
+      "learning_rate": 0.0001318164520344223,
+      "logits/chosen": -0.6018912196159363,
+      "logits/rejected": -0.6174960136413574,
+      "logps/chosen": -7.36622428894043,
+      "logps/rejected": -38.56849670410156,
+      "loss": 0.6065197587013245,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4719517230987549,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.33680105209350586,
+      "rewards/margins": 3.2080583572387695,
+      "rewards/rejected": -2.8712568283081055,
+      "step": 264,
+      "train_speed(iter/s)": 0.011249
+    },
+    {
+      "epoch": 1.281468001213224,
+      "grad_norm": 1.8414616584777832,
+      "learning_rate": 0.0001313086152479909,
+      "logits/chosen": -0.5074475407600403,
+      "logits/rejected": -0.5390546917915344,
+      "logps/chosen": -8.070147514343262,
+      "logps/rejected": -59.026817321777344,
+      "loss": 0.48236605525016785,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.41621944308280945,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.36634740233421326,
+      "rewards/margins": 4.637523651123047,
+      "rewards/rejected": -4.271176338195801,
+      "step": 265,
+      "train_speed(iter/s)": 0.011248
+    },
+    {
+      "epoch": 1.286320897785866,
+      "grad_norm": 2.0310163497924805,
+      "learning_rate": 0.00013079988168062667,
+      "logits/chosen": -0.5628597736358643,
+      "logits/rejected": -0.5743812918663025,
+      "logps/chosen": -4.6434760093688965,
+      "logps/rejected": -45.32111740112305,
+      "loss": 0.477159708738327,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3573269844055176,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2367573380470276,
+      "rewards/margins": 3.900679111480713,
+      "rewards/rejected": -3.66392183303833,
+      "step": 266,
+      "train_speed(iter/s)": 0.011248
+    },
+    {
+      "epoch": 1.2911737943585078,
+      "grad_norm": 3.795938730239868,
+      "learning_rate": 0.0001302902659041194,
+      "logits/chosen": -0.509046196937561,
+      "logits/rejected": -0.58137446641922,
+      "logps/chosen": -9.875723838806152,
+      "logps/rejected": -66.20906829833984,
+      "loss": 0.6042941808700562,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.489199161529541,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.24621973931789398,
+      "rewards/margins": 5.606562614440918,
+      "rewards/rejected": -5.360342979431152,
+      "step": 267,
+      "train_speed(iter/s)": 0.011248
+    },
+    {
+      "epoch": 1.2960266909311495,
+      "grad_norm": 1.6810376644134521,
+      "learning_rate": 0.00012977978251552835,
+      "logits/chosen": -0.550262451171875,
+      "logits/rejected": -0.5889716148376465,
+      "logps/chosen": -6.327710151672363,
+      "logps/rejected": -50.49878692626953,
+      "loss": 0.48666831851005554,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.420614093542099,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6961234211921692,
+      "rewards/margins": 4.526570796966553,
+      "rewards/rejected": -3.8304474353790283,
+      "step": 268,
+      "train_speed(iter/s)": 0.011248
+    },
+    {
+      "epoch": 1.3008795875037913,
+      "grad_norm": 16.308164596557617,
+      "learning_rate": 0.000129268446136764,
+      "logits/chosen": -0.4653426706790924,
+      "logits/rejected": -0.5483728647232056,
+      "logps/chosen": -12.072315216064453,
+      "logps/rejected": -41.71879577636719,
+      "loss": 1.1896687746047974,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.8476365804672241,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.23016682267189026,
+      "rewards/margins": 3.016373634338379,
+      "rewards/rejected": -2.7862064838409424,
+      "step": 269,
+      "train_speed(iter/s)": 0.011248
+    },
+    {
+      "epoch": 1.305732484076433,
+      "grad_norm": 2.573183298110962,
+      "learning_rate": 0.00012875627141416927,
+      "logits/chosen": -0.5348551273345947,
+      "logits/rejected": -0.6181008219718933,
+      "logps/chosen": -8.181488037109375,
+      "logps/rejected": -42.61102294921875,
+      "loss": 0.47875651717185974,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.37093132734298706,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.44617781043052673,
+      "rewards/margins": 3.2320666313171387,
+      "rewards/rejected": -2.785888671875,
+      "step": 270,
+      "train_speed(iter/s)": 0.011248
+    },
+    {
+      "epoch": 1.310585380649075,
+      "grad_norm": 1.3156378269195557,
+      "learning_rate": 0.00012824327301809993,
+      "logits/chosen": -0.5920026898384094,
+      "logits/rejected": -0.5561531186103821,
+      "logps/chosen": -6.337456226348877,
+      "logps/rejected": -42.275054931640625,
+      "loss": 0.437936931848526,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3473253846168518,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5198031067848206,
+      "rewards/margins": 3.6137380599975586,
+      "rewards/rejected": -3.093935251235962,
+      "step": 271,
+      "train_speed(iter/s)": 0.011248
+    },
+    {
+      "epoch": 1.3154382772217166,
+      "grad_norm": 3.2668862342834473,
+      "learning_rate": 0.00012772946564250454,
+      "logits/chosen": -0.5389162302017212,
+      "logits/rejected": -0.6585366725921631,
+      "logps/chosen": -5.637282371520996,
+      "logps/rejected": -69.4276351928711,
+      "loss": 0.4279293417930603,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.35049596428871155,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2999909818172455,
+      "rewards/margins": 5.595761299133301,
+      "rewards/rejected": -5.295770645141602,
+      "step": 272,
+      "train_speed(iter/s)": 0.011248
+    },
+    {
+      "epoch": 1.3202911737943586,
+      "grad_norm": 2.0276119709014893,
+      "learning_rate": 0.0001272148640045034,
+      "logits/chosen": -0.4903223514556885,
+      "logits/rejected": -0.6459658741950989,
+      "logps/chosen": -6.910153388977051,
+      "logps/rejected": -53.252685546875,
+      "loss": 0.5313152074813843,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.44600358605384827,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3201812505722046,
+      "rewards/margins": 4.161154270172119,
+      "rewards/rejected": -3.840973138809204,
+      "step": 273,
+      "train_speed(iter/s)": 0.011248
+    },
+    {
+      "epoch": 1.3251440703670003,
+      "grad_norm": 2.30670166015625,
+      "learning_rate": 0.00012669948284396707,
+      "logits/chosen": -0.4254702925682068,
+      "logits/rejected": -0.584973156452179,
+      "logps/chosen": -6.449925422668457,
+      "logps/rejected": -67.56501770019531,
+      "loss": 0.4670742154121399,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3590107858181,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.38133683800697327,
+      "rewards/margins": 5.280056953430176,
+      "rewards/rejected": -4.8987202644348145,
+      "step": 274,
+      "train_speed(iter/s)": 0.011248
+    },
+    {
+      "epoch": 1.329996966939642,
+      "grad_norm": 2.108753204345703,
+      "learning_rate": 0.00012618333692309425,
+      "logits/chosen": -0.4286295175552368,
+      "logits/rejected": -0.5902926325798035,
+      "logps/chosen": -8.96891975402832,
+      "logps/rejected": -69.50025939941406,
+      "loss": 0.611693799495697,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.538907527923584,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.41572242975234985,
+      "rewards/margins": 5.492770195007324,
+      "rewards/rejected": -5.077048301696777,
+      "step": 275,
+      "train_speed(iter/s)": 0.011248
+    },
+    {
+      "epoch": 1.3348498635122839,
+      "grad_norm": 2.288952350616455,
+      "learning_rate": 0.00012566644102598876,
+      "logits/chosen": -0.551298201084137,
+      "logits/rejected": -0.6482630968093872,
+      "logps/chosen": -4.394012928009033,
+      "logps/rejected": -64.0866470336914,
+      "loss": 0.3978855609893799,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3059917390346527,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.37667152285575867,
+      "rewards/margins": 5.563392639160156,
+      "rewards/rejected": -5.1867218017578125,
+      "step": 276,
+      "train_speed(iter/s)": 0.011248
+    },
+    {
+      "epoch": 1.3397027600849256,
+      "grad_norm": 1.6208826303482056,
+      "learning_rate": 0.00012514880995823633,
+      "logits/chosen": -0.5507360100746155,
+      "logits/rejected": -0.5011016130447388,
+      "logps/chosen": -8.466445922851562,
+      "logps/rejected": -62.78064727783203,
+      "loss": 0.4081018269062042,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3593176305294037,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6712185144424438,
+      "rewards/margins": 5.426558494567871,
+      "rewards/rejected": -4.755340576171875,
+      "step": 277,
+      "train_speed(iter/s)": 0.011248
+    },
+    {
+      "epoch": 1.3445556566575676,
+      "grad_norm": 1.3175307512283325,
+      "learning_rate": 0.00012463045854648028,
+      "logits/chosen": -0.47354656457901,
+      "logits/rejected": -0.5843176245689392,
+      "logps/chosen": -9.277057647705078,
+      "logps/rejected": -67.9296646118164,
+      "loss": 0.4239290654659271,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3504438102245331,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.31161031126976013,
+      "rewards/margins": 5.217825889587402,
+      "rewards/rejected": -4.906215190887451,
+      "step": 278,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.3494085532302094,
+      "grad_norm": 2.984900712966919,
+      "learning_rate": 0.0001241114016379969,
+      "logits/chosen": -0.5064204335212708,
+      "logits/rejected": -0.6357665061950684,
+      "logps/chosen": -5.173352241516113,
+      "logps/rejected": -70.36311340332031,
+      "loss": 0.3665909767150879,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3104407489299774,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.28970491886138916,
+      "rewards/margins": 5.905008316040039,
+      "rewards/rejected": -5.6153035163879395,
+      "step": 279,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.3542614498028511,
+      "grad_norm": 3.8592402935028076,
+      "learning_rate": 0.00012359165410027038,
+      "logits/chosen": -0.5974986553192139,
+      "logits/rejected": -0.5683308243751526,
+      "logps/chosen": -7.685509204864502,
+      "logps/rejected": -68.6963119506836,
+      "loss": 0.6189377903938293,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5725656747817993,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.017608020454645157,
+      "rewards/margins": 5.929078578948975,
+      "rewards/rejected": -5.911470890045166,
+      "step": 280,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.3591143463754929,
+      "grad_norm": 1.8517260551452637,
+      "learning_rate": 0.0001230712308205666,
+      "logits/chosen": -0.5278875827789307,
+      "logits/rejected": -0.5831331610679626,
+      "logps/chosen": -6.993500709533691,
+      "logps/rejected": -71.447265625,
+      "loss": 0.5337494611740112,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4362339377403259,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.43342167139053345,
+      "rewards/margins": 6.075827598571777,
+      "rewards/rejected": -5.642406463623047,
+      "step": 281,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.3639672429481347,
+      "grad_norm": 8.710851669311523,
+      "learning_rate": 0.00012255014670550695,
+      "logits/chosen": -0.5506738424301147,
+      "logits/rejected": -0.6786321401596069,
+      "logps/chosen": -6.308893203735352,
+      "logps/rejected": -61.42897415161133,
+      "loss": 0.4665891230106354,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3888118863105774,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.10167393088340759,
+      "rewards/margins": 4.767309188842773,
+      "rewards/rejected": -4.665635108947754,
+      "step": 282,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.3688201395207764,
+      "grad_norm": 1.639738917350769,
+      "learning_rate": 0.00012202841668064133,
+      "logits/chosen": -0.5712087750434875,
+      "logits/rejected": -0.5994412899017334,
+      "logps/chosen": -8.841192245483398,
+      "logps/rejected": -49.02497863769531,
+      "loss": 0.49249428510665894,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3942464590072632,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.513531506061554,
+      "rewards/margins": 4.046916484832764,
+      "rewards/rejected": -3.5333847999572754,
+      "step": 283,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.3736730360934182,
+      "grad_norm": 2.5005526542663574,
+      "learning_rate": 0.0001215060556900206,
+      "logits/chosen": -0.6159342527389526,
+      "logits/rejected": -0.5261151790618896,
+      "logps/chosen": -11.058247566223145,
+      "logps/rejected": -38.02698516845703,
+      "loss": 0.6315582990646362,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.45601698756217957,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.29090481996536255,
+      "rewards/margins": 3.167656421661377,
+      "rewards/rejected": -2.8767518997192383,
+      "step": 284,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.3785259326660602,
+      "grad_norm": 2.1935603618621826,
+      "learning_rate": 0.00012098307869576857,
+      "logits/chosen": -0.5347535014152527,
+      "logits/rejected": -0.6269012093544006,
+      "logps/chosen": -5.4327263832092285,
+      "logps/rejected": -53.36666488647461,
+      "loss": 0.5734392404556274,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.48551303148269653,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3451090455055237,
+      "rewards/margins": 4.587835788726807,
+      "rewards/rejected": -4.242726802825928,
+      "step": 285,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.383378829238702,
+      "grad_norm": 1.7531166076660156,
+      "learning_rate": 0.0001204595006776533,
+      "logits/chosen": -0.5986635684967041,
+      "logits/rejected": -0.5231766104698181,
+      "logps/chosen": -6.427237510681152,
+      "logps/rejected": -44.34131622314453,
+      "loss": 0.5123925805091858,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.35561901330947876,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.5461777448654175,
+      "rewards/margins": 4.032759189605713,
+      "rewards/rejected": -3.486581563949585,
+      "step": 286,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.3882317258113437,
+      "grad_norm": 5.020232677459717,
+      "learning_rate": 0.00011993533663265825,
+      "logits/chosen": -0.530123233795166,
+      "logits/rejected": -0.5217273831367493,
+      "logps/chosen": -6.204465866088867,
+      "logps/rejected": -45.284400939941406,
+      "loss": 0.41509750485420227,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.31672149896621704,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.37450146675109863,
+      "rewards/margins": 3.961106300354004,
+      "rewards/rejected": -3.586604595184326,
+      "step": 287,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.3930846223839854,
+      "grad_norm": 1.5128018856048584,
+      "learning_rate": 0.00011941060157455255,
+      "logits/chosen": -0.4853004217147827,
+      "logits/rejected": -0.5132114291191101,
+      "logps/chosen": -5.440448760986328,
+      "logps/rejected": -52.13562774658203,
+      "loss": 0.36660510301589966,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3081294596195221,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3041588366031647,
+      "rewards/margins": 4.590592384338379,
+      "rewards/rejected": -4.286433219909668,
+      "step": 288,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.3979375189566272,
+      "grad_norm": 2.4077889919281006,
+      "learning_rate": 0.00011888531053346102,
+      "logits/chosen": -0.49237561225891113,
+      "logits/rejected": -0.6040050983428955,
+      "logps/chosen": -4.646081924438477,
+      "logps/rejected": -52.04267120361328,
+      "loss": 0.5002583861351013,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3552401661872864,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1544920951128006,
+      "rewards/margins": 3.924790859222412,
+      "rewards/rejected": -3.770298480987549,
+      "step": 289,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.402790415529269,
+      "grad_norm": 4.293540000915527,
+      "learning_rate": 0.00011835947855543357,
+      "logits/chosen": -0.49632903933525085,
+      "logits/rejected": -0.5672129988670349,
+      "logps/chosen": -6.0486836433410645,
+      "logps/rejected": -46.195953369140625,
+      "loss": 0.6360176801681519,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.41714945435523987,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.17008864879608154,
+      "rewards/margins": 3.6374118328094482,
+      "rewards/rejected": -3.4673233032226562,
+      "step": 290,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.4076433121019107,
+      "grad_norm": 1.6993286609649658,
+      "learning_rate": 0.00011783312070201449,
+      "logits/chosen": -0.4478822946548462,
+      "logits/rejected": -0.54694002866745,
+      "logps/chosen": -4.392575263977051,
+      "logps/rejected": -59.24988555908203,
+      "loss": 0.37447091937065125,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.30612316727638245,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.42798465490341187,
+      "rewards/margins": 4.448915004730225,
+      "rewards/rejected": -4.020930290222168,
+      "step": 291,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.4124962086745527,
+      "grad_norm": 1.3974409103393555,
+      "learning_rate": 0.00011730625204981067,
+      "logits/chosen": -0.5312929153442383,
+      "logits/rejected": -0.5600184202194214,
+      "logps/chosen": -6.275518894195557,
+      "logps/rejected": -45.678016662597656,
+      "loss": 0.4217968285083771,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.33802202343940735,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5175706148147583,
+      "rewards/margins": 3.6293063163757324,
+      "rewards/rejected": -3.1117360591888428,
+      "step": 292,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.4173491052471945,
+      "grad_norm": 1.445651650428772,
+      "learning_rate": 0.00011677888769006006,
+      "logits/chosen": -0.43435779213905334,
+      "logits/rejected": -0.5795896053314209,
+      "logps/chosen": -5.8951029777526855,
+      "logps/rejected": -68.92474365234375,
+      "loss": 0.43839338421821594,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3801453113555908,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5051225423812866,
+      "rewards/margins": 5.6808271408081055,
+      "rewards/rejected": -5.175704479217529,
+      "step": 293,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.4222020018198362,
+      "grad_norm": 4.672330379486084,
+      "learning_rate": 0.00011625104272819924,
+      "logits/chosen": -0.5387169122695923,
+      "logits/rejected": -0.5754097104072571,
+      "logps/chosen": -11.130827903747559,
+      "logps/rejected": -46.85166931152344,
+      "loss": 0.65695720911026,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5195148587226868,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.34614866971969604,
+      "rewards/margins": 3.703500747680664,
+      "rewards/rejected": -3.357351541519165,
+      "step": 294,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.427054898392478,
+      "grad_norm": 3.555694580078125,
+      "learning_rate": 0.00011572273228343085,
+      "logits/chosen": -0.5434516668319702,
+      "logits/rejected": -0.5415772795677185,
+      "logps/chosen": -9.660689353942871,
+      "logps/rejected": -49.3802490234375,
+      "loss": 0.6120375990867615,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.5219308733940125,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.27294304966926575,
+      "rewards/margins": 4.194357872009277,
+      "rewards/rejected": -3.921414613723755,
+      "step": 295,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.4319077949651198,
+      "grad_norm": 2.3203206062316895,
+      "learning_rate": 0.00011519397148829036,
+      "logits/chosen": -0.48395296931266785,
+      "logits/rejected": -0.5959677696228027,
+      "logps/chosen": -6.713860511779785,
+      "logps/rejected": -48.37174987792969,
+      "loss": 0.5959409475326538,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.4754820168018341,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3614426255226135,
+      "rewards/margins": 4.109649181365967,
+      "rewards/rejected": -3.748206615447998,
+      "step": 296,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.4367606915377615,
+      "grad_norm": 5.711208343505859,
+      "learning_rate": 0.00011466477548821295,
+      "logits/chosen": -0.48468971252441406,
+      "logits/rejected": -0.6009706854820251,
+      "logps/chosen": -11.339531898498535,
+      "logps/rejected": -54.938899993896484,
+      "loss": 0.7243435978889465,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.580468475818634,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2628437876701355,
+      "rewards/margins": 4.038267612457275,
+      "rewards/rejected": -3.775423288345337,
+      "step": 297,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.4416135881104033,
+      "grad_norm": 2.0594329833984375,
+      "learning_rate": 0.0001141351594410993,
+      "logits/chosen": -0.3807358741760254,
+      "logits/rejected": -0.5564501285552979,
+      "logps/chosen": -5.445425510406494,
+      "logps/rejected": -80.2322998046875,
+      "loss": 0.4422198534011841,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.34769272804260254,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.43609946966171265,
+      "rewards/margins": 6.63840389251709,
+      "rewards/rejected": -6.202303886413574,
+      "step": 298,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.4464664846830453,
+      "grad_norm": 9.533624649047852,
+      "learning_rate": 0.0001136051385168817,
+      "logits/chosen": -0.588534951210022,
+      "logits/rejected": -0.5490925312042236,
+      "logps/chosen": -8.359615325927734,
+      "logps/rejected": -47.81086349487305,
+      "loss": 0.7052288055419922,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.48559439182281494,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.325547456741333,
+      "rewards/margins": 4.019394397735596,
+      "rewards/rejected": -3.6938464641571045,
+      "step": 299,
+      "train_speed(iter/s)": 0.011247
+    },
+    {
+      "epoch": 1.451319381255687,
+      "grad_norm": 3.8685522079467773,
+      "learning_rate": 0.00011307472789708941,
+      "logits/chosen": -0.42629003524780273,
+      "logits/rejected": -0.6104705929756165,
+      "logps/chosen": -5.350531578063965,
+      "logps/rejected": -81.26776123046875,
+      "loss": 0.40768933296203613,
+      "memory(GiB)": 43.93,
+      "nll_loss": 0.3698083162307739,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2111378014087677,
+      "rewards/margins": 6.788004398345947,
+      "rewards/rejected": -6.576866149902344,
+      "step": 300,
+      "train_speed(iter/s)": 0.011247
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 618,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.613143963986821e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}