diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,3634 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5716837874050915,
+  "eval_steps": 300,
+  "global_step": 200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.002858418937025458,
+      "grad_norm": 10.349462509155273,
+      "learning_rate": 3.7735849056603773e-06,
+      "logits/chosen": -0.6422490477561951,
+      "logits/rejected": -0.6447486877441406,
+      "logps/chosen": -6.999429225921631,
+      "logps/rejected": -13.433603286743164,
+      "loss": 1.3778549432754517,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6847077012062073,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1,
+      "train_speed(iter/s)": 0.005273
+    },
+    {
+      "epoch": 0.005716837874050916,
+      "grad_norm": 11.263213157653809,
+      "learning_rate": 7.547169811320755e-06,
+      "logits/chosen": -0.6429960131645203,
+      "logits/rejected": -0.6454926133155823,
+      "logps/chosen": -9.604641914367676,
+      "logps/rejected": -13.53369140625,
+      "loss": 1.4975696802139282,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.8044224381446838,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 2,
+      "train_speed(iter/s)": 0.005366
+    },
+    {
+      "epoch": 0.008575256811076373,
+      "grad_norm": 9.09404182434082,
+      "learning_rate": 1.1320754716981132e-05,
+      "logits/chosen": -0.6486532092094421,
+      "logits/rejected": -0.6493248343467712,
+      "logps/chosen": -7.69912576675415,
+      "logps/rejected": -17.816326141357422,
+      "loss": 1.2593896389007568,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5679630637168884,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0031128099653869867,
+      "rewards/margins": 0.0035441224463284016,
+      "rewards/rejected": -0.00043131227721460164,
+      "step": 3,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.011433675748101831,
+      "grad_norm": 13.314420700073242,
+      "learning_rate": 1.509433962264151e-05,
+      "logits/chosen": -0.6668453812599182,
+      "logits/rejected": -0.6718658804893494,
+      "logps/chosen": -6.061755180358887,
+      "logps/rejected": -13.643918991088867,
+      "loss": 1.4406771659851074,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.7477570176124573,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.008774133399128914,
+      "rewards/margins": 0.0006008772179484367,
+      "rewards/rejected": 0.008173256181180477,
+      "step": 4,
+      "train_speed(iter/s)": 0.005446
+    },
+    {
+      "epoch": 0.014292094685127288,
+      "grad_norm": 12.340804100036621,
+      "learning_rate": 1.8867924528301888e-05,
+      "logits/chosen": -0.6321276426315308,
+      "logits/rejected": -0.6321861743927002,
+      "logps/chosen": -11.234216690063477,
+      "logps/rejected": -11.82645034790039,
+      "loss": 1.3690662384033203,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6870464086532593,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.032103873789310455,
+      "rewards/margins": 0.022960105910897255,
+      "rewards/rejected": 0.0091437678784132,
+      "step": 5,
+      "train_speed(iter/s)": 0.005444
+    },
+    {
+      "epoch": 0.017150513622152745,
+      "grad_norm": 13.995262145996094,
+      "learning_rate": 2.2641509433962265e-05,
+      "logits/chosen": -0.6530164480209351,
+      "logits/rejected": -0.6550691723823547,
+      "logps/chosen": -8.742744445800781,
+      "logps/rejected": -12.622247695922852,
+      "loss": 1.3672001361846924,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.702350378036499,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.0800265222787857,
+      "rewards/margins": 0.06047351285815239,
+      "rewards/rejected": 0.019553007557988167,
+      "step": 6,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.020008932559178204,
+      "grad_norm": 10.019857406616211,
+      "learning_rate": 2.641509433962264e-05,
+      "logits/chosen": -0.6211694478988647,
+      "logits/rejected": -0.6214967370033264,
+      "logps/chosen": -6.935382843017578,
+      "logps/rejected": -13.613279342651367,
+      "loss": 1.2412631511688232,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5651571154594421,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.13799861073493958,
+      "rewards/margins": 0.053141627460718155,
+      "rewards/rejected": 0.08485697209835052,
+      "step": 7,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.022867351496203663,
+      "grad_norm": 6.375326633453369,
+      "learning_rate": 3.018867924528302e-05,
+      "logits/chosen": -0.6666778326034546,
+      "logits/rejected": -0.6678147315979004,
+      "logps/chosen": -8.114872932434082,
+      "logps/rejected": -13.392560958862305,
+      "loss": 1.1162137985229492,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4446101784706116,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.20063960552215576,
+      "rewards/margins": 0.09723334014415741,
+      "rewards/rejected": 0.10340625792741776,
+      "step": 8,
+      "train_speed(iter/s)": 0.005426
+    },
+    {
+      "epoch": 0.02572577043322912,
+      "grad_norm": 4.046472549438477,
+      "learning_rate": 3.39622641509434e-05,
+      "logits/chosen": -0.5842044949531555,
+      "logits/rejected": -0.5892153382301331,
+      "logps/chosen": -13.877073287963867,
+      "logps/rejected": -14.116576194763184,
+      "loss": 1.259205937385559,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5605449676513672,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.1392103135585785,
+      "rewards/margins": 0.09627523273229599,
+      "rewards/rejected": 0.0429350808262825,
+      "step": 9,
+      "train_speed(iter/s)": 0.005422
+    },
+    {
+      "epoch": 0.028584189370254576,
+      "grad_norm": 7.625533103942871,
+      "learning_rate": 3.7735849056603776e-05,
+      "logits/chosen": -0.6210353374481201,
+      "logits/rejected": -0.6224341988563538,
+      "logps/chosen": -8.32575798034668,
+      "logps/rejected": -11.715011596679688,
+      "loss": 1.2772942781448364,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.48597079515457153,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.043191827833652496,
+      "rewards/margins": 0.029505327343940735,
+      "rewards/rejected": -0.07269717007875443,
+      "step": 10,
+      "train_speed(iter/s)": 0.005438
+    },
+    {
+      "epoch": 0.03144260830728004,
+      "grad_norm": 11.961934089660645,
+      "learning_rate": 4.150943396226415e-05,
+      "logits/chosen": -0.6265634298324585,
+      "logits/rejected": -0.6289808750152588,
+      "logps/chosen": -7.129470348358154,
+      "logps/rejected": -11.932583808898926,
+      "loss": 1.447466254234314,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6552329659461975,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.00290237320587039,
+      "rewards/margins": 0.06162097677588463,
+      "rewards/rejected": -0.0645233541727066,
+      "step": 11,
+      "train_speed(iter/s)": 0.005473
+    },
+    {
+      "epoch": 0.03430102724430549,
+      "grad_norm": 10.154479026794434,
+      "learning_rate": 4.528301886792453e-05,
+      "logits/chosen": -0.6388624906539917,
+      "logits/rejected": -0.639370858669281,
+      "logps/chosen": -9.490407943725586,
+      "logps/rejected": -12.88676643371582,
+      "loss": 1.4875458478927612,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6468713283538818,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.10463112592697144,
+      "rewards/margins": -0.059477299451828,
+      "rewards/rejected": -0.04515381157398224,
+      "step": 12,
+      "train_speed(iter/s)": 0.005465
+    },
+    {
+      "epoch": 0.03715944618133095,
+      "grad_norm": 4.091860771179199,
+      "learning_rate": 4.9056603773584906e-05,
+      "logits/chosen": -0.6680133938789368,
+      "logits/rejected": -0.6711105108261108,
+      "logps/chosen": -5.56148099899292,
+      "logps/rejected": -13.865943908691406,
+      "loss": 1.0154482126235962,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.35444653034210205,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.12190410494804382,
+      "rewards/margins": 0.3293726444244385,
+      "rewards/rejected": -0.20746850967407227,
+      "step": 13,
+      "train_speed(iter/s)": 0.00546
+    },
+    {
+      "epoch": 0.04001786511835641,
+      "grad_norm": 3.039910316467285,
+      "learning_rate": 5.283018867924528e-05,
+      "logits/chosen": -0.6245446801185608,
+      "logits/rejected": -0.6280595660209656,
+      "logps/chosen": -7.042474269866943,
+      "logps/rejected": -14.88280963897705,
+      "loss": 0.9986085295677185,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.32772910594940186,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.08946945518255234,
+      "rewards/margins": 0.2548743188381195,
+      "rewards/rejected": -0.16540484130382538,
+      "step": 14,
+      "train_speed(iter/s)": 0.005463
+    },
+    {
+      "epoch": 0.042876284055381866,
+      "grad_norm": 2.8388617038726807,
+      "learning_rate": 5.660377358490566e-05,
+      "logits/chosen": -0.613956868648529,
+      "logits/rejected": -0.6150951385498047,
+      "logps/chosen": -5.398593902587891,
+      "logps/rejected": -10.984697341918945,
+      "loss": 0.9217166900634766,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21436992287635803,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.2490297257900238,
+      "rewards/margins": 0.12685683369636536,
+      "rewards/rejected": 0.12217291444540024,
+      "step": 15,
+      "train_speed(iter/s)": 0.005464
+    },
+    {
+      "epoch": 0.045734702992407325,
+      "grad_norm": 3.623399257659912,
+      "learning_rate": 6.037735849056604e-05,
+      "logits/chosen": -0.5890456438064575,
+      "logits/rejected": -0.5907201766967773,
+      "logps/chosen": -7.65773868560791,
+      "logps/rejected": -15.24480152130127,
+      "loss": 1.130638837814331,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.33376482129096985,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 0.14319562911987305,
+      "rewards/margins": 0.01443251222372055,
+      "rewards/rejected": 0.1287631392478943,
+      "step": 16,
+      "train_speed(iter/s)": 0.00546
+    },
+    {
+      "epoch": 0.048593121929432784,
+      "grad_norm": 2.9850075244903564,
+      "learning_rate": 6.415094339622641e-05,
+      "logits/chosen": -0.5228931903839111,
+      "logits/rejected": -0.524716854095459,
+      "logps/chosen": -10.135262489318848,
+      "logps/rejected": -14.072588920593262,
+      "loss": 1.2770482301712036,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.48515522480010986,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": 0.13618236780166626,
+      "rewards/margins": -0.09462341666221619,
+      "rewards/rejected": 0.23080575466156006,
+      "step": 17,
+      "train_speed(iter/s)": 0.005473
+    },
+    {
+      "epoch": 0.05145154086645824,
+      "grad_norm": 2.2746193408966064,
+      "learning_rate": 6.79245283018868e-05,
+      "logits/chosen": -0.5764032006263733,
+      "logits/rejected": -0.5762047171592712,
+      "logps/chosen": -11.675098419189453,
+      "logps/rejected": -10.994297981262207,
+      "loss": 1.2022521495819092,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4671342372894287,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 0.1593710482120514,
+      "rewards/margins": -0.00844080001115799,
+      "rewards/rejected": 0.1678118258714676,
+      "step": 18,
+      "train_speed(iter/s)": 0.005468
+    },
+    {
+      "epoch": 0.0543099598034837,
+      "grad_norm": 2.324984312057495,
+      "learning_rate": 7.169811320754717e-05,
+      "logits/chosen": -0.567690908908844,
+      "logits/rejected": -0.569066047668457,
+      "logps/chosen": -9.0728120803833,
+      "logps/rejected": -12.268661499023438,
+      "loss": 1.1060843467712402,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.41300466656684875,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.23081830143928528,
+      "rewards/margins": 0.0710553377866745,
+      "rewards/rejected": 0.15976294875144958,
+      "step": 19,
+      "train_speed(iter/s)": 0.005463
+    },
+    {
+      "epoch": 0.05716837874050915,
+      "grad_norm": 2.0991458892822266,
+      "learning_rate": 7.547169811320755e-05,
+      "logits/chosen": -0.6091416478157043,
+      "logits/rejected": -0.6100034117698669,
+      "logps/chosen": -4.26762056350708,
+      "logps/rejected": -11.32187557220459,
+      "loss": 1.0021699666976929,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2933175265789032,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.15655732154846191,
+      "rewards/margins": 0.008277412503957748,
+      "rewards/rejected": 0.14827993512153625,
+      "step": 20,
+      "train_speed(iter/s)": 0.005467
+    },
+    {
+      "epoch": 0.06002679767753461,
+      "grad_norm": 3.4958455562591553,
+      "learning_rate": 7.924528301886794e-05,
+      "logits/chosen": -0.571921706199646,
+      "logits/rejected": -0.572878360748291,
+      "logps/chosen": -9.612298965454102,
+      "logps/rejected": -12.484085083007812,
+      "loss": 1.1795949935913086,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.46760252118110657,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.14643912017345428,
+      "rewards/margins": 0.0200117826461792,
+      "rewards/rejected": 0.12642733752727509,
+      "step": 21,
+      "train_speed(iter/s)": 0.005467
+    },
+    {
+      "epoch": 0.06288521661456008,
+      "grad_norm": 3.175546169281006,
+      "learning_rate": 8.30188679245283e-05,
+      "logits/chosen": -0.6108006834983826,
+      "logits/rejected": -0.6111243367195129,
+      "logps/chosen": -6.207424163818359,
+      "logps/rejected": -7.818144798278809,
+      "loss": 1.1549935340881348,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4683927893638611,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.23285990953445435,
+      "rewards/margins": 0.06634702533483505,
+      "rewards/rejected": 0.16651292145252228,
+      "step": 22,
+      "train_speed(iter/s)": 0.00547
+    },
+    {
+      "epoch": 0.06574363555158554,
+      "grad_norm": 2.4531869888305664,
+      "learning_rate": 8.679245283018869e-05,
+      "logits/chosen": -0.6012008190155029,
+      "logits/rejected": -0.6021928191184998,
+      "logps/chosen": -5.598664283752441,
+      "logps/rejected": -13.262667655944824,
+      "loss": 0.9260164499282837,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.33532270789146423,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.3285560607910156,
+      "rewards/margins": 0.3157665431499481,
+      "rewards/rejected": 0.012789532542228699,
+      "step": 23,
+      "train_speed(iter/s)": 0.005467
+    },
+    {
+      "epoch": 0.06860205448861098,
+      "grad_norm": 2.328031301498413,
+      "learning_rate": 9.056603773584906e-05,
+      "logits/chosen": -0.567432165145874,
+      "logits/rejected": -0.5704118609428406,
+      "logps/chosen": -4.026985168457031,
+      "logps/rejected": -14.904813766479492,
+      "loss": 0.8984246253967285,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.26223480701446533,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.26638248562812805,
+      "rewards/margins": 0.22430934011936188,
+      "rewards/rejected": 0.042073119431734085,
+      "step": 24,
+      "train_speed(iter/s)": 0.005466
+    },
+    {
+      "epoch": 0.07146047342563644,
+      "grad_norm": 3.5119378566741943,
+      "learning_rate": 9.433962264150944e-05,
+      "logits/chosen": -0.5709348320960999,
+      "logits/rejected": -0.5688645243644714,
+      "logps/chosen": -7.716845989227295,
+      "logps/rejected": -7.868927955627441,
+      "loss": 1.113297462463379,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3660878837108612,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.2273256480693817,
+      "rewards/margins": -0.0022776294499635696,
+      "rewards/rejected": 0.22960326075553894,
+      "step": 25,
+      "train_speed(iter/s)": 0.005475
+    },
+    {
+      "epoch": 0.0743188923626619,
+      "grad_norm": 3.2336673736572266,
+      "learning_rate": 9.811320754716981e-05,
+      "logits/chosen": -0.589956521987915,
+      "logits/rejected": -0.587726354598999,
+      "logps/chosen": -10.072855949401855,
+      "logps/rejected": -12.857810974121094,
+      "loss": 1.0991942882537842,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.40110355615615845,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.17576897144317627,
+      "rewards/margins": 0.11343260109424591,
+      "rewards/rejected": 0.062336355447769165,
+      "step": 26,
+      "train_speed(iter/s)": 0.005466
+    },
+    {
+      "epoch": 0.07717731129968736,
+      "grad_norm": 2.6813321113586426,
+      "learning_rate": 0.0001018867924528302,
+      "logits/chosen": -0.5832359194755554,
+      "logits/rejected": -0.5860565304756165,
+      "logps/chosen": -8.197118759155273,
+      "logps/rejected": -17.45244598388672,
+      "loss": 0.8366767764091492,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2886262536048889,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.3297504186630249,
+      "rewards/margins": 0.48295390605926514,
+      "rewards/rejected": -0.15320347249507904,
+      "step": 27,
+      "train_speed(iter/s)": 0.00547
+    },
+    {
+      "epoch": 0.08003573023671282,
+      "grad_norm": 3.3800203800201416,
+      "learning_rate": 0.00010566037735849057,
+      "logits/chosen": -0.6005962491035461,
+      "logits/rejected": -0.6035608053207397,
+      "logps/chosen": -9.117904663085938,
+      "logps/rejected": -18.400161743164062,
+      "loss": 1.0143071413040161,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4276316463947296,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.271901398897171,
+      "rewards/margins": 0.4319708049297333,
+      "rewards/rejected": -0.16006940603256226,
+      "step": 28,
+      "train_speed(iter/s)": 0.005468
+    },
+    {
+      "epoch": 0.08289414917373827,
+      "grad_norm": 5.461134433746338,
+      "learning_rate": 0.00010943396226415095,
+      "logits/chosen": -0.5596082210540771,
+      "logits/rejected": -0.5635167360305786,
+      "logps/chosen": -7.607837200164795,
+      "logps/rejected": -23.174358367919922,
+      "loss": 0.803045928478241,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.32165294885635376,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.3834715783596039,
+      "rewards/margins": 0.7214637994766235,
+      "rewards/rejected": -0.33799219131469727,
+      "step": 29,
+      "train_speed(iter/s)": 0.005464
+    },
+    {
+      "epoch": 0.08575256811076373,
+      "grad_norm": 3.3377506732940674,
+      "learning_rate": 0.00011320754716981132,
+      "logits/chosen": -0.6003558039665222,
+      "logits/rejected": -0.6016712188720703,
+      "logps/chosen": -7.941157817840576,
+      "logps/rejected": -10.564404487609863,
+      "loss": 1.0737202167510986,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3314566910266876,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.1379927098751068,
+      "rewards/margins": 0.08670535683631897,
+      "rewards/rejected": 0.05128733813762665,
+      "step": 30,
+      "train_speed(iter/s)": 0.005461
+    },
+    {
+      "epoch": 0.08861098704778919,
+      "grad_norm": 3.4214563369750977,
+      "learning_rate": 0.0001169811320754717,
+      "logits/chosen": -0.5186684727668762,
+      "logits/rejected": -0.5189406275749207,
+      "logps/chosen": -6.143683910369873,
+      "logps/rejected": -25.88001251220703,
+      "loss": 0.7792238593101501,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2514691948890686,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.2757991552352905,
+      "rewards/margins": 0.5156725645065308,
+      "rewards/rejected": -0.23987337946891785,
+      "step": 31,
+      "train_speed(iter/s)": 0.005459
+    },
+    {
+      "epoch": 0.09146940598481465,
+      "grad_norm": 4.645272731781006,
+      "learning_rate": 0.00012075471698113207,
+      "logits/chosen": -0.5988867282867432,
+      "logits/rejected": -0.6016397476196289,
+      "logps/chosen": -5.776233673095703,
+      "logps/rejected": -22.404544830322266,
+      "loss": 0.8892383575439453,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4011368155479431,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.22715790569782257,
+      "rewards/margins": 0.7724226713180542,
+      "rewards/rejected": -0.545264720916748,
+      "step": 32,
+      "train_speed(iter/s)": 0.005454
+    },
+    {
+      "epoch": 0.09432782492184011,
+      "grad_norm": 4.586422443389893,
+      "learning_rate": 0.00012452830188679244,
+      "logits/chosen": -0.5782912969589233,
+      "logits/rejected": -0.5768243670463562,
+      "logps/chosen": -10.939208984375,
+      "logps/rejected": -13.486640930175781,
+      "loss": 1.1607584953308105,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5877886414527893,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.21317529678344727,
+      "rewards/margins": 0.4141784906387329,
+      "rewards/rejected": -0.20100319385528564,
+      "step": 33,
+      "train_speed(iter/s)": 0.005457
+    },
+    {
+      "epoch": 0.09718624385886557,
+      "grad_norm": 4.306042671203613,
+      "learning_rate": 0.00012830188679245283,
+      "logits/chosen": -0.6019578576087952,
+      "logits/rejected": -0.6082974076271057,
+      "logps/chosen": -7.5170578956604,
+      "logps/rejected": -24.889406204223633,
+      "loss": 0.9509016275405884,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4247076213359833,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.10510760545730591,
+      "rewards/margins": 0.8037927150726318,
+      "rewards/rejected": -0.6986850500106812,
+      "step": 34,
+      "train_speed(iter/s)": 0.005456
+    },
+    {
+      "epoch": 0.10004466279589103,
+      "grad_norm": 3.002274751663208,
+      "learning_rate": 0.0001320754716981132,
+      "logits/chosen": -0.6133092641830444,
+      "logits/rejected": -0.6127456426620483,
+      "logps/chosen": -8.859593391418457,
+      "logps/rejected": -17.02096939086914,
+      "loss": 0.8848217129707336,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3480006456375122,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.25863486528396606,
+      "rewards/margins": 0.5611037015914917,
+      "rewards/rejected": -0.302468866109848,
+      "step": 35,
+      "train_speed(iter/s)": 0.005457
+    },
+    {
+      "epoch": 0.10290308173291648,
+      "grad_norm": 4.299287796020508,
+      "learning_rate": 0.0001358490566037736,
+      "logits/chosen": -0.5725032091140747,
+      "logits/rejected": -0.5725142359733582,
+      "logps/chosen": -8.371686935424805,
+      "logps/rejected": -17.657852172851562,
+      "loss": 0.8825864791870117,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3603641390800476,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.23154468834400177,
+      "rewards/margins": 0.6240193843841553,
+      "rewards/rejected": -0.3924746811389923,
+      "step": 36,
+      "train_speed(iter/s)": 0.005457
+    },
+    {
+      "epoch": 0.10576150066994194,
+      "grad_norm": 4.354315757751465,
+      "learning_rate": 0.00013962264150943395,
+      "logits/chosen": -0.5943324565887451,
+      "logits/rejected": -0.5940991044044495,
+      "logps/chosen": -7.657552719116211,
+      "logps/rejected": -16.183917999267578,
+      "loss": 0.8160735964775085,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3588046133518219,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.27544379234313965,
+      "rewards/margins": 0.7471473217010498,
+      "rewards/rejected": -0.4717034101486206,
+      "step": 37,
+      "train_speed(iter/s)": 0.005452
+    },
+    {
+      "epoch": 0.1086199196069674,
+      "grad_norm": 7.916697978973389,
+      "learning_rate": 0.00014339622641509434,
+      "logits/chosen": -0.5676888227462769,
+      "logits/rejected": -0.5714080929756165,
+      "logps/chosen": -6.698662757873535,
+      "logps/rejected": -17.954235076904297,
+      "loss": 1.052018642425537,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.49374452233314514,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.16338011622428894,
+      "rewards/margins": 0.42942774295806885,
+      "rewards/rejected": -0.2660475969314575,
+      "step": 38,
+      "train_speed(iter/s)": 0.005452
+    },
+    {
+      "epoch": 0.11147833854399285,
+      "grad_norm": 3.7271363735198975,
+      "learning_rate": 0.00014716981132075472,
+      "logits/chosen": -0.6097766160964966,
+      "logits/rejected": -0.6135396957397461,
+      "logps/chosen": -7.30006217956543,
+      "logps/rejected": -22.168495178222656,
+      "loss": 0.7729511857032776,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3125499188899994,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.28066012263298035,
+      "rewards/margins": 0.8108583092689514,
+      "rewards/rejected": -0.5301981568336487,
+      "step": 39,
+      "train_speed(iter/s)": 0.00545
+    },
+    {
+      "epoch": 0.1143367574810183,
+      "grad_norm": 2.960145950317383,
+      "learning_rate": 0.0001509433962264151,
+      "logits/chosen": -0.6132208108901978,
+      "logits/rejected": -0.6163071990013123,
+      "logps/chosen": -7.024638652801514,
+      "logps/rejected": -21.298852920532227,
+      "loss": 0.8106734752655029,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.37238579988479614,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.39458420872688293,
+      "rewards/margins": 0.8476477265357971,
+      "rewards/rejected": -0.45306357741355896,
+      "step": 40,
+      "train_speed(iter/s)": 0.005445
+    },
+    {
+      "epoch": 0.11719517641804376,
+      "grad_norm": 4.3203630447387695,
+      "learning_rate": 0.0001547169811320755,
+      "logits/chosen": -0.5976248979568481,
+      "logits/rejected": -0.6002449989318848,
+      "logps/chosen": -5.656615257263184,
+      "logps/rejected": -16.032827377319336,
+      "loss": 0.9440045952796936,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.33253738284111023,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.14414586126804352,
+      "rewards/margins": 0.5006409287452698,
+      "rewards/rejected": -0.35649505257606506,
+      "step": 41,
+      "train_speed(iter/s)": 0.005445
+    },
+    {
+      "epoch": 0.12005359535506922,
+      "grad_norm": 4.646491527557373,
+      "learning_rate": 0.00015849056603773587,
+      "logits/chosen": -0.5956608653068542,
+      "logits/rejected": -0.5956603288650513,
+      "logps/chosen": -11.719463348388672,
+      "logps/rejected": -27.070579528808594,
+      "loss": 1.0076483488082886,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5070124268531799,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.15291725099086761,
+      "rewards/margins": 0.7710469961166382,
+      "rewards/rejected": -0.6181297898292542,
+      "step": 42,
+      "train_speed(iter/s)": 0.005446
+    },
+    {
+      "epoch": 0.12291201429209468,
+      "grad_norm": 4.248917102813721,
+      "learning_rate": 0.00016226415094339625,
+      "logits/chosen": -0.6100150942802429,
+      "logits/rejected": -0.6142972111701965,
+      "logps/chosen": -6.201865196228027,
+      "logps/rejected": -19.82131004333496,
+      "loss": 0.7969534993171692,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.31225427985191345,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.20218679308891296,
+      "rewards/margins": 0.705444872379303,
+      "rewards/rejected": -0.5032580494880676,
+      "step": 43,
+      "train_speed(iter/s)": 0.005448
+    },
+    {
+      "epoch": 0.12577043322912015,
+      "grad_norm": 3.1864066123962402,
+      "learning_rate": 0.0001660377358490566,
+      "logits/chosen": -0.6078452467918396,
+      "logits/rejected": -0.6100336909294128,
+      "logps/chosen": -8.217609405517578,
+      "logps/rejected": -18.432086944580078,
+      "loss": 0.945781409740448,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4447736442089081,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.16747131943702698,
+      "rewards/margins": 0.6352076530456543,
+      "rewards/rejected": -0.46773630380630493,
+      "step": 44,
+      "train_speed(iter/s)": 0.005451
+    },
+    {
+      "epoch": 0.1286288521661456,
+      "grad_norm": 2.8247904777526855,
+      "learning_rate": 0.000169811320754717,
+      "logits/chosen": -0.6424505710601807,
+      "logits/rejected": -0.6455634236335754,
+      "logps/chosen": -4.626930236816406,
+      "logps/rejected": -18.966289520263672,
+      "loss": 0.6525614261627197,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20949943363666534,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1935967355966568,
+      "rewards/margins": 0.7181073427200317,
+      "rewards/rejected": -0.5245105624198914,
+      "step": 45,
+      "train_speed(iter/s)": 0.005449
+    },
+    {
+      "epoch": 0.13148727110317107,
+      "grad_norm": 5.167090892791748,
+      "learning_rate": 0.00017358490566037738,
+      "logits/chosen": -0.6390554308891296,
+      "logits/rejected": -0.6455981731414795,
+      "logps/chosen": -7.457905292510986,
+      "logps/rejected": -22.238759994506836,
+      "loss": 0.7052859663963318,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3145278990268707,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3406547009944916,
+      "rewards/margins": 0.9184964895248413,
+      "rewards/rejected": -0.5778417587280273,
+      "step": 46,
+      "train_speed(iter/s)": 0.005446
+    },
+    {
+      "epoch": 0.1343456900401965,
+      "grad_norm": 3.110797166824341,
+      "learning_rate": 0.00017735849056603776,
+      "logits/chosen": -0.6957284212112427,
+      "logits/rejected": -0.6984925866127014,
+      "logps/chosen": -6.335500717163086,
+      "logps/rejected": -26.946069717407227,
+      "loss": 0.6684616804122925,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.27696001529693604,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.36519789695739746,
+      "rewards/margins": 1.261197805404663,
+      "rewards/rejected": -0.8960000276565552,
+      "step": 47,
+      "train_speed(iter/s)": 0.005445
+    },
+    {
+      "epoch": 0.13720410897722196,
+      "grad_norm": 5.592209815979004,
+      "learning_rate": 0.00018113207547169812,
+      "logits/chosen": -0.6557337641716003,
+      "logits/rejected": -0.6595371961593628,
+      "logps/chosen": -6.262232780456543,
+      "logps/rejected": -22.550495147705078,
+      "loss": 0.844598650932312,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.32042089104652405,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.098446786403656,
+      "rewards/margins": 1.1042643785476685,
+      "rewards/rejected": -1.0058175325393677,
+      "step": 48,
+      "train_speed(iter/s)": 0.005444
+    },
+    {
+      "epoch": 0.14006252791424742,
+      "grad_norm": 5.518631458282471,
+      "learning_rate": 0.0001849056603773585,
+      "logits/chosen": -0.7261903285980225,
+      "logits/rejected": -0.7333111763000488,
+      "logps/chosen": -6.155282020568848,
+      "logps/rejected": -28.635177612304688,
+      "loss": 0.7839219570159912,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3167538046836853,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.18123595416545868,
+      "rewards/margins": 1.343546986579895,
+      "rewards/rejected": -1.1623109579086304,
+      "step": 49,
+      "train_speed(iter/s)": 0.00544
+    },
+    {
+      "epoch": 0.14292094685127288,
+      "grad_norm": 5.368433475494385,
+      "learning_rate": 0.00018867924528301889,
+      "logits/chosen": -0.7460846304893494,
+      "logits/rejected": -0.7433331608772278,
+      "logps/chosen": -8.476225852966309,
+      "logps/rejected": -20.479339599609375,
+      "loss": 1.041689395904541,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5162184834480286,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.07575411349534988,
+      "rewards/margins": 0.8544278144836426,
+      "rewards/rejected": -0.7786736488342285,
+      "step": 50,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.14577936578829834,
+      "grad_norm": 12.666760444641113,
+      "learning_rate": 0.00019245283018867927,
+      "logits/chosen": -0.7423916459083557,
+      "logits/rejected": -0.7536158561706543,
+      "logps/chosen": -4.518327713012695,
+      "logps/rejected": -25.71565818786621,
+      "loss": 0.8285016417503357,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.38353338837623596,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.25002920627593994,
+      "rewards/margins": 1.0534642934799194,
+      "rewards/rejected": -0.8034350872039795,
+      "step": 51,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.1486377847253238,
+      "grad_norm": 5.63993501663208,
+      "learning_rate": 0.00019622641509433963,
+      "logits/chosen": -0.7661195993423462,
+      "logits/rejected": -0.7684649229049683,
+      "logps/chosen": -5.303972244262695,
+      "logps/rejected": -20.424297332763672,
+      "loss": 0.7760183811187744,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.30550119280815125,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.17865437269210815,
+      "rewards/margins": 0.9522275328636169,
+      "rewards/rejected": -0.7735730409622192,
+      "step": 52,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.15149620366234925,
+      "grad_norm": 3.208770275115967,
+      "learning_rate": 0.0002,
+      "logits/chosen": -0.7615375518798828,
+      "logits/rejected": -0.7639837861061096,
+      "logps/chosen": -7.793338775634766,
+      "logps/rejected": -22.267044067382812,
+      "loss": 0.6889612078666687,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2784683108329773,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.21318240463733673,
+      "rewards/margins": 1.1274638175964355,
+      "rewards/rejected": -0.9142814874649048,
+      "step": 53,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.1543546225993747,
+      "grad_norm": 4.603616714477539,
+      "learning_rate": 0.00019999950054470762,
+      "logits/chosen": -0.7416525483131409,
+      "logits/rejected": -0.7468435168266296,
+      "logps/chosen": -11.753125190734863,
+      "logps/rejected": -23.24603271484375,
+      "loss": 0.9534075856208801,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.42802760004997253,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.2138495296239853,
+      "rewards/margins": 0.8852367997169495,
+      "rewards/rejected": -0.6713871955871582,
+      "step": 54,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.15721304153640017,
+      "grad_norm": 3.226914405822754,
+      "learning_rate": 0.00019999800218381956,
+      "logits/chosen": -0.782919704914093,
+      "logits/rejected": -0.7837254405021667,
+      "logps/chosen": -10.437933921813965,
+      "logps/rejected": -21.950359344482422,
+      "loss": 0.7319959998130798,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.28649967908859253,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.17146320641040802,
+      "rewards/margins": 0.9873074889183044,
+      "rewards/rejected": -0.8158442974090576,
+      "step": 55,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.16007146047342563,
+      "grad_norm": 5.957598686218262,
+      "learning_rate": 0.00019999550493230315,
+      "logits/chosen": -0.7622566819190979,
+      "logits/rejected": -0.7720569968223572,
+      "logps/chosen": -9.299139022827148,
+      "logps/rejected": -25.902114868164062,
+      "loss": 1.0733510255813599,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5738880634307861,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1327313929796219,
+      "rewards/margins": 1.0297049283981323,
+      "rewards/rejected": -0.8969735503196716,
+      "step": 56,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.1629298794104511,
+      "grad_norm": 4.4198760986328125,
+      "learning_rate": 0.00019999200881510367,
+      "logits/chosen": -0.7777268886566162,
+      "logits/rejected": -0.7817560434341431,
+      "logps/chosen": -8.051362037658691,
+      "logps/rejected": -16.21352767944336,
+      "loss": 0.9994087815284729,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5157659649848938,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.3178602159023285,
+      "rewards/margins": 0.7738386392593384,
+      "rewards/rejected": -0.4559784233570099,
+      "step": 57,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.16578829834747655,
+      "grad_norm": 10.918800354003906,
+      "learning_rate": 0.00019998751386714417,
+      "logits/chosen": -0.7735524773597717,
+      "logits/rejected": -0.7762777209281921,
+      "logps/chosen": -6.026726245880127,
+      "logps/rejected": -20.445030212402344,
+      "loss": 0.8424608707427979,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4571954309940338,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3362487852573395,
+      "rewards/margins": 1.0223908424377441,
+      "rewards/rejected": -0.6861420273780823,
+      "step": 58,
+      "train_speed(iter/s)": 0.005438
+    },
+    {
+      "epoch": 0.168646717284502,
+      "grad_norm": 6.003457546234131,
+      "learning_rate": 0.00019998202013332526,
+      "logits/chosen": -0.7671990394592285,
+      "logits/rejected": -0.7679510712623596,
+      "logps/chosen": -7.208020210266113,
+      "logps/rejected": -16.45644187927246,
+      "loss": 0.8400717377662659,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3789638876914978,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.19236475229263306,
+      "rewards/margins": 0.8348343372344971,
+      "rewards/rejected": -0.6424695253372192,
+      "step": 59,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.17150513622152747,
+      "grad_norm": 2.5898213386535645,
+      "learning_rate": 0.00019997552766852432,
+      "logits/chosen": -0.800130307674408,
+      "logits/rejected": -0.8043981790542603,
+      "logps/chosen": -6.655481815338135,
+      "logps/rejected": -24.4234619140625,
+      "loss": 0.6945982575416565,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3340861201286316,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3088517487049103,
+      "rewards/margins": 1.2354176044464111,
+      "rewards/rejected": -0.9265658855438232,
+      "step": 60,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.17436355515855292,
+      "grad_norm": 3.626788377761841,
+      "learning_rate": 0.00019996803653759532,
+      "logits/chosen": -0.7841392159461975,
+      "logits/rejected": -0.7904000282287598,
+      "logps/chosen": -4.378767013549805,
+      "logps/rejected": -26.130855560302734,
+      "loss": 0.5890128016471863,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2802823781967163,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2776252329349518,
+      "rewards/margins": 1.491828203201294,
+      "rewards/rejected": -1.2142030000686646,
+      "step": 61,
+      "train_speed(iter/s)": 0.005438
+    },
+    {
+      "epoch": 0.17722197409557838,
+      "grad_norm": 6.747097492218018,
+      "learning_rate": 0.00019995954681536798,
+      "logits/chosen": -0.7469812035560608,
+      "logits/rejected": -0.7426640391349792,
+      "logps/chosen": -11.407801628112793,
+      "logps/rejected": -19.764530181884766,
+      "loss": 1.1509681940078735,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4816472828388214,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.023781299591064453,
+      "rewards/margins": 0.7125381231307983,
+      "rewards/rejected": -0.7363194227218628,
+      "step": 62,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.18008039303260384,
+      "grad_norm": 6.9256978034973145,
+      "learning_rate": 0.00019995005858664696,
+      "logits/chosen": -0.819698691368103,
+      "logits/rejected": -0.8221041560173035,
+      "logps/chosen": -7.0722761154174805,
+      "logps/rejected": -30.66091537475586,
+      "loss": 0.9850404262542725,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.42992380261421204,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.07954245805740356,
+      "rewards/margins": 1.697967767715454,
+      "rewards/rejected": -1.6184254884719849,
+      "step": 63,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.1829388119696293,
+      "grad_norm": 5.373042106628418,
+      "learning_rate": 0.00019993957194621128,
+      "logits/chosen": -0.8020437955856323,
+      "logits/rejected": -0.8022991418838501,
+      "logps/chosen": -12.086634635925293,
+      "logps/rejected": -21.337501525878906,
+      "loss": 0.9992907643318176,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.46714556217193604,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.11581222712993622,
+      "rewards/margins": 1.0597875118255615,
+      "rewards/rejected": -0.9439753293991089,
+      "step": 64,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.18579723090665476,
+      "grad_norm": 3.77294921875,
+      "learning_rate": 0.00019992808699881303,
+      "logits/chosen": -0.7432379722595215,
+      "logits/rejected": -0.7440240979194641,
+      "logps/chosen": -6.54131555557251,
+      "logps/rejected": -25.4437255859375,
+      "loss": 0.8025262951850891,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4345034658908844,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1653081476688385,
+      "rewards/margins": 1.261300802230835,
+      "rewards/rejected": -1.0959926843643188,
+      "step": 65,
+      "train_speed(iter/s)": 0.005442
+    },
+    {
+      "epoch": 0.18865564984368022,
+      "grad_norm": 3.9820404052734375,
+      "learning_rate": 0.0001999156038591766,
+      "logits/chosen": -0.754156231880188,
+      "logits/rejected": -0.7619881629943848,
+      "logps/chosen": -6.165823459625244,
+      "logps/rejected": -23.689210891723633,
+      "loss": 0.7558284997940063,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.35462892055511475,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.18967051804065704,
+      "rewards/margins": 0.9685844779014587,
+      "rewards/rejected": -0.7789139151573181,
+      "step": 66,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.19151406878070568,
+      "grad_norm": 4.361135482788086,
+      "learning_rate": 0.00019990212265199738,
+      "logits/chosen": -0.7575560212135315,
+      "logits/rejected": -0.7635327577590942,
+      "logps/chosen": -4.563057899475098,
+      "logps/rejected": -24.856014251708984,
+      "loss": 0.7012404203414917,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3292723000049591,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.360738068819046,
+      "rewards/margins": 1.0550967454910278,
+      "rewards/rejected": -0.6943586468696594,
+      "step": 67,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.19437248771773113,
+      "grad_norm": 8.01318359375,
+      "learning_rate": 0.00019988764351194056,
+      "logits/chosen": -0.7201259136199951,
+      "logits/rejected": -0.7213173508644104,
+      "logps/chosen": -14.019023895263672,
+      "logps/rejected": -23.463457107543945,
+      "loss": 1.155617594718933,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6643218994140625,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.16900399327278137,
+      "rewards/margins": 0.8244603276252747,
+      "rewards/rejected": -0.6554563641548157,
+      "step": 68,
+      "train_speed(iter/s)": 0.005442
+    },
+    {
+      "epoch": 0.1972309066547566,
+      "grad_norm": 4.880486488342285,
+      "learning_rate": 0.00019987216658363984,
+      "logits/chosen": -0.7603176832199097,
+      "logits/rejected": -0.7611086368560791,
+      "logps/chosen": -7.871481418609619,
+      "logps/rejected": -18.355331420898438,
+      "loss": 0.9050899744033813,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4671905040740967,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.14834776520729065,
+      "rewards/margins": 0.9282243251800537,
+      "rewards/rejected": -0.7798765897750854,
+      "step": 69,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.20008932559178205,
+      "grad_norm": 36.09653854370117,
+      "learning_rate": 0.00019985569202169584,
+      "logits/chosen": -0.826109766960144,
+      "logits/rejected": -0.8319188952445984,
+      "logps/chosen": -7.292363166809082,
+      "logps/rejected": -19.279315948486328,
+      "loss": 0.906982421875,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4692971408367157,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.17047010362148285,
+      "rewards/margins": 0.9899492859840393,
+      "rewards/rejected": -0.8194791674613953,
+      "step": 70,
+      "train_speed(iter/s)": 0.005443
+    },
+    {
+      "epoch": 0.2029477445288075,
+      "grad_norm": 5.051456451416016,
+      "learning_rate": 0.00019983821999067475,
+      "logits/chosen": -0.7659405469894409,
+      "logits/rejected": -0.773281991481781,
+      "logps/chosen": -7.996345520019531,
+      "logps/rejected": -16.987951278686523,
+      "loss": 0.9166132211685181,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.43807780742645264,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.2428373396396637,
+      "rewards/margins": 0.7172515988349915,
+      "rewards/rejected": -0.47441428899765015,
+      "step": 71,
+      "train_speed(iter/s)": 0.005444
+    },
+    {
+      "epoch": 0.20580616346583297,
+      "grad_norm": 3.3546969890594482,
+      "learning_rate": 0.00019981975066510655,
+      "logits/chosen": -0.7702159285545349,
+      "logits/rejected": -0.7756153345108032,
+      "logps/chosen": -3.096975088119507,
+      "logps/rejected": -24.7138671875,
+      "loss": 0.6350318193435669,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20447328686714172,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.23610374331474304,
+      "rewards/margins": 1.1568939685821533,
+      "rewards/rejected": -0.9207903146743774,
+      "step": 72,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.20866458240285843,
+      "grad_norm": 3.3056812286376953,
+      "learning_rate": 0.00019980028422948322,
+      "logits/chosen": -0.7785294651985168,
+      "logits/rejected": -0.7796329259872437,
+      "logps/chosen": -6.459807395935059,
+      "logps/rejected": -18.2646541595459,
+      "loss": 0.7826670408248901,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.299204021692276,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.17048794031143188,
+      "rewards/margins": 0.8538780212402344,
+      "rewards/rejected": -0.6833901405334473,
+      "step": 73,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.2115230013398839,
+      "grad_norm": 2.804248094558716,
+      "learning_rate": 0.00019977982087825713,
+      "logits/chosen": -0.7178574204444885,
+      "logits/rejected": -0.7222320437431335,
+      "logps/chosen": -5.382746696472168,
+      "logps/rejected": -21.648345947265625,
+      "loss": 0.6176108717918396,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2642488479614258,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3241485357284546,
+      "rewards/margins": 1.1783626079559326,
+      "rewards/rejected": -0.854214072227478,
+      "step": 74,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.21438142027690935,
+      "grad_norm": 2.3271028995513916,
+      "learning_rate": 0.0001997583608158388,
+      "logits/chosen": -0.7665284872055054,
+      "logits/rejected": -0.7763013243675232,
+      "logps/chosen": -3.597620725631714,
+      "logps/rejected": -28.286640167236328,
+      "loss": 0.4727986454963684,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1368505358695984,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.36978626251220703,
+      "rewards/margins": 1.460551142692566,
+      "rewards/rejected": -1.0907649993896484,
+      "step": 75,
+      "train_speed(iter/s)": 0.005442
+    },
+    {
+      "epoch": 0.2172398392139348,
+      "grad_norm": 5.8310160636901855,
+      "learning_rate": 0.00019973590425659507,
+      "logits/chosen": -0.7936246991157532,
+      "logits/rejected": -0.7950795888900757,
+      "logps/chosen": -8.558622360229492,
+      "logps/rejected": -22.39399528503418,
+      "loss": 0.7566131949424744,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4084041714668274,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.26708710193634033,
+      "rewards/margins": 1.3221920728683472,
+      "rewards/rejected": -1.0551048517227173,
+      "step": 76,
+      "train_speed(iter/s)": 0.00544
+    },
+    {
+      "epoch": 0.22009825815096026,
+      "grad_norm": 4.89293909072876,
+      "learning_rate": 0.0001997124514248469,
+      "logits/chosen": -0.8093288540840149,
+      "logits/rejected": -0.8106516003608704,
+      "logps/chosen": -12.884543418884277,
+      "logps/rejected": -22.5157413482666,
+      "loss": 1.130676507949829,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.7112829685211182,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.10567857325077057,
+      "rewards/margins": 1.2312006950378418,
+      "rewards/rejected": -1.1255221366882324,
+      "step": 77,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.2229566770879857,
+      "grad_norm": 7.881620407104492,
+      "learning_rate": 0.00019968800255486713,
+      "logits/chosen": -0.8169571161270142,
+      "logits/rejected": -0.8179275393486023,
+      "logps/chosen": -6.7093825340271,
+      "logps/rejected": -29.32290267944336,
+      "loss": 0.8154858350753784,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3944161534309387,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.16114036738872528,
+      "rewards/margins": 1.4641916751861572,
+      "rewards/rejected": -1.3030513525009155,
+      "step": 78,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.22581509602501115,
+      "grad_norm": 2.513998031616211,
+      "learning_rate": 0.00019966255789087808,
+      "logits/chosen": -0.8547464609146118,
+      "logits/rejected": -0.8608615398406982,
+      "logps/chosen": -7.297125339508057,
+      "logps/rejected": -27.81803321838379,
+      "loss": 0.6003709435462952,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.29905110597610474,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.13682395219802856,
+      "rewards/margins": 1.487520456314087,
+      "rewards/rejected": -1.3506965637207031,
+      "step": 79,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.2286735149620366,
+      "grad_norm": 22.424015045166016,
+      "learning_rate": 0.0001996361176870492,
+      "logits/chosen": -0.8499366044998169,
+      "logits/rejected": -0.863770604133606,
+      "logps/chosen": -8.103764533996582,
+      "logps/rejected": -35.18703842163086,
+      "loss": 0.7495014071464539,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5106410980224609,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2272297590970993,
+      "rewards/margins": 1.9457027912139893,
+      "rewards/rejected": -1.7184730768203735,
+      "step": 80,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.23153193389906207,
+      "grad_norm": 4.0974273681640625,
+      "learning_rate": 0.00019960868220749448,
+      "logits/chosen": -0.8633654117584229,
+      "logits/rejected": -0.8727495670318604,
+      "logps/chosen": -5.683876991271973,
+      "logps/rejected": -34.967037200927734,
+      "loss": 0.6187952756881714,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.32927608489990234,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2094997614622116,
+      "rewards/margins": 2.0794010162353516,
+      "rewards/rejected": -1.8699012994766235,
+      "step": 81,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.23439035283608753,
+      "grad_norm": 3.275705575942993,
+      "learning_rate": 0.00019958025172626986,
+      "logits/chosen": -0.8937872648239136,
+      "logits/rejected": -0.9034286737442017,
+      "logps/chosen": -11.346511840820312,
+      "logps/rejected": -35.471927642822266,
+      "loss": 0.7466462850570679,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4821043014526367,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2012697458267212,
+      "rewards/margins": 1.9956668615341187,
+      "rewards/rejected": -1.7943971157073975,
+      "step": 82,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.237248771773113,
+      "grad_norm": 8.550602912902832,
+      "learning_rate": 0.0001995508265273704,
+      "logits/chosen": -0.9247998595237732,
+      "logits/rejected": -0.934203028678894,
+      "logps/chosen": -7.0234270095825195,
+      "logps/rejected": -40.43362808227539,
+      "loss": 0.6714224219322205,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.36930039525032043,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.015845157206058502,
+      "rewards/margins": 2.3306310176849365,
+      "rewards/rejected": -2.3464760780334473,
+      "step": 83,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.24010719071013845,
+      "grad_norm": 10.380730628967285,
+      "learning_rate": 0.00019952040690472748,
+      "logits/chosen": -0.89071124792099,
+      "logits/rejected": -0.896521806716919,
+      "logps/chosen": -9.274986267089844,
+      "logps/rejected": -31.167112350463867,
+      "loss": 0.9556330442428589,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4726044833660126,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.055123914033174515,
+      "rewards/margins": 1.99386465549469,
+      "rewards/rejected": -1.9387409687042236,
+      "step": 84,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.2429656096471639,
+      "grad_norm": 59.343536376953125,
+      "learning_rate": 0.00019948899316220604,
+      "logits/chosen": -0.86629319190979,
+      "logits/rejected": -0.8767050504684448,
+      "logps/chosen": -6.04016637802124,
+      "logps/rejected": -28.20957374572754,
+      "loss": 0.725101888179779,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.40447694063186646,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.29112857580184937,
+      "rewards/margins": 1.845672369003296,
+      "rewards/rejected": -1.5545437335968018,
+      "step": 85,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.24582402858418936,
+      "grad_norm": 6.300822734832764,
+      "learning_rate": 0.0001994565856136012,
+      "logits/chosen": -0.8398523330688477,
+      "logits/rejected": -0.8439276814460754,
+      "logps/chosen": -7.2902045249938965,
+      "logps/rejected": -32.82334899902344,
+      "loss": 0.5805257558822632,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4190253019332886,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3955605626106262,
+      "rewards/margins": 2.423994302749634,
+      "rewards/rejected": -2.0284337997436523,
+      "step": 86,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.24868244752121482,
+      "grad_norm": 43.768619537353516,
+      "learning_rate": 0.0001994231845826354,
+      "logits/chosen": -0.8710224628448486,
+      "logits/rejected": -0.8721168637275696,
+      "logps/chosen": -7.397428035736084,
+      "logps/rejected": -29.24224853515625,
+      "loss": 1.1686397790908813,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.7285480499267578,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.11831004917621613,
+      "rewards/margins": 1.2224053144454956,
+      "rewards/rejected": -1.3407154083251953,
+      "step": 87,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.2515408664582403,
+      "grad_norm": 39.749202728271484,
+      "learning_rate": 0.00019938879040295508,
+      "logits/chosen": -0.8551483750343323,
+      "logits/rejected": -0.8542779684066772,
+      "logps/chosen": -10.534215927124023,
+      "logps/rejected": -26.982810974121094,
+      "loss": 1.2515333890914917,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.7181237936019897,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.03403427451848984,
+      "rewards/margins": 1.2902536392211914,
+      "rewards/rejected": -1.32428777217865,
+      "step": 88,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.25439928539526574,
+      "grad_norm": 10.098331451416016,
+      "learning_rate": 0.00019935340341812737,
+      "logits/chosen": -0.8135808706283569,
+      "logits/rejected": -0.8137494325637817,
+      "logps/chosen": -9.00113582611084,
+      "logps/rejected": -22.338396072387695,
+      "loss": 0.9040060639381409,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5397258400917053,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.04357346147298813,
+      "rewards/margins": 1.2031430006027222,
+      "rewards/rejected": -1.159569501876831,
+      "step": 89,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.2572577043322912,
+      "grad_norm": 8.489130973815918,
+      "learning_rate": 0.00019931702398163657,
+      "logits/chosen": -0.8590521216392517,
+      "logits/rejected": -0.8651595711708069,
+      "logps/chosen": -8.849437713623047,
+      "logps/rejected": -27.54432487487793,
+      "loss": 0.8707167506217957,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.48353832960128784,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.06561100482940674,
+      "rewards/margins": 1.1353689432144165,
+      "rewards/rejected": -1.0697578191757202,
+      "step": 90,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.26011612326931666,
+      "grad_norm": 5.018461227416992,
+      "learning_rate": 0.00019927965245688072,
+      "logits/chosen": -0.8591816425323486,
+      "logits/rejected": -0.8660668730735779,
+      "logps/chosen": -11.013117790222168,
+      "logps/rejected": -27.660642623901367,
+      "loss": 0.9352516531944275,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.541546106338501,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.2031691074371338,
+      "rewards/margins": 1.437905192375183,
+      "rewards/rejected": -1.2347362041473389,
+      "step": 91,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.26297454220634214,
+      "grad_norm": 6.94728946685791,
+      "learning_rate": 0.00019924128921716797,
+      "logits/chosen": -0.8400283455848694,
+      "logits/rejected": -0.8450547456741333,
+      "logps/chosen": -9.49151611328125,
+      "logps/rejected": -28.902881622314453,
+      "loss": 0.806583821773529,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3471754193305969,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.21784308552742004,
+      "rewards/margins": 1.679865837097168,
+      "rewards/rejected": -1.4620226621627808,
+      "step": 92,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.2658329611433676,
+      "grad_norm": 2.8959178924560547,
+      "learning_rate": 0.00019920193464571275,
+      "logits/chosen": -0.839047372341156,
+      "logits/rejected": -0.8450105786323547,
+      "logps/chosen": -10.59231948852539,
+      "logps/rejected": -29.6372013092041,
+      "loss": 0.7759805917739868,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5248351097106934,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3599660098552704,
+      "rewards/margins": 1.950143575668335,
+      "rewards/rejected": -1.5901774168014526,
+      "step": 93,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.268691380080393,
+      "grad_norm": 5.318148612976074,
+      "learning_rate": 0.00019916158913563207,
+      "logits/chosen": -0.8162392377853394,
+      "logits/rejected": -0.8246886730194092,
+      "logps/chosen": -5.6260833740234375,
+      "logps/rejected": -29.538055419921875,
+      "loss": 0.6956499218940735,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.38102442026138306,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.2292197048664093,
+      "rewards/margins": 1.7469617128372192,
+      "rewards/rejected": -1.5177420377731323,
+      "step": 94,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.2715497990174185,
+      "grad_norm": 4.5264787673950195,
+      "learning_rate": 0.00019912025308994148,
+      "logits/chosen": -0.8437933325767517,
+      "logits/rejected": -0.8463299870491028,
+      "logps/chosen": -8.831324577331543,
+      "logps/rejected": -26.610977172851562,
+      "loss": 0.7661141157150269,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4225902557373047,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.12178793549537659,
+      "rewards/margins": 1.467708945274353,
+      "rewards/rejected": -1.3459210395812988,
+      "step": 95,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.2744082179544439,
+      "grad_norm": 5.978212356567383,
+      "learning_rate": 0.00019907792692155113,
+      "logits/chosen": -0.7815445065498352,
+      "logits/rejected": -0.78977370262146,
+      "logps/chosen": -6.673672199249268,
+      "logps/rejected": -33.78847122192383,
+      "loss": 0.7695144414901733,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5254613757133484,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2026345133781433,
+      "rewards/margins": 1.8294543027877808,
+      "rewards/rejected": -1.6268197298049927,
+      "step": 96,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.2772666368914694,
+      "grad_norm": 9.170755386352539,
+      "learning_rate": 0.00019903461105326154,
+      "logits/chosen": -0.8260455131530762,
+      "logits/rejected": -0.830398678779602,
+      "logps/chosen": -6.689513206481934,
+      "logps/rejected": -22.671979904174805,
+      "loss": 0.7065067291259766,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.32698267698287964,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2753945589065552,
+      "rewards/margins": 1.3633196353912354,
+      "rewards/rejected": -1.0879249572753906,
+      "step": 97,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.28012505582849484,
+      "grad_norm": 3.521716594696045,
+      "learning_rate": 0.00019899030591775957,
+      "logits/chosen": -0.7963160872459412,
+      "logits/rejected": -0.8014420866966248,
+      "logps/chosen": -7.337916851043701,
+      "logps/rejected": -28.08383560180664,
+      "loss": 0.61917644739151,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3110499680042267,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.23168900609016418,
+      "rewards/margins": 1.7576744556427002,
+      "rewards/rejected": -1.525985598564148,
+      "step": 98,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.2829834747655203,
+      "grad_norm": 5.793614864349365,
+      "learning_rate": 0.00019894501195761392,
+      "logits/chosen": -0.8137191534042358,
+      "logits/rejected": -0.8227260708808899,
+      "logps/chosen": -6.954023838043213,
+      "logps/rejected": -35.75700378417969,
+      "loss": 0.6259573698043823,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.33839166164398193,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.26017990708351135,
+      "rewards/margins": 2.326916217803955,
+      "rewards/rejected": -2.0667362213134766,
+      "step": 99,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.28584189370254576,
+      "grad_norm": 5.138144493103027,
+      "learning_rate": 0.00019889872962527072,
+      "logits/chosen": -0.8491114377975464,
+      "logits/rejected": -0.8503983020782471,
+      "logps/chosen": -5.965326309204102,
+      "logps/rejected": -24.436853408813477,
+      "loss": 0.9054074287414551,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3898293972015381,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.10434619337320328,
+      "rewards/margins": 1.369442105293274,
+      "rewards/rejected": -1.265095829963684,
+      "step": 100,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.28870031263957124,
+      "grad_norm": 3.6206700801849365,
+      "learning_rate": 0.00019885145938304907,
+      "logits/chosen": -0.8245471119880676,
+      "logits/rejected": -0.8329594135284424,
+      "logps/chosen": -12.563156127929688,
+      "logps/rejected": -31.796875,
+      "loss": 0.6779420971870422,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.36515456438064575,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.2728801667690277,
+      "rewards/margins": 2.0701961517333984,
+      "rewards/rejected": -1.797316074371338,
+      "step": 101,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.2915587315765967,
+      "grad_norm": 4.129011154174805,
+      "learning_rate": 0.0001988032017031364,
+      "logits/chosen": -0.7823147773742676,
+      "logits/rejected": -0.7900741696357727,
+      "logps/chosen": -5.765602111816406,
+      "logps/rejected": -31.836132049560547,
+      "loss": 0.533344030380249,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3198188543319702,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3069157004356384,
+      "rewards/margins": 2.2244224548339844,
+      "rewards/rejected": -1.9175068140029907,
+      "step": 102,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.29441715051362216,
+      "grad_norm": 7.285937786102295,
+      "learning_rate": 0.00019875395706758388,
+      "logits/chosen": -0.8199520707130432,
+      "logits/rejected": -0.8266770839691162,
+      "logps/chosen": -6.686117172241211,
+      "logps/rejected": -35.52680969238281,
+      "loss": 0.6140994429588318,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.33540287613868713,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3744865357875824,
+      "rewards/margins": 2.1863038539886475,
+      "rewards/rejected": -1.8118171691894531,
+      "step": 103,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.2972755694506476,
+      "grad_norm": 4.08017110824585,
+      "learning_rate": 0.0001987037259683013,
+      "logits/chosen": -0.7858948707580566,
+      "logits/rejected": -0.7929791808128357,
+      "logps/chosen": -6.60341215133667,
+      "logps/rejected": -32.91640853881836,
+      "loss": 0.49151238799095154,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.26134875416755676,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3017127513885498,
+      "rewards/margins": 2.313183546066284,
+      "rewards/rejected": -2.0114707946777344,
+      "step": 104,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.3001339883876731,
+      "grad_norm": 6.110865116119385,
+      "learning_rate": 0.00019865250890705247,
+      "logits/chosen": -0.8439383506774902,
+      "logits/rejected": -0.8605507612228394,
+      "logps/chosen": -5.249790668487549,
+      "logps/rejected": -45.18266677856445,
+      "loss": 0.665439248085022,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4175432622432709,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.19819766283035278,
+      "rewards/margins": 2.974789619445801,
+      "rewards/rejected": -2.776592254638672,
+      "step": 105,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.3029924073246985,
+      "grad_norm": 8.704270362854004,
+      "learning_rate": 0.00019860030639545003,
+      "logits/chosen": -0.8581717610359192,
+      "logits/rejected": -0.8701122403144836,
+      "logps/chosen": -8.200727462768555,
+      "logps/rejected": -39.06196975708008,
+      "loss": 0.9722508788108826,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6783410310745239,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.09280453622341156,
+      "rewards/margins": 2.520831823348999,
+      "rewards/rejected": -2.4280271530151367,
+      "step": 106,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.305850826261724,
+      "grad_norm": 6.919908046722412,
+      "learning_rate": 0.00019854711895495036,
+      "logits/chosen": -0.8861989974975586,
+      "logits/rejected": -0.8968295454978943,
+      "logps/chosen": -4.962037563323975,
+      "logps/rejected": -40.77632522583008,
+      "loss": 0.5569091439247131,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.37660181522369385,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.30795052647590637,
+      "rewards/margins": 3.064462184906006,
+      "rewards/rejected": -2.756511688232422,
+      "step": 107,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.3087092451987494,
+      "grad_norm": 4.996776580810547,
+      "learning_rate": 0.00019849294711684845,
+      "logits/chosen": -0.9625644683837891,
+      "logits/rejected": -0.972130298614502,
+      "logps/chosen": -8.820281028747559,
+      "logps/rejected": -50.21597671508789,
+      "loss": 0.7489598989486694,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5024483799934387,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.26294729113578796,
+      "rewards/margins": 3.5292611122131348,
+      "rewards/rejected": -3.2663135528564453,
+      "step": 108,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.3115676641357749,
+      "grad_norm": 6.157798767089844,
+      "learning_rate": 0.00019843779142227256,
+      "logits/chosen": -0.8762018084526062,
+      "logits/rejected": -0.882946789264679,
+      "logps/chosen": -7.924221992492676,
+      "logps/rejected": -43.705806732177734,
+      "loss": 0.6497112512588501,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3160628080368042,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.10452399402856827,
+      "rewards/margins": 3.319636106491089,
+      "rewards/rejected": -3.2151122093200684,
+      "step": 109,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.31442608307280034,
+      "grad_norm": 15.488557815551758,
+      "learning_rate": 0.00019838165242217875,
+      "logits/chosen": -0.863703727722168,
+      "logits/rejected": -0.8713690042495728,
+      "logps/chosen": -6.335309028625488,
+      "logps/rejected": -46.15715026855469,
+      "loss": 0.871519923210144,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5997005701065063,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.15678179264068604,
+      "rewards/margins": 3.3330183029174805,
+      "rewards/rejected": -3.176236629486084,
+      "step": 110,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.31728450200982583,
+      "grad_norm": 4.969946384429932,
+      "learning_rate": 0.0001983245306773454,
+      "logits/chosen": -0.8029311895370483,
+      "logits/rejected": -0.8069216012954712,
+      "logps/chosen": -5.91574764251709,
+      "logps/rejected": -26.575307846069336,
+      "loss": 0.6512956619262695,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2922568917274475,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.20303387939929962,
+      "rewards/margins": 1.892715573310852,
+      "rewards/rejected": -1.6896816492080688,
+      "step": 111,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.32014292094685126,
+      "grad_norm": 6.895030498504639,
+      "learning_rate": 0.0001982664267583677,
+      "logits/chosen": -0.8168566226959229,
+      "logits/rejected": -0.8292732238769531,
+      "logps/chosen": -5.602714538574219,
+      "logps/rejected": -31.50632667541504,
+      "loss": 0.8574928641319275,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5075439810752869,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.010208956897258759,
+      "rewards/margins": 1.8891350030899048,
+      "rewards/rejected": -1.8789262771606445,
+      "step": 112,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.32300133988387675,
+      "grad_norm": 4.541954040527344,
+      "learning_rate": 0.00019820734124565182,
+      "logits/chosen": -0.7733545899391174,
+      "logits/rejected": -0.7771452069282532,
+      "logps/chosen": -6.299251079559326,
+      "logps/rejected": -27.075977325439453,
+      "loss": 0.7081068754196167,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3878493905067444,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.13715356588363647,
+      "rewards/margins": 1.6064281463623047,
+      "rewards/rejected": -1.4692747592926025,
+      "step": 113,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.3258597588209022,
+      "grad_norm": 3.1780240535736084,
+      "learning_rate": 0.00019814727472940917,
+      "logits/chosen": -0.7537604570388794,
+      "logits/rejected": -0.7559649348258972,
+      "logps/chosen": -11.733123779296875,
+      "logps/rejected": -27.673603057861328,
+      "loss": 0.71772301197052,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.42100292444229126,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.21125301718711853,
+      "rewards/margins": 1.6915286779403687,
+      "rewards/rejected": -1.4802757501602173,
+      "step": 114,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.32871817775792767,
+      "grad_norm": 3.8751778602600098,
+      "learning_rate": 0.00019808622780965066,
+      "logits/chosen": -0.7520322799682617,
+      "logits/rejected": -0.7543449401855469,
+      "logps/chosen": -7.260441303253174,
+      "logps/rejected": -29.127504348754883,
+      "loss": 0.6945881843566895,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3524607717990875,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.26789340376853943,
+      "rewards/margins": 1.691644549369812,
+      "rewards/rejected": -1.4237512350082397,
+      "step": 115,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.3315765966949531,
+      "grad_norm": 3.0139174461364746,
+      "learning_rate": 0.0001980242010961803,
+      "logits/chosen": -0.7901269793510437,
+      "logits/rejected": -0.7972939014434814,
+      "logps/chosen": -7.289261341094971,
+      "logps/rejected": -47.759063720703125,
+      "loss": 0.5331582427024841,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.38446128368377686,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3029555678367615,
+      "rewards/margins": 2.9576172828674316,
+      "rewards/rejected": -2.6546616554260254,
+      "step": 116,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.3344350156319786,
+      "grad_norm": 3.233109712600708,
+      "learning_rate": 0.00019796119520858955,
+      "logits/chosen": -0.7568274140357971,
+      "logits/rejected": -0.7630717158317566,
+      "logps/chosen": -10.064506530761719,
+      "logps/rejected": -32.50635528564453,
+      "loss": 0.789489209651947,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4839721918106079,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.3997339606285095,
+      "rewards/margins": 2.3508408069610596,
+      "rewards/rejected": -1.9511069059371948,
+      "step": 117,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.337293434569004,
+      "grad_norm": 6.901320934295654,
+      "learning_rate": 0.0001978972107762509,
+      "logits/chosen": -0.7462923526763916,
+      "logits/rejected": -0.7441357970237732,
+      "logps/chosen": -10.976001739501953,
+      "logps/rejected": -31.10492706298828,
+      "loss": 0.7512857913970947,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.43077993392944336,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1536616086959839,
+      "rewards/margins": 2.106844186782837,
+      "rewards/rejected": -1.953182578086853,
+      "step": 118,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.3401518535060295,
+      "grad_norm": 2.286742925643921,
+      "learning_rate": 0.00019783224843831162,
+      "logits/chosen": -0.7183930277824402,
+      "logits/rejected": -0.7182736396789551,
+      "logps/chosen": -8.731864929199219,
+      "logps/rejected": -33.98211669921875,
+      "loss": 0.6520930528640747,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3684110939502716,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.2964346706867218,
+      "rewards/margins": 2.4515674114227295,
+      "rewards/rejected": -2.15513277053833,
+      "step": 119,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.34301027244305493,
+      "grad_norm": 4.936792850494385,
+      "learning_rate": 0.00019776630884368738,
+      "logits/chosen": -0.7660694122314453,
+      "logits/rejected": -0.776424765586853,
+      "logps/chosen": -4.871070861816406,
+      "logps/rejected": -47.67111587524414,
+      "loss": 0.4515693187713623,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.30822524428367615,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.38517045974731445,
+      "rewards/margins": 3.7489230632781982,
+      "rewards/rejected": -3.363752841949463,
+      "step": 120,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.3458686913800804,
+      "grad_norm": 9.613821983337402,
+      "learning_rate": 0.00019769939265105573,
+      "logits/chosen": -0.7732415795326233,
+      "logits/rejected": -0.7863773703575134,
+      "logps/chosen": -4.910383701324463,
+      "logps/rejected": -44.046382904052734,
+      "loss": 0.6085942983627319,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3416149616241455,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.3152579069137573,
+      "rewards/margins": 3.380157470703125,
+      "rewards/rejected": -3.0648996829986572,
+      "step": 121,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.34872711031710585,
+      "grad_norm": 5.3062744140625,
+      "learning_rate": 0.00019763150052884966,
+      "logits/chosen": -0.7940014004707336,
+      "logits/rejected": -0.8029736280441284,
+      "logps/chosen": -3.551081895828247,
+      "logps/rejected": -38.06512451171875,
+      "loss": 0.42784249782562256,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.25387704372406006,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.37009117007255554,
+      "rewards/margins": 2.841196298599243,
+      "rewards/rejected": -2.471104860305786,
+      "step": 122,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.35158552925413133,
+      "grad_norm": 1.533804178237915,
+      "learning_rate": 0.0001975626331552507,
+      "logits/chosen": -0.8097442984580994,
+      "logits/rejected": -0.8176737427711487,
+      "logps/chosen": -3.7899668216705322,
+      "logps/rejected": -51.7594108581543,
+      "loss": 0.3041819632053375,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2387474924325943,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.35475367307662964,
+      "rewards/margins": 4.306023597717285,
+      "rewards/rejected": -3.9512698650360107,
+      "step": 123,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.35444394819115677,
+      "grad_norm": 4.373802661895752,
+      "learning_rate": 0.00019749279121818235,
+      "logits/chosen": -0.7605909705162048,
+      "logits/rejected": -0.770943284034729,
+      "logps/chosen": -8.83084774017334,
+      "logps/rejected": -59.24419021606445,
+      "loss": 0.5374789834022522,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.39307644963264465,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.14716514945030212,
+      "rewards/margins": 4.3210930824279785,
+      "rewards/rejected": -4.173928260803223,
+      "step": 124,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.3573023671281822,
+      "grad_norm": 6.3563103675842285,
+      "learning_rate": 0.0001974219754153032,
+      "logits/chosen": -0.8258897066116333,
+      "logits/rejected": -0.8272631168365479,
+      "logps/chosen": -7.77381706237793,
+      "logps/rejected": -41.343414306640625,
+      "loss": 0.6204044818878174,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.33825787901878357,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.12217142432928085,
+      "rewards/margins": 3.0349438190460205,
+      "rewards/rejected": -2.9127724170684814,
+      "step": 125,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.3601607860652077,
+      "grad_norm": 9.85932731628418,
+      "learning_rate": 0.00019735018645399967,
+      "logits/chosen": -0.8187968730926514,
+      "logits/rejected": -0.8283690214157104,
+      "logps/chosen": -4.5190534591674805,
+      "logps/rejected": -45.119789123535156,
+      "loss": 0.5073397755622864,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.33468902111053467,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.34614282846450806,
+      "rewards/margins": 3.5951905250549316,
+      "rewards/rejected": -3.2490477561950684,
+      "step": 126,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.3630192050022331,
+      "grad_norm": 3.943042039871216,
+      "learning_rate": 0.00019727742505137936,
+      "logits/chosen": -0.7813420295715332,
+      "logits/rejected": -0.7975005507469177,
+      "logps/chosen": -7.530571937561035,
+      "logps/rejected": -53.51607894897461,
+      "loss": 0.46567243337631226,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2742045819759369,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.23301076889038086,
+      "rewards/margins": 3.9482827186584473,
+      "rewards/rejected": -3.7152719497680664,
+      "step": 127,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.3658776239392586,
+      "grad_norm": 1.9621978998184204,
+      "learning_rate": 0.00019720369193426365,
+      "logits/chosen": -0.7805514335632324,
+      "logits/rejected": -0.7947923541069031,
+      "logps/chosen": -4.593271255493164,
+      "logps/rejected": -51.02714157104492,
+      "loss": 0.3543168008327484,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2695479393005371,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3205646574497223,
+      "rewards/margins": 3.938343048095703,
+      "rewards/rejected": -3.6177780628204346,
+      "step": 128,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.36873604287628403,
+      "grad_norm": 6.375858783721924,
+      "learning_rate": 0.00019712898783918038,
+      "logits/chosen": -0.7720431089401245,
+      "logits/rejected": -0.7774226069450378,
+      "logps/chosen": -3.9679479598999023,
+      "logps/rejected": -39.059242248535156,
+      "loss": 0.42859622836112976,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2718777060508728,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.335873007774353,
+      "rewards/margins": 3.0524649620056152,
+      "rewards/rejected": -2.7165920734405518,
+      "step": 129,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.3715944618133095,
+      "grad_norm": 20.90467643737793,
+      "learning_rate": 0.00019705331351235674,
+      "logits/chosen": -0.7668591737747192,
+      "logits/rejected": -0.7685776948928833,
+      "logps/chosen": -9.782769203186035,
+      "logps/rejected": -33.2745361328125,
+      "loss": 1.310682773590088,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.8834658861160278,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.039194077253341675,
+      "rewards/margins": 2.1344308853149414,
+      "rewards/rejected": -2.1736247539520264,
+      "step": 130,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.37445288075033495,
+      "grad_norm": 2.3716464042663574,
+      "learning_rate": 0.00019697666970971153,
+      "logits/chosen": -0.7852880954742432,
+      "logits/rejected": -0.7899134755134583,
+      "logps/chosen": -6.234109878540039,
+      "logps/rejected": -42.3105354309082,
+      "loss": 0.4260920286178589,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.25257518887519836,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3239039182662964,
+      "rewards/margins": 3.0808892250061035,
+      "rewards/rejected": -2.7569851875305176,
+      "step": 131,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.37731129968736044,
+      "grad_norm": 17.186786651611328,
+      "learning_rate": 0.00019689905719684782,
+      "logits/chosen": -0.7246553301811218,
+      "logits/rejected": -0.739700198173523,
+      "logps/chosen": -4.994725227355957,
+      "logps/rejected": -37.61749267578125,
+      "loss": 0.7901899814605713,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.529437243938446,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3914901614189148,
+      "rewards/margins": 2.7029402256011963,
+      "rewards/rejected": -2.311450242996216,
+      "step": 132,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.38016971862438587,
+      "grad_norm": 19.44753646850586,
+      "learning_rate": 0.00019682047674904525,
+      "logits/chosen": -0.7204717993736267,
+      "logits/rejected": -0.7237159013748169,
+      "logps/chosen": -10.331472396850586,
+      "logps/rejected": -44.90771484375,
+      "loss": 0.9498412609100342,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6981635093688965,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.13750669360160828,
+      "rewards/margins": 3.0414295196533203,
+      "rewards/rejected": -2.9039225578308105,
+      "step": 133,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.38302813756141135,
+      "grad_norm": 2.630051612854004,
+      "learning_rate": 0.00019674092915125218,
+      "logits/chosen": -0.7359256148338318,
+      "logits/rejected": -0.7523387670516968,
+      "logps/chosen": -5.2792582511901855,
+      "logps/rejected": -53.244354248046875,
+      "loss": 0.37174469232559204,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.19531044363975525,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.35951948165893555,
+      "rewards/margins": 3.9146037101745605,
+      "rewards/rejected": -3.555084228515625,
+      "step": 134,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.3858865564984368,
+      "grad_norm": 8.38878059387207,
+      "learning_rate": 0.00019666041519807802,
+      "logits/chosen": -0.7512345910072327,
+      "logits/rejected": -0.7590532302856445,
+      "logps/chosen": -7.108774662017822,
+      "logps/rejected": -49.24845886230469,
+      "loss": 0.515214741230011,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.35548365116119385,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.11580058932304382,
+      "rewards/margins": 3.6866679191589355,
+      "rewards/rejected": -3.5708670616149902,
+      "step": 135,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.38874497543546227,
+      "grad_norm": 7.971051216125488,
+      "learning_rate": 0.0001965789356937852,
+      "logits/chosen": -0.7898239493370056,
+      "logits/rejected": -0.7973160743713379,
+      "logps/chosen": -9.885854721069336,
+      "logps/rejected": -55.670047760009766,
+      "loss": 0.7229015827178955,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.43035101890563965,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.13693107664585114,
+      "rewards/margins": 3.9661924839019775,
+      "rewards/rejected": -3.829261541366577,
+      "step": 136,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.3916033943724877,
+      "grad_norm": 6.154712677001953,
+      "learning_rate": 0.00019649649145228102,
+      "logits/chosen": -0.7895013689994812,
+      "logits/rejected": -0.7943228483200073,
+      "logps/chosen": -5.192661285400391,
+      "logps/rejected": -39.91571044921875,
+      "loss": 0.5592948794364929,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22298313677310944,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.19678261876106262,
+      "rewards/margins": 2.95813250541687,
+      "rewards/rejected": -2.761349678039551,
+      "step": 137,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.3944618133095132,
+      "grad_norm": 2.1893486976623535,
+      "learning_rate": 0.0001964130832971098,
+      "logits/chosen": -0.7978731393814087,
+      "logits/rejected": -0.808353066444397,
+      "logps/chosen": -7.741363048553467,
+      "logps/rejected": -49.584014892578125,
+      "loss": 0.4424561560153961,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3187883198261261,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3768758773803711,
+      "rewards/margins": 3.4152517318725586,
+      "rewards/rejected": -3.0383756160736084,
+      "step": 138,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.3973202322465386,
+      "grad_norm": 3.585315465927124,
+      "learning_rate": 0.0001963287120614444,
+      "logits/chosen": -0.8294897079467773,
+      "logits/rejected": -0.8371224403381348,
+      "logps/chosen": -4.645942211151123,
+      "logps/rejected": -41.52901840209961,
+      "loss": 0.5509465336799622,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3694899380207062,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.15256014466285706,
+      "rewards/margins": 3.042027473449707,
+      "rewards/rejected": -2.8894670009613037,
+      "step": 139,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.4001786511835641,
+      "grad_norm": 3.1583364009857178,
+      "learning_rate": 0.00019624337858807807,
+      "logits/chosen": -0.8058665990829468,
+      "logits/rejected": -0.802970290184021,
+      "logps/chosen": -12.615983009338379,
+      "logps/rejected": -42.576560974121094,
+      "loss": 0.47237128019332886,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.28210973739624023,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.08335088193416595,
+      "rewards/margins": 2.9341583251953125,
+      "rewards/rejected": -3.0175094604492188,
+      "step": 140,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.40303707012058954,
+      "grad_norm": 1.9583823680877686,
+      "learning_rate": 0.00019615708372941587,
+      "logits/chosen": -0.7773740887641907,
+      "logits/rejected": -0.7813900113105774,
+      "logps/chosen": -7.43025016784668,
+      "logps/rejected": -48.81037521362305,
+      "loss": 0.45620426535606384,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2858808636665344,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.14979784190654755,
+      "rewards/margins": 3.327134847640991,
+      "rewards/rejected": -3.177337169647217,
+      "step": 141,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.405895489057615,
+      "grad_norm": 17.376474380493164,
+      "learning_rate": 0.00019606982834746627,
+      "logits/chosen": -0.8055461645126343,
+      "logits/rejected": -0.8150217533111572,
+      "logps/chosen": -9.388490676879883,
+      "logps/rejected": -43.86671447753906,
+      "loss": 1.0983664989471436,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.7692541480064392,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1326138973236084,
+      "rewards/margins": 2.8655741214752197,
+      "rewards/rejected": -2.7329602241516113,
+      "step": 142,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.40875390799464045,
+      "grad_norm": 35.17894744873047,
+      "learning_rate": 0.00019598161331383257,
+      "logits/chosen": -0.8625121116638184,
+      "logits/rejected": -0.871791422367096,
+      "logps/chosen": -9.16450023651123,
+      "logps/rejected": -47.45623016357422,
+      "loss": 0.7930290102958679,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.543784499168396,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.08059316873550415,
+      "rewards/margins": 3.273069381713867,
+      "rewards/rejected": -3.192476272583008,
+      "step": 143,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.41161232693166594,
+      "grad_norm": 4.112196445465088,
+      "learning_rate": 0.00019589243950970402,
+      "logits/chosen": -0.8877297043800354,
+      "logits/rejected": -0.8911035060882568,
+      "logps/chosen": -6.215106964111328,
+      "logps/rejected": -41.89563751220703,
+      "loss": 0.3530213236808777,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20008514821529388,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3117275834083557,
+      "rewards/margins": 3.073056221008301,
+      "rewards/rejected": -2.76132869720459,
+      "step": 144,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.41447074586869137,
+      "grad_norm": 12.758796691894531,
+      "learning_rate": 0.00019580230782584722,
+      "logits/chosen": -0.8928901553153992,
+      "logits/rejected": -0.8998609781265259,
+      "logps/chosen": -5.406792640686035,
+      "logps/rejected": -40.006412506103516,
+      "loss": 0.5003563165664673,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.30671095848083496,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.23732616007328033,
+      "rewards/margins": 2.932386636734009,
+      "rewards/rejected": -2.6950607299804688,
+      "step": 145,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.41732916480571686,
+      "grad_norm": 4.014595985412598,
+      "learning_rate": 0.00019571121916259706,
+      "logits/chosen": -0.887598991394043,
+      "logits/rejected": -0.8942646384239197,
+      "logps/chosen": -4.3415961265563965,
+      "logps/rejected": -46.59621810913086,
+      "loss": 0.36891594529151917,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2651711106300354,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.36169373989105225,
+      "rewards/margins": 3.6561460494995117,
+      "rewards/rejected": -3.294452428817749,
+      "step": 146,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.4201875837427423,
+      "grad_norm": 10.37033462524414,
+      "learning_rate": 0.00019561917442984788,
+      "logits/chosen": -0.8872988224029541,
+      "logits/rejected": -0.8909754753112793,
+      "logps/chosen": -7.036233901977539,
+      "logps/rejected": -44.856361389160156,
+      "loss": 0.5071041584014893,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4129847586154938,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.29683101177215576,
+      "rewards/margins": 3.6587531566619873,
+      "rewards/rejected": -3.361922025680542,
+      "step": 147,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.4230460026797678,
+      "grad_norm": 12.54053783416748,
+      "learning_rate": 0.00019552617454704428,
+      "logits/chosen": -0.909309983253479,
+      "logits/rejected": -0.9173017144203186,
+      "logps/chosen": -5.930859088897705,
+      "logps/rejected": -46.784332275390625,
+      "loss": 0.6537680625915527,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.521246075630188,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.25368478894233704,
+      "rewards/margins": 3.396470308303833,
+      "rewards/rejected": -3.1427857875823975,
+      "step": 148,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.4259044216167932,
+      "grad_norm": 2.5870954990386963,
+      "learning_rate": 0.00019543222044317188,
+      "logits/chosen": -0.8764325976371765,
+      "logits/rejected": -0.881737232208252,
+      "logps/chosen": -5.638083457946777,
+      "logps/rejected": -47.27805709838867,
+      "loss": 0.3462856113910675,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22526228427886963,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3174706697463989,
+      "rewards/margins": 3.5086669921875,
+      "rewards/rejected": -3.1911962032318115,
+      "step": 149,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.4287628405538187,
+      "grad_norm": 9.161517143249512,
+      "learning_rate": 0.00019533731305674818,
+      "logits/chosen": -0.8732975721359253,
+      "logits/rejected": -0.8769663572311401,
+      "logps/chosen": -9.518259048461914,
+      "logps/rejected": -42.666114807128906,
+      "loss": 0.9442892670631409,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6460040211677551,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.07534892857074738,
+      "rewards/margins": 3.1069083213806152,
+      "rewards/rejected": -3.0315592288970947,
+      "step": 150,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.4316212594908441,
+      "grad_norm": 17.615604400634766,
+      "learning_rate": 0.00019524145333581317,
+      "logits/chosen": -0.8912621736526489,
+      "logits/rejected": -0.8975273966789246,
+      "logps/chosen": -10.543143272399902,
+      "logps/rejected": -44.90479278564453,
+      "loss": 1.2651307582855225,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.7548849582672119,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.13807448744773865,
+      "rewards/margins": 3.1065428256988525,
+      "rewards/rejected": -3.244617223739624,
+      "step": 151,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.4344796784278696,
+      "grad_norm": 2.197153091430664,
+      "learning_rate": 0.00019514464223791965,
+      "logits/chosen": -0.8220558762550354,
+      "logits/rejected": -0.8305569291114807,
+      "logps/chosen": -7.629626274108887,
+      "logps/rejected": -46.86054229736328,
+      "loss": 0.5103957653045654,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4139241874217987,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3421666920185089,
+      "rewards/margins": 3.841639995574951,
+      "rewards/rejected": -3.4994730949401855,
+      "step": 152,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.43733809736489504,
+      "grad_norm": 2.0711119174957275,
+      "learning_rate": 0.00019504688073012397,
+      "logits/chosen": -0.8169566988945007,
+      "logits/rejected": -0.8239966034889221,
+      "logps/chosen": -6.1432342529296875,
+      "logps/rejected": -42.3114013671875,
+      "loss": 0.4369783401489258,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22974222898483276,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2705279588699341,
+      "rewards/margins": 2.976038932800293,
+      "rewards/rejected": -2.7055106163024902,
+      "step": 153,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.4401965163019205,
+      "grad_norm": 11.810593605041504,
+      "learning_rate": 0.00019494816978897616,
+      "logits/chosen": -0.8318780064582825,
+      "logits/rejected": -0.8391178846359253,
+      "logps/chosen": -7.192132472991943,
+      "logps/rejected": -39.354427337646484,
+      "loss": 0.6350818872451782,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.475485622882843,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3430967926979065,
+      "rewards/margins": 2.803071975708008,
+      "rewards/rejected": -2.459975004196167,
+      "step": 154,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.44305493523894596,
+      "grad_norm": 2.665921688079834,
+      "learning_rate": 0.0001948485104005103,
+      "logits/chosen": -0.8109292984008789,
+      "logits/rejected": -0.8165513277053833,
+      "logps/chosen": -4.813442230224609,
+      "logps/rejected": -38.313106536865234,
+      "loss": 0.6565403938293457,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4435839056968689,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.25875312089920044,
+      "rewards/margins": 3.050962209701538,
+      "rewards/rejected": -2.7922089099884033,
+      "step": 155,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.4459133541759714,
+      "grad_norm": 2.126662015914917,
+      "learning_rate": 0.00019474790356023455,
+      "logits/chosen": -0.8104051351547241,
+      "logits/rejected": -0.8165953755378723,
+      "logps/chosen": -4.292057514190674,
+      "logps/rejected": -44.862060546875,
+      "loss": 0.309725821018219,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18593546748161316,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3211231231689453,
+      "rewards/margins": 3.5143370628356934,
+      "rewards/rejected": -3.193213939666748,
+      "step": 156,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.4487717731129969,
+      "grad_norm": 3.4068145751953125,
+      "learning_rate": 0.00019464635027312128,
+      "logits/chosen": -0.8355593681335449,
+      "logits/rejected": -0.8380401730537415,
+      "logps/chosen": -6.426550388336182,
+      "logps/rejected": -44.43500518798828,
+      "loss": 0.6942108273506165,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4586290121078491,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1291281133890152,
+      "rewards/margins": 3.1570847034454346,
+      "rewards/rejected": -3.02795672416687,
+      "step": 157,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.4516301920500223,
+      "grad_norm": 4.667487621307373,
+      "learning_rate": 0.00019454385155359702,
+      "logits/chosen": -0.849906325340271,
+      "logits/rejected": -0.8568750619888306,
+      "logps/chosen": -9.276833534240723,
+      "logps/rejected": -49.35503387451172,
+      "loss": 0.4330065846443176,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2498662769794464,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3473469913005829,
+      "rewards/margins": 4.031757354736328,
+      "rewards/rejected": -3.684410572052002,
+      "step": 158,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.4544886109870478,
+      "grad_norm": 1.687165379524231,
+      "learning_rate": 0.00019444040842553237,
+      "logits/chosen": -0.890581488609314,
+      "logits/rejected": -0.9015938639640808,
+      "logps/chosen": -3.249390125274658,
+      "logps/rejected": -57.22711181640625,
+      "loss": 0.362801194190979,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22434672713279724,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2739589512348175,
+      "rewards/margins": 4.31603479385376,
+      "rewards/rejected": -4.0420756340026855,
+      "step": 159,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.4573470299240732,
+      "grad_norm": 12.536514282226562,
+      "learning_rate": 0.00019433602192223164,
+      "logits/chosen": -0.8646700382232666,
+      "logits/rejected": -0.872589111328125,
+      "logps/chosen": -6.172619342803955,
+      "logps/rejected": -48.3273811340332,
+      "loss": 0.6447067856788635,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.32002851366996765,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.09112294018268585,
+      "rewards/margins": 3.2884745597839355,
+      "rewards/rejected": -3.1973516941070557,
+      "step": 160,
+      "train_speed(iter/s)": 0.005426
+    },
+    {
+      "epoch": 0.4602054488610987,
+      "grad_norm": 3.207699775695801,
+      "learning_rate": 0.00019423069308642266,
+      "logits/chosen": -0.8848387002944946,
+      "logits/rejected": -0.8890655636787415,
+      "logps/chosen": -6.779003620147705,
+      "logps/rejected": -52.09819030761719,
+      "loss": 0.35991424322128296,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.26353347301483154,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.40888625383377075,
+      "rewards/margins": 4.320865631103516,
+      "rewards/rejected": -3.9119794368743896,
+      "step": 161,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.46306386779812414,
+      "grad_norm": 13.387569427490234,
+      "learning_rate": 0.00019412442297024637,
+      "logits/chosen": -0.8957618474960327,
+      "logits/rejected": -0.8995543718338013,
+      "logps/chosen": -8.413171768188477,
+      "logps/rejected": -39.29925537109375,
+      "loss": 0.7494506239891052,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4951186180114746,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.03331068903207779,
+      "rewards/margins": 2.7600255012512207,
+      "rewards/rejected": -2.726714849472046,
+      "step": 162,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.4659222867351496,
+      "grad_norm": 14.89892578125,
+      "learning_rate": 0.00019401721263524616,
+      "logits/chosen": -0.8836920261383057,
+      "logits/rejected": -0.8899936079978943,
+      "logps/chosen": -8.512656211853027,
+      "logps/rejected": -46.419029235839844,
+      "loss": 0.5585404634475708,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3683706820011139,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.24114097654819489,
+      "rewards/margins": 3.47017240524292,
+      "rewards/rejected": -3.229031562805176,
+      "step": 163,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.46878070567217506,
+      "grad_norm": 14.938558578491211,
+      "learning_rate": 0.0001939090631523574,
+      "logits/chosen": -0.935294508934021,
+      "logits/rejected": -0.9457290768623352,
+      "logps/chosen": -7.823429107666016,
+      "logps/rejected": -56.9812126159668,
+      "loss": 1.0811909437179565,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.7688519954681396,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.035027798265218735,
+      "rewards/margins": 3.6944613456726074,
+      "rewards/rejected": -3.729489326477051,
+      "step": 164,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.47163912460920054,
+      "grad_norm": 8.26276683807373,
+      "learning_rate": 0.00019379997560189675,
+      "logits/chosen": -0.9381538033485413,
+      "logits/rejected": -0.9450898170471191,
+      "logps/chosen": -6.668002605438232,
+      "logps/rejected": -51.126930236816406,
+      "loss": 0.5152188539505005,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4229362905025482,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.14181563258171082,
+      "rewards/margins": 4.032417297363281,
+      "rewards/rejected": -3.890601873397827,
+      "step": 165,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.474497543546226,
+      "grad_norm": 5.228875160217285,
+      "learning_rate": 0.00019368995107355133,
+      "logits/chosen": -0.9465993642807007,
+      "logits/rejected": -0.9516839981079102,
+      "logps/chosen": -6.645859718322754,
+      "logps/rejected": -49.824649810791016,
+      "loss": 0.6003601551055908,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4243951737880707,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1566632241010666,
+      "rewards/margins": 3.7477457523345947,
+      "rewards/rejected": -3.5910823345184326,
+      "step": 166,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.47735596248325146,
+      "grad_norm": 11.680534362792969,
+      "learning_rate": 0.00019357899066636773,
+      "logits/chosen": -0.9604209661483765,
+      "logits/rejected": -0.9663453102111816,
+      "logps/chosen": -11.16219711303711,
+      "logps/rejected": -54.03529357910156,
+      "loss": 0.6696478128433228,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5087156295776367,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.24885831773281097,
+      "rewards/margins": 4.16080379486084,
+      "rewards/rejected": -3.9119460582733154,
+      "step": 167,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.4802143814202769,
+      "grad_norm": 6.519625663757324,
+      "learning_rate": 0.00019346709548874127,
+      "logits/chosen": -0.9105353355407715,
+      "logits/rejected": -0.9168953895568848,
+      "logps/chosen": -7.151789665222168,
+      "logps/rejected": -51.146156311035156,
+      "loss": 0.538978099822998,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.38577884435653687,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.34396299719810486,
+      "rewards/margins": 4.075203895568848,
+      "rewards/rejected": -3.731240749359131,
+      "step": 168,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.4830728003573024,
+      "grad_norm": 11.481460571289062,
+      "learning_rate": 0.0001933542666584047,
+      "logits/chosen": -0.9098703265190125,
+      "logits/rejected": -0.9205840826034546,
+      "logps/chosen": -9.062050819396973,
+      "logps/rejected": -64.49413299560547,
+      "loss": 0.526665985584259,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.40413621068000793,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2691805362701416,
+      "rewards/margins": 5.563024044036865,
+      "rewards/rejected": -5.293843746185303,
+      "step": 169,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.4859312192943278,
+      "grad_norm": 13.25565242767334,
+      "learning_rate": 0.0001932405053024171,
+      "logits/chosen": -0.9104465246200562,
+      "logits/rejected": -0.917874276638031,
+      "logps/chosen": -11.206613540649414,
+      "logps/rejected": -57.438724517822266,
+      "loss": 1.0738518238067627,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6307643055915833,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.05714952200651169,
+      "rewards/margins": 4.167277812957764,
+      "rewards/rejected": -4.110127925872803,
+      "step": 170,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.4887896382313533,
+      "grad_norm": 17.771833419799805,
+      "learning_rate": 0.00019312581255715277,
+      "logits/chosen": -0.8956343531608582,
+      "logits/rejected": -0.9024403691291809,
+      "logps/chosen": -13.308263778686523,
+      "logps/rejected": -65.24011993408203,
+      "loss": 0.9896730184555054,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5083594918251038,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.0635133907198906,
+      "rewards/margins": 4.961539268493652,
+      "rewards/rejected": -5.025052547454834,
+      "step": 171,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.4916480571683787,
+      "grad_norm": 6.4772748947143555,
+      "learning_rate": 0.00019301018956828964,
+      "logits/chosen": -0.898830771446228,
+      "logits/rejected": -0.9118731021881104,
+      "logps/chosen": -8.135079383850098,
+      "logps/rejected": -57.8392333984375,
+      "loss": 0.5032879710197449,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3064589500427246,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2829325497150421,
+      "rewards/margins": 4.001053333282471,
+      "rewards/rejected": -3.718120574951172,
+      "step": 172,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.4945064761054042,
+      "grad_norm": 6.827486515045166,
+      "learning_rate": 0.000192893637490798,
+      "logits/chosen": -0.8352851867675781,
+      "logits/rejected": -0.8436259627342224,
+      "logps/chosen": -6.111320972442627,
+      "logps/rejected": -51.995445251464844,
+      "loss": 0.5788933038711548,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.27805906534194946,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.23611441254615784,
+      "rewards/margins": 3.686123847961426,
+      "rewards/rejected": -3.450009346008301,
+      "step": 173,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.49736489504242964,
+      "grad_norm": 3.4567549228668213,
+      "learning_rate": 0.00019277615748892885,
+      "logits/chosen": -0.785612165927887,
+      "logits/rejected": -0.7931377291679382,
+      "logps/chosen": -7.273856163024902,
+      "logps/rejected": -36.88285446166992,
+      "loss": 0.5589325428009033,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.37298017740249634,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4650301933288574,
+      "rewards/margins": 2.7481539249420166,
+      "rewards/rejected": -2.283123731613159,
+      "step": 174,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.5002233139794551,
+      "grad_norm": 4.234227180480957,
+      "learning_rate": 0.00019265775073620245,
+      "logits/chosen": -0.7631555795669556,
+      "logits/rejected": -0.7687059044837952,
+      "logps/chosen": -6.337590217590332,
+      "logps/rejected": -35.29618453979492,
+      "loss": 0.6258898377418518,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3696868121623993,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.21472924947738647,
+      "rewards/margins": 2.4024829864501953,
+      "rewards/rejected": -2.187753915786743,
+      "step": 175,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.5030817329164806,
+      "grad_norm": 2.355490207672119,
+      "learning_rate": 0.00019253841841539632,
+      "logits/chosen": -0.7690540552139282,
+      "logits/rejected": -0.7676496505737305,
+      "logps/chosen": -12.918010711669922,
+      "logps/rejected": -30.43984603881836,
+      "loss": 0.6801320910453796,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.42714056372642517,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2977921664714813,
+      "rewards/margins": 2.146160125732422,
+      "rewards/rejected": -1.8483679294586182,
+      "step": 176,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.505940151853506,
+      "grad_norm": 1.5704671144485474,
+      "learning_rate": 0.0001924181617185336,
+      "logits/chosen": -0.7616850137710571,
+      "logits/rejected": -0.767928957939148,
+      "logps/chosen": -6.664812088012695,
+      "logps/rejected": -37.411277770996094,
+      "loss": 0.42176908254623413,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2451857030391693,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.489712655544281,
+      "rewards/margins": 2.708432674407959,
+      "rewards/rejected": -2.218719959259033,
+      "step": 177,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5087985707905315,
+      "grad_norm": 2.1187188625335693,
+      "learning_rate": 0.00019229698184687128,
+      "logits/chosen": -0.7447081804275513,
+      "logits/rejected": -0.7508732080459595,
+      "logps/chosen": -5.233465671539307,
+      "logps/rejected": -35.176971435546875,
+      "loss": 0.5185775756835938,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.336780309677124,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.44237199425697327,
+      "rewards/margins": 2.7804007530212402,
+      "rewards/rejected": -2.3380286693573,
+      "step": 178,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5116569897275569,
+      "grad_norm": 3.629459857940674,
+      "learning_rate": 0.00019217488001088784,
+      "logits/chosen": -0.7853379249572754,
+      "logits/rejected": -0.8011223673820496,
+      "logps/chosen": -5.379916667938232,
+      "logps/rejected": -53.26487731933594,
+      "loss": 0.368004709482193,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2341601699590683,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.32777413725852966,
+      "rewards/margins": 3.8236327171325684,
+      "rewards/rejected": -3.4958581924438477,
+      "step": 179,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5145154086645825,
+      "grad_norm": 6.440499782562256,
+      "learning_rate": 0.00019205185743027147,
+      "logits/chosen": -0.8589999079704285,
+      "logits/rejected": -0.8642404079437256,
+      "logps/chosen": -7.5277910232543945,
+      "logps/rejected": -48.523780822753906,
+      "loss": 0.4521387219429016,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.30114930868148804,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.30447474122047424,
+      "rewards/margins": 3.6830883026123047,
+      "rewards/rejected": -3.3786139488220215,
+      "step": 180,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5173738276016079,
+      "grad_norm": 1.3223450183868408,
+      "learning_rate": 0.00019192791533390776,
+      "logits/chosen": -0.8698243498802185,
+      "logits/rejected": -0.8859831094741821,
+      "logps/chosen": -5.199261665344238,
+      "logps/rejected": -53.24699783325195,
+      "loss": 0.32750457525253296,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2843254506587982,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5321836471557617,
+      "rewards/margins": 4.558604717254639,
+      "rewards/rejected": -4.026421070098877,
+      "step": 181,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.5202322465386333,
+      "grad_norm": 6.235518932342529,
+      "learning_rate": 0.0001918030549598674,
+      "logits/chosen": -0.8939491510391235,
+      "logits/rejected": -0.9015032052993774,
+      "logps/chosen": -6.817130088806152,
+      "logps/rejected": -48.4378662109375,
+      "loss": 0.8820070028305054,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5765649676322937,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1285572648048401,
+      "rewards/margins": 3.808832883834839,
+      "rewards/rejected": -3.6802756786346436,
+      "step": 182,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.5230906654756587,
+      "grad_norm": 2.177001476287842,
+      "learning_rate": 0.00019167727755539394,
+      "logits/chosen": -0.9257232546806335,
+      "logits/rejected": -0.934444785118103,
+      "logps/chosen": -5.833935737609863,
+      "logps/rejected": -59.789955139160156,
+      "loss": 0.6103315949440002,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.42996734380722046,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.15345799922943115,
+      "rewards/margins": 5.012732982635498,
+      "rewards/rejected": -4.859274387359619,
+      "step": 183,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5259490844126843,
+      "grad_norm": 5.315007209777832,
+      "learning_rate": 0.00019155058437689114,
+      "logits/chosen": -0.9314823150634766,
+      "logits/rejected": -0.9416366815567017,
+      "logps/chosen": -6.756455421447754,
+      "logps/rejected": -56.72506332397461,
+      "loss": 0.7174702286720276,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.523749589920044,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.12303046137094498,
+      "rewards/margins": 4.29641580581665,
+      "rewards/rejected": -4.173385143280029,
+      "step": 184,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.5288075033497097,
+      "grad_norm": 3.7265548706054688,
+      "learning_rate": 0.00019142297668991055,
+      "logits/chosen": -0.9351843595504761,
+      "logits/rejected": -0.9576059579849243,
+      "logps/chosen": -5.578176021575928,
+      "logps/rejected": -59.8486328125,
+      "loss": 0.6304588317871094,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.46496981382369995,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3721676766872406,
+      "rewards/margins": 4.994194507598877,
+      "rewards/rejected": -4.622026443481445,
+      "step": 185,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5316659222867351,
+      "grad_norm": 4.928941249847412,
+      "learning_rate": 0.00019129445576913888,
+      "logits/chosen": -0.8918996453285217,
+      "logits/rejected": -0.9014327526092529,
+      "logps/chosen": -5.464908123016357,
+      "logps/rejected": -53.11490249633789,
+      "loss": 0.8896833658218384,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6313154697418213,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.12775132060050964,
+      "rewards/margins": 4.158487796783447,
+      "rewards/rejected": -4.030736923217773,
+      "step": 186,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5345243412237606,
+      "grad_norm": 7.509690284729004,
+      "learning_rate": 0.00019116502289838523,
+      "logits/chosen": -0.8859353065490723,
+      "logits/rejected": -0.9015785455703735,
+      "logps/chosen": -4.673905372619629,
+      "logps/rejected": -70.63243865966797,
+      "loss": 0.3962274491786957,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3365238308906555,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3394577205181122,
+      "rewards/margins": 5.691706657409668,
+      "rewards/rejected": -5.352249622344971,
+      "step": 187,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.537382760160786,
+      "grad_norm": 1.5954922437667847,
+      "learning_rate": 0.00019103467937056824,
+      "logits/chosen": -0.9184845685958862,
+      "logits/rejected": -0.9304262399673462,
+      "logps/chosen": -4.041568756103516,
+      "logps/rejected": -76.92243957519531,
+      "loss": 0.3016416132450104,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18911761045455933,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4253259301185608,
+      "rewards/margins": 6.5727128982543945,
+      "rewards/rejected": -6.1473870277404785,
+      "step": 188,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5402411790978116,
+      "grad_norm": 2.318643808364868,
+      "learning_rate": 0.0001909034264877032,
+      "logits/chosen": -0.8512060642242432,
+      "logits/rejected": -0.856816291809082,
+      "logps/chosen": -7.1723408699035645,
+      "logps/rejected": -48.11109161376953,
+      "loss": 0.4600585699081421,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3386399745941162,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.33302804827690125,
+      "rewards/margins": 4.015811443328857,
+      "rewards/rejected": -3.682783365249634,
+      "step": 189,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.543099598034837,
+      "grad_norm": 1.4476327896118164,
+      "learning_rate": 0.0001907712655608891,
+      "logits/chosen": -0.8637464642524719,
+      "logits/rejected": -0.8820706009864807,
+      "logps/chosen": -3.2855453491210938,
+      "logps/rejected": -71.99874114990234,
+      "loss": 0.293333500623703,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18421968817710876,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.28944161534309387,
+      "rewards/margins": 5.838719844818115,
+      "rewards/rejected": -5.549278259277344,
+      "step": 190,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.5459580169718624,
+      "grad_norm": 4.899955749511719,
+      "learning_rate": 0.0001906381979102953,
+      "logits/chosen": -0.9037999510765076,
+      "logits/rejected": -0.9121489524841309,
+      "logps/chosen": -9.670417785644531,
+      "logps/rejected": -52.208683013916016,
+      "loss": 0.5949208736419678,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.38334017992019653,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.15043097734451294,
+      "rewards/margins": 4.171907901763916,
+      "rewards/rejected": -4.021476745605469,
+      "step": 191,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.5488164359088878,
+      "grad_norm": 12.610368728637695,
+      "learning_rate": 0.00019050422486514878,
+      "logits/chosen": -0.840286374092102,
+      "logits/rejected": -0.8606438636779785,
+      "logps/chosen": -8.77206039428711,
+      "logps/rejected": -61.208587646484375,
+      "loss": 0.6860842108726501,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.49322137236595154,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2522002160549164,
+      "rewards/margins": 4.72483491897583,
+      "rewards/rejected": -4.472634792327881,
+      "step": 192,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.5516748548459134,
+      "grad_norm": 7.451924800872803,
+      "learning_rate": 0.0001903693477637204,
+      "logits/chosen": -0.8095037937164307,
+      "logits/rejected": -0.8195419311523438,
+      "logps/chosen": -4.377245903015137,
+      "logps/rejected": -58.97959518432617,
+      "loss": 0.2787761986255646,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.24894402921199799,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3338276147842407,
+      "rewards/margins": 4.592215061187744,
+      "rewards/rejected": -4.258387088775635,
+      "step": 193,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.5545332737829388,
+      "grad_norm": 1.0700743198394775,
+      "learning_rate": 0.00019023356795331182,
+      "logits/chosen": -0.8574849963188171,
+      "logits/rejected": -0.871565043926239,
+      "logps/chosen": -9.894993782043457,
+      "logps/rejected": -60.1685676574707,
+      "loss": 0.446493536233902,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3630823493003845,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3332892954349518,
+      "rewards/margins": 4.909541606903076,
+      "rewards/rejected": -4.576252460479736,
+      "step": 194,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.5573916927199642,
+      "grad_norm": 1.5138012170791626,
+      "learning_rate": 0.0001900968867902419,
+      "logits/chosen": -0.8919128775596619,
+      "logits/rejected": -0.8939546942710876,
+      "logps/chosen": -4.437774181365967,
+      "logps/rejected": -52.1574592590332,
+      "loss": 0.2619098424911499,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20320795476436615,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3197801113128662,
+      "rewards/margins": 4.577724456787109,
+      "rewards/rejected": -4.257944583892822,
+      "step": 195,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.5602501116569897,
+      "grad_norm": 3.5410804748535156,
+      "learning_rate": 0.00018995930563983334,
+      "logits/chosen": -0.8741534352302551,
+      "logits/rejected": -0.8824887871742249,
+      "logps/chosen": -5.8234734535217285,
+      "logps/rejected": -49.77042770385742,
+      "loss": 0.421186625957489,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3348409831523895,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.49023908376693726,
+      "rewards/margins": 4.286158084869385,
+      "rewards/rejected": -3.7959189414978027,
+      "step": 196,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.5631085305940152,
+      "grad_norm": 1.9928661584854126,
+      "learning_rate": 0.0001898208258763987,
+      "logits/chosen": -0.8889397382736206,
+      "logits/rejected": -0.9006186127662659,
+      "logps/chosen": -4.572671890258789,
+      "logps/rejected": -61.800926208496094,
+      "loss": 0.355736643075943,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.29381096363067627,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.31399890780448914,
+      "rewards/margins": 4.928988456726074,
+      "rewards/rejected": -4.614989757537842,
+      "step": 197,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5659669495310407,
+      "grad_norm": 14.296086311340332,
+      "learning_rate": 0.00018968144888322709,
+      "logits/chosen": -0.8595174551010132,
+      "logits/rejected": -0.861739993095398,
+      "logps/chosen": -7.95827054977417,
+      "logps/rejected": -51.00768280029297,
+      "loss": 0.9002599120140076,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5538756251335144,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.06036340817809105,
+      "rewards/margins": 3.8160269260406494,
+      "rewards/rejected": -3.7556633949279785,
+      "step": 198,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5688253684680661,
+      "grad_norm": 6.8623809814453125,
+      "learning_rate": 0.00018954117605257,
+      "logits/chosen": -0.9402197003364563,
+      "logits/rejected": -0.953208863735199,
+      "logps/chosen": -4.058620929718018,
+      "logps/rejected": -63.09252166748047,
+      "loss": 0.5629696249961853,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3906901776790619,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.26179081201553345,
+      "rewards/margins": 5.211573123931885,
+      "rewards/rejected": -4.949782371520996,
+      "step": 199,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5716837874050915,
+      "grad_norm": 7.293510913848877,
+      "learning_rate": 0.00018940000878562758,
+      "logits/chosen": -0.9181431531906128,
+      "logits/rejected": -0.9206308722496033,
+      "logps/chosen": -7.505152702331543,
+      "logps/rejected": -51.558433532714844,
+      "loss": 0.4831893742084503,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3287861943244934,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1301122009754181,
+      "rewards/margins": 4.108443737030029,
+      "rewards/rejected": -3.9783310890197754,
+      "step": 200,
+      "train_speed(iter/s)": 0.005428
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 1047,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.3999247892582236e+18,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}