diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,14468 @@
+{
+  "best_global_step": 600,
+  "best_metric": 0.26907793,
+  "best_model_checkpoint": "/data2/kdd/crag/cjz/output/task2_dpo_absgt0_taskgt6__pari0.5_no_cheat/v0-20250606-013952/checkpoint-600",
+  "epoch": 2.2858418937025458,
+  "eval_steps": 300,
+  "global_step": 800,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.002858418937025458,
+      "grad_norm": 10.349462509155273,
+      "learning_rate": 3.7735849056603773e-06,
+      "logits/chosen": -0.6422490477561951,
+      "logits/rejected": -0.6447486877441406,
+      "logps/chosen": -6.999429225921631,
+      "logps/rejected": -13.433603286743164,
+      "loss": 1.3778549432754517,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6847077012062073,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1,
+      "train_speed(iter/s)": 0.005273
+    },
+    {
+      "epoch": 0.005716837874050916,
+      "grad_norm": 11.263213157653809,
+      "learning_rate": 7.547169811320755e-06,
+      "logits/chosen": -0.6429960131645203,
+      "logits/rejected": -0.6454926133155823,
+      "logps/chosen": -9.604641914367676,
+      "logps/rejected": -13.53369140625,
+      "loss": 1.4975696802139282,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.8044224381446838,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 2,
+      "train_speed(iter/s)": 0.005366
+    },
+    {
+      "epoch": 0.008575256811076373,
+      "grad_norm": 9.09404182434082,
+      "learning_rate": 1.1320754716981132e-05,
+      "logits/chosen": -0.6486532092094421,
+      "logits/rejected": -0.6493248343467712,
+      "logps/chosen": -7.69912576675415,
+      "logps/rejected": -17.816326141357422,
+      "loss": 1.2593896389007568,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5679630637168884,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0031128099653869867,
+      "rewards/margins": 0.0035441224463284016,
+      "rewards/rejected": -0.00043131227721460164,
+      "step": 3,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.011433675748101831,
+      "grad_norm": 13.314420700073242,
+      "learning_rate": 1.509433962264151e-05,
+      "logits/chosen": -0.6668453812599182,
+      "logits/rejected": -0.6718658804893494,
+      "logps/chosen": -6.061755180358887,
+      "logps/rejected": -13.643918991088867,
+      "loss": 1.4406771659851074,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.7477570176124573,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.008774133399128914,
+      "rewards/margins": 0.0006008772179484367,
+      "rewards/rejected": 0.008173256181180477,
+      "step": 4,
+      "train_speed(iter/s)": 0.005446
+    },
+    {
+      "epoch": 0.014292094685127288,
+      "grad_norm": 12.340804100036621,
+      "learning_rate": 1.8867924528301888e-05,
+      "logits/chosen": -0.6321276426315308,
+      "logits/rejected": -0.6321861743927002,
+      "logps/chosen": -11.234216690063477,
+      "logps/rejected": -11.82645034790039,
+      "loss": 1.3690662384033203,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6870464086532593,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.032103873789310455,
+      "rewards/margins": 0.022960105910897255,
+      "rewards/rejected": 0.0091437678784132,
+      "step": 5,
+      "train_speed(iter/s)": 0.005444
+    },
+    {
+      "epoch": 0.017150513622152745,
+      "grad_norm": 13.995262145996094,
+      "learning_rate": 2.2641509433962265e-05,
+      "logits/chosen": -0.6530164480209351,
+      "logits/rejected": -0.6550691723823547,
+      "logps/chosen": -8.742744445800781,
+      "logps/rejected": -12.622247695922852,
+      "loss": 1.3672001361846924,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.702350378036499,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.0800265222787857,
+      "rewards/margins": 0.06047351285815239,
+      "rewards/rejected": 0.019553007557988167,
+      "step": 6,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.020008932559178204,
+      "grad_norm": 10.019857406616211,
+      "learning_rate": 2.641509433962264e-05,
+      "logits/chosen": -0.6211694478988647,
+      "logits/rejected": -0.6214967370033264,
+      "logps/chosen": -6.935382843017578,
+      "logps/rejected": -13.613279342651367,
+      "loss": 1.2412631511688232,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5651571154594421,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.13799861073493958,
+      "rewards/margins": 0.053141627460718155,
+      "rewards/rejected": 0.08485697209835052,
+      "step": 7,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.022867351496203663,
+      "grad_norm": 6.375326633453369,
+      "learning_rate": 3.018867924528302e-05,
+      "logits/chosen": -0.6666778326034546,
+      "logits/rejected": -0.6678147315979004,
+      "logps/chosen": -8.114872932434082,
+      "logps/rejected": -13.392560958862305,
+      "loss": 1.1162137985229492,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4446101784706116,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.20063960552215576,
+      "rewards/margins": 0.09723334014415741,
+      "rewards/rejected": 0.10340625792741776,
+      "step": 8,
+      "train_speed(iter/s)": 0.005426
+    },
+    {
+      "epoch": 0.02572577043322912,
+      "grad_norm": 4.046472549438477,
+      "learning_rate": 3.39622641509434e-05,
+      "logits/chosen": -0.5842044949531555,
+      "logits/rejected": -0.5892153382301331,
+      "logps/chosen": -13.877073287963867,
+      "logps/rejected": -14.116576194763184,
+      "loss": 1.259205937385559,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5605449676513672,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.1392103135585785,
+      "rewards/margins": 0.09627523273229599,
+      "rewards/rejected": 0.0429350808262825,
+      "step": 9,
+      "train_speed(iter/s)": 0.005422
+    },
+    {
+      "epoch": 0.028584189370254576,
+      "grad_norm": 7.625533103942871,
+      "learning_rate": 3.7735849056603776e-05,
+      "logits/chosen": -0.6210353374481201,
+      "logits/rejected": -0.6224341988563538,
+      "logps/chosen": -8.32575798034668,
+      "logps/rejected": -11.715011596679688,
+      "loss": 1.2772942781448364,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.48597079515457153,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.043191827833652496,
+      "rewards/margins": 0.029505327343940735,
+      "rewards/rejected": -0.07269717007875443,
+      "step": 10,
+      "train_speed(iter/s)": 0.005438
+    },
+    {
+      "epoch": 0.03144260830728004,
+      "grad_norm": 11.961934089660645,
+      "learning_rate": 4.150943396226415e-05,
+      "logits/chosen": -0.6265634298324585,
+      "logits/rejected": -0.6289808750152588,
+      "logps/chosen": -7.129470348358154,
+      "logps/rejected": -11.932583808898926,
+      "loss": 1.447466254234314,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6552329659461975,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.00290237320587039,
+      "rewards/margins": 0.06162097677588463,
+      "rewards/rejected": -0.0645233541727066,
+      "step": 11,
+      "train_speed(iter/s)": 0.005473
+    },
+    {
+      "epoch": 0.03430102724430549,
+      "grad_norm": 10.154479026794434,
+      "learning_rate": 4.528301886792453e-05,
+      "logits/chosen": -0.6388624906539917,
+      "logits/rejected": -0.639370858669281,
+      "logps/chosen": -9.490407943725586,
+      "logps/rejected": -12.88676643371582,
+      "loss": 1.4875458478927612,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6468713283538818,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.10463112592697144,
+      "rewards/margins": -0.059477299451828,
+      "rewards/rejected": -0.04515381157398224,
+      "step": 12,
+      "train_speed(iter/s)": 0.005465
+    },
+    {
+      "epoch": 0.03715944618133095,
+      "grad_norm": 4.091860771179199,
+      "learning_rate": 4.9056603773584906e-05,
+      "logits/chosen": -0.6680133938789368,
+      "logits/rejected": -0.6711105108261108,
+      "logps/chosen": -5.56148099899292,
+      "logps/rejected": -13.865943908691406,
+      "loss": 1.0154482126235962,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.35444653034210205,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.12190410494804382,
+      "rewards/margins": 0.3293726444244385,
+      "rewards/rejected": -0.20746850967407227,
+      "step": 13,
+      "train_speed(iter/s)": 0.00546
+    },
+    {
+      "epoch": 0.04001786511835641,
+      "grad_norm": 3.039910316467285,
+      "learning_rate": 5.283018867924528e-05,
+      "logits/chosen": -0.6245446801185608,
+      "logits/rejected": -0.6280595660209656,
+      "logps/chosen": -7.042474269866943,
+      "logps/rejected": -14.88280963897705,
+      "loss": 0.9986085295677185,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.32772910594940186,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.08946945518255234,
+      "rewards/margins": 0.2548743188381195,
+      "rewards/rejected": -0.16540484130382538,
+      "step": 14,
+      "train_speed(iter/s)": 0.005463
+    },
+    {
+      "epoch": 0.042876284055381866,
+      "grad_norm": 2.8388617038726807,
+      "learning_rate": 5.660377358490566e-05,
+      "logits/chosen": -0.613956868648529,
+      "logits/rejected": -0.6150951385498047,
+      "logps/chosen": -5.398593902587891,
+      "logps/rejected": -10.984697341918945,
+      "loss": 0.9217166900634766,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21436992287635803,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.2490297257900238,
+      "rewards/margins": 0.12685683369636536,
+      "rewards/rejected": 0.12217291444540024,
+      "step": 15,
+      "train_speed(iter/s)": 0.005464
+    },
+    {
+      "epoch": 0.045734702992407325,
+      "grad_norm": 3.623399257659912,
+      "learning_rate": 6.037735849056604e-05,
+      "logits/chosen": -0.5890456438064575,
+      "logits/rejected": -0.5907201766967773,
+      "logps/chosen": -7.65773868560791,
+      "logps/rejected": -15.24480152130127,
+      "loss": 1.130638837814331,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.33376482129096985,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 0.14319562911987305,
+      "rewards/margins": 0.01443251222372055,
+      "rewards/rejected": 0.1287631392478943,
+      "step": 16,
+      "train_speed(iter/s)": 0.00546
+    },
+    {
+      "epoch": 0.048593121929432784,
+      "grad_norm": 2.9850075244903564,
+      "learning_rate": 6.415094339622641e-05,
+      "logits/chosen": -0.5228931903839111,
+      "logits/rejected": -0.524716854095459,
+      "logps/chosen": -10.135262489318848,
+      "logps/rejected": -14.072588920593262,
+      "loss": 1.2770482301712036,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.48515522480010986,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": 0.13618236780166626,
+      "rewards/margins": -0.09462341666221619,
+      "rewards/rejected": 0.23080575466156006,
+      "step": 17,
+      "train_speed(iter/s)": 0.005473
+    },
+    {
+      "epoch": 0.05145154086645824,
+      "grad_norm": 2.2746193408966064,
+      "learning_rate": 6.79245283018868e-05,
+      "logits/chosen": -0.5764032006263733,
+      "logits/rejected": -0.5762047171592712,
+      "logps/chosen": -11.675098419189453,
+      "logps/rejected": -10.994297981262207,
+      "loss": 1.2022521495819092,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4671342372894287,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 0.1593710482120514,
+      "rewards/margins": -0.00844080001115799,
+      "rewards/rejected": 0.1678118258714676,
+      "step": 18,
+      "train_speed(iter/s)": 0.005468
+    },
+    {
+      "epoch": 0.0543099598034837,
+      "grad_norm": 2.324984312057495,
+      "learning_rate": 7.169811320754717e-05,
+      "logits/chosen": -0.567690908908844,
+      "logits/rejected": -0.569066047668457,
+      "logps/chosen": -9.0728120803833,
+      "logps/rejected": -12.268661499023438,
+      "loss": 1.1060843467712402,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.41300466656684875,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.23081830143928528,
+      "rewards/margins": 0.0710553377866745,
+      "rewards/rejected": 0.15976294875144958,
+      "step": 19,
+      "train_speed(iter/s)": 0.005463
+    },
+    {
+      "epoch": 0.05716837874050915,
+      "grad_norm": 2.0991458892822266,
+      "learning_rate": 7.547169811320755e-05,
+      "logits/chosen": -0.6091416478157043,
+      "logits/rejected": -0.6100034117698669,
+      "logps/chosen": -4.26762056350708,
+      "logps/rejected": -11.32187557220459,
+      "loss": 1.0021699666976929,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2933175265789032,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.15655732154846191,
+      "rewards/margins": 0.008277412503957748,
+      "rewards/rejected": 0.14827993512153625,
+      "step": 20,
+      "train_speed(iter/s)": 0.005467
+    },
+    {
+      "epoch": 0.06002679767753461,
+      "grad_norm": 3.4958455562591553,
+      "learning_rate": 7.924528301886794e-05,
+      "logits/chosen": -0.571921706199646,
+      "logits/rejected": -0.572878360748291,
+      "logps/chosen": -9.612298965454102,
+      "logps/rejected": -12.484085083007812,
+      "loss": 1.1795949935913086,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.46760252118110657,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.14643912017345428,
+      "rewards/margins": 0.0200117826461792,
+      "rewards/rejected": 0.12642733752727509,
+      "step": 21,
+      "train_speed(iter/s)": 0.005467
+    },
+    {
+      "epoch": 0.06288521661456008,
+      "grad_norm": 3.175546169281006,
+      "learning_rate": 8.30188679245283e-05,
+      "logits/chosen": -0.6108006834983826,
+      "logits/rejected": -0.6111243367195129,
+      "logps/chosen": -6.207424163818359,
+      "logps/rejected": -7.818144798278809,
+      "loss": 1.1549935340881348,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4683927893638611,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.23285990953445435,
+      "rewards/margins": 0.06634702533483505,
+      "rewards/rejected": 0.16651292145252228,
+      "step": 22,
+      "train_speed(iter/s)": 0.00547
+    },
+    {
+      "epoch": 0.06574363555158554,
+      "grad_norm": 2.4531869888305664,
+      "learning_rate": 8.679245283018869e-05,
+      "logits/chosen": -0.6012008190155029,
+      "logits/rejected": -0.6021928191184998,
+      "logps/chosen": -5.598664283752441,
+      "logps/rejected": -13.262667655944824,
+      "loss": 0.9260164499282837,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.33532270789146423,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.3285560607910156,
+      "rewards/margins": 0.3157665431499481,
+      "rewards/rejected": 0.012789532542228699,
+      "step": 23,
+      "train_speed(iter/s)": 0.005467
+    },
+    {
+      "epoch": 0.06860205448861098,
+      "grad_norm": 2.328031301498413,
+      "learning_rate": 9.056603773584906e-05,
+      "logits/chosen": -0.567432165145874,
+      "logits/rejected": -0.5704118609428406,
+      "logps/chosen": -4.026985168457031,
+      "logps/rejected": -14.904813766479492,
+      "loss": 0.8984246253967285,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.26223480701446533,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.26638248562812805,
+      "rewards/margins": 0.22430934011936188,
+      "rewards/rejected": 0.042073119431734085,
+      "step": 24,
+      "train_speed(iter/s)": 0.005466
+    },
+    {
+      "epoch": 0.07146047342563644,
+      "grad_norm": 3.5119378566741943,
+      "learning_rate": 9.433962264150944e-05,
+      "logits/chosen": -0.5709348320960999,
+      "logits/rejected": -0.5688645243644714,
+      "logps/chosen": -7.716845989227295,
+      "logps/rejected": -7.868927955627441,
+      "loss": 1.113297462463379,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3660878837108612,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.2273256480693817,
+      "rewards/margins": -0.0022776294499635696,
+      "rewards/rejected": 0.22960326075553894,
+      "step": 25,
+      "train_speed(iter/s)": 0.005475
+    },
+    {
+      "epoch": 0.0743188923626619,
+      "grad_norm": 3.2336673736572266,
+      "learning_rate": 9.811320754716981e-05,
+      "logits/chosen": -0.589956521987915,
+      "logits/rejected": -0.587726354598999,
+      "logps/chosen": -10.072855949401855,
+      "logps/rejected": -12.857810974121094,
+      "loss": 1.0991942882537842,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.40110355615615845,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.17576897144317627,
+      "rewards/margins": 0.11343260109424591,
+      "rewards/rejected": 0.062336355447769165,
+      "step": 26,
+      "train_speed(iter/s)": 0.005466
+    },
+    {
+      "epoch": 0.07717731129968736,
+      "grad_norm": 2.6813321113586426,
+      "learning_rate": 0.0001018867924528302,
+      "logits/chosen": -0.5832359194755554,
+      "logits/rejected": -0.5860565304756165,
+      "logps/chosen": -8.197118759155273,
+      "logps/rejected": -17.45244598388672,
+      "loss": 0.8366767764091492,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2886262536048889,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.3297504186630249,
+      "rewards/margins": 0.48295390605926514,
+      "rewards/rejected": -0.15320347249507904,
+      "step": 27,
+      "train_speed(iter/s)": 0.00547
+    },
+    {
+      "epoch": 0.08003573023671282,
+      "grad_norm": 3.3800203800201416,
+      "learning_rate": 0.00010566037735849057,
+      "logits/chosen": -0.6005962491035461,
+      "logits/rejected": -0.6035608053207397,
+      "logps/chosen": -9.117904663085938,
+      "logps/rejected": -18.400161743164062,
+      "loss": 1.0143071413040161,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4276316463947296,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.271901398897171,
+      "rewards/margins": 0.4319708049297333,
+      "rewards/rejected": -0.16006940603256226,
+      "step": 28,
+      "train_speed(iter/s)": 0.005468
+    },
+    {
+      "epoch": 0.08289414917373827,
+      "grad_norm": 5.461134433746338,
+      "learning_rate": 0.00010943396226415095,
+      "logits/chosen": -0.5596082210540771,
+      "logits/rejected": -0.5635167360305786,
+      "logps/chosen": -7.607837200164795,
+      "logps/rejected": -23.174358367919922,
+      "loss": 0.803045928478241,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.32165294885635376,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.3834715783596039,
+      "rewards/margins": 0.7214637994766235,
+      "rewards/rejected": -0.33799219131469727,
+      "step": 29,
+      "train_speed(iter/s)": 0.005464
+    },
+    {
+      "epoch": 0.08575256811076373,
+      "grad_norm": 3.3377506732940674,
+      "learning_rate": 0.00011320754716981132,
+      "logits/chosen": -0.6003558039665222,
+      "logits/rejected": -0.6016712188720703,
+      "logps/chosen": -7.941157817840576,
+      "logps/rejected": -10.564404487609863,
+      "loss": 1.0737202167510986,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3314566910266876,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.1379927098751068,
+      "rewards/margins": 0.08670535683631897,
+      "rewards/rejected": 0.05128733813762665,
+      "step": 30,
+      "train_speed(iter/s)": 0.005461
+    },
+    {
+      "epoch": 0.08861098704778919,
+      "grad_norm": 3.4214563369750977,
+      "learning_rate": 0.0001169811320754717,
+      "logits/chosen": -0.5186684727668762,
+      "logits/rejected": -0.5189406275749207,
+      "logps/chosen": -6.143683910369873,
+      "logps/rejected": -25.88001251220703,
+      "loss": 0.7792238593101501,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2514691948890686,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.2757991552352905,
+      "rewards/margins": 0.5156725645065308,
+      "rewards/rejected": -0.23987337946891785,
+      "step": 31,
+      "train_speed(iter/s)": 0.005459
+    },
+    {
+      "epoch": 0.09146940598481465,
+      "grad_norm": 4.645272731781006,
+      "learning_rate": 0.00012075471698113207,
+      "logits/chosen": -0.5988867282867432,
+      "logits/rejected": -0.6016397476196289,
+      "logps/chosen": -5.776233673095703,
+      "logps/rejected": -22.404544830322266,
+      "loss": 0.8892383575439453,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4011368155479431,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.22715790569782257,
+      "rewards/margins": 0.7724226713180542,
+      "rewards/rejected": -0.545264720916748,
+      "step": 32,
+      "train_speed(iter/s)": 0.005454
+    },
+    {
+      "epoch": 0.09432782492184011,
+      "grad_norm": 4.586422443389893,
+      "learning_rate": 0.00012452830188679244,
+      "logits/chosen": -0.5782912969589233,
+      "logits/rejected": -0.5768243670463562,
+      "logps/chosen": -10.939208984375,
+      "logps/rejected": -13.486640930175781,
+      "loss": 1.1607584953308105,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5877886414527893,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.21317529678344727,
+      "rewards/margins": 0.4141784906387329,
+      "rewards/rejected": -0.20100319385528564,
+      "step": 33,
+      "train_speed(iter/s)": 0.005457
+    },
+    {
+      "epoch": 0.09718624385886557,
+      "grad_norm": 4.306042671203613,
+      "learning_rate": 0.00012830188679245283,
+      "logits/chosen": -0.6019578576087952,
+      "logits/rejected": -0.6082974076271057,
+      "logps/chosen": -7.5170578956604,
+      "logps/rejected": -24.889406204223633,
+      "loss": 0.9509016275405884,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4247076213359833,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.10510760545730591,
+      "rewards/margins": 0.8037927150726318,
+      "rewards/rejected": -0.6986850500106812,
+      "step": 34,
+      "train_speed(iter/s)": 0.005456
+    },
+    {
+      "epoch": 0.10004466279589103,
+      "grad_norm": 3.002274751663208,
+      "learning_rate": 0.0001320754716981132,
+      "logits/chosen": -0.6133092641830444,
+      "logits/rejected": -0.6127456426620483,
+      "logps/chosen": -8.859593391418457,
+      "logps/rejected": -17.02096939086914,
+      "loss": 0.8848217129707336,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3480006456375122,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.25863486528396606,
+      "rewards/margins": 0.5611037015914917,
+      "rewards/rejected": -0.302468866109848,
+      "step": 35,
+      "train_speed(iter/s)": 0.005457
+    },
+    {
+      "epoch": 0.10290308173291648,
+      "grad_norm": 4.299287796020508,
+      "learning_rate": 0.0001358490566037736,
+      "logits/chosen": -0.5725032091140747,
+      "logits/rejected": -0.5725142359733582,
+      "logps/chosen": -8.371686935424805,
+      "logps/rejected": -17.657852172851562,
+      "loss": 0.8825864791870117,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3603641390800476,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.23154468834400177,
+      "rewards/margins": 0.6240193843841553,
+      "rewards/rejected": -0.3924746811389923,
+      "step": 36,
+      "train_speed(iter/s)": 0.005457
+    },
+    {
+      "epoch": 0.10576150066994194,
+      "grad_norm": 4.354315757751465,
+      "learning_rate": 0.00013962264150943395,
+      "logits/chosen": -0.5943324565887451,
+      "logits/rejected": -0.5940991044044495,
+      "logps/chosen": -7.657552719116211,
+      "logps/rejected": -16.183917999267578,
+      "loss": 0.8160735964775085,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3588046133518219,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.27544379234313965,
+      "rewards/margins": 0.7471473217010498,
+      "rewards/rejected": -0.4717034101486206,
+      "step": 37,
+      "train_speed(iter/s)": 0.005452
+    },
+    {
+      "epoch": 0.1086199196069674,
+      "grad_norm": 7.916697978973389,
+      "learning_rate": 0.00014339622641509434,
+      "logits/chosen": -0.5676888227462769,
+      "logits/rejected": -0.5714080929756165,
+      "logps/chosen": -6.698662757873535,
+      "logps/rejected": -17.954235076904297,
+      "loss": 1.052018642425537,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.49374452233314514,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.16338011622428894,
+      "rewards/margins": 0.42942774295806885,
+      "rewards/rejected": -0.2660475969314575,
+      "step": 38,
+      "train_speed(iter/s)": 0.005452
+    },
+    {
+      "epoch": 0.11147833854399285,
+      "grad_norm": 3.7271363735198975,
+      "learning_rate": 0.00014716981132075472,
+      "logits/chosen": -0.6097766160964966,
+      "logits/rejected": -0.6135396957397461,
+      "logps/chosen": -7.30006217956543,
+      "logps/rejected": -22.168495178222656,
+      "loss": 0.7729511857032776,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3125499188899994,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.28066012263298035,
+      "rewards/margins": 0.8108583092689514,
+      "rewards/rejected": -0.5301981568336487,
+      "step": 39,
+      "train_speed(iter/s)": 0.00545
+    },
+    {
+      "epoch": 0.1143367574810183,
+      "grad_norm": 2.960145950317383,
+      "learning_rate": 0.0001509433962264151,
+      "logits/chosen": -0.6132208108901978,
+      "logits/rejected": -0.6163071990013123,
+      "logps/chosen": -7.024638652801514,
+      "logps/rejected": -21.298852920532227,
+      "loss": 0.8106734752655029,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.37238579988479614,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.39458420872688293,
+      "rewards/margins": 0.8476477265357971,
+      "rewards/rejected": -0.45306357741355896,
+      "step": 40,
+      "train_speed(iter/s)": 0.005445
+    },
+    {
+      "epoch": 0.11719517641804376,
+      "grad_norm": 4.3203630447387695,
+      "learning_rate": 0.0001547169811320755,
+      "logits/chosen": -0.5976248979568481,
+      "logits/rejected": -0.6002449989318848,
+      "logps/chosen": -5.656615257263184,
+      "logps/rejected": -16.032827377319336,
+      "loss": 0.9440045952796936,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.33253738284111023,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.14414586126804352,
+      "rewards/margins": 0.5006409287452698,
+      "rewards/rejected": -0.35649505257606506,
+      "step": 41,
+      "train_speed(iter/s)": 0.005445
+    },
+    {
+      "epoch": 0.12005359535506922,
+      "grad_norm": 4.646491527557373,
+      "learning_rate": 0.00015849056603773587,
+      "logits/chosen": -0.5956608653068542,
+      "logits/rejected": -0.5956603288650513,
+      "logps/chosen": -11.719463348388672,
+      "logps/rejected": -27.070579528808594,
+      "loss": 1.0076483488082886,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5070124268531799,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.15291725099086761,
+      "rewards/margins": 0.7710469961166382,
+      "rewards/rejected": -0.6181297898292542,
+      "step": 42,
+      "train_speed(iter/s)": 0.005446
+    },
+    {
+      "epoch": 0.12291201429209468,
+      "grad_norm": 4.248917102813721,
+      "learning_rate": 0.00016226415094339625,
+      "logits/chosen": -0.6100150942802429,
+      "logits/rejected": -0.6142972111701965,
+      "logps/chosen": -6.201865196228027,
+      "logps/rejected": -19.82131004333496,
+      "loss": 0.7969534993171692,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.31225427985191345,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.20218679308891296,
+      "rewards/margins": 0.705444872379303,
+      "rewards/rejected": -0.5032580494880676,
+      "step": 43,
+      "train_speed(iter/s)": 0.005448
+    },
+    {
+      "epoch": 0.12577043322912015,
+      "grad_norm": 3.1864066123962402,
+      "learning_rate": 0.0001660377358490566,
+      "logits/chosen": -0.6078452467918396,
+      "logits/rejected": -0.6100336909294128,
+      "logps/chosen": -8.217609405517578,
+      "logps/rejected": -18.432086944580078,
+      "loss": 0.945781409740448,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4447736442089081,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.16747131943702698,
+      "rewards/margins": 0.6352076530456543,
+      "rewards/rejected": -0.46773630380630493,
+      "step": 44,
+      "train_speed(iter/s)": 0.005451
+    },
+    {
+      "epoch": 0.1286288521661456,
+      "grad_norm": 2.8247904777526855,
+      "learning_rate": 0.000169811320754717,
+      "logits/chosen": -0.6424505710601807,
+      "logits/rejected": -0.6455634236335754,
+      "logps/chosen": -4.626930236816406,
+      "logps/rejected": -18.966289520263672,
+      "loss": 0.6525614261627197,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20949943363666534,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1935967355966568,
+      "rewards/margins": 0.7181073427200317,
+      "rewards/rejected": -0.5245105624198914,
+      "step": 45,
+      "train_speed(iter/s)": 0.005449
+    },
+    {
+      "epoch": 0.13148727110317107,
+      "grad_norm": 5.167090892791748,
+      "learning_rate": 0.00017358490566037738,
+      "logits/chosen": -0.6390554308891296,
+      "logits/rejected": -0.6455981731414795,
+      "logps/chosen": -7.457905292510986,
+      "logps/rejected": -22.238759994506836,
+      "loss": 0.7052859663963318,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3145278990268707,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3406547009944916,
+      "rewards/margins": 0.9184964895248413,
+      "rewards/rejected": -0.5778417587280273,
+      "step": 46,
+      "train_speed(iter/s)": 0.005446
+    },
+    {
+      "epoch": 0.1343456900401965,
+      "grad_norm": 3.110797166824341,
+      "learning_rate": 0.00017735849056603776,
+      "logits/chosen": -0.6957284212112427,
+      "logits/rejected": -0.6984925866127014,
+      "logps/chosen": -6.335500717163086,
+      "logps/rejected": -26.946069717407227,
+      "loss": 0.6684616804122925,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.27696001529693604,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.36519789695739746,
+      "rewards/margins": 1.261197805404663,
+      "rewards/rejected": -0.8960000276565552,
+      "step": 47,
+      "train_speed(iter/s)": 0.005445
+    },
+    {
+      "epoch": 0.13720410897722196,
+      "grad_norm": 5.592209815979004,
+      "learning_rate": 0.00018113207547169812,
+      "logits/chosen": -0.6557337641716003,
+      "logits/rejected": -0.6595371961593628,
+      "logps/chosen": -6.262232780456543,
+      "logps/rejected": -22.550495147705078,
+      "loss": 0.844598650932312,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.32042089104652405,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.098446786403656,
+      "rewards/margins": 1.1042643785476685,
+      "rewards/rejected": -1.0058175325393677,
+      "step": 48,
+      "train_speed(iter/s)": 0.005444
+    },
+    {
+      "epoch": 0.14006252791424742,
+      "grad_norm": 5.518631458282471,
+      "learning_rate": 0.0001849056603773585,
+      "logits/chosen": -0.7261903285980225,
+      "logits/rejected": -0.7333111763000488,
+      "logps/chosen": -6.155282020568848,
+      "logps/rejected": -28.635177612304688,
+      "loss": 0.7839219570159912,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3167538046836853,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.18123595416545868,
+      "rewards/margins": 1.343546986579895,
+      "rewards/rejected": -1.1623109579086304,
+      "step": 49,
+      "train_speed(iter/s)": 0.00544
+    },
+    {
+      "epoch": 0.14292094685127288,
+      "grad_norm": 5.368433475494385,
+      "learning_rate": 0.00018867924528301889,
+      "logits/chosen": -0.7460846304893494,
+      "logits/rejected": -0.7433331608772278,
+      "logps/chosen": -8.476225852966309,
+      "logps/rejected": -20.479339599609375,
+      "loss": 1.041689395904541,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5162184834480286,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.07575411349534988,
+      "rewards/margins": 0.8544278144836426,
+      "rewards/rejected": -0.7786736488342285,
+      "step": 50,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.14577936578829834,
+      "grad_norm": 12.666760444641113,
+      "learning_rate": 0.00019245283018867927,
+      "logits/chosen": -0.7423916459083557,
+      "logits/rejected": -0.7536158561706543,
+      "logps/chosen": -4.518327713012695,
+      "logps/rejected": -25.71565818786621,
+      "loss": 0.8285016417503357,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.38353338837623596,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.25002920627593994,
+      "rewards/margins": 1.0534642934799194,
+      "rewards/rejected": -0.8034350872039795,
+      "step": 51,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.1486377847253238,
+      "grad_norm": 5.63993501663208,
+      "learning_rate": 0.00019622641509433963,
+      "logits/chosen": -0.7661195993423462,
+      "logits/rejected": -0.7684649229049683,
+      "logps/chosen": -5.303972244262695,
+      "logps/rejected": -20.424297332763672,
+      "loss": 0.7760183811187744,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.30550119280815125,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.17865437269210815,
+      "rewards/margins": 0.9522275328636169,
+      "rewards/rejected": -0.7735730409622192,
+      "step": 52,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.15149620366234925,
+      "grad_norm": 3.208770275115967,
+      "learning_rate": 0.0002,
+      "logits/chosen": -0.7615375518798828,
+      "logits/rejected": -0.7639837861061096,
+      "logps/chosen": -7.793338775634766,
+      "logps/rejected": -22.267044067382812,
+      "loss": 0.6889612078666687,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2784683108329773,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.21318240463733673,
+      "rewards/margins": 1.1274638175964355,
+      "rewards/rejected": -0.9142814874649048,
+      "step": 53,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.1543546225993747,
+      "grad_norm": 4.603616714477539,
+      "learning_rate": 0.00019999950054470762,
+      "logits/chosen": -0.7416525483131409,
+      "logits/rejected": -0.7468435168266296,
+      "logps/chosen": -11.753125190734863,
+      "logps/rejected": -23.24603271484375,
+      "loss": 0.9534075856208801,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.42802760004997253,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.2138495296239853,
+      "rewards/margins": 0.8852367997169495,
+      "rewards/rejected": -0.6713871955871582,
+      "step": 54,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.15721304153640017,
+      "grad_norm": 3.226914405822754,
+      "learning_rate": 0.00019999800218381956,
+      "logits/chosen": -0.782919704914093,
+      "logits/rejected": -0.7837254405021667,
+      "logps/chosen": -10.437933921813965,
+      "logps/rejected": -21.950359344482422,
+      "loss": 0.7319959998130798,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.28649967908859253,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.17146320641040802,
+      "rewards/margins": 0.9873074889183044,
+      "rewards/rejected": -0.8158442974090576,
+      "step": 55,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.16007146047342563,
+      "grad_norm": 5.957598686218262,
+      "learning_rate": 0.00019999550493230315,
+      "logits/chosen": -0.7622566819190979,
+      "logits/rejected": -0.7720569968223572,
+      "logps/chosen": -9.299139022827148,
+      "logps/rejected": -25.902114868164062,
+      "loss": 1.0733510255813599,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5738880634307861,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1327313929796219,
+      "rewards/margins": 1.0297049283981323,
+      "rewards/rejected": -0.8969735503196716,
+      "step": 56,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.1629298794104511,
+      "grad_norm": 4.4198760986328125,
+      "learning_rate": 0.00019999200881510367,
+      "logits/chosen": -0.7777268886566162,
+      "logits/rejected": -0.7817560434341431,
+      "logps/chosen": -8.051362037658691,
+      "logps/rejected": -16.21352767944336,
+      "loss": 0.9994087815284729,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5157659649848938,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.3178602159023285,
+      "rewards/margins": 0.7738386392593384,
+      "rewards/rejected": -0.4559784233570099,
+      "step": 57,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.16578829834747655,
+      "grad_norm": 10.918800354003906,
+      "learning_rate": 0.00019998751386714417,
+      "logits/chosen": -0.7735524773597717,
+      "logits/rejected": -0.7762777209281921,
+      "logps/chosen": -6.026726245880127,
+      "logps/rejected": -20.445030212402344,
+      "loss": 0.8424608707427979,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4571954309940338,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3362487852573395,
+      "rewards/margins": 1.0223908424377441,
+      "rewards/rejected": -0.6861420273780823,
+      "step": 58,
+      "train_speed(iter/s)": 0.005438
+    },
+    {
+      "epoch": 0.168646717284502,
+      "grad_norm": 6.003457546234131,
+      "learning_rate": 0.00019998202013332526,
+      "logits/chosen": -0.7671990394592285,
+      "logits/rejected": -0.7679510712623596,
+      "logps/chosen": -7.208020210266113,
+      "logps/rejected": -16.45644187927246,
+      "loss": 0.8400717377662659,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3789638876914978,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.19236475229263306,
+      "rewards/margins": 0.8348343372344971,
+      "rewards/rejected": -0.6424695253372192,
+      "step": 59,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.17150513622152747,
+      "grad_norm": 2.5898213386535645,
+      "learning_rate": 0.00019997552766852432,
+      "logits/chosen": -0.800130307674408,
+      "logits/rejected": -0.8043981790542603,
+      "logps/chosen": -6.655481815338135,
+      "logps/rejected": -24.4234619140625,
+      "loss": 0.6945982575416565,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3340861201286316,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3088517487049103,
+      "rewards/margins": 1.2354176044464111,
+      "rewards/rejected": -0.9265658855438232,
+      "step": 60,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.17436355515855292,
+      "grad_norm": 3.626788377761841,
+      "learning_rate": 0.00019996803653759532,
+      "logits/chosen": -0.7841392159461975,
+      "logits/rejected": -0.7904000282287598,
+      "logps/chosen": -4.378767013549805,
+      "logps/rejected": -26.130855560302734,
+      "loss": 0.5890128016471863,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2802823781967163,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2776252329349518,
+      "rewards/margins": 1.491828203201294,
+      "rewards/rejected": -1.2142030000686646,
+      "step": 61,
+      "train_speed(iter/s)": 0.005438
+    },
+    {
+      "epoch": 0.17722197409557838,
+      "grad_norm": 6.747097492218018,
+      "learning_rate": 0.00019995954681536798,
+      "logits/chosen": -0.7469812035560608,
+      "logits/rejected": -0.7426640391349792,
+      "logps/chosen": -11.407801628112793,
+      "logps/rejected": -19.764530181884766,
+      "loss": 1.1509681940078735,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4816472828388214,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.023781299591064453,
+      "rewards/margins": 0.7125381231307983,
+      "rewards/rejected": -0.7363194227218628,
+      "step": 62,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.18008039303260384,
+      "grad_norm": 6.9256978034973145,
+      "learning_rate": 0.00019995005858664696,
+      "logits/chosen": -0.819698691368103,
+      "logits/rejected": -0.8221041560173035,
+      "logps/chosen": -7.0722761154174805,
+      "logps/rejected": -30.66091537475586,
+      "loss": 0.9850404262542725,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.42992380261421204,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.07954245805740356,
+      "rewards/margins": 1.697967767715454,
+      "rewards/rejected": -1.6184254884719849,
+      "step": 63,
+      "train_speed(iter/s)": 0.005439
+    },
+    {
+      "epoch": 0.1829388119696293,
+      "grad_norm": 5.373042106628418,
+      "learning_rate": 0.00019993957194621128,
+      "logits/chosen": -0.8020437955856323,
+      "logits/rejected": -0.8022991418838501,
+      "logps/chosen": -12.086634635925293,
+      "logps/rejected": -21.337501525878906,
+      "loss": 0.9992907643318176,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.46714556217193604,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.11581222712993622,
+      "rewards/margins": 1.0597875118255615,
+      "rewards/rejected": -0.9439753293991089,
+      "step": 64,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.18579723090665476,
+      "grad_norm": 3.77294921875,
+      "learning_rate": 0.00019992808699881303,
+      "logits/chosen": -0.7432379722595215,
+      "logits/rejected": -0.7440240979194641,
+      "logps/chosen": -6.54131555557251,
+      "logps/rejected": -25.4437255859375,
+      "loss": 0.8025262951850891,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4345034658908844,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1653081476688385,
+      "rewards/margins": 1.261300802230835,
+      "rewards/rejected": -1.0959926843643188,
+      "step": 65,
+      "train_speed(iter/s)": 0.005442
+    },
+    {
+      "epoch": 0.18865564984368022,
+      "grad_norm": 3.9820404052734375,
+      "learning_rate": 0.0001999156038591766,
+      "logits/chosen": -0.754156231880188,
+      "logits/rejected": -0.7619881629943848,
+      "logps/chosen": -6.165823459625244,
+      "logps/rejected": -23.689210891723633,
+      "loss": 0.7558284997940063,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.35462892055511475,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.18967051804065704,
+      "rewards/margins": 0.9685844779014587,
+      "rewards/rejected": -0.7789139151573181,
+      "step": 66,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.19151406878070568,
+      "grad_norm": 4.361135482788086,
+      "learning_rate": 0.00019990212265199738,
+      "logits/chosen": -0.7575560212135315,
+      "logits/rejected": -0.7635327577590942,
+      "logps/chosen": -4.563057899475098,
+      "logps/rejected": -24.856014251708984,
+      "loss": 0.7012404203414917,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3292723000049591,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.360738068819046,
+      "rewards/margins": 1.0550967454910278,
+      "rewards/rejected": -0.6943586468696594,
+      "step": 67,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.19437248771773113,
+      "grad_norm": 8.01318359375,
+      "learning_rate": 0.00019988764351194056,
+      "logits/chosen": -0.7201259136199951,
+      "logits/rejected": -0.7213173508644104,
+      "logps/chosen": -14.019023895263672,
+      "logps/rejected": -23.463457107543945,
+      "loss": 1.155617594718933,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6643218994140625,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.16900399327278137,
+      "rewards/margins": 0.8244603276252747,
+      "rewards/rejected": -0.6554563641548157,
+      "step": 68,
+      "train_speed(iter/s)": 0.005442
+    },
+    {
+      "epoch": 0.1972309066547566,
+      "grad_norm": 4.880486488342285,
+      "learning_rate": 0.00019987216658363984,
+      "logits/chosen": -0.7603176832199097,
+      "logits/rejected": -0.7611086368560791,
+      "logps/chosen": -7.871481418609619,
+      "logps/rejected": -18.355331420898438,
+      "loss": 0.9050899744033813,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4671905040740967,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.14834776520729065,
+      "rewards/margins": 0.9282243251800537,
+      "rewards/rejected": -0.7798765897750854,
+      "step": 69,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.20008932559178205,
+      "grad_norm": 36.09653854370117,
+      "learning_rate": 0.00019985569202169584,
+      "logits/chosen": -0.826109766960144,
+      "logits/rejected": -0.8319188952445984,
+      "logps/chosen": -7.292363166809082,
+      "logps/rejected": -19.279315948486328,
+      "loss": 0.906982421875,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4692971408367157,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.17047010362148285,
+      "rewards/margins": 0.9899492859840393,
+      "rewards/rejected": -0.8194791674613953,
+      "step": 70,
+      "train_speed(iter/s)": 0.005443
+    },
+    {
+      "epoch": 0.2029477445288075,
+      "grad_norm": 5.051456451416016,
+      "learning_rate": 0.00019983821999067475,
+      "logits/chosen": -0.7659405469894409,
+      "logits/rejected": -0.773281991481781,
+      "logps/chosen": -7.996345520019531,
+      "logps/rejected": -16.987951278686523,
+      "loss": 0.9166132211685181,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.43807780742645264,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.2428373396396637,
+      "rewards/margins": 0.7172515988349915,
+      "rewards/rejected": -0.47441428899765015,
+      "step": 71,
+      "train_speed(iter/s)": 0.005444
+    },
+    {
+      "epoch": 0.20580616346583297,
+      "grad_norm": 3.3546969890594482,
+      "learning_rate": 0.00019981975066510655,
+      "logits/chosen": -0.7702159285545349,
+      "logits/rejected": -0.7756153345108032,
+      "logps/chosen": -3.096975088119507,
+      "logps/rejected": -24.7138671875,
+      "loss": 0.6350318193435669,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20447328686714172,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.23610374331474304,
+      "rewards/margins": 1.1568939685821533,
+      "rewards/rejected": -0.9207903146743774,
+      "step": 72,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.20866458240285843,
+      "grad_norm": 3.3056812286376953,
+      "learning_rate": 0.00019980028422948322,
+      "logits/chosen": -0.7785294651985168,
+      "logits/rejected": -0.7796329259872437,
+      "logps/chosen": -6.459807395935059,
+      "logps/rejected": -18.2646541595459,
+      "loss": 0.7826670408248901,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.299204021692276,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.17048794031143188,
+      "rewards/margins": 0.8538780212402344,
+      "rewards/rejected": -0.6833901405334473,
+      "step": 73,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.2115230013398839,
+      "grad_norm": 2.804248094558716,
+      "learning_rate": 0.00019977982087825713,
+      "logits/chosen": -0.7178574204444885,
+      "logits/rejected": -0.7222320437431335,
+      "logps/chosen": -5.382746696472168,
+      "logps/rejected": -21.648345947265625,
+      "loss": 0.6176108717918396,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2642488479614258,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3241485357284546,
+      "rewards/margins": 1.1783626079559326,
+      "rewards/rejected": -0.854214072227478,
+      "step": 74,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.21438142027690935,
+      "grad_norm": 2.3271028995513916,
+      "learning_rate": 0.0001997583608158388,
+      "logits/chosen": -0.7665284872055054,
+      "logits/rejected": -0.7763013243675232,
+      "logps/chosen": -3.597620725631714,
+      "logps/rejected": -28.286640167236328,
+      "loss": 0.4727986454963684,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1368505358695984,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.36978626251220703,
+      "rewards/margins": 1.460551142692566,
+      "rewards/rejected": -1.0907649993896484,
+      "step": 75,
+      "train_speed(iter/s)": 0.005442
+    },
+    {
+      "epoch": 0.2172398392139348,
+      "grad_norm": 5.8310160636901855,
+      "learning_rate": 0.00019973590425659507,
+      "logits/chosen": -0.7936246991157532,
+      "logits/rejected": -0.7950795888900757,
+      "logps/chosen": -8.558622360229492,
+      "logps/rejected": -22.39399528503418,
+      "loss": 0.7566131949424744,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4084041714668274,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.26708710193634033,
+      "rewards/margins": 1.3221920728683472,
+      "rewards/rejected": -1.0551048517227173,
+      "step": 76,
+      "train_speed(iter/s)": 0.00544
+    },
+    {
+      "epoch": 0.22009825815096026,
+      "grad_norm": 4.89293909072876,
+      "learning_rate": 0.0001997124514248469,
+      "logits/chosen": -0.8093288540840149,
+      "logits/rejected": -0.8106516003608704,
+      "logps/chosen": -12.884543418884277,
+      "logps/rejected": -22.5157413482666,
+      "loss": 1.130676507949829,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.7112829685211182,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.10567857325077057,
+      "rewards/margins": 1.2312006950378418,
+      "rewards/rejected": -1.1255221366882324,
+      "step": 77,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.2229566770879857,
+      "grad_norm": 7.881620407104492,
+      "learning_rate": 0.00019968800255486713,
+      "logits/chosen": -0.8169571161270142,
+      "logits/rejected": -0.8179275393486023,
+      "logps/chosen": -6.7093825340271,
+      "logps/rejected": -29.32290267944336,
+      "loss": 0.8154858350753784,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3944161534309387,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.16114036738872528,
+      "rewards/margins": 1.4641916751861572,
+      "rewards/rejected": -1.3030513525009155,
+      "step": 78,
+      "train_speed(iter/s)": 0.005441
+    },
+    {
+      "epoch": 0.22581509602501115,
+      "grad_norm": 2.513998031616211,
+      "learning_rate": 0.00019966255789087808,
+      "logits/chosen": -0.8547464609146118,
+      "logits/rejected": -0.8608615398406982,
+      "logps/chosen": -7.297125339508057,
+      "logps/rejected": -27.81803321838379,
+      "loss": 0.6003709435462952,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.29905110597610474,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.13682395219802856,
+      "rewards/margins": 1.487520456314087,
+      "rewards/rejected": -1.3506965637207031,
+      "step": 79,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.2286735149620366,
+      "grad_norm": 22.424015045166016,
+      "learning_rate": 0.0001996361176870492,
+      "logits/chosen": -0.8499366044998169,
+      "logits/rejected": -0.863770604133606,
+      "logps/chosen": -8.103764533996582,
+      "logps/rejected": -35.18703842163086,
+      "loss": 0.7495014071464539,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5106410980224609,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2272297590970993,
+      "rewards/margins": 1.9457027912139893,
+      "rewards/rejected": -1.7184730768203735,
+      "step": 80,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.23153193389906207,
+      "grad_norm": 4.0974273681640625,
+      "learning_rate": 0.00019960868220749448,
+      "logits/chosen": -0.8633654117584229,
+      "logits/rejected": -0.8727495670318604,
+      "logps/chosen": -5.683876991271973,
+      "logps/rejected": -34.967037200927734,
+      "loss": 0.6187952756881714,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.32927608489990234,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2094997614622116,
+      "rewards/margins": 2.0794010162353516,
+      "rewards/rejected": -1.8699012994766235,
+      "step": 81,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.23439035283608753,
+      "grad_norm": 3.275705575942993,
+      "learning_rate": 0.00019958025172626986,
+      "logits/chosen": -0.8937872648239136,
+      "logits/rejected": -0.9034286737442017,
+      "logps/chosen": -11.346511840820312,
+      "logps/rejected": -35.471927642822266,
+      "loss": 0.7466462850570679,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4821043014526367,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2012697458267212,
+      "rewards/margins": 1.9956668615341187,
+      "rewards/rejected": -1.7943971157073975,
+      "step": 82,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.237248771773113,
+      "grad_norm": 8.550602912902832,
+      "learning_rate": 0.0001995508265273704,
+      "logits/chosen": -0.9247998595237732,
+      "logits/rejected": -0.934203028678894,
+      "logps/chosen": -7.0234270095825195,
+      "logps/rejected": -40.43362808227539,
+      "loss": 0.6714224219322205,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.36930039525032043,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.015845157206058502,
+      "rewards/margins": 2.3306310176849365,
+      "rewards/rejected": -2.3464760780334473,
+      "step": 83,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.24010719071013845,
+      "grad_norm": 10.380730628967285,
+      "learning_rate": 0.00019952040690472748,
+      "logits/chosen": -0.89071124792099,
+      "logits/rejected": -0.896521806716919,
+      "logps/chosen": -9.274986267089844,
+      "logps/rejected": -31.167112350463867,
+      "loss": 0.9556330442428589,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4726044833660126,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.055123914033174515,
+      "rewards/margins": 1.99386465549469,
+      "rewards/rejected": -1.9387409687042236,
+      "step": 84,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.2429656096471639,
+      "grad_norm": 59.343536376953125,
+      "learning_rate": 0.00019948899316220604,
+      "logits/chosen": -0.86629319190979,
+      "logits/rejected": -0.8767050504684448,
+      "logps/chosen": -6.04016637802124,
+      "logps/rejected": -28.20957374572754,
+      "loss": 0.725101888179779,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.40447694063186646,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.29112857580184937,
+      "rewards/margins": 1.845672369003296,
+      "rewards/rejected": -1.5545437335968018,
+      "step": 85,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.24582402858418936,
+      "grad_norm": 6.300822734832764,
+      "learning_rate": 0.0001994565856136012,
+      "logits/chosen": -0.8398523330688477,
+      "logits/rejected": -0.8439276814460754,
+      "logps/chosen": -7.2902045249938965,
+      "logps/rejected": -32.82334899902344,
+      "loss": 0.5805257558822632,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4190253019332886,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3955605626106262,
+      "rewards/margins": 2.423994302749634,
+      "rewards/rejected": -2.0284337997436523,
+      "step": 86,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.24868244752121482,
+      "grad_norm": 43.768619537353516,
+      "learning_rate": 0.0001994231845826354,
+      "logits/chosen": -0.8710224628448486,
+      "logits/rejected": -0.8721168637275696,
+      "logps/chosen": -7.397428035736084,
+      "logps/rejected": -29.24224853515625,
+      "loss": 1.1686397790908813,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.7285480499267578,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.11831004917621613,
+      "rewards/margins": 1.2224053144454956,
+      "rewards/rejected": -1.3407154083251953,
+      "step": 87,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.2515408664582403,
+      "grad_norm": 39.749202728271484,
+      "learning_rate": 0.00019938879040295508,
+      "logits/chosen": -0.8551483750343323,
+      "logits/rejected": -0.8542779684066772,
+      "logps/chosen": -10.534215927124023,
+      "logps/rejected": -26.982810974121094,
+      "loss": 1.2515333890914917,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.7181237936019897,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.03403427451848984,
+      "rewards/margins": 1.2902536392211914,
+      "rewards/rejected": -1.32428777217865,
+      "step": 88,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.25439928539526574,
+      "grad_norm": 10.098331451416016,
+      "learning_rate": 0.00019935340341812737,
+      "logits/chosen": -0.8135808706283569,
+      "logits/rejected": -0.8137494325637817,
+      "logps/chosen": -9.00113582611084,
+      "logps/rejected": -22.338396072387695,
+      "loss": 0.9040060639381409,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5397258400917053,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.04357346147298813,
+      "rewards/margins": 1.2031430006027222,
+      "rewards/rejected": -1.159569501876831,
+      "step": 89,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.2572577043322912,
+      "grad_norm": 8.489130973815918,
+      "learning_rate": 0.00019931702398163657,
+      "logits/chosen": -0.8590521216392517,
+      "logits/rejected": -0.8651595711708069,
+      "logps/chosen": -8.849437713623047,
+      "logps/rejected": -27.54432487487793,
+      "loss": 0.8707167506217957,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.48353832960128784,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.06561100482940674,
+      "rewards/margins": 1.1353689432144165,
+      "rewards/rejected": -1.0697578191757202,
+      "step": 90,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.26011612326931666,
+      "grad_norm": 5.018461227416992,
+      "learning_rate": 0.00019927965245688072,
+      "logits/chosen": -0.8591816425323486,
+      "logits/rejected": -0.8660668730735779,
+      "logps/chosen": -11.013117790222168,
+      "logps/rejected": -27.660642623901367,
+      "loss": 0.9352516531944275,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.541546106338501,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.2031691074371338,
+      "rewards/margins": 1.437905192375183,
+      "rewards/rejected": -1.2347362041473389,
+      "step": 91,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.26297454220634214,
+      "grad_norm": 6.94728946685791,
+      "learning_rate": 0.00019924128921716797,
+      "logits/chosen": -0.8400283455848694,
+      "logits/rejected": -0.8450547456741333,
+      "logps/chosen": -9.49151611328125,
+      "logps/rejected": -28.902881622314453,
+      "loss": 0.806583821773529,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3471754193305969,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.21784308552742004,
+      "rewards/margins": 1.679865837097168,
+      "rewards/rejected": -1.4620226621627808,
+      "step": 92,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.2658329611433676,
+      "grad_norm": 2.8959178924560547,
+      "learning_rate": 0.00019920193464571275,
+      "logits/chosen": -0.839047372341156,
+      "logits/rejected": -0.8450105786323547,
+      "logps/chosen": -10.59231948852539,
+      "logps/rejected": -29.6372013092041,
+      "loss": 0.7759805917739868,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5248351097106934,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3599660098552704,
+      "rewards/margins": 1.950143575668335,
+      "rewards/rejected": -1.5901774168014526,
+      "step": 93,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.268691380080393,
+      "grad_norm": 5.318148612976074,
+      "learning_rate": 0.00019916158913563207,
+      "logits/chosen": -0.8162392377853394,
+      "logits/rejected": -0.8246886730194092,
+      "logps/chosen": -5.6260833740234375,
+      "logps/rejected": -29.538055419921875,
+      "loss": 0.6956499218940735,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.38102442026138306,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.2292197048664093,
+      "rewards/margins": 1.7469617128372192,
+      "rewards/rejected": -1.5177420377731323,
+      "step": 94,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.2715497990174185,
+      "grad_norm": 4.5264787673950195,
+      "learning_rate": 0.00019912025308994148,
+      "logits/chosen": -0.8437933325767517,
+      "logits/rejected": -0.8463299870491028,
+      "logps/chosen": -8.831324577331543,
+      "logps/rejected": -26.610977172851562,
+      "loss": 0.7661141157150269,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4225902557373047,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.12178793549537659,
+      "rewards/margins": 1.467708945274353,
+      "rewards/rejected": -1.3459210395812988,
+      "step": 95,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.2744082179544439,
+      "grad_norm": 5.978212356567383,
+      "learning_rate": 0.00019907792692155113,
+      "logits/chosen": -0.7815445065498352,
+      "logits/rejected": -0.78977370262146,
+      "logps/chosen": -6.673672199249268,
+      "logps/rejected": -33.78847122192383,
+      "loss": 0.7695144414901733,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5254613757133484,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2026345133781433,
+      "rewards/margins": 1.8294543027877808,
+      "rewards/rejected": -1.6268197298049927,
+      "step": 96,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.2772666368914694,
+      "grad_norm": 9.170755386352539,
+      "learning_rate": 0.00019903461105326154,
+      "logits/chosen": -0.8260455131530762,
+      "logits/rejected": -0.830398678779602,
+      "logps/chosen": -6.689513206481934,
+      "logps/rejected": -22.671979904174805,
+      "loss": 0.7065067291259766,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.32698267698287964,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2753945589065552,
+      "rewards/margins": 1.3633196353912354,
+      "rewards/rejected": -1.0879249572753906,
+      "step": 97,
+      "train_speed(iter/s)": 0.005435
+    },
+    {
+      "epoch": 0.28012505582849484,
+      "grad_norm": 3.521716594696045,
+      "learning_rate": 0.00019899030591775957,
+      "logits/chosen": -0.7963160872459412,
+      "logits/rejected": -0.8014420866966248,
+      "logps/chosen": -7.337916851043701,
+      "logps/rejected": -28.08383560180664,
+      "loss": 0.61917644739151,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3110499680042267,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.23168900609016418,
+      "rewards/margins": 1.7576744556427002,
+      "rewards/rejected": -1.525985598564148,
+      "step": 98,
+      "train_speed(iter/s)": 0.005436
+    },
+    {
+      "epoch": 0.2829834747655203,
+      "grad_norm": 5.793614864349365,
+      "learning_rate": 0.00019894501195761392,
+      "logits/chosen": -0.8137191534042358,
+      "logits/rejected": -0.8227260708808899,
+      "logps/chosen": -6.954023838043213,
+      "logps/rejected": -35.75700378417969,
+      "loss": 0.6259573698043823,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.33839166164398193,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.26017990708351135,
+      "rewards/margins": 2.326916217803955,
+      "rewards/rejected": -2.0667362213134766,
+      "step": 99,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.28584189370254576,
+      "grad_norm": 5.138144493103027,
+      "learning_rate": 0.00019889872962527072,
+      "logits/chosen": -0.8491114377975464,
+      "logits/rejected": -0.8503983020782471,
+      "logps/chosen": -5.965326309204102,
+      "logps/rejected": -24.436853408813477,
+      "loss": 0.9054074287414551,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3898293972015381,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.10434619337320328,
+      "rewards/margins": 1.369442105293274,
+      "rewards/rejected": -1.265095829963684,
+      "step": 100,
+      "train_speed(iter/s)": 0.005437
+    },
+    {
+      "epoch": 0.28870031263957124,
+      "grad_norm": 3.6206700801849365,
+      "learning_rate": 0.00019885145938304907,
+      "logits/chosen": -0.8245471119880676,
+      "logits/rejected": -0.8329594135284424,
+      "logps/chosen": -12.563156127929688,
+      "logps/rejected": -31.796875,
+      "loss": 0.6779420971870422,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.36515456438064575,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.2728801667690277,
+      "rewards/margins": 2.0701961517333984,
+      "rewards/rejected": -1.797316074371338,
+      "step": 101,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.2915587315765967,
+      "grad_norm": 4.129011154174805,
+      "learning_rate": 0.0001988032017031364,
+      "logits/chosen": -0.7823147773742676,
+      "logits/rejected": -0.7900741696357727,
+      "logps/chosen": -5.765602111816406,
+      "logps/rejected": -31.836132049560547,
+      "loss": 0.533344030380249,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3198188543319702,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3069157004356384,
+      "rewards/margins": 2.2244224548339844,
+      "rewards/rejected": -1.9175068140029907,
+      "step": 102,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.29441715051362216,
+      "grad_norm": 7.285937786102295,
+      "learning_rate": 0.00019875395706758388,
+      "logits/chosen": -0.8199520707130432,
+      "logits/rejected": -0.8266770839691162,
+      "logps/chosen": -6.686117172241211,
+      "logps/rejected": -35.52680969238281,
+      "loss": 0.6140994429588318,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.33540287613868713,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3744865357875824,
+      "rewards/margins": 2.1863038539886475,
+      "rewards/rejected": -1.8118171691894531,
+      "step": 103,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.2972755694506476,
+      "grad_norm": 4.08017110824585,
+      "learning_rate": 0.0001987037259683013,
+      "logits/chosen": -0.7858948707580566,
+      "logits/rejected": -0.7929791808128357,
+      "logps/chosen": -6.60341215133667,
+      "logps/rejected": -32.91640853881836,
+      "loss": 0.49151238799095154,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.26134875416755676,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3017127513885498,
+      "rewards/margins": 2.313183546066284,
+      "rewards/rejected": -2.0114707946777344,
+      "step": 104,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.3001339883876731,
+      "grad_norm": 6.110865116119385,
+      "learning_rate": 0.00019865250890705247,
+      "logits/chosen": -0.8439383506774902,
+      "logits/rejected": -0.8605507612228394,
+      "logps/chosen": -5.249790668487549,
+      "logps/rejected": -45.18266677856445,
+      "loss": 0.665439248085022,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4175432622432709,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.19819766283035278,
+      "rewards/margins": 2.974789619445801,
+      "rewards/rejected": -2.776592254638672,
+      "step": 105,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.3029924073246985,
+      "grad_norm": 8.704270362854004,
+      "learning_rate": 0.00019860030639545003,
+      "logits/chosen": -0.8581717610359192,
+      "logits/rejected": -0.8701122403144836,
+      "logps/chosen": -8.200727462768555,
+      "logps/rejected": -39.06196975708008,
+      "loss": 0.9722508788108826,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6783410310745239,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.09280453622341156,
+      "rewards/margins": 2.520831823348999,
+      "rewards/rejected": -2.4280271530151367,
+      "step": 106,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.305850826261724,
+      "grad_norm": 6.919908046722412,
+      "learning_rate": 0.00019854711895495036,
+      "logits/chosen": -0.8861989974975586,
+      "logits/rejected": -0.8968295454978943,
+      "logps/chosen": -4.962037563323975,
+      "logps/rejected": -40.77632522583008,
+      "loss": 0.5569091439247131,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.37660181522369385,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.30795052647590637,
+      "rewards/margins": 3.064462184906006,
+      "rewards/rejected": -2.756511688232422,
+      "step": 107,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.3087092451987494,
+      "grad_norm": 4.996776580810547,
+      "learning_rate": 0.00019849294711684845,
+      "logits/chosen": -0.9625644683837891,
+      "logits/rejected": -0.972130298614502,
+      "logps/chosen": -8.820281028747559,
+      "logps/rejected": -50.21597671508789,
+      "loss": 0.7489598989486694,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5024483799934387,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.26294729113578796,
+      "rewards/margins": 3.5292611122131348,
+      "rewards/rejected": -3.2663135528564453,
+      "step": 108,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.3115676641357749,
+      "grad_norm": 6.157798767089844,
+      "learning_rate": 0.00019843779142227256,
+      "logits/chosen": -0.8762018084526062,
+      "logits/rejected": -0.882946789264679,
+      "logps/chosen": -7.924221992492676,
+      "logps/rejected": -43.705806732177734,
+      "loss": 0.6497112512588501,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3160628080368042,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.10452399402856827,
+      "rewards/margins": 3.319636106491089,
+      "rewards/rejected": -3.2151122093200684,
+      "step": 109,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.31442608307280034,
+      "grad_norm": 15.488557815551758,
+      "learning_rate": 0.00019838165242217875,
+      "logits/chosen": -0.863703727722168,
+      "logits/rejected": -0.8713690042495728,
+      "logps/chosen": -6.335309028625488,
+      "logps/rejected": -46.15715026855469,
+      "loss": 0.871519923210144,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5997005701065063,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.15678179264068604,
+      "rewards/margins": 3.3330183029174805,
+      "rewards/rejected": -3.176236629486084,
+      "step": 110,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.31728450200982583,
+      "grad_norm": 4.969946384429932,
+      "learning_rate": 0.0001983245306773454,
+      "logits/chosen": -0.8029311895370483,
+      "logits/rejected": -0.8069216012954712,
+      "logps/chosen": -5.91574764251709,
+      "logps/rejected": -26.575307846069336,
+      "loss": 0.6512956619262695,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2922568917274475,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.20303387939929962,
+      "rewards/margins": 1.892715573310852,
+      "rewards/rejected": -1.6896816492080688,
+      "step": 111,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.32014292094685126,
+      "grad_norm": 6.895030498504639,
+      "learning_rate": 0.0001982664267583677,
+      "logits/chosen": -0.8168566226959229,
+      "logits/rejected": -0.8292732238769531,
+      "logps/chosen": -5.602714538574219,
+      "logps/rejected": -31.50632667541504,
+      "loss": 0.8574928641319275,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5075439810752869,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.010208956897258759,
+      "rewards/margins": 1.8891350030899048,
+      "rewards/rejected": -1.8789262771606445,
+      "step": 112,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.32300133988387675,
+      "grad_norm": 4.541954040527344,
+      "learning_rate": 0.00019820734124565182,
+      "logits/chosen": -0.7733545899391174,
+      "logits/rejected": -0.7771452069282532,
+      "logps/chosen": -6.299251079559326,
+      "logps/rejected": -27.075977325439453,
+      "loss": 0.7081068754196167,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3878493905067444,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.13715356588363647,
+      "rewards/margins": 1.6064281463623047,
+      "rewards/rejected": -1.4692747592926025,
+      "step": 113,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.3258597588209022,
+      "grad_norm": 3.1780240535736084,
+      "learning_rate": 0.00019814727472940917,
+      "logits/chosen": -0.7537604570388794,
+      "logits/rejected": -0.7559649348258972,
+      "logps/chosen": -11.733123779296875,
+      "logps/rejected": -27.673603057861328,
+      "loss": 0.71772301197052,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.42100292444229126,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.21125301718711853,
+      "rewards/margins": 1.6915286779403687,
+      "rewards/rejected": -1.4802757501602173,
+      "step": 114,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.32871817775792767,
+      "grad_norm": 3.8751778602600098,
+      "learning_rate": 0.00019808622780965066,
+      "logits/chosen": -0.7520322799682617,
+      "logits/rejected": -0.7543449401855469,
+      "logps/chosen": -7.260441303253174,
+      "logps/rejected": -29.127504348754883,
+      "loss": 0.6945881843566895,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3524607717990875,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.26789340376853943,
+      "rewards/margins": 1.691644549369812,
+      "rewards/rejected": -1.4237512350082397,
+      "step": 115,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.3315765966949531,
+      "grad_norm": 3.0139174461364746,
+      "learning_rate": 0.0001980242010961803,
+      "logits/chosen": -0.7901269793510437,
+      "logits/rejected": -0.7972939014434814,
+      "logps/chosen": -7.289261341094971,
+      "logps/rejected": -47.759063720703125,
+      "loss": 0.5331582427024841,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.38446128368377686,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3029555678367615,
+      "rewards/margins": 2.9576172828674316,
+      "rewards/rejected": -2.6546616554260254,
+      "step": 116,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.3344350156319786,
+      "grad_norm": 3.233109712600708,
+      "learning_rate": 0.00019796119520858955,
+      "logits/chosen": -0.7568274140357971,
+      "logits/rejected": -0.7630717158317566,
+      "logps/chosen": -10.064506530761719,
+      "logps/rejected": -32.50635528564453,
+      "loss": 0.789489209651947,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4839721918106079,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.3997339606285095,
+      "rewards/margins": 2.3508408069610596,
+      "rewards/rejected": -1.9511069059371948,
+      "step": 117,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.337293434569004,
+      "grad_norm": 6.901320934295654,
+      "learning_rate": 0.0001978972107762509,
+      "logits/chosen": -0.7462923526763916,
+      "logits/rejected": -0.7441357970237732,
+      "logps/chosen": -10.976001739501953,
+      "logps/rejected": -31.10492706298828,
+      "loss": 0.7512857913970947,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.43077993392944336,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1536616086959839,
+      "rewards/margins": 2.106844186782837,
+      "rewards/rejected": -1.953182578086853,
+      "step": 118,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.3401518535060295,
+      "grad_norm": 2.286742925643921,
+      "learning_rate": 0.00019783224843831162,
+      "logits/chosen": -0.7183930277824402,
+      "logits/rejected": -0.7182736396789551,
+      "logps/chosen": -8.731864929199219,
+      "logps/rejected": -33.98211669921875,
+      "loss": 0.6520930528640747,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3684110939502716,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.2964346706867218,
+      "rewards/margins": 2.4515674114227295,
+      "rewards/rejected": -2.15513277053833,
+      "step": 119,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.34301027244305493,
+      "grad_norm": 4.936792850494385,
+      "learning_rate": 0.00019776630884368738,
+      "logits/chosen": -0.7660694122314453,
+      "logits/rejected": -0.776424765586853,
+      "logps/chosen": -4.871070861816406,
+      "logps/rejected": -47.67111587524414,
+      "loss": 0.4515693187713623,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.30822524428367615,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.38517045974731445,
+      "rewards/margins": 3.7489230632781982,
+      "rewards/rejected": -3.363752841949463,
+      "step": 120,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.3458686913800804,
+      "grad_norm": 9.613821983337402,
+      "learning_rate": 0.00019769939265105573,
+      "logits/chosen": -0.7732415795326233,
+      "logits/rejected": -0.7863773703575134,
+      "logps/chosen": -4.910383701324463,
+      "logps/rejected": -44.046382904052734,
+      "loss": 0.6085942983627319,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3416149616241455,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.3152579069137573,
+      "rewards/margins": 3.380157470703125,
+      "rewards/rejected": -3.0648996829986572,
+      "step": 121,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.34872711031710585,
+      "grad_norm": 5.3062744140625,
+      "learning_rate": 0.00019763150052884966,
+      "logits/chosen": -0.7940014004707336,
+      "logits/rejected": -0.8029736280441284,
+      "logps/chosen": -3.551081895828247,
+      "logps/rejected": -38.06512451171875,
+      "loss": 0.42784249782562256,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.25387704372406006,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.37009117007255554,
+      "rewards/margins": 2.841196298599243,
+      "rewards/rejected": -2.471104860305786,
+      "step": 122,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.35158552925413133,
+      "grad_norm": 1.533804178237915,
+      "learning_rate": 0.0001975626331552507,
+      "logits/chosen": -0.8097442984580994,
+      "logits/rejected": -0.8176737427711487,
+      "logps/chosen": -3.7899668216705322,
+      "logps/rejected": -51.7594108581543,
+      "loss": 0.3041819632053375,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2387474924325943,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.35475367307662964,
+      "rewards/margins": 4.306023597717285,
+      "rewards/rejected": -3.9512698650360107,
+      "step": 123,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.35444394819115677,
+      "grad_norm": 4.373802661895752,
+      "learning_rate": 0.00019749279121818235,
+      "logits/chosen": -0.7605909705162048,
+      "logits/rejected": -0.770943284034729,
+      "logps/chosen": -8.83084774017334,
+      "logps/rejected": -59.24419021606445,
+      "loss": 0.5374789834022522,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.39307644963264465,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.14716514945030212,
+      "rewards/margins": 4.3210930824279785,
+      "rewards/rejected": -4.173928260803223,
+      "step": 124,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.3573023671281822,
+      "grad_norm": 6.3563103675842285,
+      "learning_rate": 0.0001974219754153032,
+      "logits/chosen": -0.8258897066116333,
+      "logits/rejected": -0.8272631168365479,
+      "logps/chosen": -7.77381706237793,
+      "logps/rejected": -41.343414306640625,
+      "loss": 0.6204044818878174,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.33825787901878357,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.12217142432928085,
+      "rewards/margins": 3.0349438190460205,
+      "rewards/rejected": -2.9127724170684814,
+      "step": 125,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.3601607860652077,
+      "grad_norm": 9.85932731628418,
+      "learning_rate": 0.00019735018645399967,
+      "logits/chosen": -0.8187968730926514,
+      "logits/rejected": -0.8283690214157104,
+      "logps/chosen": -4.5190534591674805,
+      "logps/rejected": -45.119789123535156,
+      "loss": 0.5073397755622864,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.33468902111053467,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.34614282846450806,
+      "rewards/margins": 3.5951905250549316,
+      "rewards/rejected": -3.2490477561950684,
+      "step": 126,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.3630192050022331,
+      "grad_norm": 3.943042039871216,
+      "learning_rate": 0.00019727742505137936,
+      "logits/chosen": -0.7813420295715332,
+      "logits/rejected": -0.7975005507469177,
+      "logps/chosen": -7.530571937561035,
+      "logps/rejected": -53.51607894897461,
+      "loss": 0.46567243337631226,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2742045819759369,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.23301076889038086,
+      "rewards/margins": 3.9482827186584473,
+      "rewards/rejected": -3.7152719497680664,
+      "step": 127,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.3658776239392586,
+      "grad_norm": 1.9621978998184204,
+      "learning_rate": 0.00019720369193426365,
+      "logits/chosen": -0.7805514335632324,
+      "logits/rejected": -0.7947923541069031,
+      "logps/chosen": -4.593271255493164,
+      "logps/rejected": -51.02714157104492,
+      "loss": 0.3543168008327484,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2695479393005371,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3205646574497223,
+      "rewards/margins": 3.938343048095703,
+      "rewards/rejected": -3.6177780628204346,
+      "step": 128,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.36873604287628403,
+      "grad_norm": 6.375858783721924,
+      "learning_rate": 0.00019712898783918038,
+      "logits/chosen": -0.7720431089401245,
+      "logits/rejected": -0.7774226069450378,
+      "logps/chosen": -3.9679479598999023,
+      "logps/rejected": -39.059242248535156,
+      "loss": 0.42859622836112976,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2718777060508728,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.335873007774353,
+      "rewards/margins": 3.0524649620056152,
+      "rewards/rejected": -2.7165920734405518,
+      "step": 129,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.3715944618133095,
+      "grad_norm": 20.90467643737793,
+      "learning_rate": 0.00019705331351235674,
+      "logits/chosen": -0.7668591737747192,
+      "logits/rejected": -0.7685776948928833,
+      "logps/chosen": -9.782769203186035,
+      "logps/rejected": -33.2745361328125,
+      "loss": 1.310682773590088,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.8834658861160278,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.039194077253341675,
+      "rewards/margins": 2.1344308853149414,
+      "rewards/rejected": -2.1736247539520264,
+      "step": 130,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.37445288075033495,
+      "grad_norm": 2.3716464042663574,
+      "learning_rate": 0.00019697666970971153,
+      "logits/chosen": -0.7852880954742432,
+      "logits/rejected": -0.7899134755134583,
+      "logps/chosen": -6.234109878540039,
+      "logps/rejected": -42.3105354309082,
+      "loss": 0.4260920286178589,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.25257518887519836,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3239039182662964,
+      "rewards/margins": 3.0808892250061035,
+      "rewards/rejected": -2.7569851875305176,
+      "step": 131,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.37731129968736044,
+      "grad_norm": 17.186786651611328,
+      "learning_rate": 0.00019689905719684782,
+      "logits/chosen": -0.7246553301811218,
+      "logits/rejected": -0.739700198173523,
+      "logps/chosen": -4.994725227355957,
+      "logps/rejected": -37.61749267578125,
+      "loss": 0.7901899814605713,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.529437243938446,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3914901614189148,
+      "rewards/margins": 2.7029402256011963,
+      "rewards/rejected": -2.311450242996216,
+      "step": 132,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.38016971862438587,
+      "grad_norm": 19.44753646850586,
+      "learning_rate": 0.00019682047674904525,
+      "logits/chosen": -0.7204717993736267,
+      "logits/rejected": -0.7237159013748169,
+      "logps/chosen": -10.331472396850586,
+      "logps/rejected": -44.90771484375,
+      "loss": 0.9498412609100342,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6981635093688965,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.13750669360160828,
+      "rewards/margins": 3.0414295196533203,
+      "rewards/rejected": -2.9039225578308105,
+      "step": 133,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.38302813756141135,
+      "grad_norm": 2.630051612854004,
+      "learning_rate": 0.00019674092915125218,
+      "logits/chosen": -0.7359256148338318,
+      "logits/rejected": -0.7523387670516968,
+      "logps/chosen": -5.2792582511901855,
+      "logps/rejected": -53.244354248046875,
+      "loss": 0.37174469232559204,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.19531044363975525,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.35951948165893555,
+      "rewards/margins": 3.9146037101745605,
+      "rewards/rejected": -3.555084228515625,
+      "step": 134,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.3858865564984368,
+      "grad_norm": 8.38878059387207,
+      "learning_rate": 0.00019666041519807802,
+      "logits/chosen": -0.7512345910072327,
+      "logits/rejected": -0.7590532302856445,
+      "logps/chosen": -7.108774662017822,
+      "logps/rejected": -49.24845886230469,
+      "loss": 0.515214741230011,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.35548365116119385,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.11580058932304382,
+      "rewards/margins": 3.6866679191589355,
+      "rewards/rejected": -3.5708670616149902,
+      "step": 135,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.38874497543546227,
+      "grad_norm": 7.971051216125488,
+      "learning_rate": 0.0001965789356937852,
+      "logits/chosen": -0.7898239493370056,
+      "logits/rejected": -0.7973160743713379,
+      "logps/chosen": -9.885854721069336,
+      "logps/rejected": -55.670047760009766,
+      "loss": 0.7229015827178955,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.43035101890563965,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.13693107664585114,
+      "rewards/margins": 3.9661924839019775,
+      "rewards/rejected": -3.829261541366577,
+      "step": 136,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.3916033943724877,
+      "grad_norm": 6.154712677001953,
+      "learning_rate": 0.00019649649145228102,
+      "logits/chosen": -0.7895013689994812,
+      "logits/rejected": -0.7943228483200073,
+      "logps/chosen": -5.192661285400391,
+      "logps/rejected": -39.91571044921875,
+      "loss": 0.5592948794364929,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22298313677310944,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.19678261876106262,
+      "rewards/margins": 2.95813250541687,
+      "rewards/rejected": -2.761349678039551,
+      "step": 137,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.3944618133095132,
+      "grad_norm": 2.1893486976623535,
+      "learning_rate": 0.0001964130832971098,
+      "logits/chosen": -0.7978731393814087,
+      "logits/rejected": -0.808353066444397,
+      "logps/chosen": -7.741363048553467,
+      "logps/rejected": -49.584014892578125,
+      "loss": 0.4424561560153961,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3187883198261261,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3768758773803711,
+      "rewards/margins": 3.4152517318725586,
+      "rewards/rejected": -3.0383756160736084,
+      "step": 138,
+      "train_speed(iter/s)": 0.005431
+    },
+    {
+      "epoch": 0.3973202322465386,
+      "grad_norm": 3.585315465927124,
+      "learning_rate": 0.0001963287120614444,
+      "logits/chosen": -0.8294897079467773,
+      "logits/rejected": -0.8371224403381348,
+      "logps/chosen": -4.645942211151123,
+      "logps/rejected": -41.52901840209961,
+      "loss": 0.5509465336799622,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3694899380207062,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.15256014466285706,
+      "rewards/margins": 3.042027473449707,
+      "rewards/rejected": -2.8894670009613037,
+      "step": 139,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.4001786511835641,
+      "grad_norm": 3.1583364009857178,
+      "learning_rate": 0.00019624337858807807,
+      "logits/chosen": -0.8058665990829468,
+      "logits/rejected": -0.802970290184021,
+      "logps/chosen": -12.615983009338379,
+      "logps/rejected": -42.576560974121094,
+      "loss": 0.47237128019332886,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.28210973739624023,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.08335088193416595,
+      "rewards/margins": 2.9341583251953125,
+      "rewards/rejected": -3.0175094604492188,
+      "step": 140,
+      "train_speed(iter/s)": 0.005432
+    },
+    {
+      "epoch": 0.40303707012058954,
+      "grad_norm": 1.9583823680877686,
+      "learning_rate": 0.00019615708372941587,
+      "logits/chosen": -0.7773740887641907,
+      "logits/rejected": -0.7813900113105774,
+      "logps/chosen": -7.43025016784668,
+      "logps/rejected": -48.81037521362305,
+      "loss": 0.45620426535606384,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2858808636665344,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.14979784190654755,
+      "rewards/margins": 3.327134847640991,
+      "rewards/rejected": -3.177337169647217,
+      "step": 141,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.405895489057615,
+      "grad_norm": 17.376474380493164,
+      "learning_rate": 0.00019606982834746627,
+      "logits/chosen": -0.8055461645126343,
+      "logits/rejected": -0.8150217533111572,
+      "logps/chosen": -9.388490676879883,
+      "logps/rejected": -43.86671447753906,
+      "loss": 1.0983664989471436,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.7692541480064392,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1326138973236084,
+      "rewards/margins": 2.8655741214752197,
+      "rewards/rejected": -2.7329602241516113,
+      "step": 142,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.40875390799464045,
+      "grad_norm": 35.17894744873047,
+      "learning_rate": 0.00019598161331383257,
+      "logits/chosen": -0.8625121116638184,
+      "logits/rejected": -0.871791422367096,
+      "logps/chosen": -9.16450023651123,
+      "logps/rejected": -47.45623016357422,
+      "loss": 0.7930290102958679,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.543784499168396,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.08059316873550415,
+      "rewards/margins": 3.273069381713867,
+      "rewards/rejected": -3.192476272583008,
+      "step": 143,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.41161232693166594,
+      "grad_norm": 4.112196445465088,
+      "learning_rate": 0.00019589243950970402,
+      "logits/chosen": -0.8877297043800354,
+      "logits/rejected": -0.8911035060882568,
+      "logps/chosen": -6.215106964111328,
+      "logps/rejected": -41.89563751220703,
+      "loss": 0.3530213236808777,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20008514821529388,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3117275834083557,
+      "rewards/margins": 3.073056221008301,
+      "rewards/rejected": -2.76132869720459,
+      "step": 144,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.41447074586869137,
+      "grad_norm": 12.758796691894531,
+      "learning_rate": 0.00019580230782584722,
+      "logits/chosen": -0.8928901553153992,
+      "logits/rejected": -0.8998609781265259,
+      "logps/chosen": -5.406792640686035,
+      "logps/rejected": -40.006412506103516,
+      "loss": 0.5003563165664673,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.30671095848083496,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.23732616007328033,
+      "rewards/margins": 2.932386636734009,
+      "rewards/rejected": -2.6950607299804688,
+      "step": 145,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.41732916480571686,
+      "grad_norm": 4.014595985412598,
+      "learning_rate": 0.00019571121916259706,
+      "logits/chosen": -0.887598991394043,
+      "logits/rejected": -0.8942646384239197,
+      "logps/chosen": -4.3415961265563965,
+      "logps/rejected": -46.59621810913086,
+      "loss": 0.36891594529151917,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2651711106300354,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.36169373989105225,
+      "rewards/margins": 3.6561460494995117,
+      "rewards/rejected": -3.294452428817749,
+      "step": 146,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.4201875837427423,
+      "grad_norm": 10.37033462524414,
+      "learning_rate": 0.00019561917442984788,
+      "logits/chosen": -0.8872988224029541,
+      "logits/rejected": -0.8909754753112793,
+      "logps/chosen": -7.036233901977539,
+      "logps/rejected": -44.856361389160156,
+      "loss": 0.5071041584014893,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4129847586154938,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.29683101177215576,
+      "rewards/margins": 3.6587531566619873,
+      "rewards/rejected": -3.361922025680542,
+      "step": 147,
+      "train_speed(iter/s)": 0.005434
+    },
+    {
+      "epoch": 0.4230460026797678,
+      "grad_norm": 12.54053783416748,
+      "learning_rate": 0.00019552617454704428,
+      "logits/chosen": -0.909309983253479,
+      "logits/rejected": -0.9173017144203186,
+      "logps/chosen": -5.930859088897705,
+      "logps/rejected": -46.784332275390625,
+      "loss": 0.6537680625915527,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.521246075630188,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.25368478894233704,
+      "rewards/margins": 3.396470308303833,
+      "rewards/rejected": -3.1427857875823975,
+      "step": 148,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.4259044216167932,
+      "grad_norm": 2.5870954990386963,
+      "learning_rate": 0.00019543222044317188,
+      "logits/chosen": -0.8764325976371765,
+      "logits/rejected": -0.881737232208252,
+      "logps/chosen": -5.638083457946777,
+      "logps/rejected": -47.27805709838867,
+      "loss": 0.3462856113910675,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22526228427886963,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3174706697463989,
+      "rewards/margins": 3.5086669921875,
+      "rewards/rejected": -3.1911962032318115,
+      "step": 149,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.4287628405538187,
+      "grad_norm": 9.161517143249512,
+      "learning_rate": 0.00019533731305674818,
+      "logits/chosen": -0.8732975721359253,
+      "logits/rejected": -0.8769663572311401,
+      "logps/chosen": -9.518259048461914,
+      "logps/rejected": -42.666114807128906,
+      "loss": 0.9442892670631409,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6460040211677551,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.07534892857074738,
+      "rewards/margins": 3.1069083213806152,
+      "rewards/rejected": -3.0315592288970947,
+      "step": 150,
+      "train_speed(iter/s)": 0.005433
+    },
+    {
+      "epoch": 0.4316212594908441,
+      "grad_norm": 17.615604400634766,
+      "learning_rate": 0.00019524145333581317,
+      "logits/chosen": -0.8912621736526489,
+      "logits/rejected": -0.8975273966789246,
+      "logps/chosen": -10.543143272399902,
+      "logps/rejected": -44.90479278564453,
+      "loss": 1.2651307582855225,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.7548849582672119,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.13807448744773865,
+      "rewards/margins": 3.1065428256988525,
+      "rewards/rejected": -3.244617223739624,
+      "step": 151,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.4344796784278696,
+      "grad_norm": 2.197153091430664,
+      "learning_rate": 0.00019514464223791965,
+      "logits/chosen": -0.8220558762550354,
+      "logits/rejected": -0.8305569291114807,
+      "logps/chosen": -7.629626274108887,
+      "logps/rejected": -46.86054229736328,
+      "loss": 0.5103957653045654,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4139241874217987,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3421666920185089,
+      "rewards/margins": 3.841639995574951,
+      "rewards/rejected": -3.4994730949401855,
+      "step": 152,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.43733809736489504,
+      "grad_norm": 2.0711119174957275,
+      "learning_rate": 0.00019504688073012397,
+      "logits/chosen": -0.8169566988945007,
+      "logits/rejected": -0.8239966034889221,
+      "logps/chosen": -6.1432342529296875,
+      "logps/rejected": -42.3114013671875,
+      "loss": 0.4369783401489258,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22974222898483276,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2705279588699341,
+      "rewards/margins": 2.976038932800293,
+      "rewards/rejected": -2.7055106163024902,
+      "step": 153,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.4401965163019205,
+      "grad_norm": 11.810593605041504,
+      "learning_rate": 0.00019494816978897616,
+      "logits/chosen": -0.8318780064582825,
+      "logits/rejected": -0.8391178846359253,
+      "logps/chosen": -7.192132472991943,
+      "logps/rejected": -39.354427337646484,
+      "loss": 0.6350818872451782,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.475485622882843,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3430967926979065,
+      "rewards/margins": 2.803071975708008,
+      "rewards/rejected": -2.459975004196167,
+      "step": 154,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.44305493523894596,
+      "grad_norm": 2.665921688079834,
+      "learning_rate": 0.0001948485104005103,
+      "logits/chosen": -0.8109292984008789,
+      "logits/rejected": -0.8165513277053833,
+      "logps/chosen": -4.813442230224609,
+      "logps/rejected": -38.313106536865234,
+      "loss": 0.6565403938293457,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4435839056968689,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.25875312089920044,
+      "rewards/margins": 3.050962209701538,
+      "rewards/rejected": -2.7922089099884033,
+      "step": 155,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.4459133541759714,
+      "grad_norm": 2.126662015914917,
+      "learning_rate": 0.00019474790356023455,
+      "logits/chosen": -0.8104051351547241,
+      "logits/rejected": -0.8165953755378723,
+      "logps/chosen": -4.292057514190674,
+      "logps/rejected": -44.862060546875,
+      "loss": 0.309725821018219,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18593546748161316,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3211231231689453,
+      "rewards/margins": 3.5143370628356934,
+      "rewards/rejected": -3.193213939666748,
+      "step": 156,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.4487717731129969,
+      "grad_norm": 3.4068145751953125,
+      "learning_rate": 0.00019464635027312128,
+      "logits/chosen": -0.8355593681335449,
+      "logits/rejected": -0.8380401730537415,
+      "logps/chosen": -6.426550388336182,
+      "logps/rejected": -44.43500518798828,
+      "loss": 0.6942108273506165,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4586290121078491,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1291281133890152,
+      "rewards/margins": 3.1570847034454346,
+      "rewards/rejected": -3.02795672416687,
+      "step": 157,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.4516301920500223,
+      "grad_norm": 4.667487621307373,
+      "learning_rate": 0.00019454385155359702,
+      "logits/chosen": -0.849906325340271,
+      "logits/rejected": -0.8568750619888306,
+      "logps/chosen": -9.276833534240723,
+      "logps/rejected": -49.35503387451172,
+      "loss": 0.4330065846443176,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2498662769794464,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3473469913005829,
+      "rewards/margins": 4.031757354736328,
+      "rewards/rejected": -3.684410572052002,
+      "step": 158,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.4544886109870478,
+      "grad_norm": 1.687165379524231,
+      "learning_rate": 0.00019444040842553237,
+      "logits/chosen": -0.890581488609314,
+      "logits/rejected": -0.9015938639640808,
+      "logps/chosen": -3.249390125274658,
+      "logps/rejected": -57.22711181640625,
+      "loss": 0.362801194190979,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22434672713279724,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2739589512348175,
+      "rewards/margins": 4.31603479385376,
+      "rewards/rejected": -4.0420756340026855,
+      "step": 159,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.4573470299240732,
+      "grad_norm": 12.536514282226562,
+      "learning_rate": 0.00019433602192223164,
+      "logits/chosen": -0.8646700382232666,
+      "logits/rejected": -0.872589111328125,
+      "logps/chosen": -6.172619342803955,
+      "logps/rejected": -48.3273811340332,
+      "loss": 0.6447067856788635,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.32002851366996765,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.09112294018268585,
+      "rewards/margins": 3.2884745597839355,
+      "rewards/rejected": -3.1973516941070557,
+      "step": 160,
+      "train_speed(iter/s)": 0.005426
+    },
+    {
+      "epoch": 0.4602054488610987,
+      "grad_norm": 3.207699775695801,
+      "learning_rate": 0.00019423069308642266,
+      "logits/chosen": -0.8848387002944946,
+      "logits/rejected": -0.8890655636787415,
+      "logps/chosen": -6.779003620147705,
+      "logps/rejected": -52.09819030761719,
+      "loss": 0.35991424322128296,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.26353347301483154,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.40888625383377075,
+      "rewards/margins": 4.320865631103516,
+      "rewards/rejected": -3.9119794368743896,
+      "step": 161,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.46306386779812414,
+      "grad_norm": 13.387569427490234,
+      "learning_rate": 0.00019412442297024637,
+      "logits/chosen": -0.8957618474960327,
+      "logits/rejected": -0.8995543718338013,
+      "logps/chosen": -8.413171768188477,
+      "logps/rejected": -39.29925537109375,
+      "loss": 0.7494506239891052,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4951186180114746,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.03331068903207779,
+      "rewards/margins": 2.7600255012512207,
+      "rewards/rejected": -2.726714849472046,
+      "step": 162,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.4659222867351496,
+      "grad_norm": 14.89892578125,
+      "learning_rate": 0.00019401721263524616,
+      "logits/chosen": -0.8836920261383057,
+      "logits/rejected": -0.8899936079978943,
+      "logps/chosen": -8.512656211853027,
+      "logps/rejected": -46.419029235839844,
+      "loss": 0.5585404634475708,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3683706820011139,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.24114097654819489,
+      "rewards/margins": 3.47017240524292,
+      "rewards/rejected": -3.229031562805176,
+      "step": 163,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.46878070567217506,
+      "grad_norm": 14.938558578491211,
+      "learning_rate": 0.0001939090631523574,
+      "logits/chosen": -0.935294508934021,
+      "logits/rejected": -0.9457290768623352,
+      "logps/chosen": -7.823429107666016,
+      "logps/rejected": -56.9812126159668,
+      "loss": 1.0811909437179565,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.7688519954681396,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.035027798265218735,
+      "rewards/margins": 3.6944613456726074,
+      "rewards/rejected": -3.729489326477051,
+      "step": 164,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.47163912460920054,
+      "grad_norm": 8.26276683807373,
+      "learning_rate": 0.00019379997560189675,
+      "logits/chosen": -0.9381538033485413,
+      "logits/rejected": -0.9450898170471191,
+      "logps/chosen": -6.668002605438232,
+      "logps/rejected": -51.126930236816406,
+      "loss": 0.5152188539505005,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4229362905025482,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.14181563258171082,
+      "rewards/margins": 4.032417297363281,
+      "rewards/rejected": -3.890601873397827,
+      "step": 165,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.474497543546226,
+      "grad_norm": 5.228875160217285,
+      "learning_rate": 0.00019368995107355133,
+      "logits/chosen": -0.9465993642807007,
+      "logits/rejected": -0.9516839981079102,
+      "logps/chosen": -6.645859718322754,
+      "logps/rejected": -49.824649810791016,
+      "loss": 0.6003601551055908,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4243951737880707,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1566632241010666,
+      "rewards/margins": 3.7477457523345947,
+      "rewards/rejected": -3.5910823345184326,
+      "step": 166,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.47735596248325146,
+      "grad_norm": 11.680534362792969,
+      "learning_rate": 0.00019357899066636773,
+      "logits/chosen": -0.9604209661483765,
+      "logits/rejected": -0.9663453102111816,
+      "logps/chosen": -11.16219711303711,
+      "logps/rejected": -54.03529357910156,
+      "loss": 0.6696478128433228,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5087156295776367,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.24885831773281097,
+      "rewards/margins": 4.16080379486084,
+      "rewards/rejected": -3.9119460582733154,
+      "step": 167,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.4802143814202769,
+      "grad_norm": 6.519625663757324,
+      "learning_rate": 0.00019346709548874127,
+      "logits/chosen": -0.9105353355407715,
+      "logits/rejected": -0.9168953895568848,
+      "logps/chosen": -7.151789665222168,
+      "logps/rejected": -51.146156311035156,
+      "loss": 0.538978099822998,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.38577884435653687,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.34396299719810486,
+      "rewards/margins": 4.075203895568848,
+      "rewards/rejected": -3.731240749359131,
+      "step": 168,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.4830728003573024,
+      "grad_norm": 11.481460571289062,
+      "learning_rate": 0.0001933542666584047,
+      "logits/chosen": -0.9098703265190125,
+      "logits/rejected": -0.9205840826034546,
+      "logps/chosen": -9.062050819396973,
+      "logps/rejected": -64.49413299560547,
+      "loss": 0.526665985584259,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.40413621068000793,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2691805362701416,
+      "rewards/margins": 5.563024044036865,
+      "rewards/rejected": -5.293843746185303,
+      "step": 169,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.4859312192943278,
+      "grad_norm": 13.25565242767334,
+      "learning_rate": 0.0001932405053024171,
+      "logits/chosen": -0.9104465246200562,
+      "logits/rejected": -0.917874276638031,
+      "logps/chosen": -11.206613540649414,
+      "logps/rejected": -57.438724517822266,
+      "loss": 1.0738518238067627,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6307643055915833,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.05714952200651169,
+      "rewards/margins": 4.167277812957764,
+      "rewards/rejected": -4.110127925872803,
+      "step": 170,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.4887896382313533,
+      "grad_norm": 17.771833419799805,
+      "learning_rate": 0.00019312581255715277,
+      "logits/chosen": -0.8956343531608582,
+      "logits/rejected": -0.9024403691291809,
+      "logps/chosen": -13.308263778686523,
+      "logps/rejected": -65.24011993408203,
+      "loss": 0.9896730184555054,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5083594918251038,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.0635133907198906,
+      "rewards/margins": 4.961539268493652,
+      "rewards/rejected": -5.025052547454834,
+      "step": 171,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.4916480571683787,
+      "grad_norm": 6.4772748947143555,
+      "learning_rate": 0.00019301018956828964,
+      "logits/chosen": -0.898830771446228,
+      "logits/rejected": -0.9118731021881104,
+      "logps/chosen": -8.135079383850098,
+      "logps/rejected": -57.8392333984375,
+      "loss": 0.5032879710197449,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3064589500427246,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2829325497150421,
+      "rewards/margins": 4.001053333282471,
+      "rewards/rejected": -3.718120574951172,
+      "step": 172,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.4945064761054042,
+      "grad_norm": 6.827486515045166,
+      "learning_rate": 0.000192893637490798,
+      "logits/chosen": -0.8352851867675781,
+      "logits/rejected": -0.8436259627342224,
+      "logps/chosen": -6.111320972442627,
+      "logps/rejected": -51.995445251464844,
+      "loss": 0.5788933038711548,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.27805906534194946,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.23611441254615784,
+      "rewards/margins": 3.686123847961426,
+      "rewards/rejected": -3.450009346008301,
+      "step": 173,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.49736489504242964,
+      "grad_norm": 3.4567549228668213,
+      "learning_rate": 0.00019277615748892885,
+      "logits/chosen": -0.785612165927887,
+      "logits/rejected": -0.7931377291679382,
+      "logps/chosen": -7.273856163024902,
+      "logps/rejected": -36.88285446166992,
+      "loss": 0.5589325428009033,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.37298017740249634,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4650301933288574,
+      "rewards/margins": 2.7481539249420166,
+      "rewards/rejected": -2.283123731613159,
+      "step": 174,
+      "train_speed(iter/s)": 0.00543
+    },
+    {
+      "epoch": 0.5002233139794551,
+      "grad_norm": 4.234227180480957,
+      "learning_rate": 0.00019265775073620245,
+      "logits/chosen": -0.7631555795669556,
+      "logits/rejected": -0.7687059044837952,
+      "logps/chosen": -6.337590217590332,
+      "logps/rejected": -35.29618453979492,
+      "loss": 0.6258898377418518,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3696868121623993,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.21472924947738647,
+      "rewards/margins": 2.4024829864501953,
+      "rewards/rejected": -2.187753915786743,
+      "step": 175,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.5030817329164806,
+      "grad_norm": 2.355490207672119,
+      "learning_rate": 0.00019253841841539632,
+      "logits/chosen": -0.7690540552139282,
+      "logits/rejected": -0.7676496505737305,
+      "logps/chosen": -12.918010711669922,
+      "logps/rejected": -30.43984603881836,
+      "loss": 0.6801320910453796,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.42714056372642517,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2977921664714813,
+      "rewards/margins": 2.146160125732422,
+      "rewards/rejected": -1.8483679294586182,
+      "step": 176,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.505940151853506,
+      "grad_norm": 1.5704671144485474,
+      "learning_rate": 0.0001924181617185336,
+      "logits/chosen": -0.7616850137710571,
+      "logits/rejected": -0.767928957939148,
+      "logps/chosen": -6.664812088012695,
+      "logps/rejected": -37.411277770996094,
+      "loss": 0.42176908254623413,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2451857030391693,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.489712655544281,
+      "rewards/margins": 2.708432674407959,
+      "rewards/rejected": -2.218719959259033,
+      "step": 177,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5087985707905315,
+      "grad_norm": 2.1187188625335693,
+      "learning_rate": 0.00019229698184687128,
+      "logits/chosen": -0.7447081804275513,
+      "logits/rejected": -0.7508732080459595,
+      "logps/chosen": -5.233465671539307,
+      "logps/rejected": -35.176971435546875,
+      "loss": 0.5185775756835938,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.336780309677124,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.44237199425697327,
+      "rewards/margins": 2.7804007530212402,
+      "rewards/rejected": -2.3380286693573,
+      "step": 178,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5116569897275569,
+      "grad_norm": 3.629459857940674,
+      "learning_rate": 0.00019217488001088784,
+      "logits/chosen": -0.7853379249572754,
+      "logits/rejected": -0.8011223673820496,
+      "logps/chosen": -5.379916667938232,
+      "logps/rejected": -53.26487731933594,
+      "loss": 0.368004709482193,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2341601699590683,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.32777413725852966,
+      "rewards/margins": 3.8236327171325684,
+      "rewards/rejected": -3.4958581924438477,
+      "step": 179,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5145154086645825,
+      "grad_norm": 6.440499782562256,
+      "learning_rate": 0.00019205185743027147,
+      "logits/chosen": -0.8589999079704285,
+      "logits/rejected": -0.8642404079437256,
+      "logps/chosen": -7.5277910232543945,
+      "logps/rejected": -48.523780822753906,
+      "loss": 0.4521387219429016,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.30114930868148804,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.30447474122047424,
+      "rewards/margins": 3.6830883026123047,
+      "rewards/rejected": -3.3786139488220215,
+      "step": 180,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5173738276016079,
+      "grad_norm": 1.3223450183868408,
+      "learning_rate": 0.00019192791533390776,
+      "logits/chosen": -0.8698243498802185,
+      "logits/rejected": -0.8859831094741821,
+      "logps/chosen": -5.199261665344238,
+      "logps/rejected": -53.24699783325195,
+      "loss": 0.32750457525253296,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2843254506587982,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5321836471557617,
+      "rewards/margins": 4.558604717254639,
+      "rewards/rejected": -4.026421070098877,
+      "step": 181,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.5202322465386333,
+      "grad_norm": 6.235518932342529,
+      "learning_rate": 0.0001918030549598674,
+      "logits/chosen": -0.8939491510391235,
+      "logits/rejected": -0.9015032052993774,
+      "logps/chosen": -6.817130088806152,
+      "logps/rejected": -48.4378662109375,
+      "loss": 0.8820070028305054,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5765649676322937,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1285572648048401,
+      "rewards/margins": 3.808832883834839,
+      "rewards/rejected": -3.6802756786346436,
+      "step": 182,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.5230906654756587,
+      "grad_norm": 2.177001476287842,
+      "learning_rate": 0.00019167727755539394,
+      "logits/chosen": -0.9257232546806335,
+      "logits/rejected": -0.934444785118103,
+      "logps/chosen": -5.833935737609863,
+      "logps/rejected": -59.789955139160156,
+      "loss": 0.6103315949440002,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.42996734380722046,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.15345799922943115,
+      "rewards/margins": 5.012732982635498,
+      "rewards/rejected": -4.859274387359619,
+      "step": 183,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5259490844126843,
+      "grad_norm": 5.315007209777832,
+      "learning_rate": 0.00019155058437689114,
+      "logits/chosen": -0.9314823150634766,
+      "logits/rejected": -0.9416366815567017,
+      "logps/chosen": -6.756455421447754,
+      "logps/rejected": -56.72506332397461,
+      "loss": 0.7174702286720276,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.523749589920044,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.12303046137094498,
+      "rewards/margins": 4.29641580581665,
+      "rewards/rejected": -4.173385143280029,
+      "step": 184,
+      "train_speed(iter/s)": 0.005427
+    },
+    {
+      "epoch": 0.5288075033497097,
+      "grad_norm": 3.7265548706054688,
+      "learning_rate": 0.00019142297668991055,
+      "logits/chosen": -0.9351843595504761,
+      "logits/rejected": -0.9576059579849243,
+      "logps/chosen": -5.578176021575928,
+      "logps/rejected": -59.8486328125,
+      "loss": 0.6304588317871094,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.46496981382369995,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3721676766872406,
+      "rewards/margins": 4.994194507598877,
+      "rewards/rejected": -4.622026443481445,
+      "step": 185,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5316659222867351,
+      "grad_norm": 4.928941249847412,
+      "learning_rate": 0.00019129445576913888,
+      "logits/chosen": -0.8918996453285217,
+      "logits/rejected": -0.9014327526092529,
+      "logps/chosen": -5.464908123016357,
+      "logps/rejected": -53.11490249633789,
+      "loss": 0.8896833658218384,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6313154697418213,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.12775132060050964,
+      "rewards/margins": 4.158487796783447,
+      "rewards/rejected": -4.030736923217773,
+      "step": 186,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5345243412237606,
+      "grad_norm": 7.509690284729004,
+      "learning_rate": 0.00019116502289838523,
+      "logits/chosen": -0.8859353065490723,
+      "logits/rejected": -0.9015785455703735,
+      "logps/chosen": -4.673905372619629,
+      "logps/rejected": -70.63243865966797,
+      "loss": 0.3962274491786957,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3365238308906555,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3394577205181122,
+      "rewards/margins": 5.691706657409668,
+      "rewards/rejected": -5.352249622344971,
+      "step": 187,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.537382760160786,
+      "grad_norm": 1.5954922437667847,
+      "learning_rate": 0.00019103467937056824,
+      "logits/chosen": -0.9184845685958862,
+      "logits/rejected": -0.9304262399673462,
+      "logps/chosen": -4.041568756103516,
+      "logps/rejected": -76.92243957519531,
+      "loss": 0.3016416132450104,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18911761045455933,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4253259301185608,
+      "rewards/margins": 6.5727128982543945,
+      "rewards/rejected": -6.1473870277404785,
+      "step": 188,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5402411790978116,
+      "grad_norm": 2.318643808364868,
+      "learning_rate": 0.0001909034264877032,
+      "logits/chosen": -0.8512060642242432,
+      "logits/rejected": -0.856816291809082,
+      "logps/chosen": -7.1723408699035645,
+      "logps/rejected": -48.11109161376953,
+      "loss": 0.4600585699081421,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3386399745941162,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.33302804827690125,
+      "rewards/margins": 4.015811443328857,
+      "rewards/rejected": -3.682783365249634,
+      "step": 189,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.543099598034837,
+      "grad_norm": 1.4476327896118164,
+      "learning_rate": 0.0001907712655608891,
+      "logits/chosen": -0.8637464642524719,
+      "logits/rejected": -0.8820706009864807,
+      "logps/chosen": -3.2855453491210938,
+      "logps/rejected": -71.99874114990234,
+      "loss": 0.293333500623703,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18421968817710876,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.28944161534309387,
+      "rewards/margins": 5.838719844818115,
+      "rewards/rejected": -5.549278259277344,
+      "step": 190,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.5459580169718624,
+      "grad_norm": 4.899955749511719,
+      "learning_rate": 0.0001906381979102953,
+      "logits/chosen": -0.9037999510765076,
+      "logits/rejected": -0.9121489524841309,
+      "logps/chosen": -9.670417785644531,
+      "logps/rejected": -52.208683013916016,
+      "loss": 0.5949208736419678,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.38334017992019653,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.15043097734451294,
+      "rewards/margins": 4.171907901763916,
+      "rewards/rejected": -4.021476745605469,
+      "step": 191,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.5488164359088878,
+      "grad_norm": 12.610368728637695,
+      "learning_rate": 0.00019050422486514878,
+      "logits/chosen": -0.840286374092102,
+      "logits/rejected": -0.8606438636779785,
+      "logps/chosen": -8.77206039428711,
+      "logps/rejected": -61.208587646484375,
+      "loss": 0.6860842108726501,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.49322137236595154,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2522002160549164,
+      "rewards/margins": 4.72483491897583,
+      "rewards/rejected": -4.472634792327881,
+      "step": 192,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.5516748548459134,
+      "grad_norm": 7.451924800872803,
+      "learning_rate": 0.0001903693477637204,
+      "logits/chosen": -0.8095037937164307,
+      "logits/rejected": -0.8195419311523438,
+      "logps/chosen": -4.377245903015137,
+      "logps/rejected": -58.97959518432617,
+      "loss": 0.2787761986255646,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.24894402921199799,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3338276147842407,
+      "rewards/margins": 4.592215061187744,
+      "rewards/rejected": -4.258387088775635,
+      "step": 193,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.5545332737829388,
+      "grad_norm": 1.0700743198394775,
+      "learning_rate": 0.00019023356795331182,
+      "logits/chosen": -0.8574849963188171,
+      "logits/rejected": -0.871565043926239,
+      "logps/chosen": -9.894993782043457,
+      "logps/rejected": -60.1685676574707,
+      "loss": 0.446493536233902,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3630823493003845,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3332892954349518,
+      "rewards/margins": 4.909541606903076,
+      "rewards/rejected": -4.576252460479736,
+      "step": 194,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.5573916927199642,
+      "grad_norm": 1.5138012170791626,
+      "learning_rate": 0.0001900968867902419,
+      "logits/chosen": -0.8919128775596619,
+      "logits/rejected": -0.8939546942710876,
+      "logps/chosen": -4.437774181365967,
+      "logps/rejected": -52.1574592590332,
+      "loss": 0.2619098424911499,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20320795476436615,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3197801113128662,
+      "rewards/margins": 4.577724456787109,
+      "rewards/rejected": -4.257944583892822,
+      "step": 195,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.5602501116569897,
+      "grad_norm": 3.5410804748535156,
+      "learning_rate": 0.00018995930563983334,
+      "logits/chosen": -0.8741534352302551,
+      "logits/rejected": -0.8824887871742249,
+      "logps/chosen": -5.8234734535217285,
+      "logps/rejected": -49.77042770385742,
+      "loss": 0.421186625957489,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3348409831523895,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.49023908376693726,
+      "rewards/margins": 4.286158084869385,
+      "rewards/rejected": -3.7959189414978027,
+      "step": 196,
+      "train_speed(iter/s)": 0.005429
+    },
+    {
+      "epoch": 0.5631085305940152,
+      "grad_norm": 1.9928661584854126,
+      "learning_rate": 0.0001898208258763987,
+      "logits/chosen": -0.8889397382736206,
+      "logits/rejected": -0.9006186127662659,
+      "logps/chosen": -4.572671890258789,
+      "logps/rejected": -61.800926208496094,
+      "loss": 0.355736643075943,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.29381096363067627,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.31399890780448914,
+      "rewards/margins": 4.928988456726074,
+      "rewards/rejected": -4.614989757537842,
+      "step": 197,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5659669495310407,
+      "grad_norm": 14.296086311340332,
+      "learning_rate": 0.00018968144888322709,
+      "logits/chosen": -0.8595174551010132,
+      "logits/rejected": -0.861739993095398,
+      "logps/chosen": -7.95827054977417,
+      "logps/rejected": -51.00768280029297,
+      "loss": 0.9002599120140076,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5538756251335144,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.06036340817809105,
+      "rewards/margins": 3.8160269260406494,
+      "rewards/rejected": -3.7556633949279785,
+      "step": 198,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5688253684680661,
+      "grad_norm": 6.8623809814453125,
+      "learning_rate": 0.00018954117605257,
+      "logits/chosen": -0.9402197003364563,
+      "logits/rejected": -0.953208863735199,
+      "logps/chosen": -4.058620929718018,
+      "logps/rejected": -63.09252166748047,
+      "loss": 0.5629696249961853,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3906901776790619,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.26179081201553345,
+      "rewards/margins": 5.211573123931885,
+      "rewards/rejected": -4.949782371520996,
+      "step": 199,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5716837874050915,
+      "grad_norm": 7.293510913848877,
+      "learning_rate": 0.00018940000878562758,
+      "logits/chosen": -0.9181431531906128,
+      "logits/rejected": -0.9206308722496033,
+      "logps/chosen": -7.505152702331543,
+      "logps/rejected": -51.558433532714844,
+      "loss": 0.4831893742084503,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3287861943244934,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1301122009754181,
+      "rewards/margins": 4.108443737030029,
+      "rewards/rejected": -3.9783310890197754,
+      "step": 200,
+      "train_speed(iter/s)": 0.005428
+    },
+    {
+      "epoch": 0.5745422063421171,
+      "grad_norm": 1.9978575706481934,
+      "learning_rate": 0.00018925794849253462,
+      "logits/chosen": -0.9101322889328003,
+      "logits/rejected": -0.9203266501426697,
+      "logps/chosen": -10.848203659057617,
+      "logps/rejected": -65.54148864746094,
+      "loss": 0.7732073068618774,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5795015692710876,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.06352178752422333,
+      "rewards/margins": 5.290210723876953,
+      "rewards/rejected": -5.226688861846924,
+      "step": 201,
+      "train_speed(iter/s)": 0.005425
+    },
+    {
+      "epoch": 0.5774006252791425,
+      "grad_norm": 10.420772552490234,
+      "learning_rate": 0.0001891149965923464,
+      "logits/chosen": -0.9702765941619873,
+      "logits/rejected": -0.9766021370887756,
+      "logps/chosen": -5.592143535614014,
+      "logps/rejected": -61.43619918823242,
+      "loss": 0.586257815361023,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.47159674763679504,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.17951291799545288,
+      "rewards/margins": 4.912915229797363,
+      "rewards/rejected": -4.733402252197266,
+      "step": 202,
+      "train_speed(iter/s)": 0.005424
+    },
+    {
+      "epoch": 0.5802590442161679,
+      "grad_norm": 13.509779930114746,
+      "learning_rate": 0.0001889711545130246,
+      "logits/chosen": -0.9708431959152222,
+      "logits/rejected": -0.9785487651824951,
+      "logps/chosen": -7.3830952644348145,
+      "logps/rejected": -51.72172164916992,
+      "loss": 0.526218831539154,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.334763765335083,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3495505452156067,
+      "rewards/margins": 4.381858825683594,
+      "rewards/rejected": -4.032308578491211,
+      "step": 203,
+      "train_speed(iter/s)": 0.005424
+    },
+    {
+      "epoch": 0.5831174631531933,
+      "grad_norm": 3.5766985416412354,
+      "learning_rate": 0.000188826423691423,
+      "logits/chosen": -0.9285480380058289,
+      "logits/rejected": -0.9337785243988037,
+      "logps/chosen": -4.987155437469482,
+      "logps/rejected": -48.84478759765625,
+      "loss": 0.4154307246208191,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2975158989429474,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3166441321372986,
+      "rewards/margins": 3.8094162940979004,
+      "rewards/rejected": -3.4927725791931152,
+      "step": 204,
+      "train_speed(iter/s)": 0.005425
+    },
+    {
+      "epoch": 0.5859758820902189,
+      "grad_norm": 17.63137435913086,
+      "learning_rate": 0.00018868080557327306,
+      "logits/chosen": -0.8891575336456299,
+      "logits/rejected": -0.8922260403633118,
+      "logps/chosen": -6.72401762008667,
+      "logps/rejected": -51.23278045654297,
+      "loss": 0.4788863956928253,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3527264893054962,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.29821208119392395,
+      "rewards/margins": 4.194456577301025,
+      "rewards/rejected": -3.896244525909424,
+      "step": 205,
+      "train_speed(iter/s)": 0.005425
+    },
+    {
+      "epoch": 0.5888343010272443,
+      "grad_norm": 9.541404724121094,
+      "learning_rate": 0.00018853430161316957,
+      "logits/chosen": -0.8920901417732239,
+      "logits/rejected": -0.903443455696106,
+      "logps/chosen": -5.840543270111084,
+      "logps/rejected": -56.81292724609375,
+      "loss": 0.5086037516593933,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.34491029381752014,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.27246490120887756,
+      "rewards/margins": 4.210405349731445,
+      "rewards/rejected": -3.937941074371338,
+      "step": 206,
+      "train_speed(iter/s)": 0.005425
+    },
+    {
+      "epoch": 0.5916927199642698,
+      "grad_norm": 36.082942962646484,
+      "learning_rate": 0.0001883869132745561,
+      "logits/chosen": -0.8546773195266724,
+      "logits/rejected": -0.8607960343360901,
+      "logps/chosen": -10.014286041259766,
+      "logps/rejected": -46.71479797363281,
+      "loss": 0.6521262526512146,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4141015410423279,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.14084556698799133,
+      "rewards/margins": 3.5019726753234863,
+      "rewards/rejected": -3.3611273765563965,
+      "step": 207,
+      "train_speed(iter/s)": 0.005424
+    },
+    {
+      "epoch": 0.5945511389012952,
+      "grad_norm": 3.8964920043945312,
+      "learning_rate": 0.00018823864202971042,
+      "logits/chosen": -0.8819292783737183,
+      "logits/rejected": -0.8937058448791504,
+      "logps/chosen": -9.53115177154541,
+      "logps/rejected": -55.31066131591797,
+      "loss": 0.4809289872646332,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2986198961734772,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.24837446212768555,
+      "rewards/margins": 4.183048248291016,
+      "rewards/rejected": -3.9346742630004883,
+      "step": 208,
+      "train_speed(iter/s)": 0.005424
+    },
+    {
+      "epoch": 0.5974095578383207,
+      "grad_norm": 2.320671796798706,
+      "learning_rate": 0.00018808948935972964,
+      "logits/chosen": -0.8995912671089172,
+      "logits/rejected": -0.9069520235061646,
+      "logps/chosen": -9.92770767211914,
+      "logps/rejected": -52.2730598449707,
+      "loss": 0.36179569363594055,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.29885053634643555,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2783893644809723,
+      "rewards/margins": 3.9017248153686523,
+      "rewards/rejected": -3.623335838317871,
+      "step": 209,
+      "train_speed(iter/s)": 0.005425
+    },
+    {
+      "epoch": 0.6002679767753462,
+      "grad_norm": 4.009780406951904,
+      "learning_rate": 0.00018793945675451553,
+      "logits/chosen": -0.8754842877388,
+      "logits/rejected": -0.8889762163162231,
+      "logps/chosen": -4.163789749145508,
+      "logps/rejected": -54.52180099487305,
+      "loss": 0.30726245045661926,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22840096056461334,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.38789692521095276,
+      "rewards/margins": 4.064230442047119,
+      "rewards/rejected": -3.676333427429199,
+      "step": 210,
+      "train_speed(iter/s)": 0.005425
+    },
+    {
+      "epoch": 0.6031263957123716,
+      "grad_norm": 2.2772812843322754,
+      "learning_rate": 0.00018778854571275972,
+      "logits/chosen": -0.9018064141273499,
+      "logits/rejected": -0.9058933854103088,
+      "logps/chosen": -5.221268177032471,
+      "logps/rejected": -42.75416564941406,
+      "loss": 0.31052833795547485,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2247018814086914,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3918154239654541,
+      "rewards/margins": 3.388484477996826,
+      "rewards/rejected": -2.996669292449951,
+      "step": 211,
+      "train_speed(iter/s)": 0.005425
+    },
+    {
+      "epoch": 0.605984814649397,
+      "grad_norm": 10.163191795349121,
+      "learning_rate": 0.0001876367577419286,
+      "logits/chosen": -0.850382924079895,
+      "logits/rejected": -0.864805281162262,
+      "logps/chosen": -5.846621990203857,
+      "logps/rejected": -58.741886138916016,
+      "loss": 0.4064854383468628,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3250509202480316,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.30676084756851196,
+      "rewards/margins": 4.2306623458862305,
+      "rewards/rejected": -3.9239015579223633,
+      "step": 212,
+      "train_speed(iter/s)": 0.005424
+    },
+    {
+      "epoch": 0.6088432335864226,
+      "grad_norm": 2.4588639736175537,
+      "learning_rate": 0.00018748409435824818,
+      "logits/chosen": -0.8809334635734558,
+      "logits/rejected": -0.8915649652481079,
+      "logps/chosen": -5.256155967712402,
+      "logps/rejected": -49.58497619628906,
+      "loss": 0.3395237326622009,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2899019122123718,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3255791962146759,
+      "rewards/margins": 4.010343551635742,
+      "rewards/rejected": -3.6847646236419678,
+      "step": 213,
+      "train_speed(iter/s)": 0.005424
+    },
+    {
+      "epoch": 0.611701652523448,
+      "grad_norm": 21.554292678833008,
+      "learning_rate": 0.00018733055708668926,
+      "logits/chosen": -0.9079571962356567,
+      "logits/rejected": -0.9125982522964478,
+      "logps/chosen": -8.034017562866211,
+      "logps/rejected": -38.557186126708984,
+      "loss": 0.922869086265564,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.7172112464904785,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08057776838541031,
+      "rewards/margins": 2.6949210166931152,
+      "rewards/rejected": -2.6143431663513184,
+      "step": 214,
+      "train_speed(iter/s)": 0.005424
+    },
+    {
+      "epoch": 0.6145600714604734,
+      "grad_norm": 30.98603630065918,
+      "learning_rate": 0.00018717614746095187,
+      "logits/chosen": -0.9082990288734436,
+      "logits/rejected": -0.9197255373001099,
+      "logps/chosen": -4.883214473724365,
+      "logps/rejected": -49.50416564941406,
+      "loss": 0.4404700696468353,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.34376513957977295,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.28728392720222473,
+      "rewards/margins": 3.837834119796753,
+      "rewards/rejected": -3.5505499839782715,
+      "step": 215,
+      "train_speed(iter/s)": 0.005424
+    },
+    {
+      "epoch": 0.6174184903974989,
+      "grad_norm": 1.3132481575012207,
+      "learning_rate": 0.0001870208670234501,
+      "logits/chosen": -0.9290035963058472,
+      "logits/rejected": -0.934213399887085,
+      "logps/chosen": -10.832901000976562,
+      "logps/rejected": -55.205081939697266,
+      "loss": 0.4211485683917999,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.37757277488708496,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.22118425369262695,
+      "rewards/margins": 4.514135360717773,
+      "rewards/rejected": -4.2929511070251465,
+      "step": 216,
+      "train_speed(iter/s)": 0.005424
+    },
+    {
+      "epoch": 0.6202769093345244,
+      "grad_norm": 5.870302200317383,
+      "learning_rate": 0.00018686471732529665,
+      "logits/chosen": -0.9589085578918457,
+      "logits/rejected": -0.9713011980056763,
+      "logps/chosen": -5.64130973815918,
+      "logps/rejected": -46.44078063964844,
+      "loss": 0.2975273132324219,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.191940039396286,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.35680240392684937,
+      "rewards/margins": 3.714186668395996,
+      "rewards/rejected": -3.357384443283081,
+      "step": 217,
+      "train_speed(iter/s)": 0.005424
+    },
+    {
+      "epoch": 0.6231353282715498,
+      "grad_norm": 8.464821815490723,
+      "learning_rate": 0.00018670769992628743,
+      "logits/chosen": -0.9288697838783264,
+      "logits/rejected": -0.9361052513122559,
+      "logps/chosen": -7.309250354766846,
+      "logps/rejected": -55.75934982299805,
+      "loss": 0.5372158288955688,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3037770390510559,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2024904191493988,
+      "rewards/margins": 4.475667476654053,
+      "rewards/rejected": -4.273177146911621,
+      "step": 218,
+      "train_speed(iter/s)": 0.005424
+    },
+    {
+      "epoch": 0.6259937472085753,
+      "grad_norm": 7.969956398010254,
+      "learning_rate": 0.00018654981639488577,
+      "logits/chosen": -0.8915677070617676,
+      "logits/rejected": -0.8954809904098511,
+      "logps/chosen": -10.680267333984375,
+      "logps/rejected": -46.57345199584961,
+      "loss": 0.4946494400501251,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.25526732206344604,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.10154397785663605,
+      "rewards/margins": 3.4904913902282715,
+      "rewards/rejected": -3.3889474868774414,
+      "step": 219,
+      "train_speed(iter/s)": 0.005423
+    },
+    {
+      "epoch": 0.6288521661456007,
+      "grad_norm": 4.48678731918335,
+      "learning_rate": 0.00018639106830820712,
+      "logits/chosen": -0.8933414816856384,
+      "logits/rejected": -0.9061769247055054,
+      "logps/chosen": -5.251230239868164,
+      "logps/rejected": -57.87230682373047,
+      "loss": 0.4995880722999573,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3236704170703888,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.28504300117492676,
+      "rewards/margins": 4.6635212898254395,
+      "rewards/rejected": -4.378478527069092,
+      "step": 220,
+      "train_speed(iter/s)": 0.005423
+    },
+    {
+      "epoch": 0.6317105850826261,
+      "grad_norm": 9.096211433410645,
+      "learning_rate": 0.00018623145725200278,
+      "logits/chosen": -0.8734514713287354,
+      "logits/rejected": -0.8916375637054443,
+      "logps/chosen": -1.4930850267410278,
+      "logps/rejected": -66.85340118408203,
+      "loss": 0.18740399181842804,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13703888654708862,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.37862882018089294,
+      "rewards/margins": 5.180389881134033,
+      "rewards/rejected": -4.801761150360107,
+      "step": 221,
+      "train_speed(iter/s)": 0.005423
+    },
+    {
+      "epoch": 0.6345690040196517,
+      "grad_norm": 26.261280059814453,
+      "learning_rate": 0.0001860709848206446,
+      "logits/chosen": -0.8820620775222778,
+      "logits/rejected": -0.8966219425201416,
+      "logps/chosen": -2.2250847816467285,
+      "logps/rejected": -45.76300048828125,
+      "loss": 0.3367112874984741,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.19873520731925964,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4386615753173828,
+      "rewards/margins": 3.72398042678833,
+      "rewards/rejected": -3.2853188514709473,
+      "step": 222,
+      "train_speed(iter/s)": 0.005423
+    },
+    {
+      "epoch": 0.6374274229566771,
+      "grad_norm": 22.972919464111328,
+      "learning_rate": 0.00018590965261710855,
+      "logits/chosen": -0.8403002619743347,
+      "logits/rejected": -0.8476570248603821,
+      "logps/chosen": -8.319016456604004,
+      "logps/rejected": -53.541221618652344,
+      "loss": 0.6817492842674255,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4776597023010254,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1935867965221405,
+      "rewards/margins": 4.028127670288086,
+      "rewards/rejected": -3.834541082382202,
+      "step": 223,
+      "train_speed(iter/s)": 0.005422
+    },
+    {
+      "epoch": 0.6402858418937025,
+      "grad_norm": 2.3034980297088623,
+      "learning_rate": 0.0001857474622529592,
+      "logits/chosen": -0.8160367608070374,
+      "logits/rejected": -0.8165281414985657,
+      "logps/chosen": -12.484249114990234,
+      "logps/rejected": -37.48336410522461,
+      "loss": 0.971846878528595,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.7017231583595276,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3721400499343872,
+      "rewards/margins": 3.103872060775757,
+      "rewards/rejected": -2.73173189163208,
+      "step": 224,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.643144260830728,
+      "grad_norm": 5.584660530090332,
+      "learning_rate": 0.00018558441534833326,
+      "logits/chosen": -0.8463850021362305,
+      "logits/rejected": -0.8546481728553772,
+      "logps/chosen": -6.676924228668213,
+      "logps/rejected": -42.02546310424805,
+      "loss": 0.3610745072364807,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22932007908821106,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.23803219199180603,
+      "rewards/margins": 3.133592367172241,
+      "rewards/rejected": -2.8955600261688232,
+      "step": 225,
+      "train_speed(iter/s)": 0.005422
+    },
+    {
+      "epoch": 0.6460026797677535,
+      "grad_norm": 19.920928955078125,
+      "learning_rate": 0.0001854205135319235,
+      "logits/chosen": -0.804871678352356,
+      "logits/rejected": -0.8120392560958862,
+      "logps/chosen": -4.961920738220215,
+      "logps/rejected": -35.829368591308594,
+      "loss": 0.5139291286468506,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.36350739002227783,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3230186402797699,
+      "rewards/margins": 2.9854631423950195,
+      "rewards/rejected": -2.662444591522217,
+      "step": 226,
+      "train_speed(iter/s)": 0.005422
+    },
+    {
+      "epoch": 0.6488610987047789,
+      "grad_norm": 4.711721420288086,
+      "learning_rate": 0.00018525575844096243,
+      "logits/chosen": -0.8603407740592957,
+      "logits/rejected": -0.8653955459594727,
+      "logps/chosen": -5.004653453826904,
+      "logps/rejected": -39.273414611816406,
+      "loss": 0.6492059230804443,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4128054976463318,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.22731980681419373,
+      "rewards/margins": 2.9322750568389893,
+      "rewards/rejected": -2.7049553394317627,
+      "step": 227,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.6517195176418044,
+      "grad_norm": 1.7484800815582275,
+      "learning_rate": 0.00018509015172120621,
+      "logits/chosen": -0.8575621843338013,
+      "logits/rejected": -0.8665353655815125,
+      "logps/chosen": -7.195815086364746,
+      "logps/rejected": -50.771728515625,
+      "loss": 0.31389814615249634,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2718980014324188,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.342627614736557,
+      "rewards/margins": 4.143442153930664,
+      "rewards/rejected": -3.8008151054382324,
+      "step": 228,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.6545779365788298,
+      "grad_norm": 2.002185344696045,
+      "learning_rate": 0.00018492369502691783,
+      "logits/chosen": -0.800703763961792,
+      "logits/rejected": -0.8078045845031738,
+      "logps/chosen": -2.682701349258423,
+      "logps/rejected": -49.849239349365234,
+      "loss": 0.30676865577697754,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22686806321144104,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.23367229104042053,
+      "rewards/margins": 4.046586036682129,
+      "rewards/rejected": -3.8129141330718994,
+      "step": 229,
+      "train_speed(iter/s)": 0.005422
+    },
+    {
+      "epoch": 0.6574363555158553,
+      "grad_norm": 1.5651706457138062,
+      "learning_rate": 0.00018475639002085088,
+      "logits/chosen": -0.8538340926170349,
+      "logits/rejected": -0.8597412109375,
+      "logps/chosen": -4.410967826843262,
+      "logps/rejected": -49.90096664428711,
+      "loss": 0.574176013469696,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3684491515159607,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.25362682342529297,
+      "rewards/margins": 4.066993713378906,
+      "rewards/rejected": -3.8133671283721924,
+      "step": 230,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.6602947744528808,
+      "grad_norm": 6.113332748413086,
+      "learning_rate": 0.00018458823837423272,
+      "logits/chosen": -0.7913177013397217,
+      "logits/rejected": -0.7997550368309021,
+      "logps/chosen": -4.025767803192139,
+      "logps/rejected": -53.99266052246094,
+      "loss": 0.2960505187511444,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18574629724025726,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2514365017414093,
+      "rewards/margins": 4.251288414001465,
+      "rewards/rejected": -3.999851703643799,
+      "step": 231,
+      "train_speed(iter/s)": 0.005422
+    },
+    {
+      "epoch": 0.6631531933899062,
+      "grad_norm": 19.446311950683594,
+      "learning_rate": 0.00018441924176674794,
+      "logits/chosen": -0.8168718218803406,
+      "logits/rejected": -0.8197526335716248,
+      "logps/chosen": -9.956727027893066,
+      "logps/rejected": -44.05134963989258,
+      "loss": 0.464748740196228,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.27714642882347107,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3305510878562927,
+      "rewards/margins": 3.2444560527801514,
+      "rewards/rejected": -2.913904905319214,
+      "step": 232,
+      "train_speed(iter/s)": 0.005422
+    },
+    {
+      "epoch": 0.6660116123269316,
+      "grad_norm": 4.284008979797363,
+      "learning_rate": 0.0001842494018865216,
+      "logits/chosen": -0.8440728187561035,
+      "logits/rejected": -0.8471089601516724,
+      "logps/chosen": -6.64088249206543,
+      "logps/rejected": -49.05445098876953,
+      "loss": 0.42048758268356323,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.28354334831237793,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.44579362869262695,
+      "rewards/margins": 4.0876898765563965,
+      "rewards/rejected": -3.6418962478637695,
+      "step": 233,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.6688700312639572,
+      "grad_norm": 2.603118896484375,
+      "learning_rate": 0.00018407872043010222,
+      "logits/chosen": -0.8436523079872131,
+      "logits/rejected": -0.8567028045654297,
+      "logps/chosen": -4.48859977722168,
+      "logps/rejected": -50.14689636230469,
+      "loss": 0.40216153860092163,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.27896279096603394,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.34187889099121094,
+      "rewards/margins": 4.033811569213867,
+      "rewards/rejected": -3.6919329166412354,
+      "step": 234,
+      "train_speed(iter/s)": 0.005422
+    },
+    {
+      "epoch": 0.6717284502009826,
+      "grad_norm": 2.3652243614196777,
+      "learning_rate": 0.00018390719910244487,
+      "logits/chosen": -0.8383113145828247,
+      "logits/rejected": -0.8405783772468567,
+      "logps/chosen": -7.634208679199219,
+      "logps/rejected": -47.9358024597168,
+      "loss": 0.44057831168174744,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3545606732368469,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4073534607887268,
+      "rewards/margins": 4.02724027633667,
+      "rewards/rejected": -3.619886875152588,
+      "step": 235,
+      "train_speed(iter/s)": 0.005423
+    },
+    {
+      "epoch": 0.674586869138008,
+      "grad_norm": 6.028226375579834,
+      "learning_rate": 0.00018373483961689433,
+      "logits/chosen": -0.791793942451477,
+      "logits/rejected": -0.7986205220222473,
+      "logps/chosen": -4.954317569732666,
+      "logps/rejected": -55.234737396240234,
+      "loss": 0.43700873851776123,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.30535393953323364,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.37826675176620483,
+      "rewards/margins": 4.826811790466309,
+      "rewards/rejected": -4.448545455932617,
+      "step": 236,
+      "train_speed(iter/s)": 0.005422
+    },
+    {
+      "epoch": 0.6774452880750335,
+      "grad_norm": 19.603124618530273,
+      "learning_rate": 0.0001835616436951677,
+      "logits/chosen": -0.8135831356048584,
+      "logits/rejected": -0.814205527305603,
+      "logps/chosen": -9.9032621383667,
+      "logps/rejected": -43.75004577636719,
+      "loss": 1.0214945077896118,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.7171285152435303,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.0824882835149765,
+      "rewards/margins": 3.4575212001800537,
+      "rewards/rejected": -3.375033378601074,
+      "step": 237,
+      "train_speed(iter/s)": 0.005422
+    },
+    {
+      "epoch": 0.680303707012059,
+      "grad_norm": 3.10172176361084,
+      "learning_rate": 0.0001833876130673374,
+      "logits/chosen": -0.8079448342323303,
+      "logits/rejected": -0.8127983808517456,
+      "logps/chosen": -5.635651588439941,
+      "logps/rejected": -47.28111267089844,
+      "loss": 0.7593626976013184,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5331795811653137,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1643129289150238,
+      "rewards/margins": 3.6442363262176514,
+      "rewards/rejected": -3.4799234867095947,
+      "step": 238,
+      "train_speed(iter/s)": 0.005422
+    },
+    {
+      "epoch": 0.6831621259490844,
+      "grad_norm": 27.822246551513672,
+      "learning_rate": 0.0001832127494718138,
+      "logits/chosen": -0.7551109194755554,
+      "logits/rejected": -0.7638161778450012,
+      "logps/chosen": -10.555828094482422,
+      "logps/rejected": -53.38033676147461,
+      "loss": 1.1436967849731445,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.7946538329124451,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.15928542613983154,
+      "rewards/margins": 3.742271900177002,
+      "rewards/rejected": -3.582986354827881,
+      "step": 239,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.6860205448861099,
+      "grad_norm": 16.83273696899414,
+      "learning_rate": 0.0001830370546553278,
+      "logits/chosen": -0.7728374600410461,
+      "logits/rejected": -0.7811495661735535,
+      "logps/chosen": -5.365185260772705,
+      "logps/rejected": -54.261924743652344,
+      "loss": 0.41165488958358765,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.31692206859588623,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3263813257217407,
+      "rewards/margins": 4.228232383728027,
+      "rewards/rejected": -3.901850938796997,
+      "step": 240,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.6888789638231353,
+      "grad_norm": 1.7355122566223145,
+      "learning_rate": 0.00018286053037291358,
+      "logits/chosen": -0.81122225522995,
+      "logits/rejected": -0.8211573362350464,
+      "logps/chosen": -7.20838737487793,
+      "logps/rejected": -61.72975158691406,
+      "loss": 0.6045428514480591,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.40412986278533936,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.11305103451013565,
+      "rewards/margins": 4.976436614990234,
+      "rewards/rejected": -4.863386154174805,
+      "step": 241,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.6917373827601608,
+      "grad_norm": 7.826045036315918,
+      "learning_rate": 0.00018268317838789088,
+      "logits/chosen": -0.7882257699966431,
+      "logits/rejected": -0.7961454391479492,
+      "logps/chosen": -9.31049919128418,
+      "logps/rejected": -60.73347473144531,
+      "loss": 0.47482648491859436,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3099362254142761,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.33366695046424866,
+      "rewards/margins": 5.084123134613037,
+      "rewards/rejected": -4.750456809997559,
+      "step": 242,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.6945958016971863,
+      "grad_norm": 4.641445636749268,
+      "learning_rate": 0.00018250500047184743,
+      "logits/chosen": -0.7924759387969971,
+      "logits/rejected": -0.8044589161872864,
+      "logps/chosen": -3.8715922832489014,
+      "logps/rejected": -84.22549438476562,
+      "loss": 0.2994554042816162,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1720796376466751,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3689011037349701,
+      "rewards/margins": 7.356629848480225,
+      "rewards/rejected": -6.987728595733643,
+      "step": 243,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.6974542206342117,
+      "grad_norm": 2.049391746520996,
+      "learning_rate": 0.00018232599840462127,
+      "logits/chosen": -0.7979660034179688,
+      "logits/rejected": -0.814594030380249,
+      "logps/chosen": -5.095368385314941,
+      "logps/rejected": -78.81214141845703,
+      "loss": 0.5495756268501282,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3513493835926056,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3352471590042114,
+      "rewards/margins": 6.755255222320557,
+      "rewards/rejected": -6.420008182525635,
+      "step": 244,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.7003126395712371,
+      "grad_norm": 6.190991401672363,
+      "learning_rate": 0.0001821461739742831,
+      "logits/chosen": -0.8227720856666565,
+      "logits/rejected": -0.8302331566810608,
+      "logps/chosen": -4.222079277038574,
+      "logps/rejected": -50.675010681152344,
+      "loss": 0.564222514629364,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.33342495560646057,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3969876766204834,
+      "rewards/margins": 4.388568878173828,
+      "rewards/rejected": -3.9915812015533447,
+      "step": 245,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.7031710585082627,
+      "grad_norm": 10.364058494567871,
+      "learning_rate": 0.00018196552897711805,
+      "logits/chosen": -0.8048458099365234,
+      "logits/rejected": -0.8072474002838135,
+      "logps/chosen": -10.411101341247559,
+      "logps/rejected": -56.306983947753906,
+      "loss": 1.0959556102752686,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.575545072555542,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.013596116565167904,
+      "rewards/margins": 4.414645671844482,
+      "rewards/rejected": -4.40104866027832,
+      "step": 246,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.7060294774452881,
+      "grad_norm": 1.150504469871521,
+      "learning_rate": 0.0001817840652176082,
+      "logits/chosen": -0.7576855421066284,
+      "logits/rejected": -0.7742222547531128,
+      "logps/chosen": -2.4694085121154785,
+      "logps/rejected": -71.76143646240234,
+      "loss": 0.3189219832420349,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16295407712459564,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4176994264125824,
+      "rewards/margins": 6.149940490722656,
+      "rewards/rejected": -5.732240676879883,
+      "step": 247,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.7088878963823135,
+      "grad_norm": 5.994204044342041,
+      "learning_rate": 0.00018160178450841424,
+      "logits/chosen": -0.866169273853302,
+      "logits/rejected": -0.8704244494438171,
+      "logps/chosen": -6.661141872406006,
+      "logps/rejected": -49.67292022705078,
+      "loss": 0.6027306914329529,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.34626996517181396,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.22591182589530945,
+      "rewards/margins": 4.158686637878418,
+      "rewards/rejected": -3.9327750205993652,
+      "step": 248,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.711746315319339,
+      "grad_norm": 5.770801067352295,
+      "learning_rate": 0.00018141868867035745,
+      "logits/chosen": -0.8344917893409729,
+      "logits/rejected": -0.8478953838348389,
+      "logps/chosen": -3.321021318435669,
+      "logps/rejected": -65.89788818359375,
+      "loss": 0.37574970722198486,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22786659002304077,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.46830183267593384,
+      "rewards/margins": 5.406087398529053,
+      "rewards/rejected": -4.9377851486206055,
+      "step": 249,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.7146047342563644,
+      "grad_norm": 10.37423038482666,
+      "learning_rate": 0.00018123477953240153,
+      "logits/chosen": -0.8937637805938721,
+      "logits/rejected": -0.8977770805358887,
+      "logps/chosen": -8.490730285644531,
+      "logps/rejected": -55.414119720458984,
+      "loss": 1.2775684595108032,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.8742450475692749,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.01779785007238388,
+      "rewards/margins": 4.217215061187744,
+      "rewards/rejected": -4.235013008117676,
+      "step": 250,
+      "train_speed(iter/s)": 0.005421
+    },
+    {
+      "epoch": 0.7174631531933899,
+      "grad_norm": 1.7192703485488892,
+      "learning_rate": 0.00018105005893163435,
+      "logits/chosen": -0.8390921354293823,
+      "logits/rejected": -0.8509503602981567,
+      "logps/chosen": -6.030505657196045,
+      "logps/rejected": -67.4354248046875,
+      "loss": 0.47158071398735046,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3380236029624939,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.40208595991134644,
+      "rewards/margins": 5.5573272705078125,
+      "rewards/rejected": -5.155241012573242,
+      "step": 251,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.7203215721304154,
+      "grad_norm": 2.977877140045166,
+      "learning_rate": 0.00018086452871324954,
+      "logits/chosen": -0.7667251825332642,
+      "logits/rejected": -0.7768008708953857,
+      "logps/chosen": -4.949960231781006,
+      "logps/rejected": -50.58625030517578,
+      "loss": 0.45571526885032654,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2604752779006958,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3296346664428711,
+      "rewards/margins": 3.986636161804199,
+      "rewards/rejected": -3.657001495361328,
+      "step": 252,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.7231799910674408,
+      "grad_norm": 5.2092461585998535,
+      "learning_rate": 0.0001806781907305281,
+      "logits/chosen": -0.7798765897750854,
+      "logits/rejected": -0.7910524606704712,
+      "logps/chosen": -5.809662818908691,
+      "logps/rejected": -53.34002685546875,
+      "loss": 0.8320525288581848,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5776523947715759,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.23829270899295807,
+      "rewards/margins": 4.524988174438477,
+      "rewards/rejected": -4.28669548034668,
+      "step": 253,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.7260384100044662,
+      "grad_norm": 1.3411738872528076,
+      "learning_rate": 0.00018049104684481984,
+      "logits/chosen": -0.7575161457061768,
+      "logits/rejected": -0.7671946287155151,
+      "logps/chosen": -2.434175729751587,
+      "logps/rejected": -57.82784652709961,
+      "loss": 0.20352256298065186,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.12845022976398468,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4981456398963928,
+      "rewards/margins": 5.003716945648193,
+      "rewards/rejected": -4.505571365356445,
+      "step": 254,
+      "train_speed(iter/s)": 0.00542
+    },
+    {
+      "epoch": 0.7288968289414918,
+      "grad_norm": 2.5868351459503174,
+      "learning_rate": 0.00018030309892552485,
+      "logits/chosen": -0.7499576210975647,
+      "logits/rejected": -0.7592740058898926,
+      "logps/chosen": -6.130841255187988,
+      "logps/rejected": -57.28644561767578,
+      "loss": 0.48349788784980774,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2539648711681366,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.3308817446231842,
+      "rewards/margins": 4.663580894470215,
+      "rewards/rejected": -4.332699775695801,
+      "step": 255,
+      "train_speed(iter/s)": 0.00542
+    },
+    {
+      "epoch": 0.7317552478785172,
+      "grad_norm": 4.378026962280273,
+      "learning_rate": 0.00018011434885007482,
+      "logits/chosen": -0.7099967002868652,
+      "logits/rejected": -0.7259005308151245,
+      "logps/chosen": -5.300999641418457,
+      "logps/rejected": -75.72197723388672,
+      "loss": 0.6189821362495422,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4664699137210846,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.32499435544013977,
+      "rewards/margins": 6.257512092590332,
+      "rewards/rejected": -5.932518482208252,
+      "step": 256,
+      "train_speed(iter/s)": 0.00542
+    },
+    {
+      "epoch": 0.7346136668155426,
+      "grad_norm": 5.1314287185668945,
+      "learning_rate": 0.00017992479850391417,
+      "logits/chosen": -0.7900992035865784,
+      "logits/rejected": -0.7963463068008423,
+      "logps/chosen": -7.288159370422363,
+      "logps/rejected": -65.37578582763672,
+      "loss": 0.7073066234588623,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.522219181060791,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.01863168179988861,
+      "rewards/margins": 5.2057108879089355,
+      "rewards/rejected": -5.224342346191406,
+      "step": 257,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.7374720857525681,
+      "grad_norm": 14.985835075378418,
+      "learning_rate": 0.0001797344497804814,
+      "logits/chosen": -0.8092799186706543,
+      "logits/rejected": -0.8284256458282471,
+      "logps/chosen": -3.5805835723876953,
+      "logps/rejected": -70.46089935302734,
+      "loss": 0.5027480125427246,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3328987956047058,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.34573665261268616,
+      "rewards/margins": 5.720893859863281,
+      "rewards/rejected": -5.375156879425049,
+      "step": 258,
+      "train_speed(iter/s)": 0.00542
+    },
+    {
+      "epoch": 0.7403305046895936,
+      "grad_norm": 1.8163223266601562,
+      "learning_rate": 0.0001795433045811901,
+      "logits/chosen": -0.7964171171188354,
+      "logits/rejected": -0.8145956993103027,
+      "logps/chosen": -6.411382675170898,
+      "logps/rejected": -82.747314453125,
+      "loss": 0.3530154526233673,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.27328839898109436,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3884195387363434,
+      "rewards/margins": 7.130291938781738,
+      "rewards/rejected": -6.741872787475586,
+      "step": 259,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.743188923626619,
+      "grad_norm": 6.400819301605225,
+      "learning_rate": 0.00017935136481540983,
+      "logits/chosen": -0.7889119386672974,
+      "logits/rejected": -0.7964754700660706,
+      "logps/chosen": -5.224812984466553,
+      "logps/rejected": -72.13189697265625,
+      "loss": 0.41807201504707336,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2358081340789795,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3295401632785797,
+      "rewards/margins": 6.076530933380127,
+      "rewards/rejected": -5.746991157531738,
+      "step": 260,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.7460473425636445,
+      "grad_norm": 12.328303337097168,
+      "learning_rate": 0.00017915863240044728,
+      "logits/chosen": -0.7348450422286987,
+      "logits/rejected": -0.7534111738204956,
+      "logps/chosen": -5.4054975509643555,
+      "logps/rejected": -68.88573455810547,
+      "loss": 0.7026913166046143,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.44665706157684326,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.20527832210063934,
+      "rewards/margins": 5.571951866149902,
+      "rewards/rejected": -5.366673469543457,
+      "step": 261,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.7489057615006699,
+      "grad_norm": 3.061285972595215,
+      "learning_rate": 0.0001789651092615269,
+      "logits/chosen": -0.7372769117355347,
+      "logits/rejected": -0.7543134689331055,
+      "logps/chosen": -6.071784973144531,
+      "logps/rejected": -75.79733276367188,
+      "loss": 0.5492207407951355,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.390069842338562,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.43368661403656006,
+      "rewards/margins": 6.3423871994018555,
+      "rewards/rejected": -5.908700466156006,
+      "step": 262,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.7517641804376954,
+      "grad_norm": 2.7509682178497314,
+      "learning_rate": 0.00017877079733177184,
+      "logits/chosen": -0.7882805466651917,
+      "logits/rejected": -0.7959960699081421,
+      "logps/chosen": -4.451222896575928,
+      "logps/rejected": -65.09577178955078,
+      "loss": 0.46701446175575256,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.29663312435150146,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2672671675682068,
+      "rewards/margins": 5.1345319747924805,
+      "rewards/rejected": -4.867265224456787,
+      "step": 263,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.7546225993747209,
+      "grad_norm": 7.0340471267700195,
+      "learning_rate": 0.0001785756985521845,
+      "logits/chosen": -0.7474167346954346,
+      "logits/rejected": -0.7586854696273804,
+      "logps/chosen": -2.838711738586426,
+      "logps/rejected": -66.19261169433594,
+      "loss": 0.2907945513725281,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1855563074350357,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3911718726158142,
+      "rewards/margins": 5.468771457672119,
+      "rewards/rejected": -5.077599048614502,
+      "step": 264,
+      "train_speed(iter/s)": 0.00542
+    },
+    {
+      "epoch": 0.7574810183117463,
+      "grad_norm": 6.862280368804932,
+      "learning_rate": 0.00017837981487162728,
+      "logits/chosen": -0.784568190574646,
+      "logits/rejected": -0.7910401821136475,
+      "logps/chosen": -5.12929630279541,
+      "logps/rejected": -56.361732482910156,
+      "loss": 0.40424007177352905,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.25972405076026917,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.37295520305633545,
+      "rewards/margins": 4.483123779296875,
+      "rewards/rejected": -4.110168933868408,
+      "step": 265,
+      "train_speed(iter/s)": 0.00542
+    },
+    {
+      "epoch": 0.7603394372487717,
+      "grad_norm": 4.63895845413208,
+      "learning_rate": 0.000178183148246803,
+      "logits/chosen": -0.7499387860298157,
+      "logits/rejected": -0.7595155239105225,
+      "logps/chosen": -5.115823268890381,
+      "logps/rejected": -58.23040771484375,
+      "loss": 0.36365264654159546,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2652009129524231,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5545862913131714,
+      "rewards/margins": 5.043620586395264,
+      "rewards/rejected": -4.489034652709961,
+      "step": 266,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.7631978561857973,
+      "grad_norm": 3.7681679725646973,
+      "learning_rate": 0.00017798570064223533,
+      "logits/chosen": -0.7498815059661865,
+      "logits/rejected": -0.7551140189170837,
+      "logps/chosen": -8.723041534423828,
+      "logps/rejected": -57.88478088378906,
+      "loss": 0.5662537813186646,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4090389013290405,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.42151832580566406,
+      "rewards/margins": 4.748777866363525,
+      "rewards/rejected": -4.327259540557861,
+      "step": 267,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.7660562751228227,
+      "grad_norm": 3.2269599437713623,
+      "learning_rate": 0.00017778747403024938,
+      "logits/chosen": -0.7659600973129272,
+      "logits/rejected": -0.771249532699585,
+      "logps/chosen": -4.629357814788818,
+      "logps/rejected": -65.65526580810547,
+      "loss": 0.35117611289024353,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2736473083496094,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4568191468715668,
+      "rewards/margins": 5.550890922546387,
+      "rewards/rejected": -5.094072341918945,
+      "step": 268,
+      "train_speed(iter/s)": 0.00542
+    },
+    {
+      "epoch": 0.7689146940598481,
+      "grad_norm": 2.8312392234802246,
+      "learning_rate": 0.00017758847039095168,
+      "logits/chosen": -0.798125684261322,
+      "logits/rejected": -0.8037543892860413,
+      "logps/chosen": -6.867776393890381,
+      "logps/rejected": -60.64920425415039,
+      "loss": 0.32864660024642944,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2230987399816513,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3507767915725708,
+      "rewards/margins": 5.041985988616943,
+      "rewards/rejected": -4.691208839416504,
+      "step": 269,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.7717731129968736,
+      "grad_norm": 13.008737564086914,
+      "learning_rate": 0.00017738869171221068,
+      "logits/chosen": -0.8288258910179138,
+      "logits/rejected": -0.8406282067298889,
+      "logps/chosen": -5.507926940917969,
+      "logps/rejected": -61.57422637939453,
+      "loss": 0.5156294703483582,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.35812243819236755,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3736186921596527,
+      "rewards/margins": 4.971932411193848,
+      "rewards/rejected": -4.59831428527832,
+      "step": 270,
+      "train_speed(iter/s)": 0.00542
+    },
+    {
+      "epoch": 0.7746315319338991,
+      "grad_norm": 3.5630056858062744,
+      "learning_rate": 0.00017718813998963677,
+      "logits/chosen": -0.8012739419937134,
+      "logits/rejected": -0.8092758059501648,
+      "logps/chosen": -3.510615587234497,
+      "logps/rejected": -64.04873657226562,
+      "loss": 0.32887405157089233,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22269977629184723,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4387923777103424,
+      "rewards/margins": 5.713708877563477,
+      "rewards/rejected": -5.274916648864746,
+      "step": 271,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.7774899508709245,
+      "grad_norm": 4.515592575073242,
+      "learning_rate": 0.0001769868172265623,
+      "logits/chosen": -0.7991639971733093,
+      "logits/rejected": -0.8151663541793823,
+      "logps/chosen": -6.522274017333984,
+      "logps/rejected": -66.93218231201172,
+      "loss": 0.36452019214630127,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22390373051166534,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.42787954211235046,
+      "rewards/margins": 5.180395603179932,
+      "rewards/rejected": -4.75251579284668,
+      "step": 272,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.78034836980795,
+      "grad_norm": 5.741221904754639,
+      "learning_rate": 0.00017678472543402167,
+      "logits/chosen": -0.8168489336967468,
+      "logits/rejected": -0.8267149329185486,
+      "logps/chosen": -7.215493679046631,
+      "logps/rejected": -74.48379516601562,
+      "loss": 0.43839821219444275,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.27808046340942383,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.10016904771327972,
+      "rewards/margins": 6.203993797302246,
+      "rewards/rejected": -6.103825092315674,
+      "step": 273,
+      "train_speed(iter/s)": 0.005419
+    },
+    {
+      "epoch": 0.7832067887449754,
+      "grad_norm": 32.49665451049805,
+      "learning_rate": 0.0001765818666307312,
+      "logits/chosen": -0.829395592212677,
+      "logits/rejected": -0.8386520147323608,
+      "logps/chosen": -11.311494827270508,
+      "logps/rejected": -73.18070983886719,
+      "loss": 0.5425660014152527,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.36316078901290894,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.15256868302822113,
+      "rewards/margins": 6.144671440124512,
+      "rewards/rejected": -5.992103099822998,
+      "step": 274,
+      "train_speed(iter/s)": 0.005418
+    },
+    {
+      "epoch": 0.7860652076820009,
+      "grad_norm": 3.230365753173828,
+      "learning_rate": 0.00017637824284306896,
+      "logits/chosen": -0.8549230098724365,
+      "logits/rejected": -0.8643885850906372,
+      "logps/chosen": -6.326109409332275,
+      "logps/rejected": -79.64250183105469,
+      "loss": 0.40403613448143005,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.24930235743522644,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.4288281202316284,
+      "rewards/margins": 7.1657514572143555,
+      "rewards/rejected": -6.7369232177734375,
+      "step": 275,
+      "train_speed(iter/s)": 0.005418
+    },
+    {
+      "epoch": 0.7889236266190264,
+      "grad_norm": 13.746660232543945,
+      "learning_rate": 0.00017617385610505445,
+      "logits/chosen": -0.8277623653411865,
+      "logits/rejected": -0.8511622548103333,
+      "logps/chosen": -5.98537015914917,
+      "logps/rejected": -88.5182876586914,
+      "loss": 0.43429312109947205,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3185354769229889,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.29758548736572266,
+      "rewards/margins": 7.396449089050293,
+      "rewards/rejected": -7.09886360168457,
+      "step": 276,
+      "train_speed(iter/s)": 0.005417
+    },
+    {
+      "epoch": 0.7917820455560518,
+      "grad_norm": 6.405646800994873,
+      "learning_rate": 0.0001759687084583285,
+      "logits/chosen": -0.7691607475280762,
+      "logits/rejected": -0.7741835713386536,
+      "logps/chosen": -9.308018684387207,
+      "logps/rejected": -51.25666046142578,
+      "loss": 0.8975799679756165,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5866126418113708,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1858431100845337,
+      "rewards/margins": 4.487247467041016,
+      "rewards/rejected": -4.30140495300293,
+      "step": 277,
+      "train_speed(iter/s)": 0.005417
+    },
+    {
+      "epoch": 0.7946404644930772,
+      "grad_norm": 2.4748454093933105,
+      "learning_rate": 0.00017576280195213265,
+      "logits/chosen": -0.7853768467903137,
+      "logits/rejected": -0.7936210036277771,
+      "logps/chosen": -5.454078674316406,
+      "logps/rejected": -75.30870056152344,
+      "loss": 0.2936534583568573,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18900159001350403,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4082258343696594,
+      "rewards/margins": 6.508139610290527,
+      "rewards/rejected": -6.099913597106934,
+      "step": 278,
+      "train_speed(iter/s)": 0.005417
+    },
+    {
+      "epoch": 0.7974988834301028,
+      "grad_norm": 34.00475311279297,
+      "learning_rate": 0.00017555613864328877,
+      "logits/chosen": -0.7811096906661987,
+      "logits/rejected": -0.7873689532279968,
+      "logps/chosen": -6.8495683670043945,
+      "logps/rejected": -55.13431930541992,
+      "loss": 0.637625515460968,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3317420780658722,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.15362443029880524,
+      "rewards/margins": 4.4968953132629395,
+      "rewards/rejected": -4.343270778656006,
+      "step": 279,
+      "train_speed(iter/s)": 0.005417
+    },
+    {
+      "epoch": 0.8003573023671282,
+      "grad_norm": 9.747983932495117,
+      "learning_rate": 0.00017534872059617854,
+      "logits/chosen": -0.7341008186340332,
+      "logits/rejected": -0.7383145093917847,
+      "logps/chosen": -9.441812515258789,
+      "logps/rejected": -52.60177993774414,
+      "loss": 0.7760030031204224,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5588037371635437,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.16166743636131287,
+      "rewards/margins": 4.042746067047119,
+      "rewards/rejected": -3.8810782432556152,
+      "step": 280,
+      "train_speed(iter/s)": 0.005417
+    },
+    {
+      "epoch": 0.8032157213041536,
+      "grad_norm": 3.125258684158325,
+      "learning_rate": 0.00017514054988272278,
+      "logits/chosen": -0.76849764585495,
+      "logits/rejected": -0.7703225016593933,
+      "logps/chosen": -7.879859924316406,
+      "logps/rejected": -46.87977600097656,
+      "loss": 0.40426021814346313,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22946256399154663,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1921083629131317,
+      "rewards/margins": 3.796980142593384,
+      "rewards/rejected": -3.6048717498779297,
+      "step": 281,
+      "train_speed(iter/s)": 0.005418
+    },
+    {
+      "epoch": 0.8060741402411791,
+      "grad_norm": 12.288455963134766,
+      "learning_rate": 0.00017493162858236077,
+      "logits/chosen": -0.7909262776374817,
+      "logits/rejected": -0.7904006242752075,
+      "logps/chosen": -6.4545183181762695,
+      "logps/rejected": -44.307891845703125,
+      "loss": 0.7655720114707947,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.49028268456459045,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.30812394618988037,
+      "rewards/margins": 3.476095199584961,
+      "rewards/rejected": -3.167971134185791,
+      "step": 282,
+      "train_speed(iter/s)": 0.005418
+    },
+    {
+      "epoch": 0.8089325591782045,
+      "grad_norm": 4.012885093688965,
+      "learning_rate": 0.00017472195878202954,
+      "logits/chosen": -0.7621877193450928,
+      "logits/rejected": -0.7748852968215942,
+      "logps/chosen": -5.457606315612793,
+      "logps/rejected": -58.349586486816406,
+      "loss": 0.5497166514396667,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3570767641067505,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3271387219429016,
+      "rewards/margins": 4.170994281768799,
+      "rewards/rejected": -3.843855857849121,
+      "step": 283,
+      "train_speed(iter/s)": 0.005417
+    },
+    {
+      "epoch": 0.81179097811523,
+      "grad_norm": 5.237916469573975,
+      "learning_rate": 0.00017451154257614287,
+      "logits/chosen": -0.7418099641799927,
+      "logits/rejected": -0.7571959495544434,
+      "logps/chosen": -3.4067182540893555,
+      "logps/rejected": -56.671165466308594,
+      "loss": 0.4136762320995331,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.28384339809417725,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3589616119861603,
+      "rewards/margins": 4.685504913330078,
+      "rewards/rejected": -4.32654333114624,
+      "step": 284,
+      "train_speed(iter/s)": 0.005417
+    },
+    {
+      "epoch": 0.8146493970522555,
+      "grad_norm": 1.4427282810211182,
+      "learning_rate": 0.0001743003820665705,
+      "logits/chosen": -0.7895207405090332,
+      "logits/rejected": -0.7963113188743591,
+      "logps/chosen": -5.939748764038086,
+      "logps/rejected": -61.93427276611328,
+      "loss": 0.37020552158355713,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.25463271141052246,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4543488621711731,
+      "rewards/margins": 5.030240058898926,
+      "rewards/rejected": -4.575891494750977,
+      "step": 285,
+      "train_speed(iter/s)": 0.005417
+    },
+    {
+      "epoch": 0.8175078159892809,
+      "grad_norm": 1.0243815183639526,
+      "learning_rate": 0.00017408847936261718,
+      "logits/chosen": -0.7718304395675659,
+      "logits/rejected": -0.7830629348754883,
+      "logps/chosen": -6.110411643981934,
+      "logps/rejected": -74.25471496582031,
+      "loss": 0.2691524624824524,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1934249997138977,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.37820330262184143,
+      "rewards/margins": 5.837207794189453,
+      "rewards/rejected": -5.4590044021606445,
+      "step": 286,
+      "train_speed(iter/s)": 0.005416
+    },
+    {
+      "epoch": 0.8203662349263063,
+      "grad_norm": 3.313394784927368,
+      "learning_rate": 0.00017387583658100142,
+      "logits/chosen": -0.8310220837593079,
+      "logits/rejected": -0.8387812376022339,
+      "logps/chosen": -4.55296516418457,
+      "logps/rejected": -61.40394592285156,
+      "loss": 0.4389451742172241,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.27197274565696716,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.37971678376197815,
+      "rewards/margins": 5.221004009246826,
+      "rewards/rejected": -4.841287136077881,
+      "step": 287,
+      "train_speed(iter/s)": 0.005416
+    },
+    {
+      "epoch": 0.8232246538633319,
+      "grad_norm": 2.1154696941375732,
+      "learning_rate": 0.0001736624558458344,
+      "logits/chosen": -0.8145501017570496,
+      "logits/rejected": -0.8243207931518555,
+      "logps/chosen": -5.528270721435547,
+      "logps/rejected": -68.98825073242188,
+      "loss": 0.3845535218715668,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3077916204929352,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2718258500099182,
+      "rewards/margins": 5.621387004852295,
+      "rewards/rejected": -5.34956169128418,
+      "step": 288,
+      "train_speed(iter/s)": 0.005416
+    },
+    {
+      "epoch": 0.8260830728003573,
+      "grad_norm": 24.870929718017578,
+      "learning_rate": 0.00017344833928859902,
+      "logits/chosen": -0.8270291090011597,
+      "logits/rejected": -0.835890531539917,
+      "logps/chosen": -9.923492431640625,
+      "logps/rejected": -67.6069107055664,
+      "loss": 0.7075707316398621,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5235861539840698,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.13183848559856415,
+      "rewards/margins": 5.321423530578613,
+      "rewards/rejected": -5.189585208892822,
+      "step": 289,
+      "train_speed(iter/s)": 0.005416
+    },
+    {
+      "epoch": 0.8289414917373827,
+      "grad_norm": 44.18141555786133,
+      "learning_rate": 0.00017323348904812807,
+      "logits/chosen": -0.849445641040802,
+      "logits/rejected": -0.8580566644668579,
+      "logps/chosen": -6.3443121910095215,
+      "logps/rejected": -70.16064453125,
+      "loss": 0.8740846514701843,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6279979944229126,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.03094465658068657,
+      "rewards/margins": 5.86611795425415,
+      "rewards/rejected": -5.835173606872559,
+      "step": 290,
+      "train_speed(iter/s)": 0.005416
+    },
+    {
+      "epoch": 0.8317999106744082,
+      "grad_norm": 2.5713086128234863,
+      "learning_rate": 0.00017301790727058345,
+      "logits/chosen": -0.8377387523651123,
+      "logits/rejected": -0.8420360088348389,
+      "logps/chosen": -8.347972869873047,
+      "logps/rejected": -60.33742141723633,
+      "loss": 0.5106192827224731,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.41129249334335327,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3251681327819824,
+      "rewards/margins": 5.310262680053711,
+      "rewards/rejected": -4.985095024108887,
+      "step": 291,
+      "train_speed(iter/s)": 0.005416
+    },
+    {
+      "epoch": 0.8346583296114337,
+      "grad_norm": 7.104504108428955,
+      "learning_rate": 0.0001728015961094343,
+      "logits/chosen": -0.8455878496170044,
+      "logits/rejected": -0.8570619821548462,
+      "logps/chosen": -6.6951751708984375,
+      "logps/rejected": -81.03128814697266,
+      "loss": 0.44324812293052673,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.32767486572265625,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.036870718002319336,
+      "rewards/margins": 6.7979583740234375,
+      "rewards/rejected": -6.761086940765381,
+      "step": 292,
+      "train_speed(iter/s)": 0.005416
+    },
+    {
+      "epoch": 0.8375167485484591,
+      "grad_norm": 4.131809234619141,
+      "learning_rate": 0.00017258455772543573,
+      "logits/chosen": -0.8040283918380737,
+      "logits/rejected": -0.811447024345398,
+      "logps/chosen": -8.506138801574707,
+      "logps/rejected": -61.88532638549805,
+      "loss": 0.36702975630760193,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2761310636997223,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.37248650193214417,
+      "rewards/margins": 5.04264497756958,
+      "rewards/rejected": -4.670158386230469,
+      "step": 293,
+      "train_speed(iter/s)": 0.005415
+    },
+    {
+      "epoch": 0.8403751674854846,
+      "grad_norm": 15.525777816772461,
+      "learning_rate": 0.00017236679428660719,
+      "logits/chosen": -0.8311742544174194,
+      "logits/rejected": -0.8395902514457703,
+      "logps/chosen": -5.339737892150879,
+      "logps/rejected": -54.9642333984375,
+      "loss": 0.6541984677314758,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4498509168624878,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.268288254737854,
+      "rewards/margins": 4.510727882385254,
+      "rewards/rejected": -4.242439270019531,
+      "step": 294,
+      "train_speed(iter/s)": 0.005415
+    },
+    {
+      "epoch": 0.84323358642251,
+      "grad_norm": 2.6126153469085693,
+      "learning_rate": 0.0001721483079682106,
+      "logits/chosen": -0.8357363939285278,
+      "logits/rejected": -0.8407280445098877,
+      "logps/chosen": -6.462512969970703,
+      "logps/rejected": -66.7704086303711,
+      "loss": 0.35049688816070557,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.23473332822322845,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3646930754184723,
+      "rewards/margins": 5.821950912475586,
+      "rewards/rejected": -5.4572577476501465,
+      "step": 295,
+      "train_speed(iter/s)": 0.005415
+    },
+    {
+      "epoch": 0.8460920053595355,
+      "grad_norm": 1.6905546188354492,
+      "learning_rate": 0.00017192910095272898,
+      "logits/chosen": -0.8126969337463379,
+      "logits/rejected": -0.8227717876434326,
+      "logps/chosen": -4.0240278244018555,
+      "logps/rejected": -69.86576080322266,
+      "loss": 0.36738839745521545,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2895388901233673,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3507424294948578,
+      "rewards/margins": 5.908805847167969,
+      "rewards/rejected": -5.558063507080078,
+      "step": 296,
+      "train_speed(iter/s)": 0.005415
+    },
+    {
+      "epoch": 0.848950424296561,
+      "grad_norm": 0.7303423881530762,
+      "learning_rate": 0.00017170917542984443,
+      "logits/chosen": -0.8312556743621826,
+      "logits/rejected": -0.8409414291381836,
+      "logps/chosen": -3.1676483154296875,
+      "logps/rejected": -75.8035888671875,
+      "loss": 0.19972439110279083,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.17977482080459595,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2756307125091553,
+      "rewards/margins": 6.446622848510742,
+      "rewards/rejected": -6.170992851257324,
+      "step": 297,
+      "train_speed(iter/s)": 0.005415
+    },
+    {
+      "epoch": 0.8518088432335864,
+      "grad_norm": 2.5636391639709473,
+      "learning_rate": 0.00017148853359641626,
+      "logits/chosen": -0.839911699295044,
+      "logits/rejected": -0.8532472848892212,
+      "logps/chosen": -4.941699028015137,
+      "logps/rejected": -74.04802703857422,
+      "loss": 0.6130093932151794,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4153917729854584,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.273116797208786,
+      "rewards/margins": 6.132874965667725,
+      "rewards/rejected": -5.859758377075195,
+      "step": 298,
+      "train_speed(iter/s)": 0.005415
+    },
+    {
+      "epoch": 0.8546672621706118,
+      "grad_norm": 5.344427108764648,
+      "learning_rate": 0.00017126717765645908,
+      "logits/chosen": -0.8634560704231262,
+      "logits/rejected": -0.8785987496376038,
+      "logps/chosen": -5.115147590637207,
+      "logps/rejected": -77.06375885009766,
+      "loss": 0.3565768003463745,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2940167486667633,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4229351878166199,
+      "rewards/margins": 6.207124710083008,
+      "rewards/rejected": -5.7841901779174805,
+      "step": 299,
+      "train_speed(iter/s)": 0.005414
+    },
+    {
+      "epoch": 0.8575256811076374,
+      "grad_norm": 4.262563228607178,
+      "learning_rate": 0.00017104510982112085,
+      "logits/chosen": -0.8862661719322205,
+      "logits/rejected": -0.8961150646209717,
+      "logps/chosen": -6.4602837562561035,
+      "logps/rejected": -62.53819274902344,
+      "loss": 0.3667752146720886,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2564036250114441,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.39489609003067017,
+      "rewards/margins": 5.051397800445557,
+      "rewards/rejected": -4.656500816345215,
+      "step": 300,
+      "train_speed(iter/s)": 0.005414
+    },
+    {
+      "epoch": 0.8575256811076374,
+      "eval_logits/chosen": -0.8734048008918762,
+      "eval_logits/rejected": -0.8801189661026001,
+      "eval_logps/chosen": -5.852853298187256,
+      "eval_logps/rejected": -64.62750244140625,
+      "eval_loss": 0.3423595726490021,
+      "eval_nll_loss": 0.26759958267211914,
+      "eval_rewards/accuracies": 0.9734513163566589,
+      "eval_rewards/chosen": 0.39209461212158203,
+      "eval_rewards/margins": 5.445755481719971,
+      "eval_rewards/rejected": -5.0536603927612305,
+      "eval_runtime": 267.2493,
+      "eval_samples_per_second": 0.423,
+      "eval_steps_per_second": 0.423,
+      "step": 300
+    },
+    {
+      "epoch": 0.8603841000446628,
+      "grad_norm": 20.735666275024414,
+      "learning_rate": 0.00017082233230866062,
+      "logits/chosen": -0.8673299551010132,
+      "logits/rejected": -0.876368522644043,
+      "logps/chosen": -9.548802375793457,
+      "logps/rejected": -63.38462829589844,
+      "loss": 0.6720445156097412,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4585096538066864,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.06072264164686203,
+      "rewards/margins": 5.162600517272949,
+      "rewards/rejected": -5.1018781661987305,
+      "step": 301,
+      "train_speed(iter/s)": 0.005387
+    },
+    {
+      "epoch": 0.8632425189816882,
+      "grad_norm": 1.758367657661438,
+      "learning_rate": 0.00017059884734442658,
+      "logits/chosen": -0.8942989706993103,
+      "logits/rejected": -0.9041098356246948,
+      "logps/chosen": -4.6497111320495605,
+      "logps/rejected": -70.84129333496094,
+      "loss": 0.4637173116207123,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.33638495206832886,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3641985356807709,
+      "rewards/margins": 5.980964183807373,
+      "rewards/rejected": -5.61676549911499,
+      "step": 302,
+      "train_speed(iter/s)": 0.005387
+    },
+    {
+      "epoch": 0.8661009379187137,
+      "grad_norm": 1.3300881385803223,
+      "learning_rate": 0.0001703746571608337,
+      "logits/chosen": -0.9206355214118958,
+      "logits/rejected": -0.927955687046051,
+      "logps/chosen": -4.747969627380371,
+      "logps/rejected": -60.195621490478516,
+      "loss": 0.2779415249824524,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2364496886730194,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.32591214776039124,
+      "rewards/margins": 5.419829845428467,
+      "rewards/rejected": -5.093916893005371,
+      "step": 303,
+      "train_speed(iter/s)": 0.005387
+    },
+    {
+      "epoch": 0.8689593568557392,
+      "grad_norm": 8.9483003616333,
+      "learning_rate": 0.00017014976399734144,
+      "logits/chosen": -0.9002286195755005,
+      "logits/rejected": -0.9120714068412781,
+      "logps/chosen": -3.8572702407836914,
+      "logps/rejected": -74.53612518310547,
+      "loss": 0.3219146728515625,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.24406777322292328,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3388945460319519,
+      "rewards/margins": 6.05419397354126,
+      "rewards/rejected": -5.715299129486084,
+      "step": 304,
+      "train_speed(iter/s)": 0.005387
+    },
+    {
+      "epoch": 0.8718177757927646,
+      "grad_norm": 24.502073287963867,
+      "learning_rate": 0.00016992417010043142,
+      "logits/chosen": -0.9110714197158813,
+      "logits/rejected": -0.9145501255989075,
+      "logps/chosen": -5.770493984222412,
+      "logps/rejected": -52.358863830566406,
+      "loss": 0.3569178283214569,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2734857201576233,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3025267720222473,
+      "rewards/margins": 4.277225971221924,
+      "rewards/rejected": -3.974699020385742,
+      "step": 305,
+      "train_speed(iter/s)": 0.005387
+    },
+    {
+      "epoch": 0.8746761947297901,
+      "grad_norm": 1.132225513458252,
+      "learning_rate": 0.00016969787772358493,
+      "logits/chosen": -0.883800208568573,
+      "logits/rejected": -0.8920851945877075,
+      "logps/chosen": -5.719818592071533,
+      "logps/rejected": -62.382938385009766,
+      "loss": 0.33317288756370544,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.236215278506279,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.10508742928504944,
+      "rewards/margins": 5.275176048278809,
+      "rewards/rejected": -5.170088768005371,
+      "step": 306,
+      "train_speed(iter/s)": 0.005387
+    },
+    {
+      "epoch": 0.8775346136668155,
+      "grad_norm": 2.1382904052734375,
+      "learning_rate": 0.00016947088912726052,
+      "logits/chosen": -0.8931231498718262,
+      "logits/rejected": -0.9039135575294495,
+      "logps/chosen": -6.410434246063232,
+      "logps/rejected": -71.76957702636719,
+      "loss": 0.36886727809906006,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2760961651802063,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.46529263257980347,
+      "rewards/margins": 6.17685079574585,
+      "rewards/rejected": -5.7115583419799805,
+      "step": 307,
+      "train_speed(iter/s)": 0.005388
+    },
+    {
+      "epoch": 0.880393032603841,
+      "grad_norm": 8.12493896484375,
+      "learning_rate": 0.00016924320657887127,
+      "logits/chosen": -0.8791784644126892,
+      "logits/rejected": -0.8861318230628967,
+      "logps/chosen": -5.161705017089844,
+      "logps/rejected": -60.26618576049805,
+      "loss": 0.4572223126888275,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.33235517144203186,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3299446403980255,
+      "rewards/margins": 5.036650657653809,
+      "rewards/rejected": -4.7067060470581055,
+      "step": 308,
+      "train_speed(iter/s)": 0.005388
+    },
+    {
+      "epoch": 0.8832514515408665,
+      "grad_norm": 1.3289014101028442,
+      "learning_rate": 0.00016901483235276228,
+      "logits/chosen": -0.8941358327865601,
+      "logits/rejected": -0.9037779569625854,
+      "logps/chosen": -6.27401876449585,
+      "logps/rejected": -71.07161712646484,
+      "loss": 0.3748854994773865,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2852364182472229,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4290279448032379,
+      "rewards/margins": 5.811855792999268,
+      "rewards/rejected": -5.382828235626221,
+      "step": 309,
+      "train_speed(iter/s)": 0.005388
+    },
+    {
+      "epoch": 0.8861098704778919,
+      "grad_norm": 4.328617095947266,
+      "learning_rate": 0.00016878576873018787,
+      "logits/chosen": -0.8742256760597229,
+      "logits/rejected": -0.8770206570625305,
+      "logps/chosen": -5.228196144104004,
+      "logps/rejected": -53.101741790771484,
+      "loss": 0.4242691397666931,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3180267810821533,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.34552180767059326,
+      "rewards/margins": 4.556669235229492,
+      "rewards/rejected": -4.211147308349609,
+      "step": 310,
+      "train_speed(iter/s)": 0.005388
+    },
+    {
+      "epoch": 0.8889682894149173,
+      "grad_norm": 1.7425999641418457,
+      "learning_rate": 0.00016855601799928876,
+      "logits/chosen": -0.9284868240356445,
+      "logits/rejected": -0.9364843368530273,
+      "logps/chosen": -3.691480875015259,
+      "logps/rejected": -63.92913818359375,
+      "loss": 0.22003982961177826,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15120568871498108,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5339685082435608,
+      "rewards/margins": 5.319494247436523,
+      "rewards/rejected": -4.785525321960449,
+      "step": 311,
+      "train_speed(iter/s)": 0.005388
+    },
+    {
+      "epoch": 0.8918267083519428,
+      "grad_norm": 0.9392454624176025,
+      "learning_rate": 0.00016832558245506935,
+      "logits/chosen": -0.8813795447349548,
+      "logits/rejected": -0.889714777469635,
+      "logps/chosen": -6.073534965515137,
+      "logps/rejected": -59.29630661010742,
+      "loss": 0.35096725821495056,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2775135636329651,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.43473172187805176,
+      "rewards/margins": 4.894845008850098,
+      "rewards/rejected": -4.460113048553467,
+      "step": 312,
+      "train_speed(iter/s)": 0.005388
+    },
+    {
+      "epoch": 0.8946851272889683,
+      "grad_norm": 1.2475719451904297,
+      "learning_rate": 0.0001680944643993747,
+      "logits/chosen": -0.9736749529838562,
+      "logits/rejected": -0.9871997237205505,
+      "logps/chosen": -4.467031478881836,
+      "logps/rejected": -61.235137939453125,
+      "loss": 0.2823341488838196,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.24649259448051453,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.43429625034332275,
+      "rewards/margins": 5.264223575592041,
+      "rewards/rejected": -4.829927921295166,
+      "step": 313,
+      "train_speed(iter/s)": 0.005389
+    },
+    {
+      "epoch": 0.8975435462259937,
+      "grad_norm": 3.567657709121704,
+      "learning_rate": 0.00016786266614086755,
+      "logits/chosen": -0.9802843332290649,
+      "logits/rejected": -0.9897909164428711,
+      "logps/chosen": -6.650794982910156,
+      "logps/rejected": -61.503173828125,
+      "loss": 0.5666725635528564,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.44352883100509644,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4951842427253723,
+      "rewards/margins": 5.488836765289307,
+      "rewards/rejected": -4.99365234375,
+      "step": 314,
+      "train_speed(iter/s)": 0.005389
+    },
+    {
+      "epoch": 0.9004019651630192,
+      "grad_norm": 1.4415922164916992,
+      "learning_rate": 0.00016763018999500518,
+      "logits/chosen": -0.9978210926055908,
+      "logits/rejected": -1.0054134130477905,
+      "logps/chosen": -4.759300231933594,
+      "logps/rejected": -55.411705017089844,
+      "loss": 0.3095281720161438,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.23518188297748566,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.41955995559692383,
+      "rewards/margins": 4.839236259460449,
+      "rewards/rejected": -4.419675827026367,
+      "step": 315,
+      "train_speed(iter/s)": 0.00539
+    },
+    {
+      "epoch": 0.9032603841000446,
+      "grad_norm": 1.454599142074585,
+      "learning_rate": 0.00016739703828401643,
+      "logits/chosen": -1.015478253364563,
+      "logits/rejected": -1.0180584192276,
+      "logps/chosen": -6.854709625244141,
+      "logps/rejected": -54.203224182128906,
+      "loss": 0.448214590549469,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.34830421209335327,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2672179639339447,
+      "rewards/margins": 4.653902530670166,
+      "rewards/rejected": -4.386684894561768,
+      "step": 316,
+      "train_speed(iter/s)": 0.00539
+    },
+    {
+      "epoch": 0.9061188030370702,
+      "grad_norm": 5.461782932281494,
+      "learning_rate": 0.00016716321333687848,
+      "logits/chosen": -0.986311674118042,
+      "logits/rejected": -1.0029571056365967,
+      "logps/chosen": -9.523213386535645,
+      "logps/rejected": -70.22506713867188,
+      "loss": 0.7210484147071838,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5311657190322876,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.31339961290359497,
+      "rewards/margins": 5.7375407218933105,
+      "rewards/rejected": -5.4241414070129395,
+      "step": 317,
+      "train_speed(iter/s)": 0.005391
+    },
+    {
+      "epoch": 0.9089772219740956,
+      "grad_norm": 5.448202610015869,
+      "learning_rate": 0.0001669287174892934,
+      "logits/chosen": -1.0209075212478638,
+      "logits/rejected": -1.0312987565994263,
+      "logps/chosen": -6.145677089691162,
+      "logps/rejected": -76.97433471679688,
+      "loss": 0.49676480889320374,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3716152012348175,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1940852701663971,
+      "rewards/margins": 6.112122535705566,
+      "rewards/rejected": -5.9180378913879395,
+      "step": 318,
+      "train_speed(iter/s)": 0.005391
+    },
+    {
+      "epoch": 0.911835640911121,
+      "grad_norm": 4.858055591583252,
+      "learning_rate": 0.0001666935530836651,
+      "logits/chosen": -1.0205280780792236,
+      "logits/rejected": -1.027929663658142,
+      "logps/chosen": -8.096317291259766,
+      "logps/rejected": -72.40553283691406,
+      "loss": 0.4033167362213135,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3082987666130066,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.33014655113220215,
+      "rewards/margins": 6.211818218231201,
+      "rewards/rejected": -5.881671905517578,
+      "step": 319,
+      "train_speed(iter/s)": 0.005391
+    },
+    {
+      "epoch": 0.9146940598481464,
+      "grad_norm": 25.689199447631836,
+      "learning_rate": 0.00016645772246907568,
+      "logits/chosen": -1.0175788402557373,
+      "logits/rejected": -1.0250442028045654,
+      "logps/chosen": -5.779062271118164,
+      "logps/rejected": -67.13630676269531,
+      "loss": 0.4840015172958374,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3824375867843628,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3088567554950714,
+      "rewards/margins": 5.571361064910889,
+      "rewards/rejected": -5.2625041007995605,
+      "step": 320,
+      "train_speed(iter/s)": 0.005391
+    },
+    {
+      "epoch": 0.917552478785172,
+      "grad_norm": 2.411203145980835,
+      "learning_rate": 0.00016622122800126208,
+      "logits/chosen": -1.0164726972579956,
+      "logits/rejected": -1.0213780403137207,
+      "logps/chosen": -8.589577674865723,
+      "logps/rejected": -71.54415130615234,
+      "loss": 0.32722774147987366,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.26191988587379456,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.30120402574539185,
+      "rewards/margins": 5.828619956970215,
+      "rewards/rejected": -5.527415752410889,
+      "step": 321,
+      "train_speed(iter/s)": 0.005391
+    },
+    {
+      "epoch": 0.9204108977221974,
+      "grad_norm": 10.201706886291504,
+      "learning_rate": 0.0001659840720425926,
+      "logits/chosen": -1.0485953092575073,
+      "logits/rejected": -1.0498896837234497,
+      "logps/chosen": -11.473772048950195,
+      "logps/rejected": -59.5853271484375,
+      "loss": 0.7703381776809692,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6231206059455872,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.014922700822353363,
+      "rewards/margins": 4.745216369628906,
+      "rewards/rejected": -4.730293273925781,
+      "step": 322,
+      "train_speed(iter/s)": 0.005391
+    },
+    {
+      "epoch": 0.9232693166592228,
+      "grad_norm": 8.294495582580566,
+      "learning_rate": 0.00016574625696204326,
+      "logits/chosen": -1.0103312730789185,
+      "logits/rejected": -1.0260792970657349,
+      "logps/chosen": -6.177521228790283,
+      "logps/rejected": -68.23878479003906,
+      "loss": 0.4851343035697937,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.31362757086753845,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.342265784740448,
+      "rewards/margins": 5.5506744384765625,
+      "rewards/rejected": -5.208408355712891,
+      "step": 323,
+      "train_speed(iter/s)": 0.005392
+    },
+    {
+      "epoch": 0.9261277355962483,
+      "grad_norm": 1.835939884185791,
+      "learning_rate": 0.000165507785135174,
+      "logits/chosen": -0.9879414439201355,
+      "logits/rejected": -0.9929082989692688,
+      "logps/chosen": -6.7657294273376465,
+      "logps/rejected": -73.79751586914062,
+      "loss": 0.42144203186035156,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.33192363381385803,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4221758246421814,
+      "rewards/margins": 6.4959306716918945,
+      "rewards/rejected": -6.07375431060791,
+      "step": 324,
+      "train_speed(iter/s)": 0.005392
+    },
+    {
+      "epoch": 0.9289861545332738,
+      "grad_norm": 1.5187759399414062,
+      "learning_rate": 0.00016526865894410524,
+      "logits/chosen": -0.9831660389900208,
+      "logits/rejected": -0.9866008758544922,
+      "logps/chosen": -1.8688807487487793,
+      "logps/rejected": -69.01907348632812,
+      "loss": 0.14090491831302643,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.10596515983343124,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5663707852363586,
+      "rewards/margins": 6.12815523147583,
+      "rewards/rejected": -5.561784267425537,
+      "step": 325,
+      "train_speed(iter/s)": 0.005392
+    },
+    {
+      "epoch": 0.9318445734702993,
+      "grad_norm": 9.269432067871094,
+      "learning_rate": 0.0001650288807774937,
+      "logits/chosen": -0.9924958348274231,
+      "logits/rejected": -0.9997655749320984,
+      "logps/chosen": -4.4954833984375,
+      "logps/rejected": -59.65481948852539,
+      "loss": 0.3288981020450592,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.27178075909614563,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.41092053055763245,
+      "rewards/margins": 5.2412567138671875,
+      "rewards/rejected": -4.830336093902588,
+      "step": 326,
+      "train_speed(iter/s)": 0.005393
+    },
+    {
+      "epoch": 0.9347029924073247,
+      "grad_norm": 3.1839118003845215,
+      "learning_rate": 0.0001647884530305089,
+      "logits/chosen": -1.017769455909729,
+      "logits/rejected": -1.0204215049743652,
+      "logps/chosen": -9.779426574707031,
+      "logps/rejected": -52.900611877441406,
+      "loss": 0.5431086421012878,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4328044354915619,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.40310975909233093,
+      "rewards/margins": 4.863814830780029,
+      "rewards/rejected": -4.460704803466797,
+      "step": 327,
+      "train_speed(iter/s)": 0.005393
+    },
+    {
+      "epoch": 0.9375614113443501,
+      "grad_norm": 14.360014915466309,
+      "learning_rate": 0.00016454737810480912,
+      "logits/chosen": -1.045483112335205,
+      "logits/rejected": -1.0522551536560059,
+      "logps/chosen": -6.289290428161621,
+      "logps/rejected": -57.062992095947266,
+      "loss": 0.8400169610977173,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6423981785774231,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.01210833340883255,
+      "rewards/margins": 4.530099391937256,
+      "rewards/rejected": -4.517991065979004,
+      "step": 328,
+      "train_speed(iter/s)": 0.005392
+    },
+    {
+      "epoch": 0.9404198302813757,
+      "grad_norm": 0.8805162906646729,
+      "learning_rate": 0.00016430565840851722,
+      "logits/chosen": -1.0371475219726562,
+      "logits/rejected": -1.0404698848724365,
+      "logps/chosen": -6.789125442504883,
+      "logps/rejected": -70.53579711914062,
+      "loss": 0.2656117379665375,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.24182505905628204,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2723756730556488,
+      "rewards/margins": 5.958362102508545,
+      "rewards/rejected": -5.685986518859863,
+      "step": 329,
+      "train_speed(iter/s)": 0.005392
+    },
+    {
+      "epoch": 0.9432782492184011,
+      "grad_norm": 33.674015045166016,
+      "learning_rate": 0.00016406329635619687,
+      "logits/chosen": -1.0247559547424316,
+      "logits/rejected": -1.0370457172393799,
+      "logps/chosen": -4.966681003570557,
+      "logps/rejected": -79.33086395263672,
+      "loss": 0.40524396300315857,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2835578918457031,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.25815021991729736,
+      "rewards/margins": 6.6955718994140625,
+      "rewards/rejected": -6.437422275543213,
+      "step": 330,
+      "train_speed(iter/s)": 0.005392
+    },
+    {
+      "epoch": 0.9461366681554265,
+      "grad_norm": 8.032293319702148,
+      "learning_rate": 0.00016382029436882826,
+      "logits/chosen": -0.9656038284301758,
+      "logits/rejected": -0.9722874760627747,
+      "logps/chosen": -9.293785095214844,
+      "logps/rejected": -55.016483306884766,
+      "loss": 0.7841230630874634,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.49341529607772827,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.19789664447307587,
+      "rewards/margins": 4.488459587097168,
+      "rewards/rejected": -4.290563106536865,
+      "step": 331,
+      "train_speed(iter/s)": 0.005392
+    },
+    {
+      "epoch": 0.948995087092452,
+      "grad_norm": 4.4950785636901855,
+      "learning_rate": 0.00016357665487378397,
+      "logits/chosen": -0.9956451654434204,
+      "logits/rejected": -1.0024536848068237,
+      "logps/chosen": -7.6442084312438965,
+      "logps/rejected": -71.45169830322266,
+      "loss": 0.5794447660446167,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4559825658798218,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3677746653556824,
+      "rewards/margins": 5.780731201171875,
+      "rewards/rejected": -5.412956714630127,
+      "step": 332,
+      "train_speed(iter/s)": 0.005392
+    },
+    {
+      "epoch": 0.9518535060294775,
+      "grad_norm": 10.342886924743652,
+      "learning_rate": 0.0001633323803048047,
+      "logits/chosen": -1.0820869207382202,
+      "logits/rejected": -1.0929756164550781,
+      "logps/chosen": -4.995148181915283,
+      "logps/rejected": -73.86285400390625,
+      "loss": 0.8048845529556274,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6948299407958984,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.15656176209449768,
+      "rewards/margins": 6.374897480010986,
+      "rewards/rejected": -6.2183356285095215,
+      "step": 333,
+      "train_speed(iter/s)": 0.005392
+    },
+    {
+      "epoch": 0.9547119249665029,
+      "grad_norm": 3.3627877235412598,
+      "learning_rate": 0.000163087473101975,
+      "logits/chosen": -0.9950565695762634,
+      "logits/rejected": -1.0016112327575684,
+      "logps/chosen": -5.096999645233154,
+      "logps/rejected": -79.18199920654297,
+      "loss": 0.2442929893732071,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20111380517482758,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4255572557449341,
+      "rewards/margins": 7.013650417327881,
+      "rewards/rejected": -6.588093280792236,
+      "step": 334,
+      "train_speed(iter/s)": 0.005392
+    },
+    {
+      "epoch": 0.9575703439035284,
+      "grad_norm": 2.3392751216888428,
+      "learning_rate": 0.00016284193571169877,
+      "logits/chosen": -0.942111611366272,
+      "logits/rejected": -0.9454970359802246,
+      "logps/chosen": -5.703181743621826,
+      "logps/rejected": -78.9365463256836,
+      "loss": 0.23718298971652985,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21918217837810516,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3546026349067688,
+      "rewards/margins": 7.023008346557617,
+      "rewards/rejected": -6.668405055999756,
+      "step": 335,
+      "train_speed(iter/s)": 0.005392
+    },
+    {
+      "epoch": 0.9604287628405538,
+      "grad_norm": 1.6322503089904785,
+      "learning_rate": 0.0001625957705866751,
+      "logits/chosen": -0.9369142055511475,
+      "logits/rejected": -0.9375166893005371,
+      "logps/chosen": -8.422727584838867,
+      "logps/rejected": -71.02118682861328,
+      "loss": 0.29250597953796387,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21621529757976532,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.24125435948371887,
+      "rewards/margins": 6.201101303100586,
+      "rewards/rejected": -5.9598469734191895,
+      "step": 336,
+      "train_speed(iter/s)": 0.005392
+    },
+    {
+      "epoch": 0.9632871817775793,
+      "grad_norm": 1.4241256713867188,
+      "learning_rate": 0.00016234898018587337,
+      "logits/chosen": -0.919441282749176,
+      "logits/rejected": -0.9279767274856567,
+      "logps/chosen": -3.0357580184936523,
+      "logps/rejected": -74.14045715332031,
+      "loss": 0.2309810072183609,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20462879538536072,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.45710915327072144,
+      "rewards/margins": 6.547056198120117,
+      "rewards/rejected": -6.08994722366333,
+      "step": 337,
+      "train_speed(iter/s)": 0.005392
+    },
+    {
+      "epoch": 0.9661456007146048,
+      "grad_norm": 1.0136852264404297,
+      "learning_rate": 0.0001621015669745091,
+      "logits/chosen": -0.9102113842964172,
+      "logits/rejected": -0.9235545992851257,
+      "logps/chosen": -5.8284711837768555,
+      "logps/rejected": -70.81057739257812,
+      "loss": 0.2786170244216919,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.19428883492946625,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2921997904777527,
+      "rewards/margins": 5.950136661529541,
+      "rewards/rejected": -5.6579365730285645,
+      "step": 338,
+      "train_speed(iter/s)": 0.005392
+    },
+    {
+      "epoch": 0.9690040196516302,
+      "grad_norm": 15.279269218444824,
+      "learning_rate": 0.00016185353342401896,
+      "logits/chosen": -0.8982729315757751,
+      "logits/rejected": -0.9046964049339294,
+      "logps/chosen": -7.766308307647705,
+      "logps/rejected": -62.051937103271484,
+      "loss": 0.6271102428436279,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.48667430877685547,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.0021721050143241882,
+      "rewards/margins": 5.078856945037842,
+      "rewards/rejected": -5.076685428619385,
+      "step": 339,
+      "train_speed(iter/s)": 0.005393
+    },
+    {
+      "epoch": 0.9718624385886556,
+      "grad_norm": 1.1338287591934204,
+      "learning_rate": 0.00016160488201203644,
+      "logits/chosen": -0.8646394610404968,
+      "logits/rejected": -0.8734477758407593,
+      "logps/chosen": -5.671806335449219,
+      "logps/rejected": -69.74577331542969,
+      "loss": 0.29183241724967957,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20666053891181946,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3366142809391022,
+      "rewards/margins": 5.2625555992126465,
+      "rewards/rejected": -4.925941467285156,
+      "step": 340,
+      "train_speed(iter/s)": 0.005393
+    },
+    {
+      "epoch": 0.9747208575256812,
+      "grad_norm": 1.3026493787765503,
+      "learning_rate": 0.00016135561522236672,
+      "logits/chosen": -0.8204034566879272,
+      "logits/rejected": -0.8284322023391724,
+      "logps/chosen": -4.833341598510742,
+      "logps/rejected": -69.78504180908203,
+      "loss": 0.3127586245536804,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2100476622581482,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.242780864238739,
+      "rewards/margins": 5.5957465171813965,
+      "rewards/rejected": -5.35296630859375,
+      "step": 341,
+      "train_speed(iter/s)": 0.005392
+    },
+    {
+      "epoch": 0.9775792764627066,
+      "grad_norm": 2.748405694961548,
+      "learning_rate": 0.00016110573554496224,
+      "logits/chosen": -0.8463813066482544,
+      "logits/rejected": -0.8463882803916931,
+      "logps/chosen": -5.698624134063721,
+      "logps/rejected": -55.9860954284668,
+      "loss": 0.4039191007614136,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.24814251065254211,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.21434757113456726,
+      "rewards/margins": 4.836075782775879,
+      "rewards/rejected": -4.62172794342041,
+      "step": 342,
+      "train_speed(iter/s)": 0.005392
+    },
+    {
+      "epoch": 0.980437695399732,
+      "grad_norm": 10.635896682739258,
+      "learning_rate": 0.00016085524547589745,
+      "logits/chosen": -0.834613561630249,
+      "logits/rejected": -0.8402217030525208,
+      "logps/chosen": -8.695450782775879,
+      "logps/rejected": -54.68165969848633,
+      "loss": 0.581217885017395,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4611096978187561,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.46262139081954956,
+      "rewards/margins": 4.636106491088867,
+      "rewards/rejected": -4.173485279083252,
+      "step": 343,
+      "train_speed(iter/s)": 0.005393
+    },
+    {
+      "epoch": 0.9832961143367575,
+      "grad_norm": 1.466161847114563,
+      "learning_rate": 0.00016060414751734427,
+      "logits/chosen": -0.8554599285125732,
+      "logits/rejected": -0.8568660020828247,
+      "logps/chosen": -8.234001159667969,
+      "logps/rejected": -64.19937133789062,
+      "loss": 0.43428340554237366,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.38572317361831665,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.16825053095817566,
+      "rewards/margins": 5.233675956726074,
+      "rewards/rejected": -5.065425395965576,
+      "step": 344,
+      "train_speed(iter/s)": 0.005393
+    },
+    {
+      "epoch": 0.9861545332737829,
+      "grad_norm": 5.699708938598633,
+      "learning_rate": 0.00016035244417754666,
+      "logits/chosen": -0.8450981974601746,
+      "logits/rejected": -0.8527222275733948,
+      "logps/chosen": -3.89296555519104,
+      "logps/rejected": -66.04846954345703,
+      "loss": 0.3113130033016205,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2075911909341812,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.31323832273483276,
+      "rewards/margins": 5.465184688568115,
+      "rewards/rejected": -5.151946067810059,
+      "step": 345,
+      "train_speed(iter/s)": 0.005393
+    },
+    {
+      "epoch": 0.9890129522108084,
+      "grad_norm": 3.284147262573242,
+      "learning_rate": 0.00016010013797079604,
+      "logits/chosen": -0.875758707523346,
+      "logits/rejected": -0.8854641318321228,
+      "logps/chosen": -8.347525596618652,
+      "logps/rejected": -64.02723693847656,
+      "loss": 0.3474237620830536,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.30704212188720703,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6182546615600586,
+      "rewards/margins": 5.860507965087891,
+      "rewards/rejected": -5.24225378036499,
+      "step": 346,
+      "train_speed(iter/s)": 0.005393
+    },
+    {
+      "epoch": 0.9918713711478339,
+      "grad_norm": 2.5723087787628174,
+      "learning_rate": 0.00015984723141740576,
+      "logits/chosen": -0.8617424964904785,
+      "logits/rejected": -0.8713058233261108,
+      "logps/chosen": -4.935853958129883,
+      "logps/rejected": -62.30400466918945,
+      "loss": 0.2456638514995575,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21370640397071838,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5748140811920166,
+      "rewards/margins": 5.38767671585083,
+      "rewards/rejected": -4.812862873077393,
+      "step": 347,
+      "train_speed(iter/s)": 0.005393
+    },
+    {
+      "epoch": 0.9947297900848593,
+      "grad_norm": 3.616013765335083,
+      "learning_rate": 0.00015959372704368613,
+      "logits/chosen": -0.882293701171875,
+      "logits/rejected": -0.8994702696800232,
+      "logps/chosen": -4.042830944061279,
+      "logps/rejected": -71.71359252929688,
+      "loss": 0.3966720402240753,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.30810311436653137,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.33603835105895996,
+      "rewards/margins": 5.718041896820068,
+      "rewards/rejected": -5.382004261016846,
+      "step": 348,
+      "train_speed(iter/s)": 0.005393
+    },
+    {
+      "epoch": 0.9975882090218847,
+      "grad_norm": 6.144766330718994,
+      "learning_rate": 0.00015933962738191922,
+      "logits/chosen": -0.9038645625114441,
+      "logits/rejected": -0.9091448187828064,
+      "logps/chosen": -5.831871032714844,
+      "logps/rejected": -66.433349609375,
+      "loss": 0.47910255193710327,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.41736844182014465,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3490181863307953,
+      "rewards/margins": 5.82589054107666,
+      "rewards/rejected": -5.476872444152832,
+      "step": 349,
+      "train_speed(iter/s)": 0.005394
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 6.144766330718994,
+      "learning_rate": 0.00015908493497033342,
+      "logits/chosen": -0.9684196710586548,
+      "logits/rejected": -0.9691399931907654,
+      "logps/chosen": -8.71323299407959,
+      "logps/rejected": -60.70466613769531,
+      "loss": 0.5765681862831116,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.48567596077919006,
+      "rewards/accuracies": 0.9629629850387573,
+      "rewards/chosen": 0.1075638011097908,
+      "rewards/margins": 4.990283966064453,
+      "rewards/rejected": -4.882720947265625,
+      "step": 350,
+      "train_speed(iter/s)": 0.005396
+    },
+    {
+      "epoch": 1.0028584189370255,
+      "grad_norm": 12.870558738708496,
+      "learning_rate": 0.0001588296523530782,
+      "logits/chosen": -0.9231064915657043,
+      "logits/rejected": -0.9311851263046265,
+      "logps/chosen": -2.7042253017425537,
+      "logps/rejected": -82.00444793701172,
+      "loss": 0.1967441439628601,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13856354355812073,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3296782970428467,
+      "rewards/margins": 7.18233585357666,
+      "rewards/rejected": -6.852657318115234,
+      "step": 351,
+      "train_speed(iter/s)": 0.005395
+    },
+    {
+      "epoch": 1.0057168378740509,
+      "grad_norm": 1.8271692991256714,
+      "learning_rate": 0.00015857378208019863,
+      "logits/chosen": -0.97520911693573,
+      "logits/rejected": -0.9849916696548462,
+      "logps/chosen": -4.477419853210449,
+      "logps/rejected": -86.62374877929688,
+      "loss": 0.3155742287635803,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2099926471710205,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.43573394417762756,
+      "rewards/margins": 7.636806488037109,
+      "rewards/rejected": -7.2010722160339355,
+      "step": 352,
+      "train_speed(iter/s)": 0.005396
+    },
+    {
+      "epoch": 1.0085752568110764,
+      "grad_norm": 7.060220718383789,
+      "learning_rate": 0.00015831732670761,
+      "logits/chosen": -0.9630932807922363,
+      "logits/rejected": -0.9717562198638916,
+      "logps/chosen": -5.999786376953125,
+      "logps/rejected": -72.86624145507812,
+      "loss": 0.37415221333503723,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.30320242047309875,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5031129121780396,
+      "rewards/margins": 6.387660503387451,
+      "rewards/rejected": -5.884547233581543,
+      "step": 353,
+      "train_speed(iter/s)": 0.005396
+    },
+    {
+      "epoch": 1.0114336757481017,
+      "grad_norm": 3.706690788269043,
+      "learning_rate": 0.0001580602887970721,
+      "logits/chosen": -0.9575849771499634,
+      "logits/rejected": -0.9724946022033691,
+      "logps/chosen": -5.294210433959961,
+      "logps/rejected": -91.92877960205078,
+      "loss": 0.2967577278614044,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2625669240951538,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6673180460929871,
+      "rewards/margins": 8.382440567016602,
+      "rewards/rejected": -7.715121746063232,
+      "step": 354,
+      "train_speed(iter/s)": 0.005396
+    },
+    {
+      "epoch": 1.0142920946851273,
+      "grad_norm": 1.285293698310852,
+      "learning_rate": 0.00015780267091616384,
+      "logits/chosen": -0.9639118313789368,
+      "logits/rejected": -0.9653413891792297,
+      "logps/chosen": -5.7249932289123535,
+      "logps/rejected": -69.57672119140625,
+      "loss": 0.32318511605262756,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22393131256103516,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4273739159107208,
+      "rewards/margins": 6.277651309967041,
+      "rewards/rejected": -5.850277423858643,
+      "step": 355,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.0171505136221528,
+      "grad_norm": 2.779635429382324,
+      "learning_rate": 0.00015754447563825753,
+      "logits/chosen": -1.0046184062957764,
+      "logits/rejected": -1.0239185094833374,
+      "logps/chosen": -3.307408571243286,
+      "logps/rejected": -97.91881561279297,
+      "loss": 0.3016938269138336,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.24211131036281586,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.39448609948158264,
+      "rewards/margins": 8.36285400390625,
+      "rewards/rejected": -7.968369007110596,
+      "step": 356,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.0200089325591781,
+      "grad_norm": 0.8398857116699219,
+      "learning_rate": 0.00015728570554249312,
+      "logits/chosen": -0.9939976334571838,
+      "logits/rejected": -1.0019322633743286,
+      "logps/chosen": -4.930360317230225,
+      "logps/rejected": -91.99279022216797,
+      "loss": 0.27104535698890686,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.23959386348724365,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6068980693817139,
+      "rewards/margins": 8.438031196594238,
+      "rewards/rejected": -7.831132888793945,
+      "step": 357,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.0228673514962037,
+      "grad_norm": 13.871009826660156,
+      "learning_rate": 0.00015702636321375247,
+      "logits/chosen": -1.0234308242797852,
+      "logits/rejected": -1.0354745388031006,
+      "logps/chosen": -2.2346346378326416,
+      "logps/rejected": -83.26295471191406,
+      "loss": 0.2160538136959076,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16140007972717285,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5340418219566345,
+      "rewards/margins": 7.688697814941406,
+      "rewards/rejected": -7.154655456542969,
+      "step": 358,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.0257257704332292,
+      "grad_norm": 0.8609360456466675,
+      "learning_rate": 0.0001567664512426336,
+      "logits/chosen": -1.0916311740875244,
+      "logits/rejected": -1.097614049911499,
+      "logps/chosen": -6.557211875915527,
+      "logps/rejected": -84.11883544921875,
+      "loss": 0.4536070227622986,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.39151808619499207,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4971507489681244,
+      "rewards/margins": 7.762202739715576,
+      "rewards/rejected": -7.26505184173584,
+      "step": 359,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.0285841893702545,
+      "grad_norm": 2.303415060043335,
+      "learning_rate": 0.0001565059722254246,
+      "logits/chosen": -1.0779087543487549,
+      "logits/rejected": -1.0951520204544067,
+      "logps/chosen": -3.8818159103393555,
+      "logps/rejected": -106.29981231689453,
+      "loss": 0.22627738118171692,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2000780701637268,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6046518683433533,
+      "rewards/margins": 9.897167205810547,
+      "rewards/rejected": -9.292513847351074,
+      "step": 360,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.03144260830728,
+      "grad_norm": 1.3675633668899536,
+      "learning_rate": 0.0001562449287640781,
+      "logits/chosen": -1.0752346515655518,
+      "logits/rejected": -1.0898910760879517,
+      "logps/chosen": -6.105413913726807,
+      "logps/rejected": -89.15643310546875,
+      "loss": 0.2978951036930084,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.283588171005249,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3898117244243622,
+      "rewards/margins": 7.911345481872559,
+      "rewards/rejected": -7.521533966064453,
+      "step": 361,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.0343010272443054,
+      "grad_norm": 6.952116012573242,
+      "learning_rate": 0.00015598332346618472,
+      "logits/chosen": -1.1288803815841675,
+      "logits/rejected": -1.1454200744628906,
+      "logps/chosen": -4.83287239074707,
+      "logps/rejected": -105.67863464355469,
+      "loss": 0.24368441104888916,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18769891560077667,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.40421319007873535,
+      "rewards/margins": 9.764554023742676,
+      "rewards/rejected": -9.36034107208252,
+      "step": 362,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.037159446181331,
+      "grad_norm": 1.075246810913086,
+      "learning_rate": 0.00015572115894494753,
+      "logits/chosen": -1.1167821884155273,
+      "logits/rejected": -1.126716136932373,
+      "logps/chosen": -5.83702278137207,
+      "logps/rejected": -83.78435516357422,
+      "loss": 0.6275767087936401,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.49619928002357483,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2739230990409851,
+      "rewards/margins": 7.508330345153809,
+      "rewards/rejected": -7.2344069480896,
+      "step": 363,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.0400178651183565,
+      "grad_norm": 2.651705026626587,
+      "learning_rate": 0.0001554584378191557,
+      "logits/chosen": -1.118470549583435,
+      "logits/rejected": -1.1314526796340942,
+      "logps/chosen": -4.532167911529541,
+      "logps/rejected": -109.85540771484375,
+      "loss": 0.33839932084083557,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.29753658175468445,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.300693541765213,
+      "rewards/margins": 10.05015754699707,
+      "rewards/rejected": -9.74946403503418,
+      "step": 364,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.0428762840553818,
+      "grad_norm": 2.843981981277466,
+      "learning_rate": 0.00015519516271315833,
+      "logits/chosen": -1.1179580688476562,
+      "logits/rejected": -1.1344995498657227,
+      "logps/chosen": -3.897614002227783,
+      "logps/rejected": -110.58882904052734,
+      "loss": 0.21827109158039093,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1813371628522873,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3872532844543457,
+      "rewards/margins": 9.676642417907715,
+      "rewards/rejected": -9.289389610290527,
+      "step": 365,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.0457347029924073,
+      "grad_norm": 0.8926439881324768,
+      "learning_rate": 0.00015493133625683832,
+      "logits/chosen": -1.0487337112426758,
+      "logits/rejected": -1.0591659545898438,
+      "logps/chosen": -5.423582553863525,
+      "logps/rejected": -103.98403930664062,
+      "loss": 0.2396513819694519,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1920444220304489,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.36182424426078796,
+      "rewards/margins": 9.282267570495605,
+      "rewards/rejected": -8.920443534851074,
+      "step": 366,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.0485931219294329,
+      "grad_norm": 5.607733726501465,
+      "learning_rate": 0.00015466696108558611,
+      "logits/chosen": -1.0868735313415527,
+      "logits/rejected": -1.1046335697174072,
+      "logps/chosen": -2.328686475753784,
+      "logps/rejected": -110.52572631835938,
+      "loss": 0.2047678381204605,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15454038977622986,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.42064905166625977,
+      "rewards/margins": 10.248493194580078,
+      "rewards/rejected": -9.82784366607666,
+      "step": 367,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.0514515408664582,
+      "grad_norm": 3.384974956512451,
+      "learning_rate": 0.00015440203984027324,
+      "logits/chosen": -1.0706456899642944,
+      "logits/rejected": -1.0734856128692627,
+      "logps/chosen": -8.40677261352539,
+      "logps/rejected": -90.43633270263672,
+      "loss": 0.3508933186531067,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.28089454770088196,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6354080438613892,
+      "rewards/margins": 8.334720611572266,
+      "rewards/rejected": -7.699313640594482,
+      "step": 368,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.0543099598034837,
+      "grad_norm": 8.536555290222168,
+      "learning_rate": 0.00015413657516722608,
+      "logits/chosen": -1.0756679773330688,
+      "logits/rejected": -1.0896883010864258,
+      "logps/chosen": -2.9808621406555176,
+      "logps/rejected": -115.16674041748047,
+      "loss": 0.19086603820323944,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.11452573537826538,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4531559944152832,
+      "rewards/margins": 10.596439361572266,
+      "rewards/rejected": -10.14328384399414,
+      "step": 369,
+      "train_speed(iter/s)": 0.005396
+    },
+    {
+      "epoch": 1.057168378740509,
+      "grad_norm": 1.940089464187622,
+      "learning_rate": 0.0001538705697181993,
+      "logits/chosen": -1.0724232196807861,
+      "logits/rejected": -1.0894471406936646,
+      "logps/chosen": -5.458188533782959,
+      "logps/rejected": -92.23328399658203,
+      "loss": 0.3626580834388733,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1961795538663864,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.302205353975296,
+      "rewards/margins": 8.254514694213867,
+      "rewards/rejected": -7.952309608459473,
+      "step": 370,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.0600267976775346,
+      "grad_norm": 7.417636394500732,
+      "learning_rate": 0.00015360402615034957,
+      "logits/chosen": -1.0472326278686523,
+      "logits/rejected": -1.049625277519226,
+      "logps/chosen": -8.179616928100586,
+      "logps/rejected": -63.44961166381836,
+      "loss": 0.6974677443504333,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5029023289680481,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.29456627368927,
+      "rewards/margins": 5.530611038208008,
+      "rewards/rejected": -5.236044883728027,
+      "step": 371,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.0628852166145601,
+      "grad_norm": 4.58155632019043,
+      "learning_rate": 0.00015333694712620877,
+      "logits/chosen": -1.0107908248901367,
+      "logits/rejected": -1.03103506565094,
+      "logps/chosen": -3.0122299194335938,
+      "logps/rejected": -87.87651062011719,
+      "loss": 0.2748883366584778,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.19327932596206665,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3658236265182495,
+      "rewards/margins": 7.511802673339844,
+      "rewards/rejected": -7.145978927612305,
+      "step": 372,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.0657436355515855,
+      "grad_norm": 1.9569977521896362,
+      "learning_rate": 0.00015306933531365746,
+      "logits/chosen": -1.0274032354354858,
+      "logits/rejected": -1.0408649444580078,
+      "logps/chosen": -4.216259956359863,
+      "logps/rejected": -77.02142333984375,
+      "loss": 0.1719209998846054,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1202804297208786,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.498973548412323,
+      "rewards/margins": 6.682814598083496,
+      "rewards/rejected": -6.183840274810791,
+      "step": 373,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.068602054488611,
+      "grad_norm": 1.8889274597167969,
+      "learning_rate": 0.00015280119338589848,
+      "logits/chosen": -0.9627959728240967,
+      "logits/rejected": -0.9700111150741577,
+      "logps/chosen": -4.429068565368652,
+      "logps/rejected": -58.58879089355469,
+      "loss": 0.31505152583122253,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.24982158839702606,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.48795652389526367,
+      "rewards/margins": 5.2674665451049805,
+      "rewards/rejected": -4.779509544372559,
+      "step": 374,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.0714604734256366,
+      "grad_norm": 1.3401929140090942,
+      "learning_rate": 0.00015253252402142988,
+      "logits/chosen": -0.9444267749786377,
+      "logits/rejected": -0.9516254663467407,
+      "logps/chosen": -5.844239711761475,
+      "logps/rejected": -56.261043548583984,
+      "loss": 0.5727149844169617,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4068922996520996,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3124135732650757,
+      "rewards/margins": 4.691495895385742,
+      "rewards/rejected": -4.379082202911377,
+      "step": 375,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.0743188923626619,
+      "grad_norm": 3.1873483657836914,
+      "learning_rate": 0.00015226332990401839,
+      "logits/chosen": -0.9815166592597961,
+      "logits/rejected": -0.9949151277542114,
+      "logps/chosen": -4.578482151031494,
+      "logps/rejected": -61.80070877075195,
+      "loss": 0.3074871897697449,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2443370521068573,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.37748435139656067,
+      "rewards/margins": 5.022083759307861,
+      "rewards/rejected": -4.644599437713623,
+      "step": 376,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.0771773112996874,
+      "grad_norm": 3.210460662841797,
+      "learning_rate": 0.00015199361372267252,
+      "logits/chosen": -0.9962713718414307,
+      "logits/rejected": -1.007851243019104,
+      "logps/chosen": -4.6215314865112305,
+      "logps/rejected": -68.99275970458984,
+      "loss": 0.33263853192329407,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2563571631908417,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3635731339454651,
+      "rewards/margins": 5.339460372924805,
+      "rewards/rejected": -4.975887775421143,
+      "step": 377,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.0800357302367127,
+      "grad_norm": 1.1795045137405396,
+      "learning_rate": 0.0001517233781716158,
+      "logits/chosen": -0.9740598201751709,
+      "logits/rejected": -0.9786965250968933,
+      "logps/chosen": -3.0450801849365234,
+      "logps/rejected": -58.28498840332031,
+      "loss": 0.20879173278808594,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1391848474740982,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3902430832386017,
+      "rewards/margins": 4.98201847076416,
+      "rewards/rejected": -4.591774940490723,
+      "step": 378,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.0828941491737383,
+      "grad_norm": 0.8478718996047974,
+      "learning_rate": 0.0001514526259502597,
+      "logits/chosen": -1.014763355255127,
+      "logits/rejected": -1.0234493017196655,
+      "logps/chosen": -5.755362033843994,
+      "logps/rejected": -56.58897018432617,
+      "loss": 0.29453325271606445,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.24886007606983185,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.6119623184204102,
+      "rewards/margins": 4.830573558807373,
+      "rewards/rejected": -4.218611240386963,
+      "step": 379,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.0857525681107638,
+      "grad_norm": 1.2000700235366821,
+      "learning_rate": 0.0001511813597631768,
+      "logits/chosen": -1.08107328414917,
+      "logits/rejected": -1.0919263362884521,
+      "logps/chosen": -2.3718998432159424,
+      "logps/rejected": -70.58430480957031,
+      "loss": 0.14146487414836884,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.11342191696166992,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.48773208260536194,
+      "rewards/margins": 5.9527997970581055,
+      "rewards/rejected": -5.4650678634643555,
+      "step": 380,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.0886109870477891,
+      "grad_norm": 0.994286835193634,
+      "learning_rate": 0.00015090958232007382,
+      "logits/chosen": -1.0723847150802612,
+      "logits/rejected": -1.081552505493164,
+      "logps/chosen": -4.142213344573975,
+      "logps/rejected": -59.78821563720703,
+      "loss": 0.31306585669517517,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.19472181797027588,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.36915597319602966,
+      "rewards/margins": 5.192928314208984,
+      "rewards/rejected": -4.823772430419922,
+      "step": 381,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.0914694059848147,
+      "grad_norm": 1.2460722923278809,
+      "learning_rate": 0.0001506372963357644,
+      "logits/chosen": -1.1130002737045288,
+      "logits/rejected": -1.1146095991134644,
+      "logps/chosen": -9.743265151977539,
+      "logps/rejected": -66.38863372802734,
+      "loss": 0.35199907422065735,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3197867274284363,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6031234264373779,
+      "rewards/margins": 5.4538679122924805,
+      "rewards/rejected": -4.850744247436523,
+      "step": 382,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.09432782492184,
+      "grad_norm": 1.542410969734192,
+      "learning_rate": 0.000150364504530142,
+      "logits/chosen": -1.1817418336868286,
+      "logits/rejected": -1.2013391256332397,
+      "logps/chosen": -5.177500247955322,
+      "logps/rejected": -78.95667266845703,
+      "loss": 0.2238728106021881,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20672012865543365,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6603532433509827,
+      "rewards/margins": 7.047100067138672,
+      "rewards/rejected": -6.386747360229492,
+      "step": 383,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.0971862438588655,
+      "grad_norm": 10.618709564208984,
+      "learning_rate": 0.0001500912096281529,
+      "logits/chosen": -1.1675359010696411,
+      "logits/rejected": -1.1800576448440552,
+      "logps/chosen": -6.0596442222595215,
+      "logps/rejected": -82.39451599121094,
+      "loss": 0.3103340268135071,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2941910922527313,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.23652604222297668,
+      "rewards/margins": 6.875463485717773,
+      "rewards/rejected": -6.638937950134277,
+      "step": 384,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.100044662795891,
+      "grad_norm": 1.3329583406448364,
+      "learning_rate": 0.0001498174143597688,
+      "logits/chosen": -1.1695022583007812,
+      "logits/rejected": -1.1745272874832153,
+      "logps/chosen": -4.853740692138672,
+      "logps/rejected": -73.12336730957031,
+      "loss": 0.24817080795764923,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2323508858680725,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5231865048408508,
+      "rewards/margins": 6.650723457336426,
+      "rewards/rejected": -6.127537250518799,
+      "step": 385,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.1029030817329164,
+      "grad_norm": 1.2939307689666748,
+      "learning_rate": 0.0001495431214599596,
+      "logits/chosen": -1.2455010414123535,
+      "logits/rejected": -1.2690339088439941,
+      "logps/chosen": -4.194414138793945,
+      "logps/rejected": -100.75020599365234,
+      "loss": 0.4531107246875763,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3706914484500885,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.35981571674346924,
+      "rewards/margins": 8.57312297821045,
+      "rewards/rejected": -8.213308334350586,
+      "step": 386,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.105761500669942,
+      "grad_norm": 2.0912704467773438,
+      "learning_rate": 0.0001492683336686661,
+      "logits/chosen": -1.1914197206497192,
+      "logits/rejected": -1.2101380825042725,
+      "logps/chosen": -1.7182801961898804,
+      "logps/rejected": -87.42121124267578,
+      "loss": 0.15902560949325562,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.12242907285690308,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.45987534523010254,
+      "rewards/margins": 7.868042945861816,
+      "rewards/rejected": -7.408167839050293,
+      "step": 387,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.1086199196069675,
+      "grad_norm": 1.1176247596740723,
+      "learning_rate": 0.00014899305373077267,
+      "logits/chosen": -1.2531603574752808,
+      "logits/rejected": -1.2684531211853027,
+      "logps/chosen": -2.117079734802246,
+      "logps/rejected": -87.74042510986328,
+      "loss": 0.18193872272968292,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15015342831611633,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4387218952178955,
+      "rewards/margins": 7.757615089416504,
+      "rewards/rejected": -7.3188934326171875,
+      "step": 388,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.1114783385439928,
+      "grad_norm": 0.972709059715271,
+      "learning_rate": 0.00014871728439607966,
+      "logits/chosen": -1.2104300260543823,
+      "logits/rejected": -1.2282990217208862,
+      "logps/chosen": -3.3613080978393555,
+      "logps/rejected": -94.52223205566406,
+      "loss": 0.1493372619152069,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14292338490486145,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5973756909370422,
+      "rewards/margins": 8.051233291625977,
+      "rewards/rejected": -7.453857421875,
+      "step": 389,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.1143367574810183,
+      "grad_norm": 1.1937053203582764,
+      "learning_rate": 0.00014844102841927623,
+      "logits/chosen": -1.2560189962387085,
+      "logits/rejected": -1.2647898197174072,
+      "logps/chosen": -3.7985305786132812,
+      "logps/rejected": -75.83130645751953,
+      "loss": 0.23384623229503632,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18301275372505188,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2770909070968628,
+      "rewards/margins": 6.391443729400635,
+      "rewards/rejected": -6.114353179931641,
+      "step": 390,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.1171951764180437,
+      "grad_norm": 1.0092942714691162,
+      "learning_rate": 0.00014816428855991256,
+      "logits/chosen": -1.2596346139907837,
+      "logits/rejected": -1.2714850902557373,
+      "logps/chosen": -6.020136833190918,
+      "logps/rejected": -68.42110443115234,
+      "loss": 0.32318103313446045,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2444128394126892,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.38019418716430664,
+      "rewards/margins": 6.058952808380127,
+      "rewards/rejected": -5.6787590980529785,
+      "step": 391,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.1200535953550692,
+      "grad_norm": 12.279372215270996,
+      "learning_rate": 0.00014788706758237237,
+      "logits/chosen": -1.2402408123016357,
+      "logits/rejected": -1.2524936199188232,
+      "logps/chosen": -9.088288307189941,
+      "logps/rejected": -83.71780395507812,
+      "loss": 0.5434684157371521,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3825962245464325,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3981444835662842,
+      "rewards/margins": 7.234827518463135,
+      "rewards/rejected": -6.836682319641113,
+      "step": 392,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.1229120142920948,
+      "grad_norm": 11.173020362854004,
+      "learning_rate": 0.00014760936825584535,
+      "logits/chosen": -1.3199270963668823,
+      "logits/rejected": -1.3250387907028198,
+      "logps/chosen": -7.1048784255981445,
+      "logps/rejected": -64.97396850585938,
+      "loss": 0.38601505756378174,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2887519598007202,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.37940722703933716,
+      "rewards/margins": 5.784918785095215,
+      "rewards/rejected": -5.405511856079102,
+      "step": 393,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.12577043322912,
+      "grad_norm": 3.846090078353882,
+      "learning_rate": 0.00014733119335429948,
+      "logits/chosen": -1.273574948310852,
+      "logits/rejected": -1.304163932800293,
+      "logps/chosen": -3.3201277256011963,
+      "logps/rejected": -104.27872467041016,
+      "loss": 0.12292776256799698,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.11733995378017426,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4839162826538086,
+      "rewards/margins": 8.810001373291016,
+      "rewards/rejected": -8.326085090637207,
+      "step": 394,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.1286288521661456,
+      "grad_norm": 0.7386897802352905,
+      "learning_rate": 0.00014705254565645334,
+      "logits/chosen": -1.2930562496185303,
+      "logits/rejected": -1.3070048093795776,
+      "logps/chosen": -3.3808460235595703,
+      "logps/rejected": -87.94336700439453,
+      "loss": 0.15408602356910706,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14752119779586792,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4112515151500702,
+      "rewards/margins": 7.620075225830078,
+      "rewards/rejected": -7.208824157714844,
+      "step": 395,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.1314872711031712,
+      "grad_norm": 0.8473145365715027,
+      "learning_rate": 0.00014677342794574817,
+      "logits/chosen": -1.2756608724594116,
+      "logits/rejected": -1.3103110790252686,
+      "logps/chosen": -2.1913681030273438,
+      "logps/rejected": -98.67115020751953,
+      "loss": 0.1846894919872284,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.180893674492836,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4858021140098572,
+      "rewards/margins": 8.420744895935059,
+      "rewards/rejected": -7.934943199157715,
+      "step": 396,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.1343456900401965,
+      "grad_norm": 1.5317679643630981,
+      "learning_rate": 0.00014649384301032044,
+      "logits/chosen": -1.2814905643463135,
+      "logits/rejected": -1.3005256652832031,
+      "logps/chosen": -1.5174694061279297,
+      "logps/rejected": -87.35885620117188,
+      "loss": 0.12383262068033218,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.10835046321153641,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.44735920429229736,
+      "rewards/margins": 7.7608489990234375,
+      "rewards/rejected": -7.313488960266113,
+      "step": 397,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.137204108977222,
+      "grad_norm": 1.6518385410308838,
+      "learning_rate": 0.0001462137936429736,
+      "logits/chosen": -1.3178036212921143,
+      "logits/rejected": -1.3269606828689575,
+      "logps/chosen": -3.7774932384490967,
+      "logps/rejected": -70.43819427490234,
+      "loss": 0.24924607574939728,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.19433541595935822,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2736770212650299,
+      "rewards/margins": 6.038897514343262,
+      "rewards/rejected": -5.765220642089844,
+      "step": 398,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.1400625279142473,
+      "grad_norm": 1.1277555227279663,
+      "learning_rate": 0.00014593328264115042,
+      "logits/chosen": -1.3218518495559692,
+      "logits/rejected": -1.338102102279663,
+      "logps/chosen": -5.340864181518555,
+      "logps/rejected": -76.87016296386719,
+      "loss": 0.3493357300758362,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.25427284836769104,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3287370502948761,
+      "rewards/margins": 6.68165397644043,
+      "rewards/rejected": -6.352917194366455,
+      "step": 399,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.1429209468512729,
+      "grad_norm": 4.7161102294921875,
+      "learning_rate": 0.000145652312806905,
+      "logits/chosen": -1.2902297973632812,
+      "logits/rejected": -1.2931489944458008,
+      "logps/chosen": -5.8022236824035645,
+      "logps/rejected": -77.25333404541016,
+      "loss": 0.3741215467453003,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2942506968975067,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3449039161205292,
+      "rewards/margins": 6.666882514953613,
+      "rewards/rejected": -6.3219780921936035,
+      "step": 400,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.1457793657882984,
+      "grad_norm": 13.246755599975586,
+      "learning_rate": 0.00014537088694687475,
+      "logits/chosen": -1.3303780555725098,
+      "logits/rejected": -1.3321895599365234,
+      "logps/chosen": -5.475297927856445,
+      "logps/rejected": -84.30738830566406,
+      "loss": 0.2737537622451782,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.252432644367218,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.400252103805542,
+      "rewards/margins": 7.612529754638672,
+      "rewards/rejected": -7.212278366088867,
+      "step": 401,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.1486377847253237,
+      "grad_norm": 1.5183851718902588,
+      "learning_rate": 0.0001450890078722524,
+      "logits/chosen": -1.2768632173538208,
+      "logits/rejected": -1.2883963584899902,
+      "logps/chosen": -6.603586196899414,
+      "logps/rejected": -75.68502044677734,
+      "loss": 0.2976156175136566,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.26498278975486755,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6158945560455322,
+      "rewards/margins": 6.887779235839844,
+      "rewards/rejected": -6.271884441375732,
+      "step": 402,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.1514962036623493,
+      "grad_norm": 1.2054356336593628,
+      "learning_rate": 0.00014480667839875786,
+      "logits/chosen": -1.289682149887085,
+      "logits/rejected": -1.304053544998169,
+      "logps/chosen": -4.174172878265381,
+      "logps/rejected": -92.87583923339844,
+      "loss": 0.29464054107666016,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.24246171116828918,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5281142592430115,
+      "rewards/margins": 8.316887855529785,
+      "rewards/rejected": -7.788773536682129,
+      "step": 403,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.1543546225993748,
+      "grad_norm": 1.2773363590240479,
+      "learning_rate": 0.0001445239013466101,
+      "logits/chosen": -1.3230645656585693,
+      "logits/rejected": -1.3197563886642456,
+      "logps/chosen": -4.9347124099731445,
+      "logps/rejected": -85.14564514160156,
+      "loss": 0.1755044162273407,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16279785335063934,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4090490937232971,
+      "rewards/margins": 7.78640604019165,
+      "rewards/rejected": -7.377357482910156,
+      "step": 404,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.1572130415364001,
+      "grad_norm": 27.927875518798828,
+      "learning_rate": 0.00014424067954049903,
+      "logits/chosen": -1.3196823596954346,
+      "logits/rejected": -1.3405616283416748,
+      "logps/chosen": -4.248622894287109,
+      "logps/rejected": -94.78376770019531,
+      "loss": 0.4967496693134308,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.37453967332839966,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2362315058708191,
+      "rewards/margins": 7.911720275878906,
+      "rewards/rejected": -7.6754889488220215,
+      "step": 405,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.1600714604734257,
+      "grad_norm": 1.4067364931106567,
+      "learning_rate": 0.00014395701580955723,
+      "logits/chosen": -1.2486661672592163,
+      "logits/rejected": -1.269938349723816,
+      "logps/chosen": -3.7746410369873047,
+      "logps/rejected": -95.56806945800781,
+      "loss": 0.2633632719516754,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21375316381454468,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5535802841186523,
+      "rewards/margins": 8.59311580657959,
+      "rewards/rejected": -8.039534568786621,
+      "step": 406,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.162929879410451,
+      "grad_norm": 4.405129909515381,
+      "learning_rate": 0.00014367291298733178,
+      "logits/chosen": -1.2451786994934082,
+      "logits/rejected": -1.2523881196975708,
+      "logps/chosen": -7.29015588760376,
+      "logps/rejected": -84.21794128417969,
+      "loss": 0.31225040555000305,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2892824411392212,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.44759562611579895,
+      "rewards/margins": 7.486978530883789,
+      "rewards/rejected": -7.0393829345703125,
+      "step": 407,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.1657882983474765,
+      "grad_norm": 1.350124716758728,
+      "learning_rate": 0.00014338837391175582,
+      "logits/chosen": -1.2725536823272705,
+      "logits/rejected": -1.2896746397018433,
+      "logps/chosen": -3.57419490814209,
+      "logps/rejected": -94.1634750366211,
+      "loss": 0.3036032021045685,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.27703234553337097,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4453233480453491,
+      "rewards/margins": 8.459342002868652,
+      "rewards/rejected": -8.014019966125488,
+      "step": 408,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.168646717284502,
+      "grad_norm": 2.680598497390747,
+      "learning_rate": 0.00014310340142512028,
+      "logits/chosen": -1.2726447582244873,
+      "logits/rejected": -1.2849714756011963,
+      "logps/chosen": -4.487876892089844,
+      "logps/rejected": -105.97708129882812,
+      "loss": 0.17320063710212708,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14207664132118225,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2871077358722687,
+      "rewards/margins": 9.345542907714844,
+      "rewards/rejected": -9.058435440063477,
+      "step": 409,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.1715051362215274,
+      "grad_norm": 1.2977643013000488,
+      "learning_rate": 0.00014281799837404552,
+      "logits/chosen": -1.2369587421417236,
+      "logits/rejected": -1.2519880533218384,
+      "logps/chosen": -3.1722519397735596,
+      "logps/rejected": -87.6946029663086,
+      "loss": 0.27433717250823975,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1756080687046051,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.43238213658332825,
+      "rewards/margins": 7.913693428039551,
+      "rewards/rejected": -7.4813103675842285,
+      "step": 410,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.174363555158553,
+      "grad_norm": 4.359915256500244,
+      "learning_rate": 0.00014253216760945283,
+      "logits/chosen": -1.212538719177246,
+      "logits/rejected": -1.2241911888122559,
+      "logps/chosen": -3.465891122817993,
+      "logps/rejected": -95.1599349975586,
+      "loss": 0.2568645179271698,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.25212615728378296,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.29171866178512573,
+      "rewards/margins": 8.463183403015137,
+      "rewards/rejected": -8.171464920043945,
+      "step": 411,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.1772219740955783,
+      "grad_norm": 6.41582727432251,
+      "learning_rate": 0.00014224591198653595,
+      "logits/chosen": -1.2008682489395142,
+      "logits/rejected": -1.210702657699585,
+      "logps/chosen": -7.061485290527344,
+      "logps/rejected": -72.23047637939453,
+      "loss": 0.8553668260574341,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5689457654953003,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.3008919954299927,
+      "rewards/margins": 6.426004886627197,
+      "rewards/rejected": -6.125113010406494,
+      "step": 412,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 1.1800803930326038,
+      "grad_norm": 17.078575134277344,
+      "learning_rate": 0.00014195923436473258,
+      "logits/chosen": -1.128623127937317,
+      "logits/rejected": -1.1534677743911743,
+      "logps/chosen": -2.973088026046753,
+      "logps/rejected": -94.05462646484375,
+      "loss": 0.19254088401794434,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16183622181415558,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.37168949842453003,
+      "rewards/margins": 8.012508392333984,
+      "rewards/rejected": -7.6408185958862305,
+      "step": 413,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 1.1829388119696294,
+      "grad_norm": 0.7061187624931335,
+      "learning_rate": 0.00014167213760769588,
+      "logits/chosen": -1.1624064445495605,
+      "logits/rejected": -1.1710572242736816,
+      "logps/chosen": -6.01300048828125,
+      "logps/rejected": -81.90544891357422,
+      "loss": 0.28161901235580444,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.24062369763851166,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3918932378292084,
+      "rewards/margins": 7.115993499755859,
+      "rewards/rejected": -6.724100112915039,
+      "step": 414,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.1857972309066547,
+      "grad_norm": 0.8015528321266174,
+      "learning_rate": 0.00014138462458326568,
+      "logits/chosen": -1.1726088523864746,
+      "logits/rejected": -1.1833385229110718,
+      "logps/chosen": -6.184147834777832,
+      "logps/rejected": -88.518310546875,
+      "loss": 0.28720641136169434,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2499961256980896,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5268177390098572,
+      "rewards/margins": 7.87477970123291,
+      "rewards/rejected": -7.347961902618408,
+      "step": 415,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 1.1886556498436802,
+      "grad_norm": 1.2144743204116821,
+      "learning_rate": 0.00014109669816344004,
+      "logits/chosen": -1.1292197704315186,
+      "logits/rejected": -1.1369071006774902,
+      "logps/chosen": -5.179091930389404,
+      "logps/rejected": -90.02409362792969,
+      "loss": 0.289948970079422,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.23118160665035248,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5325126051902771,
+      "rewards/margins": 8.058305740356445,
+      "rewards/rejected": -7.525793552398682,
+      "step": 416,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.1915140687807058,
+      "grad_norm": 1.0072659254074097,
+      "learning_rate": 0.0001408083612243465,
+      "logits/chosen": -1.1827200651168823,
+      "logits/rejected": -1.2069463729858398,
+      "logps/chosen": -4.6436967849731445,
+      "logps/rejected": -106.73895263671875,
+      "loss": 0.3984032869338989,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3136874735355377,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.41814562678337097,
+      "rewards/margins": 9.197477340698242,
+      "rewards/rejected": -8.779330253601074,
+      "step": 417,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 1.194372487717731,
+      "grad_norm": 53.66329574584961,
+      "learning_rate": 0.00014051961664621318,
+      "logits/chosen": -1.1490533351898193,
+      "logits/rejected": -1.1688562631607056,
+      "logps/chosen": -4.423655986785889,
+      "logps/rejected": -97.36763000488281,
+      "loss": 0.2450760006904602,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2312186360359192,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6164323091506958,
+      "rewards/margins": 8.828019142150879,
+      "rewards/rejected": -8.211586952209473,
+      "step": 418,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 1.1972309066547566,
+      "grad_norm": 3.1319069862365723,
+      "learning_rate": 0.00014023046731334032,
+      "logits/chosen": -1.1919033527374268,
+      "logits/rejected": -1.2038265466690063,
+      "logps/chosen": -4.850213527679443,
+      "logps/rejected": -79.25477600097656,
+      "loss": 0.2641419470310211,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2136654555797577,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6766065955162048,
+      "rewards/margins": 7.204361438751221,
+      "rewards/rejected": -6.527754783630371,
+      "step": 419,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 1.2000893255917822,
+      "grad_norm": 1.0065001249313354,
+      "learning_rate": 0.00013994091611407112,
+      "logits/chosen": -1.1787667274475098,
+      "logits/rejected": -1.2045613527297974,
+      "logps/chosen": -2.8472421169281006,
+      "logps/rejected": -105.08712005615234,
+      "loss": 0.1212020143866539,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1145675852894783,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3931896686553955,
+      "rewards/margins": 9.183510780334473,
+      "rewards/rejected": -8.790321350097656,
+      "step": 420,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 1.2029477445288075,
+      "grad_norm": 2.516078472137451,
+      "learning_rate": 0.0001396509659407632,
+      "logits/chosen": -1.2433277368545532,
+      "logits/rejected": -1.2500237226486206,
+      "logps/chosen": -3.4099931716918945,
+      "logps/rejected": -81.68355560302734,
+      "loss": 0.21662066876888275,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18431250751018524,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4809565544128418,
+      "rewards/margins": 7.408727169036865,
+      "rewards/rejected": -6.927770614624023,
+      "step": 421,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 1.205806163465833,
+      "grad_norm": 1.070208191871643,
+      "learning_rate": 0.00013936061968975957,
+      "logits/chosen": -1.2363523244857788,
+      "logits/rejected": -1.2570617198944092,
+      "logps/chosen": -4.615940570831299,
+      "logps/rejected": -93.6010513305664,
+      "loss": 0.24221770465373993,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2085483819246292,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5028698444366455,
+      "rewards/margins": 8.165315628051758,
+      "rewards/rejected": -7.662446022033691,
+      "step": 422,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 1.2086645824028583,
+      "grad_norm": 0.9709579944610596,
+      "learning_rate": 0.00013906988026135956,
+      "logits/chosen": -1.2579833269119263,
+      "logits/rejected": -1.2686238288879395,
+      "logps/chosen": -5.4705376625061035,
+      "logps/rejected": -85.68134307861328,
+      "loss": 0.3086671531200409,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2807824909687042,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.42558395862579346,
+      "rewards/margins": 7.466774940490723,
+      "rewards/rejected": -7.041190147399902,
+      "step": 423,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 1.2115230013398839,
+      "grad_norm": 1.1192522048950195,
+      "learning_rate": 0.00013877875055979023,
+      "logits/chosen": -1.2901450395584106,
+      "logits/rejected": -1.2925304174423218,
+      "logps/chosen": -7.668773651123047,
+      "logps/rejected": -70.98661804199219,
+      "loss": 0.3631104528903961,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.31808874011039734,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4034600853919983,
+      "rewards/margins": 6.177216053009033,
+      "rewards/rejected": -5.7737555503845215,
+      "step": 424,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.2143814202769094,
+      "grad_norm": 1.203405499458313,
+      "learning_rate": 0.00013848723349317688,
+      "logits/chosen": -1.271082878112793,
+      "logits/rejected": -1.293760061264038,
+      "logps/chosen": -3.0228474140167236,
+      "logps/rejected": -106.1806640625,
+      "loss": 0.17642824351787567,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16339841485023499,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4982394278049469,
+      "rewards/margins": 9.563376426696777,
+      "rewards/rejected": -9.065135955810547,
+      "step": 425,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.2172398392139347,
+      "grad_norm": 0.8518316745758057,
+      "learning_rate": 0.00013819533197351438,
+      "logits/chosen": -1.3219232559204102,
+      "logits/rejected": -1.3383479118347168,
+      "logps/chosen": -4.452436447143555,
+      "logps/rejected": -97.7171859741211,
+      "loss": 0.2837337553501129,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2675948739051819,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5727283954620361,
+      "rewards/margins": 9.178853988647461,
+      "rewards/rejected": -8.606125831604004,
+      "step": 426,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.2200982581509603,
+      "grad_norm": 1.294920563697815,
+      "learning_rate": 0.00013790304891663792,
+      "logits/chosen": -1.3122917413711548,
+      "logits/rejected": -1.330684781074524,
+      "logps/chosen": -2.5727081298828125,
+      "logps/rejected": -97.04032897949219,
+      "loss": 0.23420938849449158,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.17399996519088745,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.49123436212539673,
+      "rewards/margins": 8.5830659866333,
+      "rewards/rejected": -8.09183120727539,
+      "step": 427,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.2229566770879856,
+      "grad_norm": 1.3210828304290771,
+      "learning_rate": 0.00013761038724219388,
+      "logits/chosen": -1.295900821685791,
+      "logits/rejected": -1.3054581880569458,
+      "logps/chosen": -6.279821395874023,
+      "logps/rejected": -95.39372253417969,
+      "loss": 0.32580864429473877,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2732087969779968,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3629838228225708,
+      "rewards/margins": 8.550825119018555,
+      "rewards/rejected": -8.187841415405273,
+      "step": 428,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.2258150960250112,
+      "grad_norm": 2.9985227584838867,
+      "learning_rate": 0.0001373173498736107,
+      "logits/chosen": -1.288704752922058,
+      "logits/rejected": -1.3120356798171997,
+      "logps/chosen": -4.108103275299072,
+      "logps/rejected": -98.43057250976562,
+      "loss": 0.2225966602563858,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20878230035305023,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6132968664169312,
+      "rewards/margins": 9.20932388305664,
+      "rewards/rejected": -8.596027374267578,
+      "step": 429,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.2286735149620367,
+      "grad_norm": 2.3420937061309814,
+      "learning_rate": 0.00013702393973806967,
+      "logits/chosen": -1.2286124229431152,
+      "logits/rejected": -1.2492600679397583,
+      "logps/chosen": -5.495055675506592,
+      "logps/rejected": -83.2303237915039,
+      "loss": 0.22717006504535675,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20513629913330078,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5127463936805725,
+      "rewards/margins": 7.726105690002441,
+      "rewards/rejected": -7.213359832763672,
+      "step": 430,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.231531933899062,
+      "grad_norm": 0.8680245280265808,
+      "learning_rate": 0.00013673015976647568,
+      "logits/chosen": -1.2850875854492188,
+      "logits/rejected": -1.2913092374801636,
+      "logps/chosen": -6.471378803253174,
+      "logps/rejected": -87.43525695800781,
+      "loss": 0.29818040132522583,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.25542551279067993,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6377506852149963,
+      "rewards/margins": 8.0104398727417,
+      "rewards/rejected": -7.372689247131348,
+      "step": 431,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.2343903528360876,
+      "grad_norm": 1.0749446153640747,
+      "learning_rate": 0.00013643601289342803,
+      "logits/chosen": -1.2721213102340698,
+      "logits/rejected": -1.2773966789245605,
+      "logps/chosen": -3.4057223796844482,
+      "logps/rejected": -91.22700500488281,
+      "loss": 0.1517515331506729,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14677660167217255,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.40595370531082153,
+      "rewards/margins": 8.496070861816406,
+      "rewards/rejected": -8.090116500854492,
+      "step": 432,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.237248771773113,
+      "grad_norm": 0.8742938041687012,
+      "learning_rate": 0.00013614150205719085,
+      "logits/chosen": -1.3097087144851685,
+      "logits/rejected": -1.3255422115325928,
+      "logps/chosen": -3.3428828716278076,
+      "logps/rejected": -112.81401824951172,
+      "loss": 0.14485226571559906,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.12631869316101074,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.47337648272514343,
+      "rewards/margins": 10.406147956848145,
+      "rewards/rejected": -9.932769775390625,
+      "step": 433,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.2401071907101384,
+      "grad_norm": 0.7771469354629517,
+      "learning_rate": 0.00013584663019966417,
+      "logits/chosen": -1.2172707319259644,
+      "logits/rejected": -1.2154616117477417,
+      "logps/chosen": -7.059861183166504,
+      "logps/rejected": -87.25150299072266,
+      "loss": 0.34314751625061035,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.28513815999031067,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.28648173809051514,
+      "rewards/margins": 7.758938789367676,
+      "rewards/rejected": -7.472456932067871,
+      "step": 434,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.242965609647164,
+      "grad_norm": 6.615478038787842,
+      "learning_rate": 0.00013555140026635414,
+      "logits/chosen": -1.2565209865570068,
+      "logits/rejected": -1.264001488685608,
+      "logps/chosen": -7.7678656578063965,
+      "logps/rejected": -102.24252319335938,
+      "loss": 0.13871826231479645,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13072122633457184,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6000977754592896,
+      "rewards/margins": 9.453453063964844,
+      "rewards/rejected": -8.853355407714844,
+      "step": 435,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.2458240285841893,
+      "grad_norm": 0.5910543203353882,
+      "learning_rate": 0.00013525581520634381,
+      "logits/chosen": -1.194077491760254,
+      "logits/rejected": -1.211206078529358,
+      "logps/chosen": -6.643336296081543,
+      "logps/rejected": -95.53460693359375,
+      "loss": 0.3519570529460907,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.33618804812431335,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7547551989555359,
+      "rewards/margins": 8.722167015075684,
+      "rewards/rejected": -7.967411518096924,
+      "step": 436,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.2486824475212148,
+      "grad_norm": 1.195341944694519,
+      "learning_rate": 0.0001349598779722636,
+      "logits/chosen": -1.2183115482330322,
+      "logits/rejected": -1.2236500978469849,
+      "logps/chosen": -6.876954555511475,
+      "logps/rejected": -88.98007202148438,
+      "loss": 0.2813359200954437,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2684388756752014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3353041410446167,
+      "rewards/margins": 7.750663757324219,
+      "rewards/rejected": -7.4153594970703125,
+      "step": 437,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.2515408664582404,
+      "grad_norm": 0.872073233127594,
+      "learning_rate": 0.00013466359152026195,
+      "logits/chosen": -1.2435060739517212,
+      "logits/rejected": -1.2493698596954346,
+      "logps/chosen": -7.454791069030762,
+      "logps/rejected": -85.75321960449219,
+      "loss": 0.2809821367263794,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2567462623119354,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3945457637310028,
+      "rewards/margins": 7.449971675872803,
+      "rewards/rejected": -7.055425643920898,
+      "step": 438,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.2543992853952657,
+      "grad_norm": 1.6541284322738647,
+      "learning_rate": 0.00013436695880997553,
+      "logits/chosen": -1.256369948387146,
+      "logits/rejected": -1.2683699131011963,
+      "logps/chosen": -5.130362510681152,
+      "logps/rejected": -94.84146881103516,
+      "loss": 0.3861837387084961,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.35058313608169556,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4951009154319763,
+      "rewards/margins": 8.740049362182617,
+      "rewards/rejected": -8.24494743347168,
+      "step": 439,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.2572577043322912,
+      "grad_norm": 0.8177284002304077,
+      "learning_rate": 0.00013406998280449987,
+      "logits/chosen": -1.2140347957611084,
+      "logits/rejected": -1.222559928894043,
+      "logps/chosen": -4.193092346191406,
+      "logps/rejected": -86.39996337890625,
+      "loss": 0.27863699197769165,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.25425273180007935,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.48477423191070557,
+      "rewards/margins": 8.047656059265137,
+      "rewards/rejected": -7.562881946563721,
+      "step": 440,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.2601161232693165,
+      "grad_norm": 1.945068120956421,
+      "learning_rate": 0.00013377266647035974,
+      "logits/chosen": -1.201490879058838,
+      "logits/rejected": -1.2190073728561401,
+      "logps/chosen": -3.4713282585144043,
+      "logps/rejected": -91.55570983886719,
+      "loss": 0.24121132493019104,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.19668132066726685,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4186343848705292,
+      "rewards/margins": 8.065241813659668,
+      "rewards/rejected": -7.646607398986816,
+      "step": 441,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.262974542206342,
+      "grad_norm": 29.868745803833008,
+      "learning_rate": 0.00013347501277747955,
+      "logits/chosen": -1.241868495941162,
+      "logits/rejected": -1.255920171737671,
+      "logps/chosen": -2.4936649799346924,
+      "logps/rejected": -94.12937927246094,
+      "loss": 0.22337190806865692,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1589697301387787,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3206178545951843,
+      "rewards/margins": 8.269994735717773,
+      "rewards/rejected": -7.949377536773682,
+      "step": 442,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.2658329611433676,
+      "grad_norm": 0.6225547194480896,
+      "learning_rate": 0.0001331770246991534,
+      "logits/chosen": -1.2288497686386108,
+      "logits/rejected": -1.2424108982086182,
+      "logps/chosen": -7.412257194519043,
+      "logps/rejected": -98.60154724121094,
+      "loss": 0.3624575436115265,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3274103105068207,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.6840518116950989,
+      "rewards/margins": 9.37939739227295,
+      "rewards/rejected": -8.695345878601074,
+      "step": 443,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.268691380080393,
+      "grad_norm": 40.07108688354492,
+      "learning_rate": 0.00013287870521201583,
+      "logits/chosen": -1.2206084728240967,
+      "logits/rejected": -1.2286522388458252,
+      "logps/chosen": -5.912662982940674,
+      "logps/rejected": -107.58220672607422,
+      "loss": 0.2583864629268646,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22890999913215637,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.540444552898407,
+      "rewards/margins": 9.795575141906738,
+      "rewards/rejected": -9.255130767822266,
+      "step": 444,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.2715497990174185,
+      "grad_norm": 0.7341138124465942,
+      "learning_rate": 0.00013258005729601177,
+      "logits/chosen": -1.211508870124817,
+      "logits/rejected": -1.2124762535095215,
+      "logps/chosen": -6.125351428985596,
+      "logps/rejected": -97.78089904785156,
+      "loss": 0.2630341053009033,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21238940954208374,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3658996820449829,
+      "rewards/margins": 9.0274019241333,
+      "rewards/rejected": -8.66150188446045,
+      "step": 445,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.274408217954444,
+      "grad_norm": 1.2155957221984863,
+      "learning_rate": 0.00013228108393436685,
+      "logits/chosen": -1.1720253229141235,
+      "logits/rejected": -1.182066798210144,
+      "logps/chosen": -5.76838493347168,
+      "logps/rejected": -82.00324249267578,
+      "loss": 0.4643992483615875,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.40862181782722473,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5143325924873352,
+      "rewards/margins": 7.507829666137695,
+      "rewards/rejected": -6.993496894836426,
+      "step": 446,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.2772666368914694,
+      "grad_norm": 6.123267650604248,
+      "learning_rate": 0.0001319817881135576,
+      "logits/chosen": -1.2267619371414185,
+      "logits/rejected": -1.2333106994628906,
+      "logps/chosen": -5.525717258453369,
+      "logps/rejected": -93.48272705078125,
+      "loss": 0.4229140281677246,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.36744216084480286,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.34462982416152954,
+      "rewards/margins": 8.51840591430664,
+      "rewards/rejected": -8.173775672912598,
+      "step": 447,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.280125055828495,
+      "grad_norm": 36.35102462768555,
+      "learning_rate": 0.00013168217282328174,
+      "logits/chosen": -1.244513988494873,
+      "logits/rejected": -1.2543288469314575,
+      "logps/chosen": -3.243824005126953,
+      "logps/rejected": -103.88795471191406,
+      "loss": 0.19224901497364044,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13676151633262634,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4578501582145691,
+      "rewards/margins": 9.796440124511719,
+      "rewards/rejected": -9.33858871459961,
+      "step": 448,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.2829834747655204,
+      "grad_norm": 0.7684857249259949,
+      "learning_rate": 0.00013138224105642803,
+      "logits/chosen": -1.2897181510925293,
+      "logits/rejected": -1.3069959878921509,
+      "logps/chosen": -2.2601726055145264,
+      "logps/rejected": -117.00337219238281,
+      "loss": 0.12171882390975952,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.10878898203372955,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.47879016399383545,
+      "rewards/margins": 10.791764259338379,
+      "rewards/rejected": -10.312973976135254,
+      "step": 449,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.2858418937025458,
+      "grad_norm": 1.1565505266189575,
+      "learning_rate": 0.00013108199580904667,
+      "logits/chosen": -1.2609070539474487,
+      "logits/rejected": -1.2832667827606201,
+      "logps/chosen": -3.7138025760650635,
+      "logps/rejected": -120.03974914550781,
+      "loss": 0.2552796006202698,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2516552209854126,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3557589054107666,
+      "rewards/margins": 11.129087448120117,
+      "rewards/rejected": -10.773327827453613,
+      "step": 450,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.2887003126395713,
+      "grad_norm": 1.058509111404419,
+      "learning_rate": 0.00013078144008031923,
+      "logits/chosen": -1.2376177310943604,
+      "logits/rejected": -1.254594087600708,
+      "logps/chosen": -4.495041370391846,
+      "logps/rejected": -103.95249938964844,
+      "loss": 0.24078714847564697,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22126168012619019,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6826871037483215,
+      "rewards/margins": 9.435651779174805,
+      "rewards/rejected": -8.752964973449707,
+      "step": 451,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 1.2915587315765966,
+      "grad_norm": 1.2536028623580933,
+      "learning_rate": 0.00013048057687252865,
+      "logits/chosen": -1.2278149127960205,
+      "logits/rejected": -1.2435705661773682,
+      "logps/chosen": -5.1287713050842285,
+      "logps/rejected": -108.75577545166016,
+      "loss": 0.2327510118484497,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1934833973646164,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3743947446346283,
+      "rewards/margins": 9.591495513916016,
+      "rewards/rejected": -9.217100143432617,
+      "step": 452,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 1.2944171505136222,
+      "grad_norm": 1.1801437139511108,
+      "learning_rate": 0.0001301794091910294,
+      "logits/chosen": -1.1996853351593018,
+      "logits/rejected": -1.212232232093811,
+      "logps/chosen": -5.606900691986084,
+      "logps/rejected": -97.51220703125,
+      "loss": 0.226387158036232,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18147651851177216,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.35863998532295227,
+      "rewards/margins": 8.822105407714844,
+      "rewards/rejected": -8.46346664428711,
+      "step": 453,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.2972755694506475,
+      "grad_norm": 0.8085012435913086,
+      "learning_rate": 0.00012987794004421738,
+      "logits/chosen": -1.2282167673110962,
+      "logits/rejected": -1.2483009099960327,
+      "logps/chosen": -6.084074974060059,
+      "logps/rejected": -116.71503448486328,
+      "loss": 0.7732155919075012,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.7598755359649658,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.24433200061321259,
+      "rewards/margins": 10.193436622619629,
+      "rewards/rejected": -9.949104309082031,
+      "step": 454,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.300133988387673,
+      "grad_norm": 6.514425754547119,
+      "learning_rate": 0.0001295761724434997,
+      "logits/chosen": -1.2512998580932617,
+      "logits/rejected": -1.2541574239730835,
+      "logps/chosen": -4.655600547790527,
+      "logps/rejected": -101.07491302490234,
+      "loss": 0.23648858070373535,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20019900798797607,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5059707760810852,
+      "rewards/margins": 9.593456268310547,
+      "rewards/rejected": -9.087486267089844,
+      "step": 455,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.3029924073246986,
+      "grad_norm": 0.8636030554771423,
+      "learning_rate": 0.00012927410940326487,
+      "logits/chosen": -1.207863211631775,
+      "logits/rejected": -1.2212624549865723,
+      "logps/chosen": -7.982148170471191,
+      "logps/rejected": -98.51473999023438,
+      "loss": 0.35097599029541016,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3070108890533447,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5857585072517395,
+      "rewards/margins": 9.116013526916504,
+      "rewards/rejected": -8.530255317687988,
+      "step": 456,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.3058508262617239,
+      "grad_norm": 0.9966700673103333,
+      "learning_rate": 0.00012897175394085267,
+      "logits/chosen": -1.1829395294189453,
+      "logits/rejected": -1.2041140794754028,
+      "logps/chosen": -5.4352827072143555,
+      "logps/rejected": -98.21609497070312,
+      "loss": 0.21705986559391022,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20047500729560852,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.58429354429245,
+      "rewards/margins": 9.176396369934082,
+      "rewards/rejected": -8.592103004455566,
+      "step": 457,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.3087092451987494,
+      "grad_norm": 2.171597480773926,
+      "learning_rate": 0.00012866910907652373,
+      "logits/chosen": -1.1633059978485107,
+      "logits/rejected": -1.1853255033493042,
+      "logps/chosen": -5.978604316711426,
+      "logps/rejected": -105.99617004394531,
+      "loss": 0.2683447301387787,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2629088759422302,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.42330706119537354,
+      "rewards/margins": 9.704293251037598,
+      "rewards/rejected": -9.280985832214355,
+      "step": 458,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.311567664135775,
+      "grad_norm": 1.2772369384765625,
+      "learning_rate": 0.0001283661778334297,
+      "logits/chosen": -1.1277825832366943,
+      "logits/rejected": -1.134521245956421,
+      "logps/chosen": -5.945641994476318,
+      "logps/rejected": -83.91475677490234,
+      "loss": 0.22780606150627136,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1669580340385437,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.39180228114128113,
+      "rewards/margins": 7.6563615798950195,
+      "rewards/rejected": -7.264558792114258,
+      "step": 459,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.3144260830728003,
+      "grad_norm": 1.3344827890396118,
+      "learning_rate": 0.00012806296323758274,
+      "logits/chosen": -1.1059662103652954,
+      "logits/rejected": -1.1256768703460693,
+      "logps/chosen": -4.297776699066162,
+      "logps/rejected": -115.46197509765625,
+      "loss": 0.21040144562721252,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16639991104602814,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3483891189098358,
+      "rewards/margins": 10.405756950378418,
+      "rewards/rejected": -10.057367324829102,
+      "step": 460,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.3172845020098258,
+      "grad_norm": 0.9670249223709106,
+      "learning_rate": 0.00012775946831782565,
+      "logits/chosen": -1.1543110609054565,
+      "logits/rejected": -1.1647398471832275,
+      "logps/chosen": -3.428140878677368,
+      "logps/rejected": -97.68719482421875,
+      "loss": 0.1907050460577011,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.17807704210281372,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5219950079917908,
+      "rewards/margins": 9.163191795349121,
+      "rewards/rejected": -8.641196250915527,
+      "step": 461,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 1.3201429209468514,
+      "grad_norm": 2.3520402908325195,
+      "learning_rate": 0.0001274556961058012,
+      "logits/chosen": -1.158630132675171,
+      "logits/rejected": -1.1693530082702637,
+      "logps/chosen": -5.620915412902832,
+      "logps/rejected": -87.42037200927734,
+      "loss": 0.20385050773620605,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1741238236427307,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.455614298582077,
+      "rewards/margins": 8.124811172485352,
+      "rewards/rejected": -7.669197082519531,
+      "step": 462,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 1.3230013398838767,
+      "grad_norm": 2.3216702938079834,
+      "learning_rate": 0.00012715164963592228,
+      "logits/chosen": -1.0936330556869507,
+      "logits/rejected": -1.1035964488983154,
+      "logps/chosen": -4.416335582733154,
+      "logps/rejected": -96.47249603271484,
+      "loss": 0.2630074620246887,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18668687343597412,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4110918939113617,
+      "rewards/margins": 8.527597427368164,
+      "rewards/rejected": -8.116504669189453,
+      "step": 463,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 1.3258597588209022,
+      "grad_norm": 0.8408026099205017,
+      "learning_rate": 0.0001268473319453412,
+      "logits/chosen": -1.1004345417022705,
+      "logits/rejected": -1.1184501647949219,
+      "logps/chosen": -2.8537232875823975,
+      "logps/rejected": -102.44168853759766,
+      "loss": 0.23740537464618683,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1830265074968338,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4311729073524475,
+      "rewards/margins": 9.219640731811523,
+      "rewards/rejected": -8.788467407226562,
+      "step": 464,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.3287181777579278,
+      "grad_norm": 1.450626254081726,
+      "learning_rate": 0.00012654274607391957,
+      "logits/chosen": -1.1033786535263062,
+      "logits/rejected": -1.1105170249938965,
+      "logps/chosen": -6.705106735229492,
+      "logps/rejected": -87.78504943847656,
+      "loss": 0.46575862169265747,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4206477999687195,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.7023568153381348,
+      "rewards/margins": 8.287105560302734,
+      "rewards/rejected": -7.584749221801758,
+      "step": 465,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.331576596694953,
+      "grad_norm": 12.825742721557617,
+      "learning_rate": 0.0001262378950641979,
+      "logits/chosen": -1.1069098711013794,
+      "logits/rejected": -1.1303209066390991,
+      "logps/chosen": -3.534313678741455,
+      "logps/rejected": -100.05030059814453,
+      "loss": 0.22773195803165436,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.19116878509521484,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.40268319845199585,
+      "rewards/margins": 8.56247615814209,
+      "rewards/rejected": -8.159791946411133,
+      "step": 466,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 1.3344350156319786,
+      "grad_norm": 1.016136884689331,
+      "learning_rate": 0.00012593278196136525,
+      "logits/chosen": -1.1097503900527954,
+      "logits/rejected": -1.1294606924057007,
+      "logps/chosen": -5.523367404937744,
+      "logps/rejected": -90.66973876953125,
+      "loss": 0.28550341725349426,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.23537136614322662,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3222227394580841,
+      "rewards/margins": 7.817280292510986,
+      "rewards/rejected": -7.495058059692383,
+      "step": 467,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 1.337293434569004,
+      "grad_norm": 1.0535191297531128,
+      "learning_rate": 0.00012562740981322864,
+      "logits/chosen": -1.1334688663482666,
+      "logits/rejected": -1.1453745365142822,
+      "logps/chosen": -3.900404214859009,
+      "logps/rejected": -112.88008880615234,
+      "loss": 0.22159262001514435,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1639210730791092,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5866712331771851,
+      "rewards/margins": 10.50623607635498,
+      "rewards/rejected": -9.919564247131348,
+      "step": 468,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 1.3401518535060295,
+      "grad_norm": 1.8240563869476318,
+      "learning_rate": 0.00012532178167018283,
+      "logits/chosen": -1.0918580293655396,
+      "logits/rejected": -1.0988843441009521,
+      "logps/chosen": -5.854885101318359,
+      "logps/rejected": -100.22276306152344,
+      "loss": 0.2839359641075134,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2657544016838074,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5278545022010803,
+      "rewards/margins": 9.356338500976562,
+      "rewards/rejected": -8.828485488891602,
+      "step": 469,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 1.3430102724430548,
+      "grad_norm": 0.8436217904090881,
+      "learning_rate": 0.00012501590058517964,
+      "logits/chosen": -1.1500940322875977,
+      "logits/rejected": -1.1596763134002686,
+      "logps/chosen": -3.293513298034668,
+      "logps/rejected": -106.26544189453125,
+      "loss": 0.13652381300926208,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13459700345993042,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5411785244941711,
+      "rewards/margins": 9.85307502746582,
+      "rewards/rejected": -9.311896324157715,
+      "step": 470,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 1.3458686913800804,
+      "grad_norm": 0.550817608833313,
+      "learning_rate": 0.00012470976961369764,
+      "logits/chosen": -1.1057863235473633,
+      "logits/rejected": -1.1126328706741333,
+      "logps/chosen": -5.491030693054199,
+      "logps/rejected": -106.27851104736328,
+      "loss": 0.19622550904750824,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1904832124710083,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.49577879905700684,
+      "rewards/margins": 9.822453498840332,
+      "rewards/rejected": -9.32667350769043,
+      "step": 471,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 1.348727110317106,
+      "grad_norm": 4.487310409545898,
+      "learning_rate": 0.00012440339181371148,
+      "logits/chosen": -1.1322486400604248,
+      "logits/rejected": -1.1392649412155151,
+      "logps/chosen": -3.9112868309020996,
+      "logps/rejected": -90.61821746826172,
+      "loss": 0.28306886553764343,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.27220627665519714,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5206503868103027,
+      "rewards/margins": 8.518290519714355,
+      "rewards/rejected": -7.9976396560668945,
+      "step": 472,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 1.3515855292541312,
+      "grad_norm": 1.0256805419921875,
+      "learning_rate": 0.00012409677024566144,
+      "logits/chosen": -1.157659888267517,
+      "logits/rejected": -1.160758376121521,
+      "logps/chosen": -5.508733749389648,
+      "logps/rejected": -99.47381591796875,
+      "loss": 0.3204554319381714,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2864115238189697,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5594931840896606,
+      "rewards/margins": 9.383455276489258,
+      "rewards/rejected": -8.823962211608887,
+      "step": 473,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 1.3544439481911568,
+      "grad_norm": 68.41111755371094,
+      "learning_rate": 0.00012378990797242285,
+      "logits/chosen": -1.1831049919128418,
+      "logits/rejected": -1.1902449131011963,
+      "logps/chosen": -2.946296453475952,
+      "logps/rejected": -92.81370544433594,
+      "loss": 0.22099502384662628,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20321857929229736,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.44311076402664185,
+      "rewards/margins": 8.712024688720703,
+      "rewards/rejected": -8.268914222717285,
+      "step": 474,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 1.3573023671281823,
+      "grad_norm": 1.3782514333724976,
+      "learning_rate": 0.00012348280805927542,
+      "logits/chosen": -1.1564871072769165,
+      "logits/rejected": -1.1731786727905273,
+      "logps/chosen": -5.935349941253662,
+      "logps/rejected": -107.916748046875,
+      "loss": 0.23125052452087402,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22499611973762512,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4560752511024475,
+      "rewards/margins": 9.845476150512695,
+      "rewards/rejected": -9.389402389526367,
+      "step": 475,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 1.3601607860652076,
+      "grad_norm": 0.8205835819244385,
+      "learning_rate": 0.00012317547357387263,
+      "logits/chosen": -1.1861202716827393,
+      "logits/rejected": -1.2027050256729126,
+      "logps/chosen": -2.852313280105591,
+      "logps/rejected": -117.19053649902344,
+      "loss": 0.1630171239376068,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13181191682815552,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2925358712673187,
+      "rewards/margins": 10.786885261535645,
+      "rewards/rejected": -10.49435043334961,
+      "step": 476,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.3630192050022332,
+      "grad_norm": 0.634852409362793,
+      "learning_rate": 0.00012286790758621132,
+      "logits/chosen": -1.1698333024978638,
+      "logits/rejected": -1.1889692544937134,
+      "logps/chosen": -2.567922592163086,
+      "logps/rejected": -109.34284210205078,
+      "loss": 0.1461510807275772,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13811776041984558,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5183812379837036,
+      "rewards/margins": 10.025300979614258,
+      "rewards/rejected": -9.506919860839844,
+      "step": 477,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.3658776239392587,
+      "grad_norm": 0.9790384769439697,
+      "learning_rate": 0.00012256011316860057,
+      "logits/chosen": -1.1823296546936035,
+      "logits/rejected": -1.196981430053711,
+      "logps/chosen": -5.172409534454346,
+      "logps/rejected": -119.36044311523438,
+      "loss": 0.30066153407096863,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.29738298058509827,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.40812885761260986,
+      "rewards/margins": 10.934396743774414,
+      "rewards/rejected": -10.526268005371094,
+      "step": 478,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.368736042876284,
+      "grad_norm": 0.8576183915138245,
+      "learning_rate": 0.00012225209339563145,
+      "logits/chosen": -1.208139181137085,
+      "logits/rejected": -1.2124953269958496,
+      "logps/chosen": -3.6746914386749268,
+      "logps/rejected": -101.1004867553711,
+      "loss": 0.3288338780403137,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.24615323543548584,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3690415024757385,
+      "rewards/margins": 9.413076400756836,
+      "rewards/rejected": -9.044034957885742,
+      "step": 479,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.3715944618133096,
+      "grad_norm": 9.197681427001953,
+      "learning_rate": 0.00012194385134414608,
+      "logits/chosen": -1.2143265008926392,
+      "logits/rejected": -1.2214550971984863,
+      "logps/chosen": -5.161376476287842,
+      "logps/rejected": -125.9130630493164,
+      "loss": 0.2579571604728699,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1970270872116089,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.6949900388717651,
+      "rewards/margins": 11.783352851867676,
+      "rewards/rejected": -11.088363647460938,
+      "step": 480,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.374452880750335,
+      "grad_norm": 1.8511130809783936,
+      "learning_rate": 0.0001216353900932069,
+      "logits/chosen": -1.1562559604644775,
+      "logits/rejected": -1.168640375137329,
+      "logps/chosen": -5.439113140106201,
+      "logps/rejected": -96.33733367919922,
+      "loss": 0.3344132602214813,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.27547821402549744,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4759036898612976,
+      "rewards/margins": 8.769606590270996,
+      "rewards/rejected": -8.293703079223633,
+      "step": 481,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 1.3773112996873604,
+      "grad_norm": 1.9160902500152588,
+      "learning_rate": 0.00012132671272406604,
+      "logits/chosen": -1.194909691810608,
+      "logits/rejected": -1.2107833623886108,
+      "logps/chosen": -2.7277703285217285,
+      "logps/rejected": -127.53395080566406,
+      "loss": 0.1612652838230133,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14807657897472382,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6826515197753906,
+      "rewards/margins": 12.036901473999023,
+      "rewards/rejected": -11.354249954223633,
+      "step": 482,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.3801697186243858,
+      "grad_norm": 1.5948599576950073,
+      "learning_rate": 0.00012101782232013436,
+      "logits/chosen": -1.220342755317688,
+      "logits/rejected": -1.2318834066390991,
+      "logps/chosen": -3.2254557609558105,
+      "logps/rejected": -120.2594985961914,
+      "loss": 0.247829407453537,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.17633874714374542,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3721011281013489,
+      "rewards/margins": 11.143303871154785,
+      "rewards/rejected": -10.771202087402344,
+      "step": 483,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.3830281375614113,
+      "grad_norm": 29.203384399414062,
+      "learning_rate": 0.00012070872196695089,
+      "logits/chosen": -1.218416690826416,
+      "logits/rejected": -1.2325859069824219,
+      "logps/chosen": -4.859478950500488,
+      "logps/rejected": -124.26551818847656,
+      "loss": 0.25711649656295776,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2555963695049286,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.731487512588501,
+      "rewards/margins": 11.709962844848633,
+      "rewards/rejected": -10.978474617004395,
+      "step": 484,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.3858865564984368,
+      "grad_norm": 0.7737511396408081,
+      "learning_rate": 0.00012039941475215169,
+      "logits/chosen": -1.2052494287490845,
+      "logits/rejected": -1.2190189361572266,
+      "logps/chosen": -5.294344425201416,
+      "logps/rejected": -117.8834457397461,
+      "loss": 0.22346262633800507,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.19474081695079803,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3596336841583252,
+      "rewards/margins": 10.600529670715332,
+      "rewards/rejected": -10.240897178649902,
+      "step": 485,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.3887449754354622,
+      "grad_norm": 0.8315601944923401,
+      "learning_rate": 0.0001200899037654393,
+      "logits/chosen": -1.2053380012512207,
+      "logits/rejected": -1.216891884803772,
+      "logps/chosen": -4.783374786376953,
+      "logps/rejected": -118.34504699707031,
+      "loss": 0.2385406792163849,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21562349796295166,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.41789931058883667,
+      "rewards/margins": 10.679019927978516,
+      "rewards/rejected": -10.261120796203613,
+      "step": 486,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.3916033943724877,
+      "grad_norm": 0.9283787608146667,
+      "learning_rate": 0.00011978019209855174,
+      "logits/chosen": -1.2077966928482056,
+      "logits/rejected": -1.2322336435317993,
+      "logps/chosen": -4.940117359161377,
+      "logps/rejected": -145.7559051513672,
+      "loss": 0.19998067617416382,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.19920994341373444,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.44787928462028503,
+      "rewards/margins": 13.026336669921875,
+      "rewards/rejected": -12.57845687866211,
+      "step": 487,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.3944618133095132,
+      "grad_norm": 0.5662634372711182,
+      "learning_rate": 0.0001194702828452316,
+      "logits/chosen": -1.1853432655334473,
+      "logits/rejected": -1.1943426132202148,
+      "logps/chosen": -7.898364067077637,
+      "logps/rejected": -121.61571502685547,
+      "loss": 0.2524011433124542,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22587642073631287,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3780076503753662,
+      "rewards/margins": 10.995363235473633,
+      "rewards/rejected": -10.617354393005371,
+      "step": 488,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.3973202322465386,
+      "grad_norm": 0.8601922988891602,
+      "learning_rate": 0.00011916017910119524,
+      "logits/chosen": -1.1929762363433838,
+      "logits/rejected": -1.2090179920196533,
+      "logps/chosen": -2.50203800201416,
+      "logps/rejected": -121.39696502685547,
+      "loss": 0.23803545534610748,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1482609063386917,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.4478917121887207,
+      "rewards/margins": 11.308920860290527,
+      "rewards/rejected": -10.861029624938965,
+      "step": 489,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.400178651183564,
+      "grad_norm": 0.647240161895752,
+      "learning_rate": 0.0001188498839641018,
+      "logits/chosen": -1.2096933126449585,
+      "logits/rejected": -1.2204294204711914,
+      "logps/chosen": -2.4534082412719727,
+      "logps/rejected": -122.79830932617188,
+      "loss": 0.1649751216173172,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1564660668373108,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.44028615951538086,
+      "rewards/margins": 11.581570625305176,
+      "rewards/rejected": -11.141283988952637,
+      "step": 490,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.4030370701205896,
+      "grad_norm": 0.7389304041862488,
+      "learning_rate": 0.0001185394005335222,
+      "logits/chosen": -1.220829963684082,
+      "logits/rejected": -1.2447903156280518,
+      "logps/chosen": -8.018527030944824,
+      "logps/rejected": -111.50892639160156,
+      "loss": 0.4398428201675415,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.376664400100708,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3774624168872833,
+      "rewards/margins": 10.216268539428711,
+      "rewards/rejected": -9.838805198669434,
+      "step": 491,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.405895489057615,
+      "grad_norm": 2.56008243560791,
+      "learning_rate": 0.00011822873191090833,
+      "logits/chosen": -1.2262274026870728,
+      "logits/rejected": -1.2465121746063232,
+      "logps/chosen": -5.929446220397949,
+      "logps/rejected": -121.01249694824219,
+      "loss": 0.2873122990131378,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2807019352912903,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.8018953204154968,
+      "rewards/margins": 11.267487525939941,
+      "rewards/rejected": -10.465591430664062,
+      "step": 492,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.4087539079946405,
+      "grad_norm": 2.76714825630188,
+      "learning_rate": 0.0001179178811995619,
+      "logits/chosen": -1.245976448059082,
+      "logits/rejected": -1.2665190696716309,
+      "logps/chosen": -2.6952478885650635,
+      "logps/rejected": -131.0043182373047,
+      "loss": 0.13283787667751312,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13157594203948975,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4866657853126526,
+      "rewards/margins": 12.227144241333008,
+      "rewards/rejected": -11.740479469299316,
+      "step": 493,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.411612326931666,
+      "grad_norm": 0.5778344869613647,
+      "learning_rate": 0.00011760685150460362,
+      "logits/chosen": -1.2286800146102905,
+      "logits/rejected": -1.2474695444107056,
+      "logps/chosen": -3.507291316986084,
+      "logps/rejected": -118.04898071289062,
+      "loss": 0.19644924998283386,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1547624170780182,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5787097215652466,
+      "rewards/margins": 10.566407203674316,
+      "rewards/rejected": -9.98769760131836,
+      "step": 494,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.4144707458686914,
+      "grad_norm": 0.9482585787773132,
+      "learning_rate": 0.00011729564593294202,
+      "logits/chosen": -1.2618703842163086,
+      "logits/rejected": -1.2679365873336792,
+      "logps/chosen": -6.621640205383301,
+      "logps/rejected": -90.37300872802734,
+      "loss": 0.3605477511882782,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2856094539165497,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.6681168079376221,
+      "rewards/margins": 8.50804328918457,
+      "rewards/rejected": -7.839926719665527,
+      "step": 495,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.417329164805717,
+      "grad_norm": 1.035208821296692,
+      "learning_rate": 0.0001169842675932425,
+      "logits/chosen": -1.2108497619628906,
+      "logits/rejected": -1.2162362337112427,
+      "logps/chosen": -11.794290542602539,
+      "logps/rejected": -116.32825469970703,
+      "loss": 1.0830345153808594,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.856827974319458,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.44471532106399536,
+      "rewards/margins": 10.643159866333008,
+      "rewards/rejected": -10.198444366455078,
+      "step": 496,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.4201875837427422,
+      "grad_norm": 89.72486877441406,
+      "learning_rate": 0.00011667271959589623,
+      "logits/chosen": -1.2603520154953003,
+      "logits/rejected": -1.280098557472229,
+      "logps/chosen": -5.925300121307373,
+      "logps/rejected": -112.99394226074219,
+      "loss": 0.4769900143146515,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.32546478509902954,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5293501615524292,
+      "rewards/margins": 10.35472297668457,
+      "rewards/rejected": -9.825372695922852,
+      "step": 497,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.4230460026797678,
+      "grad_norm": 23.98358154296875,
+      "learning_rate": 0.00011636100505298917,
+      "logits/chosen": -1.1377421617507935,
+      "logits/rejected": -1.155126690864563,
+      "logps/chosen": -5.637790203094482,
+      "logps/rejected": -107.21414947509766,
+      "loss": 0.2201746255159378,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.19926278293132782,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.39236244559288025,
+      "rewards/margins": 9.218107223510742,
+      "rewards/rejected": -8.82574462890625,
+      "step": 498,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.425904421616793,
+      "grad_norm": 1.3702232837677002,
+      "learning_rate": 0.00011604912707827083,
+      "logits/chosen": -1.1983606815338135,
+      "logits/rejected": -1.2028909921646118,
+      "logps/chosen": -4.518444538116455,
+      "logps/rejected": -85.0959701538086,
+      "loss": 0.41947945952415466,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3087552785873413,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.33132612705230713,
+      "rewards/margins": 7.8110456466674805,
+      "rewards/rejected": -7.479720115661621,
+      "step": 499,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.4287628405538186,
+      "grad_norm": 47.98428726196289,
+      "learning_rate": 0.00011573708878712334,
+      "logits/chosen": -1.1333822011947632,
+      "logits/rejected": -1.1404212713241577,
+      "logps/chosen": -5.369108200073242,
+      "logps/rejected": -88.99859619140625,
+      "loss": 0.2577302157878876,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.24132868647575378,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.563679575920105,
+      "rewards/margins": 8.241194725036621,
+      "rewards/rejected": -7.677515029907227,
+      "step": 500,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.4316212594908442,
+      "grad_norm": 0.9259874224662781,
+      "learning_rate": 0.00011542489329653024,
+      "logits/chosen": -1.208865761756897,
+      "logits/rejected": -1.2148855924606323,
+      "logps/chosen": -3.9202871322631836,
+      "logps/rejected": -83.33064270019531,
+      "loss": 0.23122766613960266,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1727379858493805,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.31920304894447327,
+      "rewards/margins": 7.403501033782959,
+      "rewards/rejected": -7.084297180175781,
+      "step": 501,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.4344796784278695,
+      "grad_norm": 1.125532627105713,
+      "learning_rate": 0.00011511254372504531,
+      "logits/chosen": -1.1986217498779297,
+      "logits/rejected": -1.2092276811599731,
+      "logps/chosen": -3.3089041709899902,
+      "logps/rejected": -96.10748291015625,
+      "loss": 0.15806397795677185,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14296883344650269,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.48432743549346924,
+      "rewards/margins": 8.464982032775879,
+      "rewards/rejected": -7.980654716491699,
+      "step": 502,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.437338097364895,
+      "grad_norm": 1.2107607126235962,
+      "learning_rate": 0.00011480004319276146,
+      "logits/chosen": -1.139688491821289,
+      "logits/rejected": -1.1440945863723755,
+      "logps/chosen": -5.1867241859436035,
+      "logps/rejected": -73.35174560546875,
+      "loss": 0.3085225820541382,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.24397456645965576,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.36033540964126587,
+      "rewards/margins": 6.39411735534668,
+      "rewards/rejected": -6.0337815284729,
+      "step": 503,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.4401965163019206,
+      "grad_norm": 5.9218292236328125,
+      "learning_rate": 0.00011448739482127964,
+      "logits/chosen": -1.1547845602035522,
+      "logits/rejected": -1.1625118255615234,
+      "logps/chosen": -4.0660505294799805,
+      "logps/rejected": -88.60430145263672,
+      "loss": 0.22802509367465973,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20198072493076324,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5234844088554382,
+      "rewards/margins": 7.882270336151123,
+      "rewards/rejected": -7.358785629272461,
+      "step": 504,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.443054935238946,
+      "grad_norm": 1.762481689453125,
+      "learning_rate": 0.00011417460173367749,
+      "logits/chosen": -1.1733976602554321,
+      "logits/rejected": -1.1826503276824951,
+      "logps/chosen": -2.589683771133423,
+      "logps/rejected": -87.47466278076172,
+      "loss": 0.18850570917129517,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14593477547168732,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5842423439025879,
+      "rewards/margins": 8.038538932800293,
+      "rewards/rejected": -7.454297065734863,
+      "step": 505,
+      "train_speed(iter/s)": 0.005404
+    },
+    {
+      "epoch": 1.4459133541759714,
+      "grad_norm": 0.8148120641708374,
+      "learning_rate": 0.00011386166705447822,
+      "logits/chosen": -1.1997144222259521,
+      "logits/rejected": -1.2027227878570557,
+      "logps/chosen": -4.592447280883789,
+      "logps/rejected": -76.30319213867188,
+      "loss": 0.31897130608558655,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.279499351978302,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.44331854581832886,
+      "rewards/margins": 6.904688358306885,
+      "rewards/rejected": -6.461369514465332,
+      "step": 506,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.448771773112997,
+      "grad_norm": 1.2374672889709473,
+      "learning_rate": 0.00011354859390961958,
+      "logits/chosen": -1.1905529499053955,
+      "logits/rejected": -1.2038395404815674,
+      "logps/chosen": -4.858891487121582,
+      "logps/rejected": -92.95069122314453,
+      "loss": 0.2493872344493866,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22953400015830994,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5827652812004089,
+      "rewards/margins": 8.668669700622559,
+      "rewards/rejected": -8.085904121398926,
+      "step": 507,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.4516301920500223,
+      "grad_norm": 1.2931541204452515,
+      "learning_rate": 0.00011323538542642227,
+      "logits/chosen": -1.110109806060791,
+      "logits/rejected": -1.1193634271621704,
+      "logps/chosen": -4.625039100646973,
+      "logps/rejected": -80.9062728881836,
+      "loss": 0.20781661570072174,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18247604370117188,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5312872529029846,
+      "rewards/margins": 7.455812454223633,
+      "rewards/rejected": -6.924524784088135,
+      "step": 508,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.4544886109870478,
+      "grad_norm": 1.048569679260254,
+      "learning_rate": 0.00011292204473355897,
+      "logits/chosen": -1.2067433595657349,
+      "logits/rejected": -1.2175577878952026,
+      "logps/chosen": -5.1852498054504395,
+      "logps/rejected": -94.29670715332031,
+      "loss": 0.27535420656204224,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2682512700557709,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4187074303627014,
+      "rewards/margins": 8.540119171142578,
+      "rewards/rejected": -8.12141227722168,
+      "step": 509,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.4573470299240732,
+      "grad_norm": 4.128075122833252,
+      "learning_rate": 0.00011260857496102308,
+      "logits/chosen": -1.1686019897460938,
+      "logits/rejected": -1.181231141090393,
+      "logps/chosen": -3.8015005588531494,
+      "logps/rejected": -83.12324523925781,
+      "loss": 0.25129976868629456,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.23475927114486694,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5385525226593018,
+      "rewards/margins": 7.2553253173828125,
+      "rewards/rejected": -6.716772079467773,
+      "step": 510,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.4602054488610987,
+      "grad_norm": 1.234870195388794,
+      "learning_rate": 0.00011229497924009731,
+      "logits/chosen": -1.1313613653182983,
+      "logits/rejected": -1.1462265253067017,
+      "logps/chosen": -4.359948635101318,
+      "logps/rejected": -81.96128845214844,
+      "loss": 0.29682213068008423,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2458692491054535,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.29223746061325073,
+      "rewards/margins": 7.110762119293213,
+      "rewards/rejected": -6.818524360656738,
+      "step": 511,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.463063867798124,
+      "grad_norm": 3.6918282508850098,
+      "learning_rate": 0.00011198126070332253,
+      "logits/chosen": -1.1628670692443848,
+      "logits/rejected": -1.1829794645309448,
+      "logps/chosen": -3.5093777179718018,
+      "logps/rejected": -99.69490814208984,
+      "loss": 0.1665382832288742,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1274329423904419,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5227256417274475,
+      "rewards/margins": 8.994987487792969,
+      "rewards/rejected": -8.472262382507324,
+      "step": 512,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.4659222867351496,
+      "grad_norm": 0.6721384525299072,
+      "learning_rate": 0.0001116674224844664,
+      "logits/chosen": -1.1727080345153809,
+      "logits/rejected": -1.1891160011291504,
+      "logps/chosen": -2.945639133453369,
+      "logps/rejected": -103.56698608398438,
+      "loss": 0.20559175312519073,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13938431441783905,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4741285741329193,
+      "rewards/margins": 9.249117851257324,
+      "rewards/rejected": -8.774988174438477,
+      "step": 513,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.4687807056721751,
+      "grad_norm": 6.535090923309326,
+      "learning_rate": 0.00011135346771849209,
+      "logits/chosen": -1.1876343488693237,
+      "logits/rejected": -1.195608139038086,
+      "logps/chosen": -11.41011905670166,
+      "logps/rejected": -87.87091064453125,
+      "loss": 0.8029640316963196,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6119750142097473,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16716431081295013,
+      "rewards/margins": 7.507657051086426,
+      "rewards/rejected": -7.340492248535156,
+      "step": 514,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.4716391246092004,
+      "grad_norm": 1.00004243850708,
+      "learning_rate": 0.000111039399541527,
+      "logits/chosen": -1.2257870435714722,
+      "logits/rejected": -1.247442364692688,
+      "logps/chosen": -3.210059404373169,
+      "logps/rejected": -114.74640655517578,
+      "loss": 0.12618578970432281,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.11988845467567444,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.361129492521286,
+      "rewards/margins": 10.119677543640137,
+      "rewards/rejected": -9.75854778289795,
+      "step": 515,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.474497543546226,
+      "grad_norm": 1.151992678642273,
+      "learning_rate": 0.00011072522109083136,
+      "logits/chosen": -1.2160663604736328,
+      "logits/rejected": -1.223649263381958,
+      "logps/chosen": -5.680351734161377,
+      "logps/rejected": -93.8958511352539,
+      "loss": 0.21817338466644287,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20911143720149994,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34994593262672424,
+      "rewards/margins": 8.294967651367188,
+      "rewards/rejected": -7.945021629333496,
+      "step": 516,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.4773559624832515,
+      "grad_norm": 0.7956191897392273,
+      "learning_rate": 0.00011041093550476707,
+      "logits/chosen": -1.1727287769317627,
+      "logits/rejected": -1.197590708732605,
+      "logps/chosen": -6.0897536277771,
+      "logps/rejected": -114.73556518554688,
+      "loss": 0.22368358075618744,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.17487390339374542,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.42228540778160095,
+      "rewards/margins": 10.215088844299316,
+      "rewards/rejected": -9.792802810668945,
+      "step": 517,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.4802143814202768,
+      "grad_norm": 0.774364709854126,
+      "learning_rate": 0.00011009654592276602,
+      "logits/chosen": -1.2067716121673584,
+      "logits/rejected": -1.2088109254837036,
+      "logps/chosen": -6.949159145355225,
+      "logps/rejected": -90.97496795654297,
+      "loss": 0.20918475091457367,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20520548522472382,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5352796912193298,
+      "rewards/margins": 8.326306343078613,
+      "rewards/rejected": -7.7910261154174805,
+      "step": 518,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.4830728003573024,
+      "grad_norm": 3.37893009185791,
+      "learning_rate": 0.00010978205548529901,
+      "logits/chosen": -1.2004296779632568,
+      "logits/rejected": -1.2116055488586426,
+      "logps/chosen": -3.89436411857605,
+      "logps/rejected": -95.7611312866211,
+      "loss": 0.4085824489593506,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.35265466570854187,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.37127387523651123,
+      "rewards/margins": 8.661355972290039,
+      "rewards/rejected": -8.290082931518555,
+      "step": 519,
+      "train_speed(iter/s)": 0.005405
+    },
+    {
+      "epoch": 1.485931219294328,
+      "grad_norm": 16.25596046447754,
+      "learning_rate": 0.00010946746733384438,
+      "logits/chosen": -1.2258524894714355,
+      "logits/rejected": -1.2375496625900269,
+      "logps/chosen": -5.31617546081543,
+      "logps/rejected": -109.71346282958984,
+      "loss": 0.13790036737918854,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13614803552627563,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4368322193622589,
+      "rewards/margins": 10.197646141052246,
+      "rewards/rejected": -9.76081371307373,
+      "step": 520,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.4887896382313532,
+      "grad_norm": 0.502770185470581,
+      "learning_rate": 0.0001091527846108565,
+      "logits/chosen": -1.254034399986267,
+      "logits/rejected": -1.2636607885360718,
+      "logps/chosen": -3.331085205078125,
+      "logps/rejected": -97.47418975830078,
+      "loss": 0.22000513970851898,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16505232453346252,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5080281496047974,
+      "rewards/margins": 9.02621078491211,
+      "rewards/rejected": -8.518182754516602,
+      "step": 521,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.4916480571683788,
+      "grad_norm": 1.0531229972839355,
+      "learning_rate": 0.00010883801045973425,
+      "logits/chosen": -1.1826633214950562,
+      "logits/rejected": -1.19447660446167,
+      "logps/chosen": -4.556516647338867,
+      "logps/rejected": -107.08341979980469,
+      "loss": 0.25091084837913513,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.24824769794940948,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.473566472530365,
+      "rewards/margins": 9.908580780029297,
+      "rewards/rejected": -9.435013771057129,
+      "step": 522,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.4945064761054043,
+      "grad_norm": 0.7932538986206055,
+      "learning_rate": 0.00010852314802479009,
+      "logits/chosen": -1.2365505695343018,
+      "logits/rejected": -1.248761534690857,
+      "logps/chosen": -5.015833854675293,
+      "logps/rejected": -103.74222564697266,
+      "loss": 0.2333514541387558,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.23029278218746185,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4175027012825012,
+      "rewards/margins": 9.160269737243652,
+      "rewards/rejected": -8.742767333984375,
+      "step": 523,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.4973648950424296,
+      "grad_norm": 1.06013822555542,
+      "learning_rate": 0.00010820820045121813,
+      "logits/chosen": -1.222113013267517,
+      "logits/rejected": -1.2319118976593018,
+      "logps/chosen": -5.657748699188232,
+      "logps/rejected": -99.86978149414062,
+      "loss": 0.24467575550079346,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2282891571521759,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4060324430465698,
+      "rewards/margins": 9.202392578125,
+      "rewards/rejected": -8.796361923217773,
+      "step": 524,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.500223313979455,
+      "grad_norm": 2.980257987976074,
+      "learning_rate": 0.00010789317088506305,
+      "logits/chosen": -1.248826265335083,
+      "logits/rejected": -1.2513175010681152,
+      "logps/chosen": -5.99161958694458,
+      "logps/rejected": -85.02299499511719,
+      "loss": 0.2671952545642853,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20313005149364471,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.5677384734153748,
+      "rewards/margins": 8.085262298583984,
+      "rewards/rejected": -7.517524719238281,
+      "step": 525,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5030817329164807,
+      "grad_norm": 2.408642053604126,
+      "learning_rate": 0.00010757806247318851,
+      "logits/chosen": -1.23759925365448,
+      "logits/rejected": -1.2442384958267212,
+      "logps/chosen": -5.353574275970459,
+      "logps/rejected": -89.49260711669922,
+      "loss": 0.3324205279350281,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2696898877620697,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.6186475157737732,
+      "rewards/margins": 8.384135246276855,
+      "rewards/rejected": -7.765487194061279,
+      "step": 526,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.505940151853506,
+      "grad_norm": 1.4788644313812256,
+      "learning_rate": 0.00010726287836324582,
+      "logits/chosen": -1.2341465950012207,
+      "logits/rejected": -1.2422211170196533,
+      "logps/chosen": -4.538509845733643,
+      "logps/rejected": -102.45213317871094,
+      "loss": 0.2807333469390869,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2563856840133667,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.48188358545303345,
+      "rewards/margins": 9.553081512451172,
+      "rewards/rejected": -9.071197509765625,
+      "step": 527,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5087985707905314,
+      "grad_norm": 0.7929016947746277,
+      "learning_rate": 0.00010694762170364242,
+      "logits/chosen": -1.2391407489776611,
+      "logits/rejected": -1.249765157699585,
+      "logps/chosen": -2.7391014099121094,
+      "logps/rejected": -103.79396057128906,
+      "loss": 0.17492415010929108,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.17373135685920715,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.45701760053634644,
+      "rewards/margins": 9.713203430175781,
+      "rewards/rejected": -9.256185531616211,
+      "step": 528,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.511656989727557,
+      "grad_norm": 0.791999340057373,
+      "learning_rate": 0.00010663229564351041,
+      "logits/chosen": -1.2260950803756714,
+      "logits/rejected": -1.2434712648391724,
+      "logps/chosen": -5.709774017333984,
+      "logps/rejected": -98.61321258544922,
+      "loss": 0.23840050399303436,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.23402667045593262,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4017852544784546,
+      "rewards/margins": 8.891060829162598,
+      "rewards/rejected": -8.489274978637695,
+      "step": 529,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5145154086645825,
+      "grad_norm": 5.886305809020996,
+      "learning_rate": 0.00010631690333267521,
+      "logits/chosen": -1.1947157382965088,
+      "logits/rejected": -1.2065891027450562,
+      "logps/chosen": -5.48460054397583,
+      "logps/rejected": -95.61801147460938,
+      "loss": 0.3014085590839386,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.29200366139411926,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.46417590975761414,
+      "rewards/margins": 8.575323104858398,
+      "rewards/rejected": -8.1111478805542,
+      "step": 530,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5173738276016078,
+      "grad_norm": 2.0325398445129395,
+      "learning_rate": 0.00010600144792162399,
+      "logits/chosen": -1.151494026184082,
+      "logits/rejected": -1.1706622838974,
+      "logps/chosen": -2.6873748302459717,
+      "logps/rejected": -96.88280487060547,
+      "loss": 0.1879945695400238,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18690775334835052,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.49974530935287476,
+      "rewards/margins": 8.668235778808594,
+      "rewards/rejected": -8.168490409851074,
+      "step": 531,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5202322465386333,
+      "grad_norm": 1.101829171180725,
+      "learning_rate": 0.00010568593256147421,
+      "logits/chosen": -1.1817059516906738,
+      "logits/rejected": -1.1975884437561035,
+      "logps/chosen": -3.4241294860839844,
+      "logps/rejected": -99.36744689941406,
+      "loss": 0.16117167472839355,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15802986919879913,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5246546864509583,
+      "rewards/margins": 8.936569213867188,
+      "rewards/rejected": -8.411913871765137,
+      "step": 532,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5230906654756589,
+      "grad_norm": 0.7341797947883606,
+      "learning_rate": 0.00010537036040394226,
+      "logits/chosen": -1.1905285120010376,
+      "logits/rejected": -1.2094738483428955,
+      "logps/chosen": -3.8606913089752197,
+      "logps/rejected": -100.25830078125,
+      "loss": 0.15881170332431793,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1546865999698639,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6509500741958618,
+      "rewards/margins": 9.299416542053223,
+      "rewards/rejected": -8.648467063903809,
+      "step": 533,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5259490844126842,
+      "grad_norm": 1.048133373260498,
+      "learning_rate": 0.00010505473460131182,
+      "logits/chosen": -1.1491801738739014,
+      "logits/rejected": -1.1666440963745117,
+      "logps/chosen": -3.386275291442871,
+      "logps/rejected": -95.7393569946289,
+      "loss": 0.1639423668384552,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14697596430778503,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.41195330023765564,
+      "rewards/margins": 8.413421630859375,
+      "rewards/rejected": -8.00146770477295,
+      "step": 534,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5288075033497097,
+      "grad_norm": 0.7253103256225586,
+      "learning_rate": 0.00010473905830640238,
+      "logits/chosen": -1.2081520557403564,
+      "logits/rejected": -1.2300549745559692,
+      "logps/chosen": -4.6544599533081055,
+      "logps/rejected": -102.84630584716797,
+      "loss": 0.2066834568977356,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18399091064929962,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5013687610626221,
+      "rewards/margins": 9.11780834197998,
+      "rewards/rejected": -8.616438865661621,
+      "step": 535,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5316659222867353,
+      "grad_norm": 0.8472306132316589,
+      "learning_rate": 0.00010442333467253789,
+      "logits/chosen": -1.1342350244522095,
+      "logits/rejected": -1.1452178955078125,
+      "logps/chosen": -6.043959140777588,
+      "logps/rejected": -86.95714569091797,
+      "loss": 0.30448004603385925,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.29562440514564514,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6062089800834656,
+      "rewards/margins": 7.933745861053467,
+      "rewards/rejected": -7.327536582946777,
+      "step": 536,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5345243412237606,
+      "grad_norm": 3.631605863571167,
+      "learning_rate": 0.00010410756685351517,
+      "logits/chosen": -1.1697407960891724,
+      "logits/rejected": -1.1903973817825317,
+      "logps/chosen": -2.9826488494873047,
+      "logps/rejected": -102.60981750488281,
+      "loss": 0.1628805547952652,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1444227546453476,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4331924617290497,
+      "rewards/margins": 9.25413703918457,
+      "rewards/rejected": -8.820945739746094,
+      "step": 537,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.537382760160786,
+      "grad_norm": 4.747304439544678,
+      "learning_rate": 0.0001037917580035724,
+      "logits/chosen": -1.1395968198776245,
+      "logits/rejected": -1.1540745496749878,
+      "logps/chosen": -4.616944789886475,
+      "logps/rejected": -96.46205139160156,
+      "loss": 0.18046514689922333,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15388183295726776,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4708944261074066,
+      "rewards/margins": 8.634149551391602,
+      "rewards/rejected": -8.163254737854004,
+      "step": 538,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5402411790978117,
+      "grad_norm": 1.0566221475601196,
+      "learning_rate": 0.00010347591127735754,
+      "logits/chosen": -1.1123058795928955,
+      "logits/rejected": -1.1218605041503906,
+      "logps/chosen": -4.521425247192383,
+      "logps/rejected": -98.85065460205078,
+      "loss": 0.25702419877052307,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2428530752658844,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4769361615180969,
+      "rewards/margins": 9.206279754638672,
+      "rewards/rejected": -8.72934341430664,
+      "step": 539,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.543099598034837,
+      "grad_norm": 0.8213050365447998,
+      "learning_rate": 0.00010316002982989704,
+      "logits/chosen": -1.1132948398590088,
+      "logits/rejected": -1.129378080368042,
+      "logps/chosen": -2.907618522644043,
+      "logps/rejected": -92.56205749511719,
+      "loss": 0.19241493940353394,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16153347492218018,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5939139723777771,
+      "rewards/margins": 8.283432006835938,
+      "rewards/rejected": -7.689518451690674,
+      "step": 540,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5459580169718623,
+      "grad_norm": 0.8551529049873352,
+      "learning_rate": 0.00010284411681656408,
+      "logits/chosen": -1.2044731378555298,
+      "logits/rejected": -1.209726333618164,
+      "logps/chosen": -6.703289985656738,
+      "logps/rejected": -88.3790283203125,
+      "loss": 0.3231280446052551,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.30079561471939087,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6794237494468689,
+      "rewards/margins": 8.502208709716797,
+      "rewards/rejected": -7.822785377502441,
+      "step": 541,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5488164359088878,
+      "grad_norm": 1.2103654146194458,
+      "learning_rate": 0.00010252817539304718,
+      "logits/chosen": -1.1682829856872559,
+      "logits/rejected": -1.1821935176849365,
+      "logps/chosen": -8.161247253417969,
+      "logps/rejected": -91.14198303222656,
+      "loss": 0.7169775366783142,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.5195838212966919,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2014901340007782,
+      "rewards/margins": 8.132518768310547,
+      "rewards/rejected": -7.931027412414551,
+      "step": 542,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5516748548459134,
+      "grad_norm": 16.268648147583008,
+      "learning_rate": 0.00010221220871531869,
+      "logits/chosen": -1.1647533178329468,
+      "logits/rejected": -1.1727814674377441,
+      "logps/chosen": -4.000942707061768,
+      "logps/rejected": -95.975341796875,
+      "loss": 0.23619498312473297,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20583844184875488,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.6236611604690552,
+      "rewards/margins": 9.150650978088379,
+      "rewards/rejected": -8.526989936828613,
+      "step": 543,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5545332737829387,
+      "grad_norm": 0.8984525799751282,
+      "learning_rate": 0.00010189621993960319,
+      "logits/chosen": -1.1389577388763428,
+      "logits/rejected": -1.1588479280471802,
+      "logps/chosen": -2.0922861099243164,
+      "logps/rejected": -99.55612182617188,
+      "loss": 0.15487171709537506,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.11652800440788269,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4093736708164215,
+      "rewards/margins": 8.549884796142578,
+      "rewards/rejected": -8.140511512756348,
+      "step": 544,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5573916927199642,
+      "grad_norm": 8.225284576416016,
+      "learning_rate": 0.00010158021222234602,
+      "logits/chosen": -1.2096428871154785,
+      "logits/rejected": -1.2262201309204102,
+      "logps/chosen": -4.548305511474609,
+      "logps/rejected": -99.0030517578125,
+      "loss": 0.16668973863124847,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15765738487243652,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5047243237495422,
+      "rewards/margins": 8.951805114746094,
+      "rewards/rejected": -8.447081565856934,
+      "step": 545,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5602501116569898,
+      "grad_norm": 0.8983806371688843,
+      "learning_rate": 0.00010126418872018169,
+      "logits/chosen": -1.1991511583328247,
+      "logits/rejected": -1.209883689880371,
+      "logps/chosen": -4.1333327293396,
+      "logps/rejected": -94.91553497314453,
+      "loss": 0.22168555855751038,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20513451099395752,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5711698532104492,
+      "rewards/margins": 8.50210189819336,
+      "rewards/rejected": -7.930931568145752,
+      "step": 546,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.563108530594015,
+      "grad_norm": 1.7599843740463257,
+      "learning_rate": 0.00010094815258990241,
+      "logits/chosen": -1.1996725797653198,
+      "logits/rejected": -1.2069827318191528,
+      "logps/chosen": -7.075511455535889,
+      "logps/rejected": -77.37564086914062,
+      "loss": 0.34534481167793274,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.29777780175209045,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.7596228122711182,
+      "rewards/margins": 7.05924129486084,
+      "rewards/rejected": -6.299618244171143,
+      "step": 547,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5659669495310407,
+      "grad_norm": 1.2366605997085571,
+      "learning_rate": 0.00010063210698842655,
+      "logits/chosen": -1.1746978759765625,
+      "logits/rejected": -1.1860934495925903,
+      "logps/chosen": -3.1920042037963867,
+      "logps/rejected": -96.23448181152344,
+      "loss": 0.13993731141090393,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13228188455104828,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5295498967170715,
+      "rewards/margins": 8.722616195678711,
+      "rewards/rejected": -8.193065643310547,
+      "step": 548,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5688253684680662,
+      "grad_norm": 0.6894782185554504,
+      "learning_rate": 0.00010031605507276705,
+      "logits/chosen": -1.2104339599609375,
+      "logits/rejected": -1.2190818786621094,
+      "logps/chosen": -3.9462296962738037,
+      "logps/rejected": -91.4450912475586,
+      "loss": 0.21633976697921753,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18217234313488007,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.48840969800949097,
+      "rewards/margins": 8.675142288208008,
+      "rewards/rejected": -8.18673324584961,
+      "step": 549,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5716837874050915,
+      "grad_norm": 0.7373618483543396,
+      "learning_rate": 0.0001,
+      "logits/chosen": -1.2142421007156372,
+      "logits/rejected": -1.2380558252334595,
+      "logps/chosen": -2.7597246170043945,
+      "logps/rejected": -105.89647674560547,
+      "loss": 0.20728927850723267,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20096683502197266,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.515934944152832,
+      "rewards/margins": 9.024059295654297,
+      "rewards/rejected": -8.508124351501465,
+      "step": 550,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.574542206342117,
+      "grad_norm": 1.0780363082885742,
+      "learning_rate": 9.968394492723299e-05,
+      "logits/chosen": -1.1850985288619995,
+      "logits/rejected": -1.19618821144104,
+      "logps/chosen": -3.918487787246704,
+      "logps/rejected": -99.82266998291016,
+      "loss": 0.18428170680999756,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1705784946680069,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.309816837310791,
+      "rewards/margins": 8.807823181152344,
+      "rewards/rejected": -8.498005867004395,
+      "step": 551,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5774006252791426,
+      "grad_norm": 0.6045752167701721,
+      "learning_rate": 9.936789301157347e-05,
+      "logits/chosen": -1.200334072113037,
+      "logits/rejected": -1.2195042371749878,
+      "logps/chosen": -3.4113171100616455,
+      "logps/rejected": -101.41400909423828,
+      "loss": 0.1890278160572052,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16223224997520447,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6845565438270569,
+      "rewards/margins": 9.446554183959961,
+      "rewards/rejected": -8.761998176574707,
+      "step": 552,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.580259044216168,
+      "grad_norm": 0.6226416230201721,
+      "learning_rate": 9.905184741009764e-05,
+      "logits/chosen": -1.1828663349151611,
+      "logits/rejected": -1.2045037746429443,
+      "logps/chosen": -4.39558219909668,
+      "logps/rejected": -95.02586364746094,
+      "loss": 0.20744438469409943,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.19546304643154144,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3974071741104126,
+      "rewards/margins": 8.272377967834473,
+      "rewards/rejected": -7.874969959259033,
+      "step": 553,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.5831174631531932,
+      "grad_norm": 0.8041784763336182,
+      "learning_rate": 9.873581127981834e-05,
+      "logits/chosen": -1.2021336555480957,
+      "logits/rejected": -1.215855598449707,
+      "logps/chosen": -5.235670566558838,
+      "logps/rejected": -98.94778442382812,
+      "loss": 0.3087376654148102,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.30778488516807556,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.508550226688385,
+      "rewards/margins": 9.163681030273438,
+      "rewards/rejected": -8.655131340026855,
+      "step": 554,
+      "train_speed(iter/s)": 0.005406
+    },
+    {
+      "epoch": 1.585975882090219,
+      "grad_norm": 0.8672412633895874,
+      "learning_rate": 9.841978777765402e-05,
+      "logits/chosen": -1.2232412099838257,
+      "logits/rejected": -1.2419785261154175,
+      "logps/chosen": -4.191494464874268,
+      "logps/rejected": -115.4002914428711,
+      "loss": 0.14720436930656433,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1216704249382019,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4956449568271637,
+      "rewards/margins": 10.168661117553711,
+      "rewards/rejected": -9.673015594482422,
+      "step": 555,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 1.5888343010272443,
+      "grad_norm": 0.7310967445373535,
+      "learning_rate": 9.81037800603968e-05,
+      "logits/chosen": -1.2371845245361328,
+      "logits/rejected": -1.2543206214904785,
+      "logps/chosen": -3.877095937728882,
+      "logps/rejected": -100.69743347167969,
+      "loss": 0.1970527470111847,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1900380700826645,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4485085904598236,
+      "rewards/margins": 9.28439712524414,
+      "rewards/rejected": -8.835887908935547,
+      "step": 556,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 1.5916927199642696,
+      "grad_norm": 1.4273149967193604,
+      "learning_rate": 9.778779128468132e-05,
+      "logits/chosen": -1.2515287399291992,
+      "logits/rejected": -1.2682404518127441,
+      "logps/chosen": -3.310947895050049,
+      "logps/rejected": -96.26271057128906,
+      "loss": 0.1776140332221985,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.17431208491325378,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.46884265542030334,
+      "rewards/margins": 8.55155086517334,
+      "rewards/rejected": -8.082708358764648,
+      "step": 557,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 1.5945511389012952,
+      "grad_norm": 0.7328734993934631,
+      "learning_rate": 9.747182460695282e-05,
+      "logits/chosen": -1.2264363765716553,
+      "logits/rejected": -1.2469466924667358,
+      "logps/chosen": -3.252176284790039,
+      "logps/rejected": -101.14380645751953,
+      "loss": 0.16898810863494873,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1429700255393982,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.537392258644104,
+      "rewards/margins": 9.478887557983398,
+      "rewards/rejected": -8.941495895385742,
+      "step": 558,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 1.5974095578383207,
+      "grad_norm": 0.5761097073554993,
+      "learning_rate": 9.715588318343593e-05,
+      "logits/chosen": -1.2373993396759033,
+      "logits/rejected": -1.2531373500823975,
+      "logps/chosen": -3.8606765270233154,
+      "logps/rejected": -99.5583724975586,
+      "loss": 0.17141491174697876,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15882301330566406,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5505474209785461,
+      "rewards/margins": 8.967745780944824,
+      "rewards/rejected": -8.417198181152344,
+      "step": 559,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 1.600267976775346,
+      "grad_norm": 0.9028222560882568,
+      "learning_rate": 9.6839970170103e-05,
+      "logits/chosen": -1.205949068069458,
+      "logits/rejected": -1.2180453538894653,
+      "logps/chosen": -5.940152645111084,
+      "logps/rejected": -84.56391906738281,
+      "loss": 0.2950308620929718,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.23100607097148895,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.702341616153717,
+      "rewards/margins": 7.944380760192871,
+      "rewards/rejected": -7.242038726806641,
+      "step": 560,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 1.6031263957123716,
+      "grad_norm": 1.0543800592422485,
+      "learning_rate": 9.652408872264249e-05,
+      "logits/chosen": -1.1957297325134277,
+      "logits/rejected": -1.2104666233062744,
+      "logps/chosen": -4.604339599609375,
+      "logps/rejected": -101.80133819580078,
+      "loss": 0.238393172621727,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22904415428638458,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5547606348991394,
+      "rewards/margins": 9.415586471557617,
+      "rewards/rejected": -8.860825538635254,
+      "step": 561,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 1.6059848146493971,
+      "grad_norm": 0.9146276116371155,
+      "learning_rate": 9.620824199642764e-05,
+      "logits/chosen": -1.228825569152832,
+      "logits/rejected": -1.2460359334945679,
+      "logps/chosen": -6.999305725097656,
+      "logps/rejected": -101.86087036132812,
+      "loss": 0.29198122024536133,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.27209144830703735,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5028784871101379,
+      "rewards/margins": 8.82616901397705,
+      "rewards/rejected": -8.32328987121582,
+      "step": 562,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 1.6088432335864224,
+      "grad_norm": 0.9446074366569519,
+      "learning_rate": 9.589243314648482e-05,
+      "logits/chosen": -1.2695645093917847,
+      "logits/rejected": -1.2769098281860352,
+      "logps/chosen": -3.131113290786743,
+      "logps/rejected": -102.22225952148438,
+      "loss": 0.23388896882534027,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18364675343036652,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.32783710956573486,
+      "rewards/margins": 9.313203811645508,
+      "rewards/rejected": -8.985366821289062,
+      "step": 563,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 1.611701652523448,
+      "grad_norm": 0.8410360217094421,
+      "learning_rate": 9.557666532746213e-05,
+      "logits/chosen": -1.2425626516342163,
+      "logits/rejected": -1.257200837135315,
+      "logps/chosen": -4.220077991485596,
+      "logps/rejected": -108.97372436523438,
+      "loss": 0.21360714733600616,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18227827548980713,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5547472834587097,
+      "rewards/margins": 9.62452220916748,
+      "rewards/rejected": -9.069774627685547,
+      "step": 564,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 1.6145600714604735,
+      "grad_norm": 3.650477409362793,
+      "learning_rate": 9.526094169359765e-05,
+      "logits/chosen": -1.2563040256500244,
+      "logits/rejected": -1.2690678834915161,
+      "logps/chosen": -4.583124160766602,
+      "logps/rejected": -89.01020050048828,
+      "loss": 0.3627120852470398,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.30041414499282837,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4141501188278198,
+      "rewards/margins": 8.265939712524414,
+      "rewards/rejected": -7.8517889976501465,
+      "step": 565,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 1.6174184903974989,
+      "grad_norm": 9.177145957946777,
+      "learning_rate": 9.494526539868822e-05,
+      "logits/chosen": -1.217350721359253,
+      "logits/rejected": -1.2367099523544312,
+      "logps/chosen": -4.153363227844238,
+      "logps/rejected": -106.2103500366211,
+      "loss": 0.24249762296676636,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.23106533288955688,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5973933339118958,
+      "rewards/margins": 9.575693130493164,
+      "rewards/rejected": -8.978300094604492,
+      "step": 566,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 1.6202769093345244,
+      "grad_norm": 0.7864315509796143,
+      "learning_rate": 9.462963959605778e-05,
+      "logits/chosen": -1.25490403175354,
+      "logits/rejected": -1.2653908729553223,
+      "logps/chosen": -7.570868015289307,
+      "logps/rejected": -98.19856262207031,
+      "loss": 0.8767191171646118,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6035685539245605,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.30315718054771423,
+      "rewards/margins": 8.980510711669922,
+      "rewards/rejected": -8.677353858947754,
+      "step": 567,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.62313532827155,
+      "grad_norm": 3.0301690101623535,
+      "learning_rate": 9.43140674385258e-05,
+      "logits/chosen": -1.2209892272949219,
+      "logits/rejected": -1.2403444051742554,
+      "logps/chosen": -1.368801474571228,
+      "logps/rejected": -107.77033996582031,
+      "loss": 0.1274924874305725,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.10486172139644623,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.48542851209640503,
+      "rewards/margins": 9.754288673400879,
+      "rewards/rejected": -9.26885986328125,
+      "step": 568,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.6259937472085753,
+      "grad_norm": 0.9174420237541199,
+      "learning_rate": 9.399855207837606e-05,
+      "logits/chosen": -1.2614572048187256,
+      "logits/rejected": -1.273358702659607,
+      "logps/chosen": -4.721066951751709,
+      "logps/rejected": -93.51985931396484,
+      "loss": 0.16965904831886292,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15142567455768585,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5274036526679993,
+      "rewards/margins": 8.461057662963867,
+      "rewards/rejected": -7.933653354644775,
+      "step": 569,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.6288521661456006,
+      "grad_norm": 1.023853063583374,
+      "learning_rate": 9.368309666732481e-05,
+      "logits/chosen": -1.2322520017623901,
+      "logits/rejected": -1.2480456829071045,
+      "logps/chosen": -6.101306438446045,
+      "logps/rejected": -101.52810668945312,
+      "loss": 0.19685791432857513,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.19015911221504211,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.41192999482154846,
+      "rewards/margins": 8.994438171386719,
+      "rewards/rejected": -8.582507133483887,
+      "step": 570,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.6317105850826261,
+      "grad_norm": 0.6543206572532654,
+      "learning_rate": 9.336770435648964e-05,
+      "logits/chosen": -1.2358951568603516,
+      "logits/rejected": -1.2526280879974365,
+      "logps/chosen": -3.6635141372680664,
+      "logps/rejected": -99.887451171875,
+      "loss": 0.20197272300720215,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16135680675506592,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5029939413070679,
+      "rewards/margins": 9.152663230895996,
+      "rewards/rejected": -8.649669647216797,
+      "step": 571,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.6345690040196517,
+      "grad_norm": 3.662933588027954,
+      "learning_rate": 9.30523782963576e-05,
+      "logits/chosen": -1.230069637298584,
+      "logits/rejected": -1.2422807216644287,
+      "logps/chosen": -10.505573272705078,
+      "logps/rejected": -85.44378662109375,
+      "loss": 0.9089314937591553,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.6872246265411377,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4905371367931366,
+      "rewards/margins": 8.058351516723633,
+      "rewards/rejected": -7.567814350128174,
+      "step": 572,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 1.637427422956677,
+      "grad_norm": 1.039322853088379,
+      "learning_rate": 9.27371216367542e-05,
+      "logits/chosen": -1.2223392724990845,
+      "logits/rejected": -1.2409743070602417,
+      "logps/chosen": -3.3646481037139893,
+      "logps/rejected": -100.68852996826172,
+      "loss": 0.13831134140491486,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.12199953198432922,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4276912212371826,
+      "rewards/margins": 8.902277946472168,
+      "rewards/rejected": -8.474586486816406,
+      "step": 573,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 1.6402858418937025,
+      "grad_norm": 0.6266536116600037,
+      "learning_rate": 9.242193752681149e-05,
+      "logits/chosen": -1.217388391494751,
+      "logits/rejected": -1.2337368726730347,
+      "logps/chosen": -10.087833404541016,
+      "logps/rejected": -106.16631317138672,
+      "loss": 0.27814164757728577,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2720891833305359,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5779958963394165,
+      "rewards/margins": 9.594115257263184,
+      "rewards/rejected": -9.016119956970215,
+      "step": 574,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.643144260830728,
+      "grad_norm": 1.939039945602417,
+      "learning_rate": 9.210682911493696e-05,
+      "logits/chosen": -1.1878530979156494,
+      "logits/rejected": -1.1841703653335571,
+      "logps/chosen": -9.380166053771973,
+      "logps/rejected": -74.41200256347656,
+      "loss": 0.4060296416282654,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.34181496500968933,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.368032842874527,
+      "rewards/margins": 6.767005920410156,
+      "rewards/rejected": -6.398972511291504,
+      "step": 575,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.6460026797677534,
+      "grad_norm": 1.4155151844024658,
+      "learning_rate": 9.179179954878191e-05,
+      "logits/chosen": -1.2296230792999268,
+      "logits/rejected": -1.2440543174743652,
+      "logps/chosen": -3.602640390396118,
+      "logps/rejected": -102.97189331054688,
+      "loss": 0.1902437061071396,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.17669150233268738,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4757131338119507,
+      "rewards/margins": 9.499579429626465,
+      "rewards/rejected": -9.023865699768066,
+      "step": 576,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 1.648861098704779,
+      "grad_norm": 0.6756049990653992,
+      "learning_rate": 9.147685197520995e-05,
+      "logits/chosen": -1.232581615447998,
+      "logits/rejected": -1.2494488954544067,
+      "logps/chosen": -3.071582317352295,
+      "logps/rejected": -104.58504486083984,
+      "loss": 0.15779809653759003,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1385500133037567,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5847654342651367,
+      "rewards/margins": 9.329889297485352,
+      "rewards/rejected": -8.745122909545898,
+      "step": 577,
+      "train_speed(iter/s)": 0.005407
+    },
+    {
+      "epoch": 1.6517195176418045,
+      "grad_norm": 0.5691357254981995,
+      "learning_rate": 9.116198954026577e-05,
+      "logits/chosen": -1.2136074304580688,
+      "logits/rejected": -1.231778860092163,
+      "logps/chosen": -3.6910195350646973,
+      "logps/rejected": -104.98346710205078,
+      "loss": 0.2058335244655609,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15693770349025726,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.24480776488780975,
+      "rewards/margins": 9.162877082824707,
+      "rewards/rejected": -8.918069839477539,
+      "step": 578,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.6545779365788298,
+      "grad_norm": 0.98762047290802,
+      "learning_rate": 9.084721538914354e-05,
+      "logits/chosen": -1.1979000568389893,
+      "logits/rejected": -1.1992981433868408,
+      "logps/chosen": -8.207277297973633,
+      "logps/rejected": -76.73429107666016,
+      "loss": 0.38480570912361145,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.32915711402893066,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.424187570810318,
+      "rewards/margins": 7.018537521362305,
+      "rewards/rejected": -6.594350337982178,
+      "step": 579,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.6574363555158553,
+      "grad_norm": 1.4615544080734253,
+      "learning_rate": 9.053253266615563e-05,
+      "logits/chosen": -1.2005051374435425,
+      "logits/rejected": -1.207005500793457,
+      "logps/chosen": -3.0953447818756104,
+      "logps/rejected": -90.17530822753906,
+      "loss": 0.20222529768943787,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16874569654464722,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4227086007595062,
+      "rewards/margins": 8.343812942504883,
+      "rewards/rejected": -7.921104907989502,
+      "step": 580,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.6602947744528809,
+      "grad_norm": 0.9963876605033875,
+      "learning_rate": 9.0217944514701e-05,
+      "logits/chosen": -1.2267546653747559,
+      "logits/rejected": -1.236647129058838,
+      "logps/chosen": -4.396671295166016,
+      "logps/rejected": -106.08937072753906,
+      "loss": 0.35176020860671997,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3066858649253845,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.40714532136917114,
+      "rewards/margins": 9.592981338500977,
+      "rewards/rejected": -9.185835838317871,
+      "step": 581,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.6631531933899062,
+      "grad_norm": 4.256789684295654,
+      "learning_rate": 8.990345407723402e-05,
+      "logits/chosen": -1.2223454713821411,
+      "logits/rejected": -1.2440755367279053,
+      "logps/chosen": -4.9550347328186035,
+      "logps/rejected": -101.04862213134766,
+      "loss": 0.22037917375564575,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21300825476646423,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4511691927909851,
+      "rewards/margins": 8.963902473449707,
+      "rewards/rejected": -8.512733459472656,
+      "step": 582,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.6660116123269315,
+      "grad_norm": 1.3565281629562378,
+      "learning_rate": 8.958906449523294e-05,
+      "logits/chosen": -1.1865954399108887,
+      "logits/rejected": -1.2029664516448975,
+      "logps/chosen": -6.465363025665283,
+      "logps/rejected": -98.9941635131836,
+      "loss": 0.5277466177940369,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4747985601425171,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5788065195083618,
+      "rewards/margins": 9.07010269165039,
+      "rewards/rejected": -8.49129581451416,
+      "step": 583,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.6688700312639573,
+      "grad_norm": 19.33406639099121,
+      "learning_rate": 8.927477890916865e-05,
+      "logits/chosen": -1.146034598350525,
+      "logits/rejected": -1.1573537588119507,
+      "logps/chosen": -5.037684440612793,
+      "logps/rejected": -94.7759780883789,
+      "loss": 0.40787991881370544,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3248973488807678,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.385827898979187,
+      "rewards/margins": 8.398825645446777,
+      "rewards/rejected": -8.0129976272583,
+      "step": 584,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.6717284502009826,
+      "grad_norm": 11.837591171264648,
+      "learning_rate": 8.896060045847304e-05,
+      "logits/chosen": -1.1754140853881836,
+      "logits/rejected": -1.1862783432006836,
+      "logps/chosen": -4.227570056915283,
+      "logps/rejected": -103.70059204101562,
+      "loss": 0.21096792817115784,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.19895918667316437,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7301295399665833,
+      "rewards/margins": 9.55334186553955,
+      "rewards/rejected": -8.823212623596191,
+      "step": 585,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.674586869138008,
+      "grad_norm": 0.8223007321357727,
+      "learning_rate": 8.864653228150794e-05,
+      "logits/chosen": -1.1963828802108765,
+      "logits/rejected": -1.2057627439498901,
+      "logps/chosen": -5.076923847198486,
+      "logps/rejected": -97.21467590332031,
+      "loss": 0.2129833847284317,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2047015130519867,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4923573136329651,
+      "rewards/margins": 9.023185729980469,
+      "rewards/rejected": -8.530828475952148,
+      "step": 586,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.6774452880750335,
+      "grad_norm": 0.682444155216217,
+      "learning_rate": 8.833257751553365e-05,
+      "logits/chosen": -1.192571759223938,
+      "logits/rejected": -1.1966464519500732,
+      "logps/chosen": -5.998629570007324,
+      "logps/rejected": -95.70146179199219,
+      "loss": 0.24226294457912445,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2067227065563202,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.6862426996231079,
+      "rewards/margins": 8.965702056884766,
+      "rewards/rejected": -8.279458999633789,
+      "step": 587,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.680303707012059,
+      "grad_norm": 0.8181328177452087,
+      "learning_rate": 8.801873929667749e-05,
+      "logits/chosen": -1.1211837530136108,
+      "logits/rejected": -1.1410701274871826,
+      "logps/chosen": -3.5135490894317627,
+      "logps/rejected": -117.64289855957031,
+      "loss": 0.18599815666675568,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14936751127243042,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3863241970539093,
+      "rewards/margins": 10.570147514343262,
+      "rewards/rejected": -10.183822631835938,
+      "step": 588,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.6831621259490843,
+      "grad_norm": 1.1397695541381836,
+      "learning_rate": 8.77050207599027e-05,
+      "logits/chosen": -1.1019999980926514,
+      "logits/rejected": -1.1149682998657227,
+      "logps/chosen": -2.0262680053710938,
+      "logps/rejected": -104.42642211914062,
+      "loss": 0.15114909410476685,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1371248960494995,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3417889177799225,
+      "rewards/margins": 9.413578033447266,
+      "rewards/rejected": -9.071789741516113,
+      "step": 589,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.6860205448861099,
+      "grad_norm": 0.9280815124511719,
+      "learning_rate": 8.739142503897693e-05,
+      "logits/chosen": -1.1994444131851196,
+      "logits/rejected": -1.2191089391708374,
+      "logps/chosen": -3.230380058288574,
+      "logps/rejected": -116.66558837890625,
+      "loss": 0.19586162269115448,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.17760922014713287,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5155038833618164,
+      "rewards/margins": 10.553391456604004,
+      "rewards/rejected": -10.037887573242188,
+      "step": 590,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.6888789638231354,
+      "grad_norm": 4.784222602844238,
+      "learning_rate": 8.707795526644106e-05,
+      "logits/chosen": -1.1569873094558716,
+      "logits/rejected": -1.1652321815490723,
+      "logps/chosen": -4.722146987915039,
+      "logps/rejected": -103.7422103881836,
+      "loss": 0.2793360948562622,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.23997855186462402,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.7041978240013123,
+      "rewards/margins": 9.703722953796387,
+      "rewards/rejected": -8.99952507019043,
+      "step": 591,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.6917373827601607,
+      "grad_norm": 0.943581759929657,
+      "learning_rate": 8.676461457357776e-05,
+      "logits/chosen": -1.1320044994354248,
+      "logits/rejected": -1.138698935508728,
+      "logps/chosen": -5.889208793640137,
+      "logps/rejected": -93.14749145507812,
+      "loss": 0.3549827039241791,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.32344335317611694,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3422737717628479,
+      "rewards/margins": 8.550951957702637,
+      "rewards/rejected": -8.20867919921875,
+      "step": 592,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.6945958016971863,
+      "grad_norm": 2.039912462234497,
+      "learning_rate": 8.645140609038046e-05,
+      "logits/chosen": -1.098019003868103,
+      "logits/rejected": -1.115633487701416,
+      "logps/chosen": -3.978429079055786,
+      "logps/rejected": -117.03365325927734,
+      "loss": 0.17504629492759705,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15153054893016815,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.606253445148468,
+      "rewards/margins": 10.49371337890625,
+      "rewards/rejected": -9.887459754943848,
+      "step": 593,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.6974542206342118,
+      "grad_norm": 1.539162039756775,
+      "learning_rate": 8.61383329455218e-05,
+      "logits/chosen": -1.1445422172546387,
+      "logits/rejected": -1.1526044607162476,
+      "logps/chosen": -4.6503682136535645,
+      "logps/rejected": -98.78849792480469,
+      "loss": 0.25876373052597046,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.24835312366485596,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.261483758687973,
+      "rewards/margins": 8.811944961547852,
+      "rewards/rejected": -8.550460815429688,
+      "step": 594,
+      "train_speed(iter/s)": 0.005408
+    },
+    {
+      "epoch": 1.7003126395712371,
+      "grad_norm": 9.246718406677246,
+      "learning_rate": 8.582539826632254e-05,
+      "logits/chosen": -1.1492161750793457,
+      "logits/rejected": -1.155732274055481,
+      "logps/chosen": -5.179446697235107,
+      "logps/rejected": -103.48001861572266,
+      "loss": 0.23773784935474396,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22704634070396423,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6112527847290039,
+      "rewards/margins": 9.949883460998535,
+      "rewards/rejected": -9.338631629943848,
+      "step": 595,
+      "train_speed(iter/s)": 0.005409
+    },
+    {
+      "epoch": 1.7031710585082627,
+      "grad_norm": 0.8107644319534302,
+      "learning_rate": 8.55126051787204e-05,
+      "logits/chosen": -1.1577305793762207,
+      "logits/rejected": -1.1765245199203491,
+      "logps/chosen": -3.527681827545166,
+      "logps/rejected": -112.77857971191406,
+      "loss": 0.14476284384727478,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14021086692810059,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6686225533485413,
+      "rewards/margins": 10.401530265808105,
+      "rewards/rejected": -9.732908248901367,
+      "step": 596,
+      "train_speed(iter/s)": 0.005409
+    },
+    {
+      "epoch": 1.7060294774452882,
+      "grad_norm": 0.5915255546569824,
+      "learning_rate": 8.519995680723854e-05,
+      "logits/chosen": -1.180711269378662,
+      "logits/rejected": -1.192284107208252,
+      "logps/chosen": -3.1985299587249756,
+      "logps/rejected": -103.07992553710938,
+      "loss": 0.14900489151477814,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1284285932779312,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5310749411582947,
+      "rewards/margins": 9.687756538391113,
+      "rewards/rejected": -9.156682014465332,
+      "step": 597,
+      "train_speed(iter/s)": 0.005409
+    },
+    {
+      "epoch": 1.7088878963823135,
+      "grad_norm": 0.5708374381065369,
+      "learning_rate": 8.488745627495471e-05,
+      "logits/chosen": -1.1305077075958252,
+      "logits/rejected": -1.1556541919708252,
+      "logps/chosen": -2.296858787536621,
+      "logps/rejected": -129.15554809570312,
+      "loss": 0.18966425955295563,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16467303037643433,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.36356276273727417,
+      "rewards/margins": 11.156661987304688,
+      "rewards/rejected": -10.793099403381348,
+      "step": 598,
+      "train_speed(iter/s)": 0.005409
+    },
+    {
+      "epoch": 1.7117463153193389,
+      "grad_norm": 2.8251218795776367,
+      "learning_rate": 8.457510670346976e-05,
+      "logits/chosen": -1.159468173980713,
+      "logits/rejected": -1.1627717018127441,
+      "logps/chosen": -4.14231014251709,
+      "logps/rejected": -90.7442855834961,
+      "loss": 0.24305839836597443,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21578028798103333,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.44796624779701233,
+      "rewards/margins": 8.427356719970703,
+      "rewards/rejected": -7.9793901443481445,
+      "step": 599,
+      "train_speed(iter/s)": 0.005409
+    },
+    {
+      "epoch": 1.7146047342563644,
+      "grad_norm": 0.8032998442649841,
+      "learning_rate": 8.426291121287668e-05,
+      "logits/chosen": -1.11725914478302,
+      "logits/rejected": -1.1244605779647827,
+      "logps/chosen": -4.1861982345581055,
+      "logps/rejected": -92.41063690185547,
+      "loss": 0.17941614985466003,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.17137563228607178,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.407539427280426,
+      "rewards/margins": 8.353867530822754,
+      "rewards/rejected": -7.946327209472656,
+      "step": 600,
+      "train_speed(iter/s)": 0.005409
+    },
+    {
+      "epoch": 1.7146047342563644,
+      "eval_logits/chosen": -1.1357148885726929,
+      "eval_logits/rejected": -1.1456408500671387,
+      "eval_logps/chosen": -5.507038593292236,
+      "eval_logps/rejected": -92.07612609863281,
+      "eval_loss": 0.26907792687416077,
+      "eval_nll_loss": 0.23485247790813446,
+      "eval_rewards/accuracies": 0.982300877571106,
+      "eval_rewards/chosen": 0.42667603492736816,
+      "eval_rewards/margins": 8.225197792053223,
+      "eval_rewards/rejected": -7.798521995544434,
+      "eval_runtime": 265.8083,
+      "eval_samples_per_second": 0.425,
+      "eval_steps_per_second": 0.425,
+      "step": 600
+    },
+    {
+      "epoch": 1.71746315319339,
+      "grad_norm": 0.5689127445220947,
+      "learning_rate": 8.395087292172921e-05,
+      "logits/chosen": -1.1845214366912842,
+      "logits/rejected": -1.2025808095932007,
+      "logps/chosen": -1.7930316925048828,
+      "logps/rejected": -115.36644744873047,
+      "loss": 0.08309746533632278,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.06800209730863571,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.46668967604637146,
+      "rewards/margins": 10.267763137817383,
+      "rewards/rejected": -9.80107307434082,
+      "step": 601,
+      "train_speed(iter/s)": 0.005396
+    },
+    {
+      "epoch": 1.7203215721304153,
+      "grad_norm": 0.4128093421459198,
+      "learning_rate": 8.363899494701086e-05,
+      "logits/chosen": -1.1646056175231934,
+      "logits/rejected": -1.172208547592163,
+      "logps/chosen": -6.8123860359191895,
+      "logps/rejected": -94.38031768798828,
+      "loss": 0.6606139540672302,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.47054457664489746,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2928743362426758,
+      "rewards/margins": 8.511747360229492,
+      "rewards/rejected": -8.218873977661133,
+      "step": 602,
+      "train_speed(iter/s)": 0.005396
+    },
+    {
+      "epoch": 1.7231799910674408,
+      "grad_norm": 17.9466609954834,
+      "learning_rate": 8.33272804041038e-05,
+      "logits/chosen": -1.1903241872787476,
+      "logits/rejected": -1.2006161212921143,
+      "logps/chosen": -1.96488618850708,
+      "logps/rejected": -86.98416900634766,
+      "loss": 0.19571945071220398,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.12956076860427856,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5292338132858276,
+      "rewards/margins": 8.149774551391602,
+      "rewards/rejected": -7.620540618896484,
+      "step": 603,
+      "train_speed(iter/s)": 0.005396
+    },
+    {
+      "epoch": 1.7260384100044663,
+      "grad_norm": 0.7187644839286804,
+      "learning_rate": 8.301573240675752e-05,
+      "logits/chosen": -1.1262531280517578,
+      "logits/rejected": -1.140869379043579,
+      "logps/chosen": -7.186244964599609,
+      "logps/rejected": -105.5767593383789,
+      "loss": 0.2748340368270874,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2181284874677658,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4821949303150177,
+      "rewards/margins": 8.686951637268066,
+      "rewards/rejected": -8.204756736755371,
+      "step": 604,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.7288968289414917,
+      "grad_norm": 1.1701804399490356,
+      "learning_rate": 8.2704354067058e-05,
+      "logits/chosen": -1.229970932006836,
+      "logits/rejected": -1.2415151596069336,
+      "logps/chosen": -3.912670612335205,
+      "logps/rejected": -96.19180297851562,
+      "loss": 0.2875363230705261,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.23937594890594482,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.41248124837875366,
+      "rewards/margins": 8.670553207397461,
+      "rewards/rejected": -8.258072853088379,
+      "step": 605,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.7317552478785172,
+      "grad_norm": 4.315155982971191,
+      "learning_rate": 8.239314849539638e-05,
+      "logits/chosen": -1.2182215452194214,
+      "logits/rejected": -1.2354011535644531,
+      "logps/chosen": -4.708987712860107,
+      "logps/rejected": -119.11360168457031,
+      "loss": 0.5458219647407532,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.40526482462882996,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.22136862576007843,
+      "rewards/margins": 10.272619247436523,
+      "rewards/rejected": -10.051248550415039,
+      "step": 606,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.7346136668155427,
+      "grad_norm": 3.1639983654022217,
+      "learning_rate": 8.208211880043812e-05,
+      "logits/chosen": -1.1820584535598755,
+      "logits/rejected": -1.2001395225524902,
+      "logps/chosen": -2.4597911834716797,
+      "logps/rejected": -120.22541809082031,
+      "loss": 0.1489083170890808,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.12029991298913956,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6557915210723877,
+      "rewards/margins": 11.194437026977539,
+      "rewards/rejected": -10.53864574432373,
+      "step": 607,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.737472085752568,
+      "grad_norm": 2.4347145557403564,
+      "learning_rate": 8.17712680890917e-05,
+      "logits/chosen": -1.212243914604187,
+      "logits/rejected": -1.22697114944458,
+      "logps/chosen": -3.6350185871124268,
+      "logps/rejected": -90.45594787597656,
+      "loss": 0.1942821741104126,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1534501016139984,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5750898122787476,
+      "rewards/margins": 8.578381538391113,
+      "rewards/rejected": -8.003292083740234,
+      "step": 608,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.7403305046895936,
+      "grad_norm": 0.6472948789596558,
+      "learning_rate": 8.146059946647782e-05,
+      "logits/chosen": -1.2153496742248535,
+      "logits/rejected": -1.2242867946624756,
+      "logps/chosen": -3.4136900901794434,
+      "logps/rejected": -82.82501220703125,
+      "loss": 0.16542817652225494,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15195731818675995,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5285678505897522,
+      "rewards/margins": 7.61013126373291,
+      "rewards/rejected": -7.081563949584961,
+      "step": 609,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.7431889236266191,
+      "grad_norm": 0.6857250928878784,
+      "learning_rate": 8.115011603589826e-05,
+      "logits/chosen": -1.1793605089187622,
+      "logits/rejected": -1.1929919719696045,
+      "logps/chosen": -3.4406254291534424,
+      "logps/rejected": -98.36747741699219,
+      "loss": 0.2728448808193207,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21860109269618988,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5499365329742432,
+      "rewards/margins": 9.20262622833252,
+      "rewards/rejected": -8.652688980102539,
+      "step": 610,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.7460473425636445,
+      "grad_norm": 1.0251792669296265,
+      "learning_rate": 8.083982089880477e-05,
+      "logits/chosen": -1.260735273361206,
+      "logits/rejected": -1.2722655534744263,
+      "logps/chosen": -2.7653603553771973,
+      "logps/rejected": -90.46491241455078,
+      "loss": 0.21464641392230988,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1605410873889923,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4746001958847046,
+      "rewards/margins": 8.401023864746094,
+      "rewards/rejected": -7.926423072814941,
+      "step": 611,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.7489057615006698,
+      "grad_norm": 1.4469356536865234,
+      "learning_rate": 8.052971715476842e-05,
+      "logits/chosen": -1.2575992345809937,
+      "logits/rejected": -1.2725930213928223,
+      "logps/chosen": -3.87493896484375,
+      "logps/rejected": -107.04155731201172,
+      "loss": 0.2182140350341797,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16839709877967834,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6470860242843628,
+      "rewards/margins": 9.724501609802246,
+      "rewards/rejected": -9.077415466308594,
+      "step": 612,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.7517641804376956,
+      "grad_norm": 0.9951202273368835,
+      "learning_rate": 8.021980790144827e-05,
+      "logits/chosen": -1.2438392639160156,
+      "logits/rejected": -1.2667877674102783,
+      "logps/chosen": -2.443464994430542,
+      "logps/rejected": -111.27606964111328,
+      "loss": 0.21300199627876282,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20494098961353302,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.511634349822998,
+      "rewards/margins": 10.146453857421875,
+      "rewards/rejected": -9.634819030761719,
+      "step": 613,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.7546225993747209,
+      "grad_norm": 1.8081765174865723,
+      "learning_rate": 7.991009623456072e-05,
+      "logits/chosen": -1.2732195854187012,
+      "logits/rejected": -1.2859796285629272,
+      "logps/chosen": -2.8509671688079834,
+      "logps/rejected": -100.18556213378906,
+      "loss": 0.18652215600013733,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14973853528499603,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5410576462745667,
+      "rewards/margins": 9.196635246276855,
+      "rewards/rejected": -8.655577659606934,
+      "step": 614,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.7574810183117462,
+      "grad_norm": 0.7432581782341003,
+      "learning_rate": 7.960058524784832e-05,
+      "logits/chosen": -1.2543963193893433,
+      "logits/rejected": -1.2693120241165161,
+      "logps/chosen": -3.452606678009033,
+      "logps/rejected": -120.65855407714844,
+      "loss": 0.14814871549606323,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.12885232269763947,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4775761365890503,
+      "rewards/margins": 11.281514167785645,
+      "rewards/rejected": -10.803937911987305,
+      "step": 615,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.7603394372487717,
+      "grad_norm": 0.7075319886207581,
+      "learning_rate": 7.929127803304915e-05,
+      "logits/chosen": -1.2688357830047607,
+      "logits/rejected": -1.276627540588379,
+      "logps/chosen": -4.918938159942627,
+      "logps/rejected": -106.51580810546875,
+      "loss": 0.2689712345600128,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21882139146327972,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3920802175998688,
+      "rewards/margins": 9.607563972473145,
+      "rewards/rejected": -9.215483665466309,
+      "step": 616,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.7631978561857973,
+      "grad_norm": 1.1403652429580688,
+      "learning_rate": 7.898217767986562e-05,
+      "logits/chosen": -1.273037314414978,
+      "logits/rejected": -1.2761380672454834,
+      "logps/chosen": -6.625821113586426,
+      "logps/rejected": -85.25454711914062,
+      "loss": 0.36789366602897644,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.28557538986206055,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.49920541048049927,
+      "rewards/margins": 7.796347618103027,
+      "rewards/rejected": -7.297142028808594,
+      "step": 617,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.7660562751228226,
+      "grad_norm": 9.335824966430664,
+      "learning_rate": 7.867328727593397e-05,
+      "logits/chosen": -1.3096352815628052,
+      "logits/rejected": -1.3141634464263916,
+      "logps/chosen": -4.814456462860107,
+      "logps/rejected": -92.47824096679688,
+      "loss": 0.36238977313041687,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.243287593126297,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.37459686398506165,
+      "rewards/margins": 8.336633682250977,
+      "rewards/rejected": -7.962037086486816,
+      "step": 618,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.7689146940598481,
+      "grad_norm": 1.5570136308670044,
+      "learning_rate": 7.836460990679313e-05,
+      "logits/chosen": -1.3636045455932617,
+      "logits/rejected": -1.3825900554656982,
+      "logps/chosen": -4.613166332244873,
+      "logps/rejected": -118.72332000732422,
+      "loss": 0.2116784155368805,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.209022656083107,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7731083035469055,
+      "rewards/margins": 10.994028091430664,
+      "rewards/rejected": -10.22092056274414,
+      "step": 619,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.7717731129968737,
+      "grad_norm": 0.7922633290290833,
+      "learning_rate": 7.805614865585396e-05,
+      "logits/chosen": -1.3091944456100464,
+      "logits/rejected": -1.316933035850525,
+      "logps/chosen": -3.8784146308898926,
+      "logps/rejected": -126.78861236572266,
+      "loss": 0.12152136117219925,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.11782631278038025,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4741920828819275,
+      "rewards/margins": 11.795244216918945,
+      "rewards/rejected": -11.321053504943848,
+      "step": 620,
+      "train_speed(iter/s)": 0.005397
+    },
+    {
+      "epoch": 1.774631531933899,
+      "grad_norm": 0.6836639642715454,
+      "learning_rate": 7.774790660436858e-05,
+      "logits/chosen": -1.3928108215332031,
+      "logits/rejected": -1.4215530157089233,
+      "logps/chosen": -3.4057624340057373,
+      "logps/rejected": -142.04086303710938,
+      "loss": 0.1845034658908844,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16980323195457458,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5302206873893738,
+      "rewards/margins": 13.118966102600098,
+      "rewards/rejected": -12.588746070861816,
+      "step": 621,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.7774899508709245,
+      "grad_norm": 6.991085052490234,
+      "learning_rate": 7.743988683139943e-05,
+      "logits/chosen": -1.3453571796417236,
+      "logits/rejected": -1.3684511184692383,
+      "logps/chosen": -5.252518653869629,
+      "logps/rejected": -133.31072998046875,
+      "loss": 0.20858323574066162,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20254190266132355,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5752936601638794,
+      "rewards/margins": 12.213183403015137,
+      "rewards/rejected": -11.637889862060547,
+      "step": 622,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.78034836980795,
+      "grad_norm": 4.359181880950928,
+      "learning_rate": 7.713209241378872e-05,
+      "logits/chosen": -1.2878490686416626,
+      "logits/rejected": -1.3060500621795654,
+      "logps/chosen": -3.2410290241241455,
+      "logps/rejected": -127.3503646850586,
+      "loss": 0.17801642417907715,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.17661628127098083,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6783971190452576,
+      "rewards/margins": 11.938961029052734,
+      "rewards/rejected": -11.260563850402832,
+      "step": 623,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.7832067887449754,
+      "grad_norm": 0.7485926747322083,
+      "learning_rate": 7.682452642612733e-05,
+      "logits/chosen": -1.323854923248291,
+      "logits/rejected": -1.3345876932144165,
+      "logps/chosen": -7.461188316345215,
+      "logps/rejected": -113.36080932617188,
+      "loss": 0.28294721245765686,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2697088122367859,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7665496468544006,
+      "rewards/margins": 10.277408599853516,
+      "rewards/rejected": -9.510859489440918,
+      "step": 624,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.786065207682001,
+      "grad_norm": 0.8652713894844055,
+      "learning_rate": 7.65171919407246e-05,
+      "logits/chosen": -1.3405261039733887,
+      "logits/rejected": -1.3571374416351318,
+      "logps/chosen": -3.0921409130096436,
+      "logps/rejected": -118.87600708007812,
+      "loss": 0.17847004532814026,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14839963614940643,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4123997986316681,
+      "rewards/margins": 10.91650676727295,
+      "rewards/rejected": -10.504107475280762,
+      "step": 625,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.7889236266190265,
+      "grad_norm": 1.3180344104766846,
+      "learning_rate": 7.621009202757719e-05,
+      "logits/chosen": -1.3654556274414062,
+      "logits/rejected": -1.3729454278945923,
+      "logps/chosen": -5.795345783233643,
+      "logps/rejected": -122.83097839355469,
+      "loss": 0.23871037364006042,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.19673317670822144,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5445254445075989,
+      "rewards/margins": 11.494521141052246,
+      "rewards/rejected": -10.949995040893555,
+      "step": 626,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.7917820455560518,
+      "grad_norm": 0.6763105392456055,
+      "learning_rate": 7.590322975433857e-05,
+      "logits/chosen": -1.3876533508300781,
+      "logits/rejected": -1.4038093090057373,
+      "logps/chosen": -4.756526947021484,
+      "logps/rejected": -113.34770965576172,
+      "loss": 0.2270170897245407,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21937969326972961,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7127946615219116,
+      "rewards/margins": 10.715826034545898,
+      "rewards/rejected": -10.003030776977539,
+      "step": 627,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.7946404644930771,
+      "grad_norm": 1.2457427978515625,
+      "learning_rate": 7.559660818628855e-05,
+      "logits/chosen": -1.3642257452011108,
+      "logits/rejected": -1.3769264221191406,
+      "logps/chosen": -3.014768123626709,
+      "logps/rejected": -121.77210235595703,
+      "loss": 0.14762888848781586,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14068850874900818,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.375095009803772,
+      "rewards/margins": 11.27329158782959,
+      "rewards/rejected": -10.89819622039795,
+      "step": 628,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.797498883430103,
+      "grad_norm": 0.8837608098983765,
+      "learning_rate": 7.529023038630238e-05,
+      "logits/chosen": -1.3430863618850708,
+      "logits/rejected": -1.3552993535995483,
+      "logps/chosen": -5.123016834259033,
+      "logps/rejected": -108.41499328613281,
+      "loss": 0.21152086555957794,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20837949216365814,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6010651588439941,
+      "rewards/margins": 9.848522186279297,
+      "rewards/rejected": -9.247457504272461,
+      "step": 629,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.8003573023671282,
+      "grad_norm": 6.133004188537598,
+      "learning_rate": 7.49840994148204e-05,
+      "logits/chosen": -1.344904899597168,
+      "logits/rejected": -1.367358922958374,
+      "logps/chosen": -4.031676769256592,
+      "logps/rejected": -112.61686706542969,
+      "loss": 0.377747505903244,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3464767038822174,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5361669063568115,
+      "rewards/margins": 10.34013557434082,
+      "rewards/rejected": -9.80396842956543,
+      "step": 630,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.8032157213041535,
+      "grad_norm": 0.8749508261680603,
+      "learning_rate": 7.467821832981719e-05,
+      "logits/chosen": -1.3188172578811646,
+      "logits/rejected": -1.335577368736267,
+      "logps/chosen": -5.043167591094971,
+      "logps/rejected": -112.44739532470703,
+      "loss": 0.2038888782262802,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.19640286266803741,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6657987236976624,
+      "rewards/margins": 10.651543617248535,
+      "rewards/rejected": -9.985745429992676,
+      "step": 631,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.806074140241179,
+      "grad_norm": 0.6289976835250854,
+      "learning_rate": 7.437259018677136e-05,
+      "logits/chosen": -1.3327033519744873,
+      "logits/rejected": -1.3480082750320435,
+      "logps/chosen": -3.615278482437134,
+      "logps/rejected": -119.2198486328125,
+      "loss": 0.20541872084140778,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1953929364681244,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3347194492816925,
+      "rewards/margins": 10.586311340332031,
+      "rewards/rejected": -10.251592636108398,
+      "step": 632,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.8089325591782046,
+      "grad_norm": 1.1460577249526978,
+      "learning_rate": 7.406721803863475e-05,
+      "logits/chosen": -1.3531547784805298,
+      "logits/rejected": -1.3722925186157227,
+      "logps/chosen": -1.7670950889587402,
+      "logps/rejected": -140.48046875,
+      "loss": 0.14490243792533875,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14003917574882507,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3310070335865021,
+      "rewards/margins": 13.185526847839355,
+      "rewards/rejected": -12.854520797729492,
+      "step": 633,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.81179097811523,
+      "grad_norm": 4.37379264831543,
+      "learning_rate": 7.376210493580212e-05,
+      "logits/chosen": -1.3406105041503906,
+      "logits/rejected": -1.3556296825408936,
+      "logps/chosen": -3.7733154296875,
+      "logps/rejected": -124.59563446044922,
+      "loss": 0.2903311848640442,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.28832513093948364,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3790757656097412,
+      "rewards/margins": 11.557982444763184,
+      "rewards/rejected": -11.178906440734863,
+      "step": 634,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.8146493970522555,
+      "grad_norm": 4.76070499420166,
+      "learning_rate": 7.345725392608048e-05,
+      "logits/chosen": -1.348390817642212,
+      "logits/rejected": -1.3591151237487793,
+      "logps/chosen": -5.138166427612305,
+      "logps/rejected": -115.6446762084961,
+      "loss": 0.3292956054210663,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2817382216453552,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.30544814467430115,
+      "rewards/margins": 10.786002159118652,
+      "rewards/rejected": -10.48055362701416,
+      "step": 635,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.817507815989281,
+      "grad_norm": 1.9394831657409668,
+      "learning_rate": 7.315266805465883e-05,
+      "logits/chosen": -1.2964552640914917,
+      "logits/rejected": -1.3200057744979858,
+      "logps/chosen": -3.403735399246216,
+      "logps/rejected": -123.78720092773438,
+      "loss": 0.17933297157287598,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.17736488580703735,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5881022214889526,
+      "rewards/margins": 11.664973258972168,
+      "rewards/rejected": -11.07686996459961,
+      "step": 636,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.8203662349263063,
+      "grad_norm": 0.8564069867134094,
+      "learning_rate": 7.284835036407776e-05,
+      "logits/chosen": -1.336266279220581,
+      "logits/rejected": -1.3451544046401978,
+      "logps/chosen": -4.740027904510498,
+      "logps/rejected": -113.02291107177734,
+      "loss": 0.27193519473075867,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21789833903312683,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5400663614273071,
+      "rewards/margins": 10.416783332824707,
+      "rewards/rejected": -9.876716613769531,
+      "step": 637,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.8232246538633319,
+      "grad_norm": 0.9509347081184387,
+      "learning_rate": 7.254430389419879e-05,
+      "logits/chosen": -1.270435094833374,
+      "logits/rejected": -1.2815449237823486,
+      "logps/chosen": -3.3484411239624023,
+      "logps/rejected": -110.3446044921875,
+      "loss": 0.18270909786224365,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16942772269248962,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.512046217918396,
+      "rewards/margins": 10.352546691894531,
+      "rewards/rejected": -9.840499877929688,
+      "step": 638,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.8260830728003574,
+      "grad_norm": 0.5768864750862122,
+      "learning_rate": 7.22405316821744e-05,
+      "logits/chosen": -1.3135160207748413,
+      "logits/rejected": -1.3353259563446045,
+      "logps/chosen": -2.1458687782287598,
+      "logps/rejected": -120.35028076171875,
+      "loss": 0.1152193695306778,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.10469970852136612,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.46638453006744385,
+      "rewards/margins": 10.977595329284668,
+      "rewards/rejected": -10.511210441589355,
+      "step": 639,
+      "train_speed(iter/s)": 0.005398
+    },
+    {
+      "epoch": 1.8289414917373827,
+      "grad_norm": 0.7836558818817139,
+      "learning_rate": 7.193703676241725e-05,
+      "logits/chosen": -1.2732415199279785,
+      "logits/rejected": -1.2796140909194946,
+      "logps/chosen": -5.664248466491699,
+      "logps/rejected": -91.83935546875,
+      "loss": 0.41713982820510864,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.32591503858566284,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4501730799674988,
+      "rewards/margins": 8.382551193237305,
+      "rewards/rejected": -7.932378768920898,
+      "step": 640,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.831799910674408,
+      "grad_norm": 0.8730625510215759,
+      "learning_rate": 7.163382216657034e-05,
+      "logits/chosen": -1.2692418098449707,
+      "logits/rejected": -1.301629662513733,
+      "logps/chosen": -1.288639783859253,
+      "logps/rejected": -160.97267150878906,
+      "loss": 0.09459640830755234,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.09011334925889969,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4853152632713318,
+      "rewards/margins": 14.271899223327637,
+      "rewards/rejected": -13.786584854125977,
+      "step": 641,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.8346583296114338,
+      "grad_norm": 0.5060243606567383,
+      "learning_rate": 7.133089092347627e-05,
+      "logits/chosen": -1.2539950609207153,
+      "logits/rejected": -1.2674380540847778,
+      "logps/chosen": -4.00560188293457,
+      "logps/rejected": -111.9576187133789,
+      "loss": 0.2435949742794037,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.24140885472297668,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6132622957229614,
+      "rewards/margins": 10.611983299255371,
+      "rewards/rejected": -9.998720169067383,
+      "step": 642,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.8375167485484591,
+      "grad_norm": 1.167800784111023,
+      "learning_rate": 7.102824605914735e-05,
+      "logits/chosen": -1.264164924621582,
+      "logits/rejected": -1.2702038288116455,
+      "logps/chosen": -5.026443958282471,
+      "logps/rejected": -101.44984436035156,
+      "loss": 0.26005586981773376,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22672684490680695,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.48070228099823,
+      "rewards/margins": 9.517464637756348,
+      "rewards/rejected": -9.036762237548828,
+      "step": 643,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.8403751674854845,
+      "grad_norm": 0.7475759983062744,
+      "learning_rate": 7.072589059673514e-05,
+      "logits/chosen": -1.3097971677780151,
+      "logits/rejected": -1.3304953575134277,
+      "logps/chosen": -3.952174186706543,
+      "logps/rejected": -125.90980529785156,
+      "loss": 0.3319244682788849,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21963773667812347,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3299381732940674,
+      "rewards/margins": 11.375055313110352,
+      "rewards/rejected": -11.04511833190918,
+      "step": 644,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.84323358642251,
+      "grad_norm": 19.21202278137207,
+      "learning_rate": 7.042382755650032e-05,
+      "logits/chosen": -1.2788704633712769,
+      "logits/rejected": -1.3028172254562378,
+      "logps/chosen": -3.9370901584625244,
+      "logps/rejected": -126.76161193847656,
+      "loss": 0.16361507773399353,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13888728618621826,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.47258222103118896,
+      "rewards/margins": 11.63074779510498,
+      "rewards/rejected": -11.158166885375977,
+      "step": 645,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.8460920053595355,
+      "grad_norm": 0.6701921820640564,
+      "learning_rate": 7.012205995578267e-05,
+      "logits/chosen": -1.2379904985427856,
+      "logits/rejected": -1.2615790367126465,
+      "logps/chosen": -2.9081027507781982,
+      "logps/rejected": -123.61541748046875,
+      "loss": 0.16233164072036743,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1606723517179489,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4661065340042114,
+      "rewards/margins": 11.18898868560791,
+      "rewards/rejected": -10.722882270812988,
+      "step": 646,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.8489504242965609,
+      "grad_norm": 0.7222447395324707,
+      "learning_rate": 6.982059080897059e-05,
+      "logits/chosen": -1.2836098670959473,
+      "logits/rejected": -1.2917486429214478,
+      "logps/chosen": -3.8639073371887207,
+      "logps/rejected": -104.71327209472656,
+      "loss": 0.21296782791614532,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1951403170824051,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5273296236991882,
+      "rewards/margins": 9.60672378540039,
+      "rewards/rejected": -9.079394340515137,
+      "step": 647,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.8518088432335864,
+      "grad_norm": 0.7563202977180481,
+      "learning_rate": 6.951942312747134e-05,
+      "logits/chosen": -1.2675378322601318,
+      "logits/rejected": -1.2755509614944458,
+      "logps/chosen": -6.645654201507568,
+      "logps/rejected": -103.13592529296875,
+      "loss": 0.27974945306777954,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2530043423175812,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.584045946598053,
+      "rewards/margins": 9.444131851196289,
+      "rewards/rejected": -8.860086441040039,
+      "step": 648,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.854667262170612,
+      "grad_norm": 1.215735912322998,
+      "learning_rate": 6.921855991968078e-05,
+      "logits/chosen": -1.2765772342681885,
+      "logits/rejected": -1.2841192483901978,
+      "logps/chosen": -4.45599889755249,
+      "logps/rejected": -110.97665405273438,
+      "loss": 0.21567204594612122,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18534396588802338,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.39356788992881775,
+      "rewards/margins": 10.287214279174805,
+      "rewards/rejected": -9.893646240234375,
+      "step": 649,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.8575256811076373,
+      "grad_norm": 2.7902543544769287,
+      "learning_rate": 6.891800419095335e-05,
+      "logits/chosen": -1.3014237880706787,
+      "logits/rejected": -1.312886118888855,
+      "logps/chosen": -6.769974708557129,
+      "logps/rejected": -116.2103042602539,
+      "loss": 0.25906240940093994,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.23928385972976685,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7981483936309814,
+      "rewards/margins": 10.857406616210938,
+      "rewards/rejected": -10.059259414672852,
+      "step": 650,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.8603841000446628,
+      "grad_norm": 1.3061541318893433,
+      "learning_rate": 6.861775894357198e-05,
+      "logits/chosen": -1.2520601749420166,
+      "logits/rejected": -1.262779712677002,
+      "logps/chosen": -5.169130802154541,
+      "logps/rejected": -106.49501037597656,
+      "loss": 0.24649611115455627,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22475162148475647,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6006129384040833,
+      "rewards/margins": 9.921269416809082,
+      "rewards/rejected": -9.320656776428223,
+      "step": 651,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.8632425189816884,
+      "grad_norm": 1.1500881910324097,
+      "learning_rate": 6.831782717671828e-05,
+      "logits/chosen": -1.3243428468704224,
+      "logits/rejected": -1.3408803939819336,
+      "logps/chosen": -3.5129246711730957,
+      "logps/rejected": -114.7535629272461,
+      "loss": 0.16800451278686523,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1650160849094391,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5265602469444275,
+      "rewards/margins": 10.674477577209473,
+      "rewards/rejected": -10.147916793823242,
+      "step": 652,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.8661009379187137,
+      "grad_norm": 0.8044180870056152,
+      "learning_rate": 6.801821188644242e-05,
+      "logits/chosen": -1.36525297164917,
+      "logits/rejected": -1.375981092453003,
+      "logps/chosen": -3.876332998275757,
+      "logps/rejected": -103.86865234375,
+      "loss": 0.2005651891231537,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.17257581651210785,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.6263987421989441,
+      "rewards/margins": 9.682666778564453,
+      "rewards/rejected": -9.056267738342285,
+      "step": 653,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.8689593568557392,
+      "grad_norm": 0.7117923498153687,
+      "learning_rate": 6.771891606563318e-05,
+      "logits/chosen": -1.2599382400512695,
+      "logits/rejected": -1.274010419845581,
+      "logps/chosen": -3.5532844066619873,
+      "logps/rejected": -107.72050476074219,
+      "loss": 0.2625138461589813,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21689662337303162,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3454374074935913,
+      "rewards/margins": 9.692204475402832,
+      "rewards/rejected": -9.34676742553711,
+      "step": 654,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.8718177757927648,
+      "grad_norm": 2.125316858291626,
+      "learning_rate": 6.741994270398826e-05,
+      "logits/chosen": -1.330776572227478,
+      "logits/rejected": -1.344359040260315,
+      "logps/chosen": -3.1112051010131836,
+      "logps/rejected": -106.1573715209961,
+      "loss": 0.16326259076595306,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16014985740184784,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.44455885887145996,
+      "rewards/margins": 9.886975288391113,
+      "rewards/rejected": -9.442416191101074,
+      "step": 655,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.87467619472979,
+      "grad_norm": 0.7575691342353821,
+      "learning_rate": 6.71212947879842e-05,
+      "logits/chosen": -1.3439936637878418,
+      "logits/rejected": -1.356628179550171,
+      "logps/chosen": -2.2696585655212402,
+      "logps/rejected": -101.44575500488281,
+      "loss": 0.14300636947155,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13764385879039764,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5373290181159973,
+      "rewards/margins": 9.317428588867188,
+      "rewards/rejected": -8.780098915100098,
+      "step": 656,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.8775346136668154,
+      "grad_norm": 0.592279314994812,
+      "learning_rate": 6.682297530084664e-05,
+      "logits/chosen": -1.3380376100540161,
+      "logits/rejected": -1.3524363040924072,
+      "logps/chosen": -6.481423377990723,
+      "logps/rejected": -113.6147689819336,
+      "loss": 0.26005515456199646,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.25132831931114197,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6584410071372986,
+      "rewards/margins": 10.201264381408691,
+      "rewards/rejected": -9.54282283782959,
+      "step": 657,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.8803930326038412,
+      "grad_norm": 0.8957412242889404,
+      "learning_rate": 6.652498722252049e-05,
+      "logits/chosen": -1.2798575162887573,
+      "logits/rejected": -1.2862627506256104,
+      "logps/chosen": -4.732370853424072,
+      "logps/rejected": -89.04149627685547,
+      "loss": 0.3374495506286621,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.30817681550979614,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4829080104827881,
+      "rewards/margins": 8.177124977111816,
+      "rewards/rejected": -7.694216728210449,
+      "step": 658,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.8832514515408665,
+      "grad_norm": 1.277299165725708,
+      "learning_rate": 6.622733352964025e-05,
+      "logits/chosen": -1.3507089614868164,
+      "logits/rejected": -1.3629826307296753,
+      "logps/chosen": -4.253856182098389,
+      "logps/rejected": -99.68025970458984,
+      "loss": 0.20950721204280853,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1871398240327835,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4054906964302063,
+      "rewards/margins": 9.246431350708008,
+      "rewards/rejected": -8.840941429138184,
+      "step": 659,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.8861098704778918,
+      "grad_norm": 0.8714025616645813,
+      "learning_rate": 6.593001719550016e-05,
+      "logits/chosen": -1.296878695487976,
+      "logits/rejected": -1.3038222789764404,
+      "logps/chosen": -4.94182014465332,
+      "logps/rejected": -94.9803466796875,
+      "loss": 0.19033871591091156,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.17941981554031372,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.42471638321876526,
+      "rewards/margins": 8.327040672302246,
+      "rewards/rejected": -7.902324676513672,
+      "step": 660,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.8889682894149173,
+      "grad_norm": 0.8253157138824463,
+      "learning_rate": 6.563304119002451e-05,
+      "logits/chosen": -1.3093528747558594,
+      "logits/rejected": -1.3244248628616333,
+      "logps/chosen": -5.993863105773926,
+      "logps/rejected": -100.63095092773438,
+      "loss": 0.5593117475509644,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4283536970615387,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5702846050262451,
+      "rewards/margins": 9.172146797180176,
+      "rewards/rejected": -8.601861000061035,
+      "step": 661,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.8918267083519429,
+      "grad_norm": 25.315296173095703,
+      "learning_rate": 6.533640847973808e-05,
+      "logits/chosen": -1.2889041900634766,
+      "logits/rejected": -1.2987148761749268,
+      "logps/chosen": -3.3531904220581055,
+      "logps/rejected": -101.64501190185547,
+      "loss": 0.15289689600467682,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13183356821537018,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6400858759880066,
+      "rewards/margins": 9.589509010314941,
+      "rewards/rejected": -8.949423789978027,
+      "step": 662,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.8946851272889682,
+      "grad_norm": 0.6394268870353699,
+      "learning_rate": 6.50401220277364e-05,
+      "logits/chosen": -1.2872529029846191,
+      "logits/rejected": -1.3098706007003784,
+      "logps/chosen": -4.84055757522583,
+      "logps/rejected": -112.51201629638672,
+      "loss": 0.5871531367301941,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.43112945556640625,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.48831021785736084,
+      "rewards/margins": 10.27175521850586,
+      "rewards/rejected": -9.783445358276367,
+      "step": 663,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.8975435462259937,
+      "grad_norm": 2.8964788913726807,
+      "learning_rate": 6.474418479365622e-05,
+      "logits/chosen": -1.282837986946106,
+      "logits/rejected": -1.285774827003479,
+      "logps/chosen": -8.157180786132812,
+      "logps/rejected": -92.5395278930664,
+      "loss": 0.2315860092639923,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2284000813961029,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7858994007110596,
+      "rewards/margins": 8.78559684753418,
+      "rewards/rejected": -7.999697685241699,
+      "step": 664,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.9004019651630193,
+      "grad_norm": 0.8511047959327698,
+      "learning_rate": 6.444859973364587e-05,
+      "logits/chosen": -1.2938660383224487,
+      "logits/rejected": -1.3118674755096436,
+      "logps/chosen": -3.9691340923309326,
+      "logps/rejected": -106.54265594482422,
+      "loss": 0.2253965139389038,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15968890488147736,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.49960803985595703,
+      "rewards/margins": 9.341988563537598,
+      "rewards/rejected": -8.842381477355957,
+      "step": 665,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.9032603841000446,
+      "grad_norm": 0.5567864775657654,
+      "learning_rate": 6.415336980033585e-05,
+      "logits/chosen": -1.30922532081604,
+      "logits/rejected": -1.328365683555603,
+      "logps/chosen": -2.8081579208374023,
+      "logps/rejected": -113.6122817993164,
+      "loss": 0.20439419150352478,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18372051417827606,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3098168969154358,
+      "rewards/margins": 10.348260879516602,
+      "rewards/rejected": -10.038445472717285,
+      "step": 666,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.9061188030370702,
+      "grad_norm": 9.81278133392334,
+      "learning_rate": 6.385849794280915e-05,
+      "logits/chosen": -1.316828727722168,
+      "logits/rejected": -1.3409326076507568,
+      "logps/chosen": -2.031130075454712,
+      "logps/rejected": -106.19338989257812,
+      "loss": 0.14540082216262817,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.11819420754909515,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6250270009040833,
+      "rewards/margins": 9.933358192443848,
+      "rewards/rejected": -9.308330535888672,
+      "step": 667,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.9089772219740957,
+      "grad_norm": 2.9741618633270264,
+      "learning_rate": 6.3563987106572e-05,
+      "logits/chosen": -1.2994283437728882,
+      "logits/rejected": -1.3059117794036865,
+      "logps/chosen": -4.3763322830200195,
+      "logps/rejected": -98.63020324707031,
+      "loss": 0.20989640057086945,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.20499569177627563,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4033859074115753,
+      "rewards/margins": 8.991562843322754,
+      "rewards/rejected": -8.588177680969238,
+      "step": 668,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.911835640911121,
+      "grad_norm": 0.934133768081665,
+      "learning_rate": 6.326984023352435e-05,
+      "logits/chosen": -1.2849228382110596,
+      "logits/rejected": -1.304962396621704,
+      "logps/chosen": -2.7252705097198486,
+      "logps/rejected": -114.20394134521484,
+      "loss": 0.16401462256908417,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16158060729503632,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6825951337814331,
+      "rewards/margins": 10.86156940460205,
+      "rewards/rejected": -10.178974151611328,
+      "step": 669,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.9146940598481463,
+      "grad_norm": 0.706195592880249,
+      "learning_rate": 6.297606026193036e-05,
+      "logits/chosen": -1.2965331077575684,
+      "logits/rejected": -1.3067684173583984,
+      "logps/chosen": -5.783702850341797,
+      "logps/rejected": -102.84695434570312,
+      "loss": 0.23226682841777802,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22864435613155365,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.47119635343551636,
+      "rewards/margins": 9.723640441894531,
+      "rewards/rejected": -9.252445220947266,
+      "step": 670,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.917552478785172,
+      "grad_norm": 0.7782325744628906,
+      "learning_rate": 6.268265012638934e-05,
+      "logits/chosen": -1.3040504455566406,
+      "logits/rejected": -1.3242979049682617,
+      "logps/chosen": -5.526919841766357,
+      "logps/rejected": -114.40222930908203,
+      "loss": 0.2729805111885071,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.26506656408309937,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4165946841239929,
+      "rewards/margins": 10.218692779541016,
+      "rewards/rejected": -9.802099227905273,
+      "step": 671,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.9204108977221974,
+      "grad_norm": 1.7584947347640991,
+      "learning_rate": 6.238961275780613e-05,
+      "logits/chosen": -1.329121470451355,
+      "logits/rejected": -1.3434271812438965,
+      "logps/chosen": -5.148283958435059,
+      "logps/rejected": -106.88273620605469,
+      "loss": 0.16379527747631073,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15229424834251404,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.41878870129585266,
+      "rewards/margins": 9.952224731445312,
+      "rewards/rejected": -9.53343677520752,
+      "step": 672,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.9232693166592227,
+      "grad_norm": 0.772023618221283,
+      "learning_rate": 6.209695108336211e-05,
+      "logits/chosen": -1.324845314025879,
+      "logits/rejected": -1.3292207717895508,
+      "logps/chosen": -3.755115032196045,
+      "logps/rejected": -98.88433837890625,
+      "loss": 0.22213026881217957,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.19585399329662323,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5098289847373962,
+      "rewards/margins": 9.438490867614746,
+      "rewards/rejected": -8.928662300109863,
+      "step": 673,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.9261277355962483,
+      "grad_norm": 0.6828939318656921,
+      "learning_rate": 6.180466802648563e-05,
+      "logits/chosen": -1.3263074159622192,
+      "logits/rejected": -1.3385473489761353,
+      "logps/chosen": -2.9277350902557373,
+      "logps/rejected": -107.61619567871094,
+      "loss": 0.18101422488689423,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15924564003944397,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3217340111732483,
+      "rewards/margins": 9.498228073120117,
+      "rewards/rejected": -9.176494598388672,
+      "step": 674,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.9289861545332738,
+      "grad_norm": 1.5087296962738037,
+      "learning_rate": 6.151276650682315e-05,
+      "logits/chosen": -1.2586640119552612,
+      "logits/rejected": -1.27299165725708,
+      "logps/chosen": -4.842021465301514,
+      "logps/rejected": -110.8651123046875,
+      "loss": 0.2104535698890686,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.19624516367912292,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6069389581680298,
+      "rewards/margins": 9.962318420410156,
+      "rewards/rejected": -9.355380058288574,
+      "step": 675,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.9318445734702991,
+      "grad_norm": 0.7660964727401733,
+      "learning_rate": 6.122124944020977e-05,
+      "logits/chosen": -1.2959116697311401,
+      "logits/rejected": -1.3225114345550537,
+      "logps/chosen": -4.718099117279053,
+      "logps/rejected": -125.47923278808594,
+      "loss": 0.5585540533065796,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.4054650068283081,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4143763482570648,
+      "rewards/margins": 10.734457969665527,
+      "rewards/rejected": -10.32008171081543,
+      "step": 676,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.9347029924073247,
+      "grad_norm": 2.9249446392059326,
+      "learning_rate": 6.0930119738640445e-05,
+      "logits/chosen": -1.282720685005188,
+      "logits/rejected": -1.2819023132324219,
+      "logps/chosen": -6.72110652923584,
+      "logps/rejected": -87.6020736694336,
+      "loss": 0.2858355939388275,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.2798173427581787,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6944171786308289,
+      "rewards/margins": 8.2020902633667,
+      "rewards/rejected": -7.5076727867126465,
+      "step": 677,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.9375614113443502,
+      "grad_norm": 3.115387439727783,
+      "learning_rate": 6.063938031024048e-05,
+      "logits/chosen": -1.2956721782684326,
+      "logits/rejected": -1.3137578964233398,
+      "logps/chosen": -4.9737958908081055,
+      "logps/rejected": -103.87081146240234,
+      "loss": 0.41333433985710144,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.32286494970321655,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4960055649280548,
+      "rewards/margins": 8.867454528808594,
+      "rewards/rejected": -8.37144947052002,
+      "step": 678,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.9404198302813755,
+      "grad_norm": 8.556899070739746,
+      "learning_rate": 6.034903405923681e-05,
+      "logits/chosen": -1.3017889261245728,
+      "logits/rejected": -1.3088685274124146,
+      "logps/chosen": -5.11128044128418,
+      "logps/rejected": -89.395263671875,
+      "loss": 0.23161855340003967,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.209635391831398,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.551887571811676,
+      "rewards/margins": 8.253191947937012,
+      "rewards/rejected": -7.7013044357299805,
+      "step": 679,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.943278249218401,
+      "grad_norm": 0.6750122308731079,
+      "learning_rate": 6.0059083885928926e-05,
+      "logits/chosen": -1.3443058729171753,
+      "logits/rejected": -1.3679203987121582,
+      "logps/chosen": -4.200978755950928,
+      "logps/rejected": -121.6429672241211,
+      "loss": 0.14882132411003113,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14047406613826752,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6916794180870056,
+      "rewards/margins": 11.149093627929688,
+      "rewards/rejected": -10.457413673400879,
+      "step": 680,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.9461366681554266,
+      "grad_norm": 3.7811436653137207,
+      "learning_rate": 5.976953268665971e-05,
+      "logits/chosen": -1.3269189596176147,
+      "logits/rejected": -1.3355252742767334,
+      "logps/chosen": -4.741105079650879,
+      "logps/rejected": -95.5803451538086,
+      "loss": 0.18303585052490234,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14606527984142303,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.8169770240783691,
+      "rewards/margins": 8.926080703735352,
+      "rewards/rejected": -8.109103202819824,
+      "step": 681,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.948995087092452,
+      "grad_norm": 0.9065592288970947,
+      "learning_rate": 5.948038335378683e-05,
+      "logits/chosen": -1.2741483449935913,
+      "logits/rejected": -1.288372278213501,
+      "logps/chosen": -3.3015213012695312,
+      "logps/rejected": -83.7618637084961,
+      "loss": 0.2943568825721741,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21783985197544098,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.5525356531143188,
+      "rewards/margins": 7.655857563018799,
+      "rewards/rejected": -7.103321552276611,
+      "step": 682,
+      "train_speed(iter/s)": 0.005399
+    },
+    {
+      "epoch": 1.9518535060294775,
+      "grad_norm": 1.0822813510894775,
+      "learning_rate": 5.91916387756535e-05,
+      "logits/chosen": -1.3422785997390747,
+      "logits/rejected": -1.353929042816162,
+      "logps/chosen": -3.696439027786255,
+      "logps/rejected": -83.76953887939453,
+      "loss": 0.2142258882522583,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.17013080418109894,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.6672707200050354,
+      "rewards/margins": 7.74099063873291,
+      "rewards/rejected": -7.073719501495361,
+      "step": 683,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.954711924966503,
+      "grad_norm": 0.7260097861289978,
+      "learning_rate": 5.8903301836559966e-05,
+      "logits/chosen": -1.370530605316162,
+      "logits/rejected": -1.3802154064178467,
+      "logps/chosen": -6.263431549072266,
+      "logps/rejected": -87.44656372070312,
+      "loss": 0.3457471430301666,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3114210367202759,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4694344103336334,
+      "rewards/margins": 7.827997207641602,
+      "rewards/rejected": -7.358563423156738,
+      "step": 684,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.9575703439035284,
+      "grad_norm": 0.98732590675354,
+      "learning_rate": 5.861537541673436e-05,
+      "logits/chosen": -1.292124629020691,
+      "logits/rejected": -1.304084062576294,
+      "logps/chosen": -4.441066265106201,
+      "logps/rejected": -99.29835510253906,
+      "loss": 0.40864118933677673,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.35176336765289307,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.3288501501083374,
+      "rewards/margins": 8.98994255065918,
+      "rewards/rejected": -8.661092758178711,
+      "step": 685,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.9604287628405537,
+      "grad_norm": 2.8306355476379395,
+      "learning_rate": 5.832786239230415e-05,
+      "logits/chosen": -1.341307282447815,
+      "logits/rejected": -1.3494575023651123,
+      "logps/chosen": -4.303823471069336,
+      "logps/rejected": -108.20519256591797,
+      "loss": 0.20767070353031158,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16874229907989502,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5772908926010132,
+      "rewards/margins": 9.876459121704102,
+      "rewards/rejected": -9.29916763305664,
+      "step": 686,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.9632871817775794,
+      "grad_norm": 1.0595661401748657,
+      "learning_rate": 5.804076563526744e-05,
+      "logits/chosen": -1.3210382461547852,
+      "logits/rejected": -1.335936427116394,
+      "logps/chosen": -1.7512412071228027,
+      "logps/rejected": -107.4802474975586,
+      "loss": 0.14855973422527313,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.10580422729253769,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.47398966550827026,
+      "rewards/margins": 10.063344955444336,
+      "rewards/rejected": -9.58935546875,
+      "step": 687,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.9661456007146048,
+      "grad_norm": 1.0690404176712036,
+      "learning_rate": 5.775408801346407e-05,
+      "logits/chosen": -1.3550838232040405,
+      "logits/rejected": -1.356623649597168,
+      "logps/chosen": -5.809085845947266,
+      "logps/rejected": -89.94126892089844,
+      "loss": 0.27439332008361816,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22611746191978455,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.6475862264633179,
+      "rewards/margins": 8.384334564208984,
+      "rewards/rejected": -7.736748695373535,
+      "step": 688,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.96900401965163,
+      "grad_norm": 2.0467631816864014,
+      "learning_rate": 5.746783239054721e-05,
+      "logits/chosen": -1.3741295337677002,
+      "logits/rejected": -1.3841922283172607,
+      "logps/chosen": -4.964568138122559,
+      "logps/rejected": -88.7358169555664,
+      "loss": 0.6663780212402344,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.607894778251648,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4510470926761627,
+      "rewards/margins": 8.151575088500977,
+      "rewards/rejected": -7.700528144836426,
+      "step": 689,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.9718624385886556,
+      "grad_norm": 9.836221694946289,
+      "learning_rate": 5.718200162595449e-05,
+      "logits/chosen": -1.363444209098816,
+      "logits/rejected": -1.374962329864502,
+      "logps/chosen": -5.092153072357178,
+      "logps/rejected": -94.88340759277344,
+      "loss": 0.20714730024337769,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.18801794946193695,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4679247736930847,
+      "rewards/margins": 8.542611122131348,
+      "rewards/rejected": -8.074687004089355,
+      "step": 690,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.9747208575256812,
+      "grad_norm": 0.7933641672134399,
+      "learning_rate": 5.6896598574879745e-05,
+      "logits/chosen": -1.2816342115402222,
+      "logits/rejected": -1.2929383516311646,
+      "logps/chosen": -3.448430299758911,
+      "logps/rejected": -86.89325714111328,
+      "loss": 0.19773589074611664,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16216182708740234,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.45908814668655396,
+      "rewards/margins": 7.683626651763916,
+      "rewards/rejected": -7.2245378494262695,
+      "step": 691,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.9775792764627065,
+      "grad_norm": 0.7447614073753357,
+      "learning_rate": 5.6611626088244194e-05,
+      "logits/chosen": -1.2754361629486084,
+      "logits/rejected": -1.2913568019866943,
+      "logps/chosen": -4.723841667175293,
+      "logps/rejected": -114.28559875488281,
+      "loss": 0.217029869556427,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21108339726924896,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.552419900894165,
+      "rewards/margins": 10.366623878479004,
+      "rewards/rejected": -9.814202308654785,
+      "step": 692,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.980437695399732,
+      "grad_norm": 0.7340989708900452,
+      "learning_rate": 5.632708701266822e-05,
+      "logits/chosen": -1.3483805656433105,
+      "logits/rejected": -1.361656904220581,
+      "logps/chosen": -3.649254322052002,
+      "logps/rejected": -94.763916015625,
+      "loss": 0.11617171764373779,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.10703612118959427,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5086368918418884,
+      "rewards/margins": 8.350646018981934,
+      "rewards/rejected": -7.842009544372559,
+      "step": 693,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.9832961143367576,
+      "grad_norm": 0.5886948704719543,
+      "learning_rate": 5.6042984190442797e-05,
+      "logits/chosen": -1.3470284938812256,
+      "logits/rejected": -1.3555852174758911,
+      "logps/chosen": -3.8740155696868896,
+      "logps/rejected": -85.09982299804688,
+      "loss": 0.24540625512599945,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21094366908073425,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6043745279312134,
+      "rewards/margins": 7.761390209197998,
+      "rewards/rejected": -7.157015800476074,
+      "step": 694,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.9861545332737829,
+      "grad_norm": 0.9088613986968994,
+      "learning_rate": 5.5759320459500996e-05,
+      "logits/chosen": -1.294097900390625,
+      "logits/rejected": -1.301591396331787,
+      "logps/chosen": -8.632216453552246,
+      "logps/rejected": -86.00621032714844,
+      "loss": 0.31499531865119934,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.30169758200645447,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7339040040969849,
+      "rewards/margins": 7.947988510131836,
+      "rewards/rejected": -7.214084625244141,
+      "step": 695,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.9890129522108084,
+      "grad_norm": 0.9410954713821411,
+      "learning_rate": 5.547609865338994e-05,
+      "logits/chosen": -1.3490209579467773,
+      "logits/rejected": -1.36653470993042,
+      "logps/chosen": -3.855334520339966,
+      "logps/rejected": -100.82417297363281,
+      "loss": 0.1946963518857956,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1695217490196228,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4291667640209198,
+      "rewards/margins": 8.582954406738281,
+      "rewards/rejected": -8.153787612915039,
+      "step": 696,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.991871371147834,
+      "grad_norm": 0.9446479678153992,
+      "learning_rate": 5.5193321601242156e-05,
+      "logits/chosen": -1.2942531108856201,
+      "logits/rejected": -1.3041220903396606,
+      "logps/chosen": -6.212667465209961,
+      "logps/rejected": -99.26578521728516,
+      "loss": 0.28405243158340454,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.24633678793907166,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.45503294467926025,
+      "rewards/margins": 9.119531631469727,
+      "rewards/rejected": -8.664498329162598,
+      "step": 697,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 1.9947297900848593,
+      "grad_norm": 0.801936686038971,
+      "learning_rate": 5.491099212774763e-05,
+      "logits/chosen": -1.3422621488571167,
+      "logits/rejected": -1.3603028059005737,
+      "logps/chosen": -2.1552841663360596,
+      "logps/rejected": -95.64045715332031,
+      "loss": 0.17218460142612457,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16264331340789795,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3500121235847473,
+      "rewards/margins": 8.44575309753418,
+      "rewards/rejected": -8.09574031829834,
+      "step": 698,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 1.9975882090218846,
+      "grad_norm": 0.893062174320221,
+      "learning_rate": 5.462911305312526e-05,
+      "logits/chosen": -1.3398520946502686,
+      "logits/rejected": -1.3459690809249878,
+      "logps/chosen": -2.4024996757507324,
+      "logps/rejected": -91.11569213867188,
+      "loss": 0.1802024245262146,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.12556976079940796,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4403124749660492,
+      "rewards/margins": 8.568432807922363,
+      "rewards/rejected": -8.128119468688965,
+      "step": 699,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 5.554687023162842,
+      "learning_rate": 5.434768719309501e-05,
+      "logits/chosen": -1.3195669651031494,
+      "logits/rejected": -1.3334029912948608,
+      "logps/chosen": -3.12284255027771,
+      "logps/rejected": -90.41193389892578,
+      "loss": 0.19962038099765778,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15631449222564697,
+      "rewards/accuracies": 0.9629629850387573,
+      "rewards/chosen": 0.4614593982696533,
+      "rewards/margins": 7.9853901863098145,
+      "rewards/rejected": -7.52393102645874,
+      "step": 700,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.0028584189370253,
+      "grad_norm": 0.7382018566131592,
+      "learning_rate": 5.406671735884958e-05,
+      "logits/chosen": -1.31937837600708,
+      "logits/rejected": -1.3208019733428955,
+      "logps/chosen": -3.603522300720215,
+      "logps/rejected": -82.04541015625,
+      "loss": 0.21211326122283936,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15621349215507507,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5101105570793152,
+      "rewards/margins": 7.389132499694824,
+      "rewards/rejected": -6.879022598266602,
+      "step": 701,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.005716837874051,
+      "grad_norm": 0.6500789523124695,
+      "learning_rate": 5.378620635702643e-05,
+      "logits/chosen": -1.3107669353485107,
+      "logits/rejected": -1.3187661170959473,
+      "logps/chosen": -2.401862382888794,
+      "logps/rejected": -74.21663665771484,
+      "loss": 0.19214877486228943,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15796902775764465,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.6314170956611633,
+      "rewards/margins": 6.921903610229492,
+      "rewards/rejected": -6.2904863357543945,
+      "step": 702,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.0085752568110764,
+      "grad_norm": 0.5643107891082764,
+      "learning_rate": 5.35061569896796e-05,
+      "logits/chosen": -1.3577075004577637,
+      "logits/rejected": -1.3637503385543823,
+      "logps/chosen": -3.0552637577056885,
+      "logps/rejected": -72.03173065185547,
+      "loss": 0.13705869019031525,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1205982118844986,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5661800503730774,
+      "rewards/margins": 6.688817024230957,
+      "rewards/rejected": -6.122636795043945,
+      "step": 703,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.0114336757481017,
+      "grad_norm": 0.8152499198913574,
+      "learning_rate": 5.322657205425183e-05,
+      "logits/chosen": -1.3671040534973145,
+      "logits/rejected": -1.3767889738082886,
+      "logps/chosen": -2.06760835647583,
+      "logps/rejected": -91.6816635131836,
+      "loss": 0.1262805312871933,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.11861655116081238,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4480833113193512,
+      "rewards/margins": 8.306568145751953,
+      "rewards/rejected": -7.858485221862793,
+      "step": 704,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.0142920946851275,
+      "grad_norm": 0.5312683582305908,
+      "learning_rate": 5.294745434354671e-05,
+      "logits/chosen": -1.3388588428497314,
+      "logits/rejected": -1.3574988842010498,
+      "logps/chosen": -1.744231939315796,
+      "logps/rejected": -107.92884826660156,
+      "loss": 0.06678034365177155,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.049260981380939484,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7083232402801514,
+      "rewards/margins": 10.067192077636719,
+      "rewards/rejected": -9.358870506286621,
+      "step": 705,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.017150513622153,
+      "grad_norm": 0.5014054775238037,
+      "learning_rate": 5.266880664570053e-05,
+      "logits/chosen": -1.3985620737075806,
+      "logits/rejected": -1.4070799350738525,
+      "logps/chosen": -3.5201594829559326,
+      "logps/rejected": -89.9655532836914,
+      "loss": 0.16251122951507568,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13327711820602417,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6243622303009033,
+      "rewards/margins": 8.346537590026855,
+      "rewards/rejected": -7.7221760749816895,
+      "step": 706,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.020008932559178,
+      "grad_norm": 0.8896200060844421,
+      "learning_rate": 5.239063174415466e-05,
+      "logits/chosen": -1.3996846675872803,
+      "logits/rejected": -1.4094620943069458,
+      "logps/chosen": -4.70017671585083,
+      "logps/rejected": -94.6190414428711,
+      "loss": 0.20163139700889587,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1798512190580368,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.0199339389801025,
+      "rewards/margins": 8.901236534118652,
+      "rewards/rejected": -7.881302356719971,
+      "step": 707,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.0228673514962034,
+      "grad_norm": 0.6634477376937866,
+      "learning_rate": 5.211293241762764e-05,
+      "logits/chosen": -1.4573795795440674,
+      "logits/rejected": -1.4657036066055298,
+      "logps/chosen": -3.595468521118164,
+      "logps/rejected": -99.15465545654297,
+      "loss": 0.1454685479402542,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1412096917629242,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.9684085249900818,
+      "rewards/margins": 9.598406791687012,
+      "rewards/rejected": -8.629999160766602,
+      "step": 708,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.025725770433229,
+      "grad_norm": 0.4990958273410797,
+      "learning_rate": 5.183571144008747e-05,
+      "logits/chosen": -1.464647650718689,
+      "logits/rejected": -1.4746804237365723,
+      "logps/chosen": -2.852874517440796,
+      "logps/rejected": -91.08935546875,
+      "loss": 0.160193532705307,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15670600533485413,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5958665013313293,
+      "rewards/margins": 8.472614288330078,
+      "rewards/rejected": -7.8767476081848145,
+      "step": 709,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.0285841893702545,
+      "grad_norm": 0.6225377321243286,
+      "learning_rate": 5.155897158072378e-05,
+      "logits/chosen": -1.5063742399215698,
+      "logits/rejected": -1.522721290588379,
+      "logps/chosen": -1.9705212116241455,
+      "logps/rejected": -121.17532348632812,
+      "loss": 0.08409148454666138,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.08197915554046631,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6765754222869873,
+      "rewards/margins": 11.44405460357666,
+      "rewards/rejected": -10.767478942871094,
+      "step": 710,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.03144260830728,
+      "grad_norm": 0.5770713686943054,
+      "learning_rate": 5.1282715603920374e-05,
+      "logits/chosen": -1.5320286750793457,
+      "logits/rejected": -1.5518614053726196,
+      "logps/chosen": -6.716513633728027,
+      "logps/rejected": -110.9957504272461,
+      "loss": 0.19623152911663055,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1891591101884842,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6402191519737244,
+      "rewards/margins": 10.306550025939941,
+      "rewards/rejected": -9.666330337524414,
+      "step": 711,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.0343010272443056,
+      "grad_norm": 0.6445726156234741,
+      "learning_rate": 5.1006946269227376e-05,
+      "logits/chosen": -1.49067223072052,
+      "logits/rejected": -1.4948961734771729,
+      "logps/chosen": -3.5928125381469727,
+      "logps/rejected": -87.59977722167969,
+      "loss": 0.14255580306053162,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14030030369758606,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7219333052635193,
+      "rewards/margins": 8.310097694396973,
+      "rewards/rejected": -7.588164329528809,
+      "step": 712,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.037159446181331,
+      "grad_norm": 0.6341351270675659,
+      "learning_rate": 5.073166633133389e-05,
+      "logits/chosen": -1.5766968727111816,
+      "logits/rejected": -1.587092399597168,
+      "logps/chosen": -2.0322654247283936,
+      "logps/rejected": -98.49361419677734,
+      "loss": 0.1262267529964447,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.11367999762296677,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6843907833099365,
+      "rewards/margins": 9.339305877685547,
+      "rewards/rejected": -8.654915809631348,
+      "step": 713,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.0400178651183563,
+      "grad_norm": 0.7600017786026001,
+      "learning_rate": 5.045687854004042e-05,
+      "logits/chosen": -1.5253121852874756,
+      "logits/rejected": -1.5327656269073486,
+      "logps/chosen": -2.819199323654175,
+      "logps/rejected": -122.22134399414062,
+      "loss": 0.11136095970869064,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1008281484246254,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.48294559121131897,
+      "rewards/margins": 11.306340217590332,
+      "rewards/rejected": -10.823394775390625,
+      "step": 714,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.042876284055382,
+      "grad_norm": 0.646041989326477,
+      "learning_rate": 5.01825856402312e-05,
+      "logits/chosen": -1.6005913019180298,
+      "logits/rejected": -1.6237785816192627,
+      "logps/chosen": -2.513141393661499,
+      "logps/rejected": -106.20286560058594,
+      "loss": 0.12050984054803848,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.11930551379919052,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6920363306999207,
+      "rewards/margins": 9.648955345153809,
+      "rewards/rejected": -8.956918716430664,
+      "step": 715,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.0457347029924073,
+      "grad_norm": 0.6046995520591736,
+      "learning_rate": 4.99087903718471e-05,
+      "logits/chosen": -1.606223464012146,
+      "logits/rejected": -1.6436808109283447,
+      "logps/chosen": -1.5272647142410278,
+      "logps/rejected": -131.58401489257812,
+      "loss": 0.11919206380844116,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.09792602807283401,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6252794861793518,
+      "rewards/margins": 11.937460899353027,
+      "rewards/rejected": -11.31218147277832,
+      "step": 716,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.0485931219294327,
+      "grad_norm": 1.2827632427215576,
+      "learning_rate": 4.963549546985799e-05,
+      "logits/chosen": -1.6043542623519897,
+      "logits/rejected": -1.6206470727920532,
+      "logps/chosen": -1.8725172281265259,
+      "logps/rejected": -102.03213500976562,
+      "loss": 0.13097313046455383,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1279985010623932,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5102192163467407,
+      "rewards/margins": 9.184639930725098,
+      "rewards/rejected": -8.674420356750488,
+      "step": 717,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.0514515408664584,
+      "grad_norm": 0.6364453434944153,
+      "learning_rate": 4.936270366423563e-05,
+      "logits/chosen": -1.6342445611953735,
+      "logits/rejected": -1.6388630867004395,
+      "logps/chosen": -3.203178644180298,
+      "logps/rejected": -110.25364685058594,
+      "loss": 0.1496572345495224,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14746828377246857,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6244829297065735,
+      "rewards/margins": 10.414778709411621,
+      "rewards/rejected": -9.790295600891113,
+      "step": 718,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.0543099598034837,
+      "grad_norm": 1.0839275121688843,
+      "learning_rate": 4.9090417679926195e-05,
+      "logits/chosen": -1.6477729082107544,
+      "logits/rejected": -1.6559243202209473,
+      "logps/chosen": -2.1751749515533447,
+      "logps/rejected": -114.43892669677734,
+      "loss": 0.10991393774747849,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.10836468636989594,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5266481637954712,
+      "rewards/margins": 10.735066413879395,
+      "rewards/rejected": -10.208417892456055,
+      "step": 719,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.057168378740509,
+      "grad_norm": 0.5046917796134949,
+      "learning_rate": 4.88186402368232e-05,
+      "logits/chosen": -1.6776890754699707,
+      "logits/rejected": -1.701670527458191,
+      "logps/chosen": -2.6740882396698,
+      "logps/rejected": -135.4667510986328,
+      "loss": 0.13255618512630463,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.12441231310367584,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6434326767921448,
+      "rewards/margins": 12.391685485839844,
+      "rewards/rejected": -11.748252868652344,
+      "step": 720,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.060026797677535,
+      "grad_norm": 0.7640891075134277,
+      "learning_rate": 4.8547374049740354e-05,
+      "logits/chosen": -1.6784427165985107,
+      "logits/rejected": -1.689367413520813,
+      "logps/chosen": -4.439858913421631,
+      "logps/rejected": -116.31710815429688,
+      "loss": 0.17832107841968536,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.17599192261695862,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7900669574737549,
+      "rewards/margins": 11.103574752807617,
+      "rewards/rejected": -10.313508033752441,
+      "step": 721,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.06288521661456,
+      "grad_norm": 0.916210412979126,
+      "learning_rate": 4.8276621828384225e-05,
+      "logits/chosen": -1.6554255485534668,
+      "logits/rejected": -1.6821229457855225,
+      "logps/chosen": -1.9292747974395752,
+      "logps/rejected": -125.80508422851562,
+      "loss": 0.10639452189207077,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.0997253805398941,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7844244241714478,
+      "rewards/margins": 11.666866302490234,
+      "rewards/rejected": -10.882440567016602,
+      "step": 722,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.0657436355515855,
+      "grad_norm": 0.5751746296882629,
+      "learning_rate": 4.800638627732751e-05,
+      "logits/chosen": -1.725898265838623,
+      "logits/rejected": -1.7359280586242676,
+      "logps/chosen": -2.285418748855591,
+      "logps/rejected": -117.50049591064453,
+      "loss": 0.1161622479557991,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.09694134443998337,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.8232448697090149,
+      "rewards/margins": 11.284551620483398,
+      "rewards/rejected": -10.461305618286133,
+      "step": 723,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.068602054488611,
+      "grad_norm": 0.6991005539894104,
+      "learning_rate": 4.773667009598163e-05,
+      "logits/chosen": -1.7288771867752075,
+      "logits/rejected": -1.7394602298736572,
+      "logps/chosen": -3.0993452072143555,
+      "logps/rejected": -114.0384521484375,
+      "loss": 0.13271521031856537,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1309974640607834,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.8902750015258789,
+      "rewards/margins": 11.224279403686523,
+      "rewards/rejected": -10.334004402160645,
+      "step": 724,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.0714604734256366,
+      "grad_norm": 0.717708170413971,
+      "learning_rate": 4.746747597857014e-05,
+      "logits/chosen": -1.7381887435913086,
+      "logits/rejected": -1.7586032152175903,
+      "logps/chosen": -3.0901057720184326,
+      "logps/rejected": -116.30864715576172,
+      "loss": 0.14192581176757812,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13648167252540588,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7617445588111877,
+      "rewards/margins": 11.020862579345703,
+      "rewards/rejected": -10.25911808013916,
+      "step": 725,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.074318892362662,
+      "grad_norm": 0.7786213159561157,
+      "learning_rate": 4.719880661410153e-05,
+      "logits/chosen": -1.6829264163970947,
+      "logits/rejected": -1.7100300788879395,
+      "logps/chosen": -1.696429967880249,
+      "logps/rejected": -116.65681457519531,
+      "loss": 0.12481532990932465,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.12021290510892868,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.47501328587532043,
+      "rewards/margins": 10.90579605102539,
+      "rewards/rejected": -10.430782318115234,
+      "step": 726,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.077177311299687,
+      "grad_norm": 0.8564444780349731,
+      "learning_rate": 4.6930664686342526e-05,
+      "logits/chosen": -1.7280608415603638,
+      "logits/rejected": -1.7375504970550537,
+      "logps/chosen": -2.874122381210327,
+      "logps/rejected": -129.02975463867188,
+      "loss": 0.13600313663482666,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1322639435529709,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5178536176681519,
+      "rewards/margins": 12.142501831054688,
+      "rewards/rejected": -11.624648094177246,
+      "step": 727,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.080035730236713,
+      "grad_norm": 2.468414068222046,
+      "learning_rate": 4.666305287379129e-05,
+      "logits/chosen": -1.6879026889801025,
+      "logits/rejected": -1.7079944610595703,
+      "logps/chosen": -3.1912660598754883,
+      "logps/rejected": -138.30685424804688,
+      "loss": 0.16840460896492004,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1648813784122467,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.2080085277557373,
+      "rewards/margins": 13.236478805541992,
+      "rewards/rejected": -12.028470993041992,
+      "step": 728,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.0828941491737383,
+      "grad_norm": 0.8098137378692627,
+      "learning_rate": 4.6395973849650456e-05,
+      "logits/chosen": -1.7033703327178955,
+      "logits/rejected": -1.719029426574707,
+      "logps/chosen": -3.1450066566467285,
+      "logps/rejected": -116.48741149902344,
+      "loss": 0.14289624989032745,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.12687478959560394,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5492764115333557,
+      "rewards/margins": 10.482539176940918,
+      "rewards/rejected": -9.933263778686523,
+      "step": 729,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.0857525681107636,
+      "grad_norm": 0.7132099866867065,
+      "learning_rate": 4.612943028180072e-05,
+      "logits/chosen": -1.6585897207260132,
+      "logits/rejected": -1.675646424293518,
+      "logps/chosen": -2.4841561317443848,
+      "logps/rejected": -118.85638427734375,
+      "loss": 0.12736377120018005,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.10308001935482025,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.508740246295929,
+      "rewards/margins": 11.070262908935547,
+      "rewards/rejected": -10.561522483825684,
+      "step": 730,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.0886109870477894,
+      "grad_norm": 0.5194377899169922,
+      "learning_rate": 4.586342483277396e-05,
+      "logits/chosen": -1.6939231157302856,
+      "logits/rejected": -1.7033421993255615,
+      "logps/chosen": -2.945922374725342,
+      "logps/rejected": -112.94274139404297,
+      "loss": 0.13661476969718933,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13415244221687317,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7902134656906128,
+      "rewards/margins": 10.974791526794434,
+      "rewards/rejected": -10.184578895568848,
+      "step": 731,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.0914694059848147,
+      "grad_norm": 0.8727378845214844,
+      "learning_rate": 4.559796015972677e-05,
+      "logits/chosen": -1.683030128479004,
+      "logits/rejected": -1.6943947076797485,
+      "logps/chosen": -3.6087183952331543,
+      "logps/rejected": -115.31521606445312,
+      "loss": 0.19977882504463196,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16409388184547424,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5767125487327576,
+      "rewards/margins": 10.678343772888184,
+      "rewards/rejected": -10.101630210876465,
+      "step": 732,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.09432782492184,
+      "grad_norm": 0.9550994038581848,
+      "learning_rate": 4.533303891441388e-05,
+      "logits/chosen": -1.7100588083267212,
+      "logits/rejected": -1.7279833555221558,
+      "logps/chosen": -2.625103235244751,
+      "logps/rejected": -116.67841339111328,
+      "loss": 0.15604126453399658,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15421541035175323,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5999993681907654,
+      "rewards/margins": 10.786684036254883,
+      "rewards/rejected": -10.186685562133789,
+      "step": 733,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.0971862438588658,
+      "grad_norm": 0.6655582189559937,
+      "learning_rate": 4.5068663743161686e-05,
+      "logits/chosen": -1.7050411701202393,
+      "logits/rejected": -1.7205008268356323,
+      "logps/chosen": -3.2177319526672363,
+      "logps/rejected": -127.66587829589844,
+      "loss": 0.13336561620235443,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1308842897415161,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.8900558948516846,
+      "rewards/margins": 12.344279289245605,
+      "rewards/rejected": -11.454222679138184,
+      "step": 734,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.100044662795891,
+      "grad_norm": 0.7084753513336182,
+      "learning_rate": 4.4804837286841686e-05,
+      "logits/chosen": -1.7348458766937256,
+      "logits/rejected": -1.74097740650177,
+      "logps/chosen": -2.4784886837005615,
+      "logps/rejected": -115.00691986083984,
+      "loss": 0.11979944258928299,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.11902526021003723,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7228187322616577,
+      "rewards/margins": 11.022592544555664,
+      "rewards/rejected": -10.299775123596191,
+      "step": 735,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.1029030817329164,
+      "grad_norm": 0.7655794024467468,
+      "learning_rate": 4.454156218084433e-05,
+      "logits/chosen": -1.7176027297973633,
+      "logits/rejected": -1.737367033958435,
+      "logps/chosen": -2.0084376335144043,
+      "logps/rejected": -125.28208923339844,
+      "loss": 0.13023331761360168,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1288643777370453,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5075401663780212,
+      "rewards/margins": 11.424022674560547,
+      "rewards/rejected": -10.916481018066406,
+      "step": 736,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.105761500669942,
+      "grad_norm": 0.7896831631660461,
+      "learning_rate": 4.427884105505251e-05,
+      "logits/chosen": -1.7016427516937256,
+      "logits/rejected": -1.7272720336914062,
+      "logps/chosen": -3.91815447807312,
+      "logps/rejected": -142.9935302734375,
+      "loss": 0.2238529473543167,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.22090935707092285,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.37484681606292725,
+      "rewards/margins": 12.707845687866211,
+      "rewards/rejected": -12.33299732208252,
+      "step": 737,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.1086199196069675,
+      "grad_norm": 12.026676177978516,
+      "learning_rate": 4.4016676533815324e-05,
+      "logits/chosen": -1.7297816276550293,
+      "logits/rejected": -1.7420827150344849,
+      "logps/chosen": -2.4720804691314697,
+      "logps/rejected": -125.89964294433594,
+      "loss": 0.16339817643165588,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16232505440711975,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5476446747779846,
+      "rewards/margins": 11.702983856201172,
+      "rewards/rejected": -11.155339241027832,
+      "step": 738,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.111478338543993,
+      "grad_norm": 1.414394736289978,
+      "learning_rate": 4.375507123592194e-05,
+      "logits/chosen": -1.6991113424301147,
+      "logits/rejected": -1.714755892753601,
+      "logps/chosen": -3.4832603931427,
+      "logps/rejected": -130.6119842529297,
+      "loss": 0.1240750402212143,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.12312482297420502,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.49840059876441956,
+      "rewards/margins": 12.143492698669434,
+      "rewards/rejected": -11.645090103149414,
+      "step": 739,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.114336757481018,
+      "grad_norm": 0.6430306434631348,
+      "learning_rate": 4.349402777457537e-05,
+      "logits/chosen": -1.6896321773529053,
+      "logits/rejected": -1.7094862461090088,
+      "logps/chosen": -2.1210155487060547,
+      "logps/rejected": -133.11016845703125,
+      "loss": 0.11661946028470993,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.11605717241764069,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5362185835838318,
+      "rewards/margins": 12.352823257446289,
+      "rewards/rejected": -11.816605567932129,
+      "step": 740,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.117195176418044,
+      "grad_norm": 0.6329697966575623,
+      "learning_rate": 4.3233548757366436e-05,
+      "logits/chosen": -1.6787524223327637,
+      "logits/rejected": -1.6913310289382935,
+      "logps/chosen": -1.8568882942199707,
+      "logps/rejected": -95.99836730957031,
+      "loss": 0.1527225375175476,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1344267576932907,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6739444136619568,
+      "rewards/margins": 8.90192985534668,
+      "rewards/rejected": -8.227986335754395,
+      "step": 741,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.120053595355069,
+      "grad_norm": 0.8138287663459778,
+      "learning_rate": 4.297363678624753e-05,
+      "logits/chosen": -1.7275301218032837,
+      "logits/rejected": -1.7589508295059204,
+      "logps/chosen": -1.4376156330108643,
+      "logps/rejected": -136.57284545898438,
+      "loss": 0.0878395289182663,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.08727831393480301,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6229580640792847,
+      "rewards/margins": 12.466249465942383,
+      "rewards/rejected": -11.843290328979492,
+      "step": 742,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.1229120142920945,
+      "grad_norm": 0.6505977511405945,
+      "learning_rate": 4.271429445750692e-05,
+      "logits/chosen": -1.7142571210861206,
+      "logits/rejected": -1.7310396432876587,
+      "logps/chosen": -1.753830909729004,
+      "logps/rejected": -121.19338989257812,
+      "loss": 0.10254449397325516,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.10117828845977783,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6088793277740479,
+      "rewards/margins": 11.494156837463379,
+      "rewards/rejected": -10.885275840759277,
+      "step": 743,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.1257704332291203,
+      "grad_norm": 0.5495070219039917,
+      "learning_rate": 4.245552436174253e-05,
+      "logits/chosen": -1.7415564060211182,
+      "logits/rejected": -1.7485125064849854,
+      "logps/chosen": -2.7452197074890137,
+      "logps/rejected": -121.05813598632812,
+      "loss": 0.12934887409210205,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.12875869870185852,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.9706099629402161,
+      "rewards/margins": 11.918776512145996,
+      "rewards/rejected": -10.948166847229004,
+      "step": 744,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.1286288521661456,
+      "grad_norm": 0.6341418027877808,
+      "learning_rate": 4.21973290838362e-05,
+      "logits/chosen": -1.7050449848175049,
+      "logits/rejected": -1.7371711730957031,
+      "logps/chosen": -2.296316146850586,
+      "logps/rejected": -126.93817138671875,
+      "loss": 0.12343169003725052,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.11587755382061005,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7295063734054565,
+      "rewards/margins": 11.936881065368652,
+      "rewards/rejected": -11.207374572753906,
+      "step": 745,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.131487271103171,
+      "grad_norm": 0.9399663209915161,
+      "learning_rate": 4.1939711202927936e-05,
+      "logits/chosen": -1.6567258834838867,
+      "logits/rejected": -1.6777889728546143,
+      "logps/chosen": -3.963951826095581,
+      "logps/rejected": -111.61817932128906,
+      "loss": 0.18954475224018097,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1747589260339737,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5586780309677124,
+      "rewards/margins": 10.387514114379883,
+      "rewards/rejected": -9.828836441040039,
+      "step": 746,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.1343456900401967,
+      "grad_norm": 0.5456673502922058,
+      "learning_rate": 4.168267329239002e-05,
+      "logits/chosen": -1.6570465564727783,
+      "logits/rejected": -1.6693735122680664,
+      "logps/chosen": -3.4366085529327393,
+      "logps/rejected": -115.15528106689453,
+      "loss": 0.1558106392621994,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14104154706001282,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.46783921122550964,
+      "rewards/margins": 10.435863494873047,
+      "rewards/rejected": -9.968024253845215,
+      "step": 747,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.137204108977222,
+      "grad_norm": 1.1644296646118164,
+      "learning_rate": 4.142621791980138e-05,
+      "logits/chosen": -1.7093998193740845,
+      "logits/rejected": -1.7273249626159668,
+      "logps/chosen": -2.527416944503784,
+      "logps/rejected": -124.19613647460938,
+      "loss": 0.11928348988294601,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.11651065200567245,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4832688570022583,
+      "rewards/margins": 11.187657356262207,
+      "rewards/rejected": -10.704388618469238,
+      "step": 748,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.1400625279142473,
+      "grad_norm": 0.6623984575271606,
+      "learning_rate": 4.11703476469218e-05,
+      "logits/chosen": -1.7346500158309937,
+      "logits/rejected": -1.7358237504959106,
+      "logps/chosen": -4.151916027069092,
+      "logps/rejected": -108.73068237304688,
+      "loss": 0.1444595456123352,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14280128479003906,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.705640435218811,
+      "rewards/margins": 10.313786506652832,
+      "rewards/rejected": -9.608145713806152,
+      "step": 749,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.142920946851273,
+      "grad_norm": 0.5999913811683655,
+      "learning_rate": 4.09150650296666e-05,
+      "logits/chosen": -1.689178228378296,
+      "logits/rejected": -1.7057077884674072,
+      "logps/chosen": -2.7739779949188232,
+      "logps/rejected": -117.21680450439453,
+      "loss": 0.11793866008520126,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1133996844291687,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6371058821678162,
+      "rewards/margins": 10.845322608947754,
+      "rewards/rejected": -10.20821762084961,
+      "step": 750,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.1457793657882984,
+      "grad_norm": 0.616504967212677,
+      "learning_rate": 4.0660372618080787e-05,
+      "logits/chosen": -1.705957055091858,
+      "logits/rejected": -1.720078706741333,
+      "logps/chosen": -1.91254460811615,
+      "logps/rejected": -104.59209442138672,
+      "loss": 0.15148289501667023,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14934603869915009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5575397610664368,
+      "rewards/margins": 9.8103666305542,
+      "rewards/rejected": -9.252824783325195,
+      "step": 751,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.1486377847253237,
+      "grad_norm": 0.6763497591018677,
+      "learning_rate": 4.0406272956313895e-05,
+      "logits/chosen": -1.73313307762146,
+      "logits/rejected": -1.739758014678955,
+      "logps/chosen": -4.059084892272949,
+      "logps/rejected": -107.08638000488281,
+      "loss": 0.13731777667999268,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1345837116241455,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.484380841255188,
+      "rewards/margins": 10.125937461853027,
+      "rewards/rejected": -9.641556739807129,
+      "step": 752,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.151496203662349,
+      "grad_norm": 0.6765409111976624,
+      "learning_rate": 4.015276858259427e-05,
+      "logits/chosen": -1.713019609451294,
+      "logits/rejected": -1.7212321758270264,
+      "logps/chosen": -2.8288121223449707,
+      "logps/rejected": -115.45738220214844,
+      "loss": 0.19622966647148132,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16696611046791077,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6339028477668762,
+      "rewards/margins": 10.791833877563477,
+      "rewards/rejected": -10.157931327819824,
+      "step": 753,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.154354622599375,
+      "grad_norm": 0.7982415556907654,
+      "learning_rate": 3.989986202920397e-05,
+      "logits/chosen": -1.7288676500320435,
+      "logits/rejected": -1.7484595775604248,
+      "logps/chosen": -2.475815534591675,
+      "logps/rejected": -108.9861068725586,
+      "loss": 0.15874387323856354,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1394469439983368,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.48281630873680115,
+      "rewards/margins": 9.926338195800781,
+      "rewards/rejected": -9.443522453308105,
+      "step": 754,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.1572130415364,
+      "grad_norm": 0.6383085250854492,
+      "learning_rate": 3.964755582245335e-05,
+      "logits/chosen": -1.7366212606430054,
+      "logits/rejected": -1.7480270862579346,
+      "logps/chosen": -3.7524828910827637,
+      "logps/rejected": -124.93280029296875,
+      "loss": 0.16495665907859802,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.16129866242408752,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7390357255935669,
+      "rewards/margins": 11.99774169921875,
+      "rewards/rejected": -11.258706092834473,
+      "step": 755,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.1600714604734255,
+      "grad_norm": 11.229231834411621,
+      "learning_rate": 3.939585248265576e-05,
+      "logits/chosen": -1.6887023448944092,
+      "logits/rejected": -1.7098785638809204,
+      "logps/chosen": -1.5743474960327148,
+      "logps/rejected": -125.05543518066406,
+      "loss": 0.15721367299556732,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.10757317394018173,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.5603189468383789,
+      "rewards/margins": 11.491580963134766,
+      "rewards/rejected": -10.931262969970703,
+      "step": 756,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.1629298794104512,
+      "grad_norm": 0.6276991963386536,
+      "learning_rate": 3.914475452410257e-05,
+      "logits/chosen": -1.7208404541015625,
+      "logits/rejected": -1.7312361001968384,
+      "logps/chosen": -2.457312822341919,
+      "logps/rejected": -105.10942840576172,
+      "loss": 0.12477224320173264,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.11921665072441101,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.8507872223854065,
+      "rewards/margins": 10.100202560424805,
+      "rewards/rejected": -9.249414443969727,
+      "step": 757,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.1657882983474765,
+      "grad_norm": 0.38204801082611084,
+      "learning_rate": 3.889426445503779e-05,
+      "logits/chosen": -1.7382487058639526,
+      "logits/rejected": -1.7489298582077026,
+      "logps/chosen": -2.2348647117614746,
+      "logps/rejected": -117.04852294921875,
+      "loss": 0.09758877009153366,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.09699364751577377,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7443951368331909,
+      "rewards/margins": 11.229491233825684,
+      "rewards/rejected": -10.485095977783203,
+      "step": 758,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.168646717284502,
+      "grad_norm": 0.5256333351135254,
+      "learning_rate": 3.864438477763327e-05,
+      "logits/chosen": -1.733298659324646,
+      "logits/rejected": -1.7572256326675415,
+      "logps/chosen": -2.88501238822937,
+      "logps/rejected": -126.08180236816406,
+      "loss": 0.10684897005558014,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.10602684319019318,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5964562892913818,
+      "rewards/margins": 11.40107536315918,
+      "rewards/rejected": -10.804619789123535,
+      "step": 759,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.1715051362215276,
+      "grad_norm": 0.5174509286880493,
+      "learning_rate": 3.839511798796357e-05,
+      "logits/chosen": -1.7838695049285889,
+      "logits/rejected": -1.799968957901001,
+      "logps/chosen": -3.1636159420013428,
+      "logps/rejected": -154.8913116455078,
+      "loss": 0.14846816658973694,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14811007678508759,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7431506514549255,
+      "rewards/margins": 14.516936302185059,
+      "rewards/rejected": -13.773785591125488,
+      "step": 760,
+      "train_speed(iter/s)": 0.005401
+    },
+    {
+      "epoch": 2.174363555158553,
+      "grad_norm": 0.7228491306304932,
+      "learning_rate": 3.814646657598104e-05,
+      "logits/chosen": -1.7531218528747559,
+      "logits/rejected": -1.777261734008789,
+      "logps/chosen": -2.1842851638793945,
+      "logps/rejected": -112.90637969970703,
+      "loss": 0.10662516206502914,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.09334062784910202,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5663318634033203,
+      "rewards/margins": 10.066019058227539,
+      "rewards/rejected": -9.499687194824219,
+      "step": 761,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.1772219740955783,
+      "grad_norm": 0.6269167065620422,
+      "learning_rate": 3.789843302549096e-05,
+      "logits/chosen": -1.7703839540481567,
+      "logits/rejected": -1.780287504196167,
+      "logps/chosen": -2.7307345867156982,
+      "logps/rejected": -122.10258483886719,
+      "loss": 0.15339156985282898,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13365858793258667,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6474071145057678,
+      "rewards/margins": 11.623534202575684,
+      "rewards/rejected": -10.976127624511719,
+      "step": 762,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.180080393032604,
+      "grad_norm": 0.6452962756156921,
+      "learning_rate": 3.7651019814126654e-05,
+      "logits/chosen": -1.6857422590255737,
+      "logits/rejected": -1.7062132358551025,
+      "logps/chosen": -3.037238836288452,
+      "logps/rejected": -132.969970703125,
+      "loss": 0.15664508938789368,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14627954363822937,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6218290328979492,
+      "rewards/margins": 12.158147811889648,
+      "rewards/rejected": -11.536317825317383,
+      "step": 763,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.1829388119696294,
+      "grad_norm": 1.2026190757751465,
+      "learning_rate": 3.740422941332496e-05,
+      "logits/chosen": -1.739600419998169,
+      "logits/rejected": -1.7670246362686157,
+      "logps/chosen": -2.065814256668091,
+      "logps/rejected": -127.63172912597656,
+      "loss": 0.12204542756080627,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.12009820342063904,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5392940044403076,
+      "rewards/margins": 11.599030494689941,
+      "rewards/rejected": -11.059735298156738,
+      "step": 764,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.1857972309066547,
+      "grad_norm": 0.539072573184967,
+      "learning_rate": 3.7158064288301244e-05,
+      "logits/chosen": -1.7427868843078613,
+      "logits/rejected": -1.7634449005126953,
+      "logps/chosen": -1.8705532550811768,
+      "logps/rejected": -125.5798110961914,
+      "loss": 0.13085120916366577,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.109225794672966,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5458089709281921,
+      "rewards/margins": 11.635404586791992,
+      "rewards/rejected": -11.089594841003418,
+      "step": 765,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.18865564984368,
+      "grad_norm": 0.9112529158592224,
+      "learning_rate": 3.691252689802502e-05,
+      "logits/chosen": -1.6936545372009277,
+      "logits/rejected": -1.7068877220153809,
+      "logps/chosen": -5.790098667144775,
+      "logps/rejected": -112.80413055419922,
+      "loss": 0.2206256240606308,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21952351927757263,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6771332025527954,
+      "rewards/margins": 10.539006233215332,
+      "rewards/rejected": -9.861872673034668,
+      "step": 766,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.1915140687807058,
+      "grad_norm": 0.8130010962486267,
+      "learning_rate": 3.6667619695195285e-05,
+      "logits/chosen": -1.7631025314331055,
+      "logits/rejected": -1.7726210355758667,
+      "logps/chosen": -3.349720001220703,
+      "logps/rejected": -105.22746276855469,
+      "loss": 0.18691176176071167,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.17924201488494873,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7146871089935303,
+      "rewards/margins": 9.890175819396973,
+      "rewards/rejected": -9.175488471984863,
+      "step": 767,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.194372487717731,
+      "grad_norm": 1.0899032354354858,
+      "learning_rate": 3.6423345126216045e-05,
+      "logits/chosen": -1.7186263799667358,
+      "logits/rejected": -1.7368674278259277,
+      "logps/chosen": -2.324446201324463,
+      "logps/rejected": -118.56622314453125,
+      "loss": 0.16899357736110687,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1636049896478653,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6314736604690552,
+      "rewards/margins": 11.031686782836914,
+      "rewards/rejected": -10.400211334228516,
+      "step": 768,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.1972309066547564,
+      "grad_norm": 0.7936761975288391,
+      "learning_rate": 3.6179705631171746e-05,
+      "logits/chosen": -1.703823208808899,
+      "logits/rejected": -1.7315338850021362,
+      "logps/chosen": -2.9620230197906494,
+      "logps/rejected": -143.7433319091797,
+      "loss": 0.13453835248947144,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.12804599106311798,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.769576370716095,
+      "rewards/margins": 13.67524528503418,
+      "rewards/rejected": -12.905670166015625,
+      "step": 769,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.200089325591782,
+      "grad_norm": 0.9569531083106995,
+      "learning_rate": 3.593670364380316e-05,
+      "logits/chosen": -1.694765567779541,
+      "logits/rejected": -1.7086044549942017,
+      "logps/chosen": -2.6550450325012207,
+      "logps/rejected": -126.60090637207031,
+      "loss": 0.22317641973495483,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1666475385427475,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5760101079940796,
+      "rewards/margins": 12.053494453430176,
+      "rewards/rejected": -11.477484703063965,
+      "step": 770,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.2029477445288075,
+      "grad_norm": 0.7008557915687561,
+      "learning_rate": 3.569434159148283e-05,
+      "logits/chosen": -1.681830644607544,
+      "logits/rejected": -1.7102689743041992,
+      "logps/chosen": -1.4984784126281738,
+      "logps/rejected": -152.89645385742188,
+      "loss": 0.08517744392156601,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.06586003303527832,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7491201162338257,
+      "rewards/margins": 14.39028263092041,
+      "rewards/rejected": -13.641162872314453,
+      "step": 771,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.205806163465833,
+      "grad_norm": 0.4644488990306854,
+      "learning_rate": 3.545262189519092e-05,
+      "logits/chosen": -1.7865322828292847,
+      "logits/rejected": -1.7923797369003296,
+      "logps/chosen": -2.8405375480651855,
+      "logps/rejected": -138.0335693359375,
+      "loss": 0.14850133657455444,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14815452694892883,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4763367772102356,
+      "rewards/margins": 13.229516983032227,
+      "rewards/rejected": -12.753180503845215,
+      "step": 772,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.2086645824028586,
+      "grad_norm": 0.8949534893035889,
+      "learning_rate": 3.5211546969491095e-05,
+      "logits/chosen": -1.751487135887146,
+      "logits/rejected": -1.7620105743408203,
+      "logps/chosen": -1.7669379711151123,
+      "logps/rejected": -119.79267883300781,
+      "loss": 0.10754434764385223,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.10662030428647995,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4996607303619385,
+      "rewards/margins": 11.162941932678223,
+      "rewards/rejected": -10.663281440734863,
+      "step": 773,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.211523001339884,
+      "grad_norm": 0.7084648609161377,
+      "learning_rate": 3.49711192225063e-05,
+      "logits/chosen": -1.682307481765747,
+      "logits/rejected": -1.7079421281814575,
+      "logps/chosen": -2.9968488216400146,
+      "logps/rejected": -134.74200439453125,
+      "loss": 0.09475664794445038,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.084076888859272,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5336630940437317,
+      "rewards/margins": 12.228374481201172,
+      "rewards/rejected": -11.694710731506348,
+      "step": 774,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.214381420276909,
+      "grad_norm": 0.5224748253822327,
+      "learning_rate": 3.4731341055894785e-05,
+      "logits/chosen": -1.72243070602417,
+      "logits/rejected": -1.7281838655471802,
+      "logps/chosen": -2.824965238571167,
+      "logps/rejected": -115.74949645996094,
+      "loss": 0.17388476431369781,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14388418197631836,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4050171971321106,
+      "rewards/margins": 10.795921325683594,
+      "rewards/rejected": -10.39090347290039,
+      "step": 775,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.217239839213935,
+      "grad_norm": 0.7625648975372314,
+      "learning_rate": 3.4492214864825965e-05,
+      "logits/chosen": -1.6454460620880127,
+      "logits/rejected": -1.677930235862732,
+      "logps/chosen": -1.8394455909729004,
+      "logps/rejected": -150.86842346191406,
+      "loss": 0.10291773825883865,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.09716804325580597,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7498948574066162,
+      "rewards/margins": 13.786081314086914,
+      "rewards/rejected": -13.036186218261719,
+      "step": 776,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.2200982581509603,
+      "grad_norm": 0.5313823819160461,
+      "learning_rate": 3.425374303795675e-05,
+      "logits/chosen": -1.7055084705352783,
+      "logits/rejected": -1.7109901905059814,
+      "logps/chosen": -3.196258068084717,
+      "logps/rejected": -104.67936706542969,
+      "loss": 0.13488231599330902,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.12887902557849884,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.43103182315826416,
+      "rewards/margins": 9.69394302368164,
+      "rewards/rejected": -9.262910842895508,
+      "step": 777,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.2229566770879856,
+      "grad_norm": 0.45489898324012756,
+      "learning_rate": 3.401592795740739e-05,
+      "logits/chosen": -1.7363502979278564,
+      "logits/rejected": -1.7455774545669556,
+      "logps/chosen": -3.791541814804077,
+      "logps/rejected": -127.68539428710938,
+      "loss": 0.15376096963882446,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15042446553707123,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6921437382698059,
+      "rewards/margins": 12.30968189239502,
+      "rewards/rejected": -11.617538452148438,
+      "step": 778,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.225815096025011,
+      "grad_norm": 0.9365268349647522,
+      "learning_rate": 3.3778771998737934e-05,
+      "logits/chosen": -1.6844940185546875,
+      "logits/rejected": -1.7032924890518188,
+      "logps/chosen": -3.275221109390259,
+      "logps/rejected": -137.47164916992188,
+      "loss": 0.15848176181316376,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15660107135772705,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5174179673194885,
+      "rewards/margins": 12.837626457214355,
+      "rewards/rejected": -12.320207595825195,
+      "step": 779,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.2286735149620367,
+      "grad_norm": 0.6778178811073303,
+      "learning_rate": 3.3542277530924374e-05,
+      "logits/chosen": -1.6570497751235962,
+      "logits/rejected": -1.7019526958465576,
+      "logps/chosen": -1.7437933683395386,
+      "logps/rejected": -138.32015991210938,
+      "loss": 0.09863787889480591,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.09677759557962418,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6153618097305298,
+      "rewards/margins": 12.391786575317383,
+      "rewards/rejected": -11.776426315307617,
+      "step": 780,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.231531933899062,
+      "grad_norm": 0.4646144509315491,
+      "learning_rate": 3.330644691633492e-05,
+      "logits/chosen": -1.6599035263061523,
+      "logits/rejected": -1.69225013256073,
+      "logps/chosen": -1.6837167739868164,
+      "logps/rejected": -164.30398559570312,
+      "loss": 0.09873135387897491,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.09822063148021698,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6354748010635376,
+      "rewards/margins": 15.040769577026367,
+      "rewards/rejected": -14.405292510986328,
+      "step": 781,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.2343903528360873,
+      "grad_norm": 0.6401258111000061,
+      "learning_rate": 3.3071282510706624e-05,
+      "logits/chosen": -1.6564258337020874,
+      "logits/rejected": -1.693245768547058,
+      "logps/chosen": -1.5007685422897339,
+      "logps/rejected": -136.3229217529297,
+      "loss": 0.09584607928991318,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.09055926650762558,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5642552375793457,
+      "rewards/margins": 12.582983016967773,
+      "rewards/rejected": -12.01872730255127,
+      "step": 782,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.237248771773113,
+      "grad_norm": 0.5455237030982971,
+      "learning_rate": 3.283678666312154e-05,
+      "logits/chosen": -1.688979148864746,
+      "logits/rejected": -1.696632981300354,
+      "logps/chosen": -3.467952013015747,
+      "logps/rejected": -122.91952514648438,
+      "loss": 0.14648161828517914,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.144728884100914,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.8166089057922363,
+      "rewards/margins": 12.023063659667969,
+      "rewards/rejected": -11.20645523071289,
+      "step": 783,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.2401071907101384,
+      "grad_norm": 0.9562914967536926,
+      "learning_rate": 3.260296171598358e-05,
+      "logits/chosen": -1.7287250757217407,
+      "logits/rejected": -1.7348930835723877,
+      "logps/chosen": -3.420701265335083,
+      "logps/rejected": -107.7225112915039,
+      "loss": 0.17758214473724365,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.15587428212165833,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6694732904434204,
+      "rewards/margins": 10.439946174621582,
+      "rewards/rejected": -9.770472526550293,
+      "step": 784,
+      "train_speed(iter/s)": 0.005402
+    },
+    {
+      "epoch": 2.2429656096471637,
+      "grad_norm": 0.6848970651626587,
+      "learning_rate": 3.236981000499485e-05,
+      "logits/chosen": -1.6658862829208374,
+      "logits/rejected": -1.6751506328582764,
+      "logps/chosen": -2.954430341720581,
+      "logps/rejected": -113.62657165527344,
+      "loss": 0.15038307011127472,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14674027264118195,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6098470687866211,
+      "rewards/margins": 10.768763542175293,
+      "rewards/rejected": -10.158916473388672,
+      "step": 785,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 2.2458240285841895,
+      "grad_norm": 0.6783968806266785,
+      "learning_rate": 3.2137333859132466e-05,
+      "logits/chosen": -1.6658713817596436,
+      "logits/rejected": -1.6860020160675049,
+      "logps/chosen": -1.8329298496246338,
+      "logps/rejected": -136.6328125,
+      "loss": 0.11154640465974808,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.11017812043428421,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7413030862808228,
+      "rewards/margins": 13.236644744873047,
+      "rewards/rejected": -12.495339393615723,
+      "step": 786,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 2.248682447521215,
+      "grad_norm": 0.5898379683494568,
+      "learning_rate": 3.1905535600625314e-05,
+      "logits/chosen": -1.676591396331787,
+      "logits/rejected": -1.6916677951812744,
+      "logps/chosen": -3.6847083568573,
+      "logps/rejected": -131.96946716308594,
+      "loss": 0.11107183992862701,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.09921784698963165,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3924238085746765,
+      "rewards/margins": 12.32601547241211,
+      "rewards/rejected": -11.933591842651367,
+      "step": 787,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 2.25154086645824,
+      "grad_norm": 9.934392929077148,
+      "learning_rate": 3.167441754493066e-05,
+      "logits/chosen": -1.6075174808502197,
+      "logits/rejected": -1.6402552127838135,
+      "logps/chosen": -5.406757354736328,
+      "logps/rejected": -125.26739501953125,
+      "loss": 0.5008838772773743,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.21078495681285858,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.32979682087898254,
+      "rewards/margins": 10.943861961364746,
+      "rewards/rejected": -10.614065170288086,
+      "step": 788,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 2.254399285395266,
+      "grad_norm": 0.6809211373329163,
+      "learning_rate": 3.144398200071128e-05,
+      "logits/chosen": -1.6855800151824951,
+      "logits/rejected": -1.6980534791946411,
+      "logps/chosen": -2.5400819778442383,
+      "logps/rejected": -114.75363159179688,
+      "loss": 0.1503981649875641,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.12278814613819122,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.6013644337654114,
+      "rewards/margins": 10.973652839660645,
+      "rewards/rejected": -10.37228775024414,
+      "step": 789,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 2.2572577043322912,
+      "grad_norm": 0.6184489727020264,
+      "learning_rate": 3.121423126981217e-05,
+      "logits/chosen": -1.644142985343933,
+      "logits/rejected": -1.659967303276062,
+      "logps/chosen": -4.191022872924805,
+      "logps/rejected": -114.24392700195312,
+      "loss": 0.1494584083557129,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.135895237326622,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.9744225740432739,
+      "rewards/margins": 11.152600288391113,
+      "rewards/rejected": -10.178176879882812,
+      "step": 790,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 2.2601161232693165,
+      "grad_norm": 0.7756046652793884,
+      "learning_rate": 3.0985167647237756e-05,
+      "logits/chosen": -1.7134742736816406,
+      "logits/rejected": -1.7250508069992065,
+      "logps/chosen": -2.833266258239746,
+      "logps/rejected": -119.01715087890625,
+      "loss": 0.1561891734600067,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13894352316856384,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5146474242210388,
+      "rewards/margins": 11.285941123962402,
+      "rewards/rejected": -10.771294593811035,
+      "step": 791,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 2.2629745422063423,
+      "grad_norm": 0.6245182752609253,
+      "learning_rate": 3.075679342112874e-05,
+      "logits/chosen": -1.640115737915039,
+      "logits/rejected": -1.6473097801208496,
+      "logps/chosen": -3.92708683013916,
+      "logps/rejected": -110.22953796386719,
+      "loss": 0.3563145697116852,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.3547799587249756,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4437691867351532,
+      "rewards/margins": 10.290136337280273,
+      "rewards/rejected": -9.846367835998535,
+      "step": 792,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 2.2658329611433676,
+      "grad_norm": 8.907173156738281,
+      "learning_rate": 3.052911087273949e-05,
+      "logits/chosen": -1.6591439247131348,
+      "logits/rejected": -1.6838045120239258,
+      "logps/chosen": -1.6061533689498901,
+      "logps/rejected": -136.19979858398438,
+      "loss": 0.10105100274085999,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.08847977221012115,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7345350980758667,
+      "rewards/margins": 12.696416854858398,
+      "rewards/rejected": -11.961880683898926,
+      "step": 793,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 2.268691380080393,
+      "grad_norm": 0.6540257334709167,
+      "learning_rate": 3.0302122276415056e-05,
+      "logits/chosen": -1.6991549730300903,
+      "logits/rejected": -1.7236639261245728,
+      "logps/chosen": -3.227992534637451,
+      "logps/rejected": -145.28456115722656,
+      "loss": 0.15280549228191376,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14838962256908417,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5916187763214111,
+      "rewards/margins": 13.367993354797363,
+      "rewards/rejected": -12.776374816894531,
+      "step": 794,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 2.2715497990174187,
+      "grad_norm": 0.7351321578025818,
+      "learning_rate": 3.0075829899568597e-05,
+      "logits/chosen": -1.6017165184020996,
+      "logits/rejected": -1.6121318340301514,
+      "logps/chosen": -3.3314077854156494,
+      "logps/rejected": -128.59197998046875,
+      "loss": 0.17674173414707184,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.13497605919837952,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.7480218410491943,
+      "rewards/margins": 12.017620086669922,
+      "rewards/rejected": -11.269598007202148,
+      "step": 795,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 2.274408217954444,
+      "grad_norm": 0.7349981069564819,
+      "learning_rate": 2.9850236002658595e-05,
+      "logits/chosen": -1.6180601119995117,
+      "logits/rejected": -1.629482388496399,
+      "logps/chosen": -3.289719581604004,
+      "logps/rejected": -130.05755615234375,
+      "loss": 0.12025503069162369,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.119521364569664,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7377493381500244,
+      "rewards/margins": 12.549837112426758,
+      "rewards/rejected": -11.812088012695312,
+      "step": 796,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 2.2772666368914694,
+      "grad_norm": 0.45565128326416016,
+      "learning_rate": 2.9625342839166316e-05,
+      "logits/chosen": -1.7096292972564697,
+      "logits/rejected": -1.7318222522735596,
+      "logps/chosen": -2.09987211227417,
+      "logps/rejected": -137.1615447998047,
+      "loss": 0.0756252259016037,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.07524615526199341,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6224530935287476,
+      "rewards/margins": 12.75358772277832,
+      "rewards/rejected": -12.131135940551758,
+      "step": 797,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 2.2801250558284947,
+      "grad_norm": 0.5187860131263733,
+      "learning_rate": 2.940115265557345e-05,
+      "logits/chosen": -1.6489043235778809,
+      "logits/rejected": -1.67991042137146,
+      "logps/chosen": -1.8865444660186768,
+      "logps/rejected": -122.9993667602539,
+      "loss": 0.1377023160457611,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.12219592928886414,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7240051627159119,
+      "rewards/margins": 11.111031532287598,
+      "rewards/rejected": -10.3870267868042,
+      "step": 798,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 2.2829834747655204,
+      "grad_norm": 0.637151837348938,
+      "learning_rate": 2.917766769133937e-05,
+      "logits/chosen": -1.6871957778930664,
+      "logits/rejected": -1.702707290649414,
+      "logps/chosen": -2.857081174850464,
+      "logps/rejected": -118.28166961669922,
+      "loss": 0.1422526240348816,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.14158561825752258,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5195335745811462,
+      "rewards/margins": 11.0984468460083,
+      "rewards/rejected": -10.578912734985352,
+      "step": 799,
+      "train_speed(iter/s)": 0.005403
+    },
+    {
+      "epoch": 2.2858418937025458,
+      "grad_norm": 0.6244106292724609,
+      "learning_rate": 2.8954890178879178e-05,
+      "logits/chosen": -1.678558588027954,
+      "logits/rejected": -1.6917924880981445,
+      "logps/chosen": -3.7977027893066406,
+      "logps/rejected": -119.4873046875,
+      "loss": 0.18403074145317078,
+      "memory(GiB)": 46.73,
+      "nll_loss": 0.1719832867383957,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.816820502281189,
+      "rewards/margins": 11.37762451171875,
+      "rewards/rejected": -10.56080436706543,
+      "step": 800,
+      "train_speed(iter/s)": 0.005404
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 1047,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.606176109986054e+18,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}