diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,5451 @@
+{
+  "best_global_step": 300,
+  "best_metric": 0.42507252,
+  "best_model_checkpoint": "/data2/kdd/crag/cjz/output/task1_dpo_round2_0dot9_0603/v1-20250603-223558/checkpoint-300",
+  "epoch": 1.7264864864864866,
+  "eval_steps": 300,
+  "global_step": 300,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.005765765765765766,
+      "grad_norm": 2.1474921703338623,
+      "learning_rate": 7.692307692307694e-06,
+      "logits/chosen": -0.4285973310470581,
+      "logits/rejected": -0.721319854259491,
+      "logps/chosen": -2.468954086303711,
+      "logps/rejected": -59.87730407714844,
+      "loss": 0.22946561872959137,
+      "memory(GiB)": 30.65,
+      "nll_loss": 0.13756254315376282,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.23724405467510223,
+      "rewards/margins": 3.9806408882141113,
+      "rewards/rejected": -3.743396759033203,
+      "step": 1,
+      "train_speed(iter/s)": 0.01022
+    },
+    {
+      "epoch": 0.011531531531531532,
+      "grad_norm": 2.328110456466675,
+      "learning_rate": 1.5384615384615387e-05,
+      "logits/chosen": -0.3982561528682709,
+      "logits/rejected": -0.7214521169662476,
+      "logps/chosen": -3.335645914077759,
+      "logps/rejected": -60.79209899902344,
+      "loss": 0.3770241439342499,
+      "memory(GiB)": 30.65,
+      "nll_loss": 0.23075783252716064,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.10687967389822006,
+      "rewards/margins": 3.9392333030700684,
+      "rewards/rejected": -3.832353353500366,
+      "step": 2,
+      "train_speed(iter/s)": 0.010488
+    },
+    {
+      "epoch": 0.017297297297297298,
+      "grad_norm": 14.568985939025879,
+      "learning_rate": 2.307692307692308e-05,
+      "logits/chosen": -0.43175598978996277,
+      "logits/rejected": -0.6921666264533997,
+      "logps/chosen": -5.190546035766602,
+      "logps/rejected": -51.33646011352539,
+      "loss": 0.9886025786399841,
+      "memory(GiB)": 34.1,
+      "nll_loss": 0.6371194124221802,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.034887176007032394,
+      "rewards/margins": 3.2021734714508057,
+      "rewards/rejected": -3.237060785293579,
+      "step": 3,
+      "train_speed(iter/s)": 0.010581
+    },
+    {
+      "epoch": 0.023063063063063063,
+      "grad_norm": 13.817249298095703,
+      "learning_rate": 3.0769230769230774e-05,
+      "logits/chosen": -0.4616771340370178,
+      "logits/rejected": -0.6264235973358154,
+      "logps/chosen": -3.2552552223205566,
+      "logps/rejected": -51.25586700439453,
+      "loss": 0.5021163821220398,
+      "memory(GiB)": 37.58,
+      "nll_loss": 0.26375898718833923,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.06816040724515915,
+      "rewards/margins": 3.5297036170959473,
+      "rewards/rejected": -3.461543083190918,
+      "step": 4,
+      "train_speed(iter/s)": 0.010625
+    },
+    {
+      "epoch": 0.02882882882882883,
+      "grad_norm": 2.363553524017334,
+      "learning_rate": 3.846153846153846e-05,
+      "logits/chosen": -0.48805952072143555,
+      "logits/rejected": -0.7040677070617676,
+      "logps/chosen": -6.89398717880249,
+      "logps/rejected": -59.37879943847656,
+      "loss": 0.39657193422317505,
+      "memory(GiB)": 37.58,
+      "nll_loss": 0.2818652391433716,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.1999940723180771,
+      "rewards/margins": 4.230030059814453,
+      "rewards/rejected": -4.030035495758057,
+      "step": 5,
+      "train_speed(iter/s)": 0.010659
+    },
+    {
+      "epoch": 0.034594594594594595,
+      "grad_norm": 4.551424026489258,
+      "learning_rate": 4.615384615384616e-05,
+      "logits/chosen": -0.39471542835235596,
+      "logits/rejected": -0.6508359909057617,
+      "logps/chosen": -6.633285999298096,
+      "logps/rejected": -58.268653869628906,
+      "loss": 0.5636770129203796,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.41472452878952026,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.1396198272705078,
+      "rewards/margins": 3.7723348140716553,
+      "rewards/rejected": -3.6327152252197266,
+      "step": 6,
+      "train_speed(iter/s)": 0.010667
+    },
+    {
+      "epoch": 0.04036036036036036,
+      "grad_norm": 4.302145481109619,
+      "learning_rate": 5.384615384615385e-05,
+      "logits/chosen": -0.42404329776763916,
+      "logits/rejected": -0.7084805369377136,
+      "logps/chosen": -4.09792947769165,
+      "logps/rejected": -69.69088745117188,
+      "loss": 0.5202179551124573,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3875095546245575,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.018997177481651306,
+      "rewards/margins": 5.0437703132629395,
+      "rewards/rejected": -5.062767505645752,
+      "step": 7,
+      "train_speed(iter/s)": 0.010685
+    },
+    {
+      "epoch": 0.04612612612612613,
+      "grad_norm": 4.475785255432129,
+      "learning_rate": 6.153846153846155e-05,
+      "logits/chosen": -0.4411894977092743,
+      "logits/rejected": -0.6594015955924988,
+      "logps/chosen": -6.067828178405762,
+      "logps/rejected": -52.161441802978516,
+      "loss": 0.8079419136047363,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.5527109503746033,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.13843362033367157,
+      "rewards/margins": 3.638244390487671,
+      "rewards/rejected": -3.4998104572296143,
+      "step": 8,
+      "train_speed(iter/s)": 0.010699
+    },
+    {
+      "epoch": 0.05189189189189189,
+      "grad_norm": 7.412286281585693,
+      "learning_rate": 6.923076923076924e-05,
+      "logits/chosen": -0.4308490753173828,
+      "logits/rejected": -0.6868806481361389,
+      "logps/chosen": -4.5601372718811035,
+      "logps/rejected": -75.88018798828125,
+      "loss": 0.3704576790332794,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.23506566882133484,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1938612163066864,
+      "rewards/margins": 5.914104461669922,
+      "rewards/rejected": -5.720243453979492,
+      "step": 9,
+      "train_speed(iter/s)": 0.010708
+    },
+    {
+      "epoch": 0.05765765765765766,
+      "grad_norm": 2.0841763019561768,
+      "learning_rate": 7.692307692307693e-05,
+      "logits/chosen": -0.4546598792076111,
+      "logits/rejected": -0.7517622113227844,
+      "logps/chosen": -1.471292495727539,
+      "logps/rejected": -70.88200378417969,
+      "loss": 0.2737431526184082,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.14067727327346802,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.13278257846832275,
+      "rewards/margins": 5.628582000732422,
+      "rewards/rejected": -5.495799541473389,
+      "step": 10,
+      "train_speed(iter/s)": 0.010721
+    },
+    {
+      "epoch": 0.06342342342342343,
+      "grad_norm": 5.912591457366943,
+      "learning_rate": 8.461538461538461e-05,
+      "logits/chosen": -0.4128813147544861,
+      "logits/rejected": -0.5927000045776367,
+      "logps/chosen": -7.319942474365234,
+      "logps/rejected": -61.27806091308594,
+      "loss": 0.5680095553398132,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3753523826599121,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.07616602629423141,
+      "rewards/margins": 4.830938816070557,
+      "rewards/rejected": -4.754772663116455,
+      "step": 11,
+      "train_speed(iter/s)": 0.010727
+    },
+    {
+      "epoch": 0.06918918918918919,
+      "grad_norm": 2.5374209880828857,
+      "learning_rate": 9.230769230769232e-05,
+      "logits/chosen": -0.349031925201416,
+      "logits/rejected": -0.6373917460441589,
+      "logps/chosen": -4.199922561645508,
+      "logps/rejected": -72.83607482910156,
+      "loss": 0.34027042984962463,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.22808796167373657,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.20608201622962952,
+      "rewards/margins": 5.424815654754639,
+      "rewards/rejected": -5.218733787536621,
+      "step": 12,
+      "train_speed(iter/s)": 0.010728
+    },
+    {
+      "epoch": 0.07495495495495495,
+      "grad_norm": 2.7130517959594727,
+      "learning_rate": 0.0001,
+      "logits/chosen": -0.3631379306316376,
+      "logits/rejected": -0.6601940989494324,
+      "logps/chosen": -3.3938565254211426,
+      "logps/rejected": -65.2313232421875,
+      "loss": 0.4514213800430298,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.30333614349365234,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.06806281208992004,
+      "rewards/margins": 4.726724624633789,
+      "rewards/rejected": -4.658661365509033,
+      "step": 13,
+      "train_speed(iter/s)": 0.010733
+    },
+    {
+      "epoch": 0.08072072072072072,
+      "grad_norm": 1.5474798679351807,
+      "learning_rate": 0.0001076923076923077,
+      "logits/chosen": -0.4096493124961853,
+      "logits/rejected": -0.6515597105026245,
+      "logps/chosen": -2.300206184387207,
+      "logps/rejected": -57.26811218261719,
+      "loss": 0.3060320019721985,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1727571338415146,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08898697793483734,
+      "rewards/margins": 4.1406354904174805,
+      "rewards/rejected": -4.0516486167907715,
+      "step": 14,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 0.08648648648648649,
+      "grad_norm": 1.628075361251831,
+      "learning_rate": 0.00011538461538461538,
+      "logits/chosen": -0.4402551054954529,
+      "logits/rejected": -0.6672070622444153,
+      "logps/chosen": -2.9995224475860596,
+      "logps/rejected": -53.87057113647461,
+      "loss": 0.3575773239135742,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2217392474412918,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.20550209283828735,
+      "rewards/margins": 3.6874935626983643,
+      "rewards/rejected": -3.4819912910461426,
+      "step": 15,
+      "train_speed(iter/s)": 0.010742
+    },
+    {
+      "epoch": 0.09225225225225225,
+      "grad_norm": 1.838375449180603,
+      "learning_rate": 0.0001230769230769231,
+      "logits/chosen": -0.3547935485839844,
+      "logits/rejected": -0.6626885533332825,
+      "logps/chosen": -4.558413982391357,
+      "logps/rejected": -64.36922454833984,
+      "loss": 0.45015937089920044,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3128659129142761,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.14241908490657806,
+      "rewards/margins": 4.83186149597168,
+      "rewards/rejected": -4.689442157745361,
+      "step": 16,
+      "train_speed(iter/s)": 0.010745
+    },
+    {
+      "epoch": 0.09801801801801802,
+      "grad_norm": 3.119417667388916,
+      "learning_rate": 0.00013076923076923077,
+      "logits/chosen": -0.3334687352180481,
+      "logits/rejected": -0.6293997168540955,
+      "logps/chosen": -5.949204444885254,
+      "logps/rejected": -74.25395965576172,
+      "loss": 0.6850419044494629,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.5004001259803772,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.07516612857580185,
+      "rewards/margins": 4.788809776306152,
+      "rewards/rejected": -4.713644027709961,
+      "step": 17,
+      "train_speed(iter/s)": 0.010741
+    },
+    {
+      "epoch": 0.10378378378378378,
+      "grad_norm": 1.6356091499328613,
+      "learning_rate": 0.00013846153846153847,
+      "logits/chosen": -0.3526935875415802,
+      "logits/rejected": -0.6357190608978271,
+      "logps/chosen": -2.911073923110962,
+      "logps/rejected": -66.46131896972656,
+      "loss": 0.32070592045783997,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.18397927284240723,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.09694083034992218,
+      "rewards/margins": 5.153117656707764,
+      "rewards/rejected": -5.056177139282227,
+      "step": 18,
+      "train_speed(iter/s)": 0.010741
+    },
+    {
+      "epoch": 0.10954954954954955,
+      "grad_norm": 2.7403504848480225,
+      "learning_rate": 0.00014615384615384615,
+      "logits/chosen": -0.3763662278652191,
+      "logits/rejected": -0.586420476436615,
+      "logps/chosen": -4.417596340179443,
+      "logps/rejected": -58.99639892578125,
+      "loss": 0.4634058177471161,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.28715017437934875,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1435782015323639,
+      "rewards/margins": 4.609218120574951,
+      "rewards/rejected": -4.465640068054199,
+      "step": 19,
+      "train_speed(iter/s)": 0.010743
+    },
+    {
+      "epoch": 0.11531531531531532,
+      "grad_norm": 1.982851266860962,
+      "learning_rate": 0.00015384615384615385,
+      "logits/chosen": -0.3701193332672119,
+      "logits/rejected": -0.6088087558746338,
+      "logps/chosen": -4.619842529296875,
+      "logps/rejected": -68.1020278930664,
+      "loss": 0.4344192445278168,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.26591166853904724,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.10610304027795792,
+      "rewards/margins": 5.044137954711914,
+      "rewards/rejected": -4.938034534454346,
+      "step": 20,
+      "train_speed(iter/s)": 0.010745
+    },
+    {
+      "epoch": 0.12108108108108108,
+      "grad_norm": 3.175961494445801,
+      "learning_rate": 0.00016153846153846155,
+      "logits/chosen": -0.40009674429893494,
+      "logits/rejected": -0.642030656337738,
+      "logps/chosen": -3.357761859893799,
+      "logps/rejected": -65.21231842041016,
+      "loss": 0.40142354369163513,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2889588177204132,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1323939859867096,
+      "rewards/margins": 5.234187602996826,
+      "rewards/rejected": -5.1017937660217285,
+      "step": 21,
+      "train_speed(iter/s)": 0.010748
+    },
+    {
+      "epoch": 0.12684684684684686,
+      "grad_norm": 1.9400721788406372,
+      "learning_rate": 0.00016923076923076923,
+      "logits/chosen": -0.3025435209274292,
+      "logits/rejected": -0.6950021386146545,
+      "logps/chosen": -2.492130994796753,
+      "logps/rejected": -107.03312683105469,
+      "loss": 0.2920421361923218,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.19772115349769592,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.19073158502578735,
+      "rewards/margins": 8.599780082702637,
+      "rewards/rejected": -8.409048080444336,
+      "step": 22,
+      "train_speed(iter/s)": 0.010746
+    },
+    {
+      "epoch": 0.13261261261261262,
+      "grad_norm": 1.3099267482757568,
+      "learning_rate": 0.00017692307692307693,
+      "logits/chosen": -0.36498135328292847,
+      "logits/rejected": -0.6359634399414062,
+      "logps/chosen": -2.421377658843994,
+      "logps/rejected": -74.05368041992188,
+      "loss": 0.20869436860084534,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.11023169010877609,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.18500684201717377,
+      "rewards/margins": 5.958514213562012,
+      "rewards/rejected": -5.773507118225098,
+      "step": 23,
+      "train_speed(iter/s)": 0.010747
+    },
+    {
+      "epoch": 0.13837837837837838,
+      "grad_norm": 1.9334064722061157,
+      "learning_rate": 0.00018461538461538463,
+      "logits/chosen": -0.45037421584129333,
+      "logits/rejected": -0.7719848155975342,
+      "logps/chosen": -2.6665430068969727,
+      "logps/rejected": -94.5350570678711,
+      "loss": 0.3057625889778137,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2374470978975296,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.09573014080524445,
+      "rewards/margins": 8.040020942687988,
+      "rewards/rejected": -7.944291114807129,
+      "step": 24,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 0.14414414414414414,
+      "grad_norm": 7.789314270019531,
+      "learning_rate": 0.00019230769230769233,
+      "logits/chosen": -0.3473276197910309,
+      "logits/rejected": -0.6793957352638245,
+      "logps/chosen": -4.235269069671631,
+      "logps/rejected": -109.01844024658203,
+      "loss": 0.42337244749069214,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.21255403757095337,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.009569963440299034,
+      "rewards/margins": 8.769816398620605,
+      "rewards/rejected": -8.760246276855469,
+      "step": 25,
+      "train_speed(iter/s)": 0.010747
+    },
+    {
+      "epoch": 0.1499099099099099,
+      "grad_norm": 1.970948338508606,
+      "learning_rate": 0.0002,
+      "logits/chosen": -0.40258127450942993,
+      "logits/rejected": -0.7479668259620667,
+      "logps/chosen": -2.0013341903686523,
+      "logps/rejected": -85.51154327392578,
+      "loss": 0.4197818636894226,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2574591636657715,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.16396881639957428,
+      "rewards/margins": 7.033238410949707,
+      "rewards/rejected": -6.869269847869873,
+      "step": 26,
+      "train_speed(iter/s)": 0.010746
+    },
+    {
+      "epoch": 0.15567567567567567,
+      "grad_norm": 11.3534574508667,
+      "learning_rate": 0.00019999796963348897,
+      "logits/chosen": -0.5246711373329163,
+      "logits/rejected": -0.7294141054153442,
+      "logps/chosen": -11.85606575012207,
+      "logps/rejected": -76.03211975097656,
+      "loss": 0.8875234127044678,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.575400710105896,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.2269383817911148,
+      "rewards/margins": 5.498793125152588,
+      "rewards/rejected": -5.725731372833252,
+      "step": 27,
+      "train_speed(iter/s)": 0.010747
+    },
+    {
+      "epoch": 0.16144144144144143,
+      "grad_norm": 2.0314371585845947,
+      "learning_rate": 0.00019999187861640362,
+      "logits/chosen": -0.5284671187400818,
+      "logits/rejected": -0.7846834063529968,
+      "logps/chosen": -4.434022426605225,
+      "logps/rejected": -87.89623260498047,
+      "loss": 0.3868662416934967,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.30320096015930176,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.07812386751174927,
+      "rewards/margins": 7.374299049377441,
+      "rewards/rejected": -7.296175479888916,
+      "step": 28,
+      "train_speed(iter/s)": 0.010748
+    },
+    {
+      "epoch": 0.16720720720720722,
+      "grad_norm": 6.089433193206787,
+      "learning_rate": 0.0001999817271960839,
+      "logits/chosen": -0.4792092442512512,
+      "logits/rejected": -0.7371792793273926,
+      "logps/chosen": -3.6791412830352783,
+      "logps/rejected": -80.48262023925781,
+      "loss": 0.34335166215896606,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.22308313846588135,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.049518827348947525,
+      "rewards/margins": 6.395692825317383,
+      "rewards/rejected": -6.445211410522461,
+      "step": 29,
+      "train_speed(iter/s)": 0.010751
+    },
+    {
+      "epoch": 0.17297297297297298,
+      "grad_norm": 1.8896535634994507,
+      "learning_rate": 0.00019996751578475186,
+      "logits/chosen": -0.44468259811401367,
+      "logits/rejected": -0.7023504972457886,
+      "logps/chosen": -3.048004388809204,
+      "logps/rejected": -80.6348648071289,
+      "loss": 0.30949145555496216,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1800393909215927,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.15738406777381897,
+      "rewards/margins": 5.8682780265808105,
+      "rewards/rejected": -5.710893630981445,
+      "step": 30,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 0.17873873873873874,
+      "grad_norm": 2.7516067028045654,
+      "learning_rate": 0.00019994924495949504,
+      "logits/chosen": -0.4754641652107239,
+      "logits/rejected": -0.7408227920532227,
+      "logps/chosen": -5.943354606628418,
+      "logps/rejected": -79.13115692138672,
+      "loss": 0.6435320973396301,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.4586246609687805,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.0629303902387619,
+      "rewards/margins": 6.110820293426514,
+      "rewards/rejected": -6.0478901863098145,
+      "step": 31,
+      "train_speed(iter/s)": 0.01075
+    },
+    {
+      "epoch": 0.1845045045045045,
+      "grad_norm": 5.854465007781982,
+      "learning_rate": 0.00019992691546224278,
+      "logits/chosen": -0.45127877593040466,
+      "logits/rejected": -0.7568551898002625,
+      "logps/chosen": -2.736678123474121,
+      "logps/rejected": -66.9885025024414,
+      "loss": 0.3652667701244354,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.16899552941322327,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.050895966589450836,
+      "rewards/margins": 4.933393478393555,
+      "rewards/rejected": -4.882497310638428,
+      "step": 32,
+      "train_speed(iter/s)": 0.01075
+    },
+    {
+      "epoch": 0.19027027027027027,
+      "grad_norm": 1.3124663829803467,
+      "learning_rate": 0.00019990052819973642,
+      "logits/chosen": -0.3465266227722168,
+      "logits/rejected": -0.6685288548469543,
+      "logps/chosen": -3.301164388656616,
+      "logps/rejected": -85.9378662109375,
+      "loss": 0.3046538233757019,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1670866757631302,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.17874348163604736,
+      "rewards/margins": 6.709518909454346,
+      "rewards/rejected": -6.530775547027588,
+      "step": 33,
+      "train_speed(iter/s)": 0.010748
+    },
+    {
+      "epoch": 0.19603603603603603,
+      "grad_norm": 2.341381549835205,
+      "learning_rate": 0.00019987008424349225,
+      "logits/chosen": -0.38037875294685364,
+      "logits/rejected": -0.699614405632019,
+      "logps/chosen": -1.5970546007156372,
+      "logps/rejected": -74.28025817871094,
+      "loss": 0.20820346474647522,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.10679290443658829,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.11616384983062744,
+      "rewards/margins": 5.825808048248291,
+      "rewards/rejected": -5.709644794464111,
+      "step": 34,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 0.2018018018018018,
+      "grad_norm": 5.144662380218506,
+      "learning_rate": 0.00019983558482975799,
+      "logits/chosen": -0.46614986658096313,
+      "logits/rejected": -0.6599275469779968,
+      "logps/chosen": -5.982501983642578,
+      "logps/rejected": -67.7139892578125,
+      "loss": 0.5879941582679749,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.4303022027015686,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16055577993392944,
+      "rewards/margins": 5.052016258239746,
+      "rewards/rejected": -4.89146089553833,
+      "step": 35,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 0.20756756756756756,
+      "grad_norm": 1.990370273590088,
+      "learning_rate": 0.00019979703135946278,
+      "logits/chosen": -0.4655105471611023,
+      "logits/rejected": -0.6668959856033325,
+      "logps/chosen": -4.062808036804199,
+      "logps/rejected": -72.87210845947266,
+      "loss": 0.4453062415122986,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.25426411628723145,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.09591292589902878,
+      "rewards/margins": 5.528655052185059,
+      "rewards/rejected": -5.432741641998291,
+      "step": 36,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 0.21333333333333335,
+      "grad_norm": 2.0748369693756104,
+      "learning_rate": 0.00019975442539816012,
+      "logits/chosen": -0.4139191210269928,
+      "logits/rejected": -0.7536109685897827,
+      "logps/chosen": -5.956673622131348,
+      "logps/rejected": -66.99488830566406,
+      "loss": 0.5514004826545715,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3969358503818512,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.33245787024497986,
+      "rewards/margins": 5.0754570960998535,
+      "rewards/rejected": -4.742999076843262,
+      "step": 37,
+      "train_speed(iter/s)": 0.01075
+    },
+    {
+      "epoch": 0.2190990990990991,
+      "grad_norm": 1.6668081283569336,
+      "learning_rate": 0.0001997077686759643,
+      "logits/chosen": -0.42306363582611084,
+      "logits/rejected": -0.6915761232376099,
+      "logps/chosen": -3.814265727996826,
+      "logps/rejected": -82.30941772460938,
+      "loss": 0.3710038363933563,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.22380009293556213,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.06087161600589752,
+      "rewards/margins": 6.076653957366943,
+      "rewards/rejected": -6.015782356262207,
+      "step": 38,
+      "train_speed(iter/s)": 0.01075
+    },
+    {
+      "epoch": 0.22486486486486487,
+      "grad_norm": 13.827637672424316,
+      "learning_rate": 0.00019965706308748028,
+      "logits/chosen": -0.49769100546836853,
+      "logits/rejected": -0.7370949983596802,
+      "logps/chosen": -3.956310272216797,
+      "logps/rejected": -79.41323852539062,
+      "loss": 0.6633445620536804,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3870386779308319,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.13494591414928436,
+      "rewards/margins": 6.147372722625732,
+      "rewards/rejected": -6.282319068908691,
+      "step": 39,
+      "train_speed(iter/s)": 0.010751
+    },
+    {
+      "epoch": 0.23063063063063063,
+      "grad_norm": 2.19854474067688,
+      "learning_rate": 0.0001996023106917267,
+      "logits/chosen": -0.44343504309654236,
+      "logits/rejected": -0.6730125546455383,
+      "logps/chosen": -2.9238576889038086,
+      "logps/rejected": -77.40494537353516,
+      "loss": 0.3453696668148041,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.19069285690784454,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.14595907926559448,
+      "rewards/margins": 6.002254486083984,
+      "rewards/rejected": -5.856295108795166,
+      "step": 40,
+      "train_speed(iter/s)": 0.010752
+    },
+    {
+      "epoch": 0.2363963963963964,
+      "grad_norm": 6.0794596672058105,
+      "learning_rate": 0.00019954351371205203,
+      "logits/chosen": -0.47579658031463623,
+      "logits/rejected": -0.6826990842819214,
+      "logps/chosen": -5.335310935974121,
+      "logps/rejected": -75.78384399414062,
+      "loss": 0.575509250164032,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.4477939307689667,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2103371024131775,
+      "rewards/margins": 6.012211322784424,
+      "rewards/rejected": -5.801875114440918,
+      "step": 41,
+      "train_speed(iter/s)": 0.010754
+    },
+    {
+      "epoch": 0.24216216216216216,
+      "grad_norm": 1.4152264595031738,
+      "learning_rate": 0.00019948067453604473,
+      "logits/chosen": -0.49620914459228516,
+      "logits/rejected": -0.7446714043617249,
+      "logps/chosen": -3.284031867980957,
+      "logps/rejected": -77.9183349609375,
+      "loss": 0.29828324913978577,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.21160344779491425,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.18678154051303864,
+      "rewards/margins": 6.247434616088867,
+      "rewards/rejected": -6.060652732849121,
+      "step": 42,
+      "train_speed(iter/s)": 0.010755
+    },
+    {
+      "epoch": 0.24792792792792792,
+      "grad_norm": 2.9591963291168213,
+      "learning_rate": 0.00019941379571543596,
+      "logits/chosen": -0.4777877628803253,
+      "logits/rejected": -0.6974171996116638,
+      "logps/chosen": -2.9466118812561035,
+      "logps/rejected": -66.2719497680664,
+      "loss": 0.35521066188812256,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.22793087363243103,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.27246373891830444,
+      "rewards/margins": 5.403186798095703,
+      "rewards/rejected": -5.130722999572754,
+      "step": 43,
+      "train_speed(iter/s)": 0.010756
+    },
+    {
+      "epoch": 0.2536936936936937,
+      "grad_norm": 4.199843883514404,
+      "learning_rate": 0.00019934287996599607,
+      "logits/chosen": -0.4372207522392273,
+      "logits/rejected": -0.700534462928772,
+      "logps/chosen": -3.31451416015625,
+      "logps/rejected": -70.0909194946289,
+      "loss": 0.3793654143810272,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.20433279871940613,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.059201404452323914,
+      "rewards/margins": 4.912087917327881,
+      "rewards/rejected": -4.852885723114014,
+      "step": 44,
+      "train_speed(iter/s)": 0.010756
+    },
+    {
+      "epoch": 0.2594594594594595,
+      "grad_norm": 0.7397028803825378,
+      "learning_rate": 0.00019926793016742435,
+      "logits/chosen": -0.42062634229660034,
+      "logits/rejected": -0.7593384981155396,
+      "logps/chosen": -1.404799222946167,
+      "logps/rejected": -102.4892349243164,
+      "loss": 0.1313902586698532,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.09521690011024475,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.12229674309492111,
+      "rewards/margins": 8.041703224182129,
+      "rewards/rejected": -7.919405937194824,
+      "step": 45,
+      "train_speed(iter/s)": 0.010754
+    },
+    {
+      "epoch": 0.26522522522522524,
+      "grad_norm": 5.14201545715332,
+      "learning_rate": 0.00019918894936323195,
+      "logits/chosen": -0.3798273205757141,
+      "logits/rejected": -0.5865642428398132,
+      "logps/chosen": -6.911247730255127,
+      "logps/rejected": -72.06941223144531,
+      "loss": 0.49299517273902893,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.27924880385398865,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.1307646483182907,
+      "rewards/margins": 5.647462368011475,
+      "rewards/rejected": -5.516696929931641,
+      "step": 46,
+      "train_speed(iter/s)": 0.010753
+    },
+    {
+      "epoch": 0.270990990990991,
+      "grad_norm": 1.6439752578735352,
+      "learning_rate": 0.00019910594076061853,
+      "logits/chosen": -0.4431299567222595,
+      "logits/rejected": -0.7268660664558411,
+      "logps/chosen": -2.0275888442993164,
+      "logps/rejected": -81.84529876708984,
+      "loss": 0.23740296065807343,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.14832313358783722,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.06923090666532516,
+      "rewards/margins": 6.1783928871154785,
+      "rewards/rejected": -6.109161376953125,
+      "step": 47,
+      "train_speed(iter/s)": 0.010753
+    },
+    {
+      "epoch": 0.27675675675675676,
+      "grad_norm": 6.499570846557617,
+      "learning_rate": 0.0001990189077303418,
+      "logits/chosen": -0.45142635703086853,
+      "logits/rejected": -0.7249264121055603,
+      "logps/chosen": -5.794652938842773,
+      "logps/rejected": -78.53363800048828,
+      "loss": 0.46158841252326965,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3398270010948181,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.24555836617946625,
+      "rewards/margins": 6.308343887329102,
+      "rewards/rejected": -6.062786102294922,
+      "step": 48,
+      "train_speed(iter/s)": 0.010753
+    },
+    {
+      "epoch": 0.2825225225225225,
+      "grad_norm": 5.58511209487915,
+      "learning_rate": 0.00019892785380658078,
+      "logits/chosen": -0.4099079370498657,
+      "logits/rejected": -0.6898340582847595,
+      "logps/chosen": -7.449429035186768,
+      "logps/rejected": -92.18098449707031,
+      "loss": 0.5815075039863586,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.41358357667922974,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.053265977650880814,
+      "rewards/margins": 7.150655269622803,
+      "rewards/rejected": -7.203920841217041,
+      "step": 49,
+      "train_speed(iter/s)": 0.010752
+    },
+    {
+      "epoch": 0.2882882882882883,
+      "grad_norm": 1.5411535501480103,
+      "learning_rate": 0.00019883278268679216,
+      "logits/chosen": -0.4670988917350769,
+      "logits/rejected": -0.6586328148841858,
+      "logps/chosen": -6.984646320343018,
+      "logps/rejected": -89.9208755493164,
+      "loss": 0.45735102891921997,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.28862154483795166,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1682412177324295,
+      "rewards/margins": 7.457679748535156,
+      "rewards/rejected": -7.289438247680664,
+      "step": 50,
+      "train_speed(iter/s)": 0.01075
+    },
+    {
+      "epoch": 0.29405405405405405,
+      "grad_norm": 6.264907360076904,
+      "learning_rate": 0.00019873369823156036,
+      "logits/chosen": -0.35443419218063354,
+      "logits/rejected": -0.7265148758888245,
+      "logps/chosen": -5.11212158203125,
+      "logps/rejected": -97.37500762939453,
+      "loss": 0.648152232170105,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.5091428756713867,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.03551238775253296,
+      "rewards/margins": 7.55380392074585,
+      "rewards/rejected": -7.589317321777344,
+      "step": 51,
+      "train_speed(iter/s)": 0.010707
+    },
+    {
+      "epoch": 0.2998198198198198,
+      "grad_norm": 2.1848013401031494,
+      "learning_rate": 0.00019863060446444054,
+      "logits/chosen": -0.5179776549339294,
+      "logits/rejected": -0.7255269289016724,
+      "logps/chosen": -6.782817363739014,
+      "logps/rejected": -86.8519515991211,
+      "loss": 0.5702924728393555,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3926149308681488,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.05346503108739853,
+      "rewards/margins": 7.021766185760498,
+      "rewards/rejected": -6.968300819396973,
+      "step": 52,
+      "train_speed(iter/s)": 0.010708
+    },
+    {
+      "epoch": 0.3055855855855856,
+      "grad_norm": 3.3478569984436035,
+      "learning_rate": 0.00019852350557179538,
+      "logits/chosen": -0.5434097051620483,
+      "logits/rejected": -0.8009167313575745,
+      "logps/chosen": -4.053906440734863,
+      "logps/rejected": -105.9520263671875,
+      "loss": 0.4421282112598419,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3492012917995453,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.1857108175754547,
+      "rewards/margins": 9.035101890563965,
+      "rewards/rejected": -8.849390983581543,
+      "step": 53,
+      "train_speed(iter/s)": 0.010709
+    },
+    {
+      "epoch": 0.31135135135135134,
+      "grad_norm": 0.8761724829673767,
+      "learning_rate": 0.00019841240590262493,
+      "logits/chosen": -0.45293307304382324,
+      "logits/rejected": -0.7608242034912109,
+      "logps/chosen": -0.6350191831588745,
+      "logps/rejected": -91.82192993164062,
+      "loss": 0.11042662709951401,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.06284812837839127,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1965743899345398,
+      "rewards/margins": 7.520632743835449,
+      "rewards/rejected": -7.324058532714844,
+      "step": 54,
+      "train_speed(iter/s)": 0.01071
+    },
+    {
+      "epoch": 0.3171171171171171,
+      "grad_norm": 5.344722747802734,
+      "learning_rate": 0.0001982973099683902,
+      "logits/chosen": -0.5431379675865173,
+      "logits/rejected": -0.8100754022598267,
+      "logps/chosen": -1.98147714138031,
+      "logps/rejected": -91.00532531738281,
+      "loss": 0.3004758059978485,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1778518110513687,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.1774783581495285,
+      "rewards/margins": 7.744819641113281,
+      "rewards/rejected": -7.567340850830078,
+      "step": 55,
+      "train_speed(iter/s)": 0.010711
+    },
+    {
+      "epoch": 0.32288288288288286,
+      "grad_norm": 5.424385070800781,
+      "learning_rate": 0.00019817822244282973,
+      "logits/chosen": -0.48769453167915344,
+      "logits/rejected": -0.7255769371986389,
+      "logps/chosen": -3.6506757736206055,
+      "logps/rejected": -84.81509399414062,
+      "loss": 0.4457959830760956,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.23988068103790283,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.11002250760793686,
+      "rewards/margins": 6.652472019195557,
+      "rewards/rejected": -6.542449951171875,
+      "step": 56,
+      "train_speed(iter/s)": 0.010711
+    },
+    {
+      "epoch": 0.3286486486486486,
+      "grad_norm": 2.3812549114227295,
+      "learning_rate": 0.00019805514816177006,
+      "logits/chosen": -0.556337833404541,
+      "logits/rejected": -0.7923259735107422,
+      "logps/chosen": -5.455251693725586,
+      "logps/rejected": -89.06196594238281,
+      "loss": 0.4395422339439392,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.29659417271614075,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.01958906650543213,
+      "rewards/margins": 6.816771984100342,
+      "rewards/rejected": -6.836360931396484,
+      "step": 57,
+      "train_speed(iter/s)": 0.01071
+    },
+    {
+      "epoch": 0.33441441441441444,
+      "grad_norm": 8.72442626953125,
+      "learning_rate": 0.00019792809212292912,
+      "logits/chosen": -0.6302639842033386,
+      "logits/rejected": -0.9231972098350525,
+      "logps/chosen": -5.753312110900879,
+      "logps/rejected": -103.71651458740234,
+      "loss": 1.1447618007659912,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.6733683943748474,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.26626041531562805,
+      "rewards/margins": 8.245807647705078,
+      "rewards/rejected": -8.512067794799805,
+      "step": 58,
+      "train_speed(iter/s)": 0.010711
+    },
+    {
+      "epoch": 0.3401801801801802,
+      "grad_norm": 2.6023576259613037,
+      "learning_rate": 0.00019779705948571346,
+      "logits/chosen": -0.5867213010787964,
+      "logits/rejected": -0.8538931608200073,
+      "logps/chosen": -3.069394111633301,
+      "logps/rejected": -96.5278091430664,
+      "loss": 0.3167837858200073,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.20217999815940857,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.10776210576295853,
+      "rewards/margins": 8.32030963897705,
+      "rewards/rejected": -8.21254825592041,
+      "step": 59,
+      "train_speed(iter/s)": 0.010712
+    },
+    {
+      "epoch": 0.34594594594594597,
+      "grad_norm": 2.678529977798462,
+      "learning_rate": 0.00019766205557100868,
+      "logits/chosen": -0.6138476133346558,
+      "logits/rejected": -0.7842435836791992,
+      "logps/chosen": -6.119746208190918,
+      "logps/rejected": -88.4565658569336,
+      "loss": 0.44386592507362366,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2780015468597412,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.06730165332555771,
+      "rewards/margins": 7.516484260559082,
+      "rewards/rejected": -7.449182987213135,
+      "step": 60,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.3517117117117117,
+      "grad_norm": 2.223165512084961,
+      "learning_rate": 0.00019752308586096326,
+      "logits/chosen": -0.5360291004180908,
+      "logits/rejected": -0.894256591796875,
+      "logps/chosen": -1.3611795902252197,
+      "logps/rejected": -122.1118392944336,
+      "loss": 0.13098187744617462,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.07775433361530304,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.12457617372274399,
+      "rewards/margins": 10.008496284484863,
+      "rewards/rejected": -9.883918762207031,
+      "step": 61,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.3574774774774775,
+      "grad_norm": 10.891358375549316,
+      "learning_rate": 0.0001973801559987661,
+      "logits/chosen": -0.7173175811767578,
+      "logits/rejected": -0.8792683482170105,
+      "logps/chosen": -12.6211519241333,
+      "logps/rejected": -73.51658630371094,
+      "loss": 1.0753252506256104,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.9100903868675232,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.495037317276001,
+      "rewards/margins": 5.504097938537598,
+      "rewards/rejected": -5.9991350173950195,
+      "step": 62,
+      "train_speed(iter/s)": 0.010715
+    },
+    {
+      "epoch": 0.36324324324324325,
+      "grad_norm": 26.432544708251953,
+      "learning_rate": 0.00019723327178841743,
+      "logits/chosen": -0.6629639267921448,
+      "logits/rejected": -0.8958379030227661,
+      "logps/chosen": -5.678650856018066,
+      "logps/rejected": -81.07159423828125,
+      "loss": 0.6158818602561951,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.4310333728790283,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.02597002126276493,
+      "rewards/margins": 6.523518085479736,
+      "rewards/rejected": -6.497547149658203,
+      "step": 63,
+      "train_speed(iter/s)": 0.010716
+    },
+    {
+      "epoch": 0.369009009009009,
+      "grad_norm": 4.643962860107422,
+      "learning_rate": 0.0001970824391944927,
+      "logits/chosen": -0.6061327457427979,
+      "logits/rejected": -0.8394734859466553,
+      "logps/chosen": -6.52767276763916,
+      "logps/rejected": -74.48380279541016,
+      "loss": 0.5795359015464783,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.414874792098999,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.0007130205631256104,
+      "rewards/margins": 5.6362080574035645,
+      "rewards/rejected": -5.635494709014893,
+      "step": 64,
+      "train_speed(iter/s)": 0.010716
+    },
+    {
+      "epoch": 0.3747747747747748,
+      "grad_norm": 1.8694281578063965,
+      "learning_rate": 0.00019692766434190105,
+      "logits/chosen": -0.6322288513183594,
+      "logits/rejected": -0.8566944003105164,
+      "logps/chosen": -1.9187999963760376,
+      "logps/rejected": -56.726871490478516,
+      "loss": 0.26561087369918823,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.12671466171741486,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.17268410325050354,
+      "rewards/margins": 4.417227268218994,
+      "rewards/rejected": -4.244543075561523,
+      "step": 65,
+      "train_speed(iter/s)": 0.010717
+    },
+    {
+      "epoch": 0.38054054054054054,
+      "grad_norm": 2.3087944984436035,
+      "learning_rate": 0.00019676895351563588,
+      "logits/chosen": -0.6048943996429443,
+      "logits/rejected": -0.8088940382003784,
+      "logps/chosen": -4.430207252502441,
+      "logps/rejected": -64.05681610107422,
+      "loss": 0.33754763007164,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.22767014801502228,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.025299668312072754,
+      "rewards/margins": 4.6934709548950195,
+      "rewards/rejected": -4.668170928955078,
+      "step": 66,
+      "train_speed(iter/s)": 0.010717
+    },
+    {
+      "epoch": 0.3863063063063063,
+      "grad_norm": 2.2111432552337646,
+      "learning_rate": 0.00019660631316052021,
+      "logits/chosen": -0.623546838760376,
+      "logits/rejected": -0.811290979385376,
+      "logps/chosen": -6.471018314361572,
+      "logps/rejected": -57.678314208984375,
+      "loss": 0.4503980875015259,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.34320953488349915,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.06837636232376099,
+      "rewards/margins": 4.10473108291626,
+      "rewards/rejected": -4.173107624053955,
+      "step": 67,
+      "train_speed(iter/s)": 0.010718
+    },
+    {
+      "epoch": 0.39207207207207206,
+      "grad_norm": 1.7001421451568604,
+      "learning_rate": 0.00019643974988094458,
+      "logits/chosen": -0.5895470380783081,
+      "logits/rejected": -0.7888103723526001,
+      "logps/chosen": -5.163002014160156,
+      "logps/rejected": -45.541908264160156,
+      "loss": 0.42964497208595276,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3103106617927551,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3456801772117615,
+      "rewards/margins": 3.432896614074707,
+      "rewards/rejected": -3.08721661567688,
+      "step": 68,
+      "train_speed(iter/s)": 0.01072
+    },
+    {
+      "epoch": 0.3978378378378378,
+      "grad_norm": 3.909360885620117,
+      "learning_rate": 0.00019626927044059914,
+      "logits/chosen": -0.6134467124938965,
+      "logits/rejected": -0.7660009860992432,
+      "logps/chosen": -5.3520307540893555,
+      "logps/rejected": -49.259185791015625,
+      "loss": 0.5181165337562561,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.30421963334083557,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.014049924910068512,
+      "rewards/margins": 3.3693478107452393,
+      "rewards/rejected": -3.355297565460205,
+      "step": 69,
+      "train_speed(iter/s)": 0.01072
+    },
+    {
+      "epoch": 0.4036036036036036,
+      "grad_norm": 2.67299485206604,
+      "learning_rate": 0.00019609488176219886,
+      "logits/chosen": -0.6107767820358276,
+      "logits/rejected": -0.7868641018867493,
+      "logps/chosen": -3.616837501525879,
+      "logps/rejected": -50.11117172241211,
+      "loss": 0.3146559000015259,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.21267317235469818,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.13358786702156067,
+      "rewards/margins": 3.8761374950408936,
+      "rewards/rejected": -3.742549419403076,
+      "step": 70,
+      "train_speed(iter/s)": 0.010721
+    },
+    {
+      "epoch": 0.40936936936936935,
+      "grad_norm": 7.013823509216309,
+      "learning_rate": 0.00019591659092720227,
+      "logits/chosen": -0.5896518230438232,
+      "logits/rejected": -0.858326256275177,
+      "logps/chosen": -2.8418002128601074,
+      "logps/rejected": -64.546630859375,
+      "loss": 0.37414392828941345,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.25448697805404663,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.024522263556718826,
+      "rewards/margins": 4.37087345123291,
+      "rewards/rejected": -4.346351623535156,
+      "step": 71,
+      "train_speed(iter/s)": 0.010722
+    },
+    {
+      "epoch": 0.4151351351351351,
+      "grad_norm": 2.0888102054595947,
+      "learning_rate": 0.00019573440517552427,
+      "logits/chosen": -0.5669695138931274,
+      "logits/rejected": -0.7572087049484253,
+      "logps/chosen": -2.418893337249756,
+      "logps/rejected": -50.858699798583984,
+      "loss": 0.3724361062049866,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.20290419459342957,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.16552519798278809,
+      "rewards/margins": 3.5489120483398438,
+      "rewards/rejected": -3.3833868503570557,
+      "step": 72,
+      "train_speed(iter/s)": 0.010723
+    },
+    {
+      "epoch": 0.4209009009009009,
+      "grad_norm": 1.9631896018981934,
+      "learning_rate": 0.00019554833190524182,
+      "logits/chosen": -0.575507402420044,
+      "logits/rejected": -0.7697166204452515,
+      "logps/chosen": -3.5829074382781982,
+      "logps/rejected": -52.79640197753906,
+      "loss": 0.400629460811615,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2976799011230469,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.17720367014408112,
+      "rewards/margins": 4.019702434539795,
+      "rewards/rejected": -3.842499017715454,
+      "step": 73,
+      "train_speed(iter/s)": 0.010724
+    },
+    {
+      "epoch": 0.4266666666666667,
+      "grad_norm": 3.227459192276001,
+      "learning_rate": 0.00019535837867229363,
+      "logits/chosen": -0.6866841316223145,
+      "logits/rejected": -0.8467439413070679,
+      "logps/chosen": -1.4898688793182373,
+      "logps/rejected": -52.35246276855469,
+      "loss": 0.2691449224948883,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.16896064579486847,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.20350010693073273,
+      "rewards/margins": 4.1118974685668945,
+      "rewards/rejected": -3.9083974361419678,
+      "step": 74,
+      "train_speed(iter/s)": 0.010726
+    },
+    {
+      "epoch": 0.43243243243243246,
+      "grad_norm": 2.0599851608276367,
+      "learning_rate": 0.0001951645531901734,
+      "logits/chosen": -0.6125411987304688,
+      "logits/rejected": -0.8178025484085083,
+      "logps/chosen": -5.753840923309326,
+      "logps/rejected": -60.490638732910156,
+      "loss": 0.4720686376094818,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2955226004123688,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.02442253567278385,
+      "rewards/margins": 4.439631462097168,
+      "rewards/rejected": -4.41520881652832,
+      "step": 75,
+      "train_speed(iter/s)": 0.010726
+    },
+    {
+      "epoch": 0.4381981981981982,
+      "grad_norm": 1.8315515518188477,
+      "learning_rate": 0.00019496686332961646,
+      "logits/chosen": -0.5511488914489746,
+      "logits/rejected": -0.8103834390640259,
+      "logps/chosen": -2.737548828125,
+      "logps/rejected": -72.79937744140625,
+      "loss": 0.2598808705806732,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.20735348761081696,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.20416659116744995,
+      "rewards/margins": 5.639776706695557,
+      "rewards/rejected": -5.435609817504883,
+      "step": 76,
+      "train_speed(iter/s)": 0.010726
+    },
+    {
+      "epoch": 0.443963963963964,
+      "grad_norm": 4.658299922943115,
+      "learning_rate": 0.00019476531711828027,
+      "logits/chosen": -0.5621539354324341,
+      "logits/rejected": -0.817601203918457,
+      "logps/chosen": -2.554128646850586,
+      "logps/rejected": -75.40179443359375,
+      "loss": 0.29839468002319336,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.21096321940422058,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.11831088364124298,
+      "rewards/margins": 5.797797679901123,
+      "rewards/rejected": -5.679486274719238,
+      "step": 77,
+      "train_speed(iter/s)": 0.010727
+    },
+    {
+      "epoch": 0.44972972972972974,
+      "grad_norm": 7.5920844078063965,
+      "learning_rate": 0.00019455992274041835,
+      "logits/chosen": -0.5446800589561462,
+      "logits/rejected": -0.7866684794425964,
+      "logps/chosen": -5.180241584777832,
+      "logps/rejected": -80.02595520019531,
+      "loss": 0.4992501735687256,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.4071677029132843,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.006200561299920082,
+      "rewards/margins": 6.354435920715332,
+      "rewards/rejected": -6.348235607147217,
+      "step": 78,
+      "train_speed(iter/s)": 0.010728
+    },
+    {
+      "epoch": 0.4554954954954955,
+      "grad_norm": 2.045180082321167,
+      "learning_rate": 0.00019435068853654807,
+      "logits/chosen": -0.5892366170883179,
+      "logits/rejected": -0.782485842704773,
+      "logps/chosen": -7.082699775695801,
+      "logps/rejected": -76.8997573852539,
+      "loss": 0.4684464931488037,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.38624832034111023,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.03203499689698219,
+      "rewards/margins": 5.84527587890625,
+      "rewards/rejected": -5.813241004943848,
+      "step": 79,
+      "train_speed(iter/s)": 0.010728
+    },
+    {
+      "epoch": 0.46126126126126127,
+      "grad_norm": 6.412923812866211,
+      "learning_rate": 0.00019413762300311182,
+      "logits/chosen": -0.6524518728256226,
+      "logits/rejected": -0.8889943957328796,
+      "logps/chosen": -6.470341682434082,
+      "logps/rejected": -86.74302673339844,
+      "loss": 0.7839629650115967,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.6858316659927368,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.10852594673633575,
+      "rewards/margins": 6.819857120513916,
+      "rewards/rejected": -6.928382873535156,
+      "step": 80,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 0.46702702702702703,
+      "grad_norm": 2.171276807785034,
+      "learning_rate": 0.00019392073479213213,
+      "logits/chosen": -0.5607702136039734,
+      "logits/rejected": -0.7560180425643921,
+      "logps/chosen": -6.120340347290039,
+      "logps/rejected": -81.5516128540039,
+      "loss": 0.575468897819519,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3079308271408081,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.05391833186149597,
+      "rewards/margins": 5.951426029205322,
+      "rewards/rejected": -6.005344390869141,
+      "step": 81,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 0.4727927927927928,
+      "grad_norm": 1.5074808597564697,
+      "learning_rate": 0.0001937000327108601,
+      "logits/chosen": -0.5817635655403137,
+      "logits/rejected": -0.8122580051422119,
+      "logps/chosen": -3.3293583393096924,
+      "logps/rejected": -76.75574493408203,
+      "loss": 0.27197888493537903,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1920165866613388,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.04472556710243225,
+      "rewards/margins": 6.159332275390625,
+      "rewards/rejected": -6.114606857299805,
+      "step": 82,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 0.47855855855855856,
+      "grad_norm": 8.971928596496582,
+      "learning_rate": 0.0001934755257214181,
+      "logits/chosen": -0.5557482838630676,
+      "logits/rejected": -0.791240930557251,
+      "logps/chosen": -5.363508701324463,
+      "logps/rejected": -71.66944122314453,
+      "loss": 0.8509294986724854,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.7254281640052795,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.1657000482082367,
+      "rewards/margins": 5.284771919250488,
+      "rewards/rejected": -5.450472354888916,
+      "step": 83,
+      "train_speed(iter/s)": 0.010731
+    },
+    {
+      "epoch": 0.4843243243243243,
+      "grad_norm": 3.7317609786987305,
+      "learning_rate": 0.00019324722294043558,
+      "logits/chosen": -0.49631816148757935,
+      "logits/rejected": -0.7238998413085938,
+      "logps/chosen": -5.0319366455078125,
+      "logps/rejected": -55.3072395324707,
+      "loss": 0.4805355668067932,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.30268925428390503,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.16919195652008057,
+      "rewards/margins": 4.183167934417725,
+      "rewards/rejected": -4.013975620269775,
+      "step": 84,
+      "train_speed(iter/s)": 0.010732
+    },
+    {
+      "epoch": 0.4900900900900901,
+      "grad_norm": 7.251592636108398,
+      "learning_rate": 0.00019301513363867895,
+      "logits/chosen": -0.4797047972679138,
+      "logits/rejected": -0.699076771736145,
+      "logps/chosen": -4.068089485168457,
+      "logps/rejected": -62.654762268066406,
+      "loss": 0.41215279698371887,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3088461756706238,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.13345223665237427,
+      "rewards/margins": 4.483903884887695,
+      "rewards/rejected": -4.350452423095703,
+      "step": 85,
+      "train_speed(iter/s)": 0.010732
+    },
+    {
+      "epoch": 0.49585585585585584,
+      "grad_norm": 1.6465262174606323,
+      "learning_rate": 0.0001927792672406751,
+      "logits/chosen": -0.4875030219554901,
+      "logits/rejected": -0.7307705879211426,
+      "logps/chosen": -2.494859218597412,
+      "logps/rejected": -57.72221374511719,
+      "loss": 0.31443703174591064,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1957024335861206,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.1970575600862503,
+      "rewards/margins": 4.039019584655762,
+      "rewards/rejected": -3.8419618606567383,
+      "step": 86,
+      "train_speed(iter/s)": 0.010732
+    },
+    {
+      "epoch": 0.5016216216216216,
+      "grad_norm": 2.362053394317627,
+      "learning_rate": 0.00019253963332432878,
+      "logits/chosen": -0.6053374409675598,
+      "logits/rejected": -0.7439757585525513,
+      "logps/chosen": -5.3451008796691895,
+      "logps/rejected": -47.488853454589844,
+      "loss": 0.4348059296607971,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.23809021711349487,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.019803959876298904,
+      "rewards/margins": 3.427215099334717,
+      "rewards/rejected": -3.447019338607788,
+      "step": 87,
+      "train_speed(iter/s)": 0.010733
+    },
+    {
+      "epoch": 0.5073873873873874,
+      "grad_norm": 2.1728515625,
+      "learning_rate": 0.00019229624162053356,
+      "logits/chosen": -0.5406431555747986,
+      "logits/rejected": -0.739841103553772,
+      "logps/chosen": -2.6260576248168945,
+      "logps/rejected": -56.7901496887207,
+      "loss": 0.4087463915348053,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2899452745914459,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.0745973214507103,
+      "rewards/margins": 4.15200662612915,
+      "rewards/rejected": -4.077408790588379,
+      "step": 88,
+      "train_speed(iter/s)": 0.010733
+    },
+    {
+      "epoch": 0.5131531531531531,
+      "grad_norm": 1.563063144683838,
+      "learning_rate": 0.00019204910201277672,
+      "logits/chosen": -0.5593241453170776,
+      "logits/rejected": -0.7506377100944519,
+      "logps/chosen": -3.1961171627044678,
+      "logps/rejected": -63.025291442871094,
+      "loss": 0.20274411141872406,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.15050008893013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.13150721788406372,
+      "rewards/margins": 4.959373950958252,
+      "rewards/rejected": -4.827866554260254,
+      "step": 89,
+      "train_speed(iter/s)": 0.010734
+    },
+    {
+      "epoch": 0.518918918918919,
+      "grad_norm": 1.8057938814163208,
+      "learning_rate": 0.0001917982245367379,
+      "logits/chosen": -0.5387026071548462,
+      "logits/rejected": -0.779965341091156,
+      "logps/chosen": -5.658284664154053,
+      "logps/rejected": -61.30659484863281,
+      "loss": 0.5316486358642578,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.4009394347667694,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2519141137599945,
+      "rewards/margins": 4.251716613769531,
+      "rewards/rejected": -3.9998018741607666,
+      "step": 90,
+      "train_speed(iter/s)": 0.010734
+    },
+    {
+      "epoch": 0.5246846846846847,
+      "grad_norm": 1.6844978332519531,
+      "learning_rate": 0.00019154361937988163,
+      "logits/chosen": -0.5544890761375427,
+      "logits/rejected": -0.7064109444618225,
+      "logps/chosen": -6.5527496337890625,
+      "logps/rejected": -65.5311050415039,
+      "loss": 0.38739749789237976,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2913617193698883,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.1942709982395172,
+      "rewards/margins": 5.296841621398926,
+      "rewards/rejected": -5.102570533752441,
+      "step": 91,
+      "train_speed(iter/s)": 0.010734
+    },
+    {
+      "epoch": 0.5304504504504505,
+      "grad_norm": 3.110102653503418,
+      "learning_rate": 0.00019128529688104364,
+      "logits/chosen": -0.5477601885795593,
+      "logits/rejected": -0.7957394123077393,
+      "logps/chosen": -2.064303398132324,
+      "logps/rejected": -68.15472412109375,
+      "loss": 0.3176850974559784,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.21008610725402832,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.18838940560817719,
+      "rewards/margins": 5.544022560119629,
+      "rewards/rejected": -5.355632781982422,
+      "step": 92,
+      "train_speed(iter/s)": 0.010735
+    },
+    {
+      "epoch": 0.5362162162162162,
+      "grad_norm": 3.8029165267944336,
+      "learning_rate": 0.00019102326753001086,
+      "logits/chosen": -0.5357910394668579,
+      "logits/rejected": -0.7163031101226807,
+      "logps/chosen": -4.390456676483154,
+      "logps/rejected": -55.20818328857422,
+      "loss": 0.4618917405605316,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.33474794030189514,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.36951008439064026,
+      "rewards/margins": 4.2895612716674805,
+      "rewards/rejected": -3.920051336288452,
+      "step": 93,
+      "train_speed(iter/s)": 0.010735
+    },
+    {
+      "epoch": 0.541981981981982,
+      "grad_norm": 1.8191365003585815,
+      "learning_rate": 0.00019075754196709572,
+      "logits/chosen": -0.5523058176040649,
+      "logits/rejected": -0.7712856531143188,
+      "logps/chosen": -2.1930363178253174,
+      "logps/rejected": -66.22615814208984,
+      "loss": 0.28284692764282227,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2161579132080078,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.14300797879695892,
+      "rewards/margins": 5.302109718322754,
+      "rewards/rejected": -5.159101486206055,
+      "step": 94,
+      "train_speed(iter/s)": 0.010735
+    },
+    {
+      "epoch": 0.5477477477477477,
+      "grad_norm": 2.65755558013916,
+      "learning_rate": 0.00019048813098270387,
+      "logits/chosen": -0.49344098567962646,
+      "logits/rejected": -0.7781980633735657,
+      "logps/chosen": -3.545104742050171,
+      "logps/rejected": -78.23759460449219,
+      "loss": 0.3401697278022766,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3010118901729584,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.07626594603061676,
+      "rewards/margins": 5.846086025238037,
+      "rewards/rejected": -5.769820213317871,
+      "step": 95,
+      "train_speed(iter/s)": 0.010736
+    },
+    {
+      "epoch": 0.5535135135135135,
+      "grad_norm": 1.6140971183776855,
+      "learning_rate": 0.0001902150455168962,
+      "logits/chosen": -0.4496663808822632,
+      "logits/rejected": -0.7431093454360962,
+      "logps/chosen": -2.8349695205688477,
+      "logps/rejected": -72.62895965576172,
+      "loss": 0.3297802209854126,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2532678544521332,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.149441659450531,
+      "rewards/margins": 5.512978553771973,
+      "rewards/rejected": -5.363536357879639,
+      "step": 96,
+      "train_speed(iter/s)": 0.010735
+    },
+    {
+      "epoch": 0.5592792792792792,
+      "grad_norm": 2.243736743927002,
+      "learning_rate": 0.0001899382966589443,
+      "logits/chosen": -0.44952693581581116,
+      "logits/rejected": -0.7028639316558838,
+      "logps/chosen": -5.277416229248047,
+      "logps/rejected": -69.73493957519531,
+      "loss": 0.41486623883247375,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.25434425473213196,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.13436982035636902,
+      "rewards/margins": 4.8991923332214355,
+      "rewards/rejected": -4.764822006225586,
+      "step": 97,
+      "train_speed(iter/s)": 0.010735
+    },
+    {
+      "epoch": 0.565045045045045,
+      "grad_norm": 4.0099101066589355,
+      "learning_rate": 0.0001896578956468805,
+      "logits/chosen": -0.5480562448501587,
+      "logits/rejected": -0.7537853717803955,
+      "logps/chosen": -4.075699806213379,
+      "logps/rejected": -61.6773681640625,
+      "loss": 0.34289711713790894,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.24206680059432983,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.09123305231332779,
+      "rewards/margins": 4.509951114654541,
+      "rewards/rejected": -4.418718338012695,
+      "step": 98,
+      "train_speed(iter/s)": 0.010736
+    },
+    {
+      "epoch": 0.5708108108108109,
+      "grad_norm": 12.38167667388916,
+      "learning_rate": 0.00018937385386704126,
+      "logits/chosen": -0.6038029789924622,
+      "logits/rejected": -0.7428975105285645,
+      "logps/chosen": -6.938250541687012,
+      "logps/rejected": -58.5086555480957,
+      "loss": 0.4984804391860962,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.37261444330215454,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.10895882546901703,
+      "rewards/margins": 4.678299903869629,
+      "rewards/rejected": -4.569341659545898,
+      "step": 99,
+      "train_speed(iter/s)": 0.010736
+    },
+    {
+      "epoch": 0.5765765765765766,
+      "grad_norm": 4.255463123321533,
+      "learning_rate": 0.00018908618285360494,
+      "logits/chosen": -0.5780411958694458,
+      "logits/rejected": -0.7493711709976196,
+      "logps/chosen": -6.087734222412109,
+      "logps/rejected": -68.05068969726562,
+      "loss": 0.5377764701843262,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.4144035577774048,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.07401561737060547,
+      "rewards/margins": 5.022759437561035,
+      "rewards/rejected": -4.94874382019043,
+      "step": 100,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 0.5823423423423424,
+      "grad_norm": 2.052121162414551,
+      "learning_rate": 0.00018879489428812334,
+      "logits/chosen": -0.5155804753303528,
+      "logits/rejected": -0.7454657554626465,
+      "logps/chosen": -4.093721389770508,
+      "logps/rejected": -73.28987121582031,
+      "loss": 0.4639092683792114,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3733065724372864,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.16241005063056946,
+      "rewards/margins": 5.644482135772705,
+      "rewards/rejected": -5.482071876525879,
+      "step": 101,
+      "train_speed(iter/s)": 0.010718
+    },
+    {
+      "epoch": 0.5881081081081081,
+      "grad_norm": 2.1512622833251953,
+      "learning_rate": 0.0001884999999990475,
+      "logits/chosen": -0.5633401274681091,
+      "logits/rejected": -0.7381998896598816,
+      "logps/chosen": -6.3428497314453125,
+      "logps/rejected": -52.1317138671875,
+      "loss": 0.4867536425590515,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.34330999851226807,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.33921706676483154,
+      "rewards/margins": 4.20556116104126,
+      "rewards/rejected": -3.8663439750671387,
+      "step": 102,
+      "train_speed(iter/s)": 0.010718
+    },
+    {
+      "epoch": 0.5938738738738739,
+      "grad_norm": 3.3250627517700195,
+      "learning_rate": 0.00018820151196124717,
+      "logits/chosen": -0.5157067179679871,
+      "logits/rejected": -0.8171908855438232,
+      "logps/chosen": -3.4731268882751465,
+      "logps/rejected": -87.99000549316406,
+      "loss": 0.5975285768508911,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.46800604462623596,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.06358014047145844,
+      "rewards/margins": 6.483148097991943,
+      "rewards/rejected": -6.419568061828613,
+      "step": 103,
+      "train_speed(iter/s)": 0.010718
+    },
+    {
+      "epoch": 0.5996396396396396,
+      "grad_norm": 3.476541757583618,
+      "learning_rate": 0.0001878994422955246,
+      "logits/chosen": -0.5131168365478516,
+      "logits/rejected": -0.8059231042861938,
+      "logps/chosen": -3.1174309253692627,
+      "logps/rejected": -86.71981811523438,
+      "loss": 0.3160158693790436,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.25080835819244385,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.28578388690948486,
+      "rewards/margins": 6.907861232757568,
+      "rewards/rejected": -6.622077941894531,
+      "step": 104,
+      "train_speed(iter/s)": 0.010718
+    },
+    {
+      "epoch": 0.6054054054054054,
+      "grad_norm": 5.32436990737915,
+      "learning_rate": 0.00018759380326812257,
+      "logits/chosen": -0.6340402960777283,
+      "logits/rejected": -0.867938756942749,
+      "logps/chosen": -2.323990821838379,
+      "logps/rejected": -71.03258514404297,
+      "loss": 0.31046488881111145,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.21423426270484924,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.011083951219916344,
+      "rewards/margins": 6.008091926574707,
+      "rewards/rejected": -6.0191755294799805,
+      "step": 105,
+      "train_speed(iter/s)": 0.010719
+    },
+    {
+      "epoch": 0.6111711711711711,
+      "grad_norm": 3.4041733741760254,
+      "learning_rate": 0.0001872846072902259,
+      "logits/chosen": -0.589099645614624,
+      "logits/rejected": -0.8120458126068115,
+      "logps/chosen": -1.5791796445846558,
+      "logps/rejected": -82.34098052978516,
+      "loss": 0.2268940657377243,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.14617523550987244,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2277921438217163,
+      "rewards/margins": 6.835321426391602,
+      "rewards/rejected": -6.607529640197754,
+      "step": 106,
+      "train_speed(iter/s)": 0.010719
+    },
+    {
+      "epoch": 0.616936936936937,
+      "grad_norm": 1.9216675758361816,
+      "learning_rate": 0.00018697186691745782,
+      "logits/chosen": -0.599471926689148,
+      "logits/rejected": -0.8271054029464722,
+      "logps/chosen": -4.24713134765625,
+      "logps/rejected": -84.08229064941406,
+      "loss": 0.3841734826564789,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.31657740473747253,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.022511344403028488,
+      "rewards/margins": 6.769836902618408,
+      "rewards/rejected": -6.747325897216797,
+      "step": 107,
+      "train_speed(iter/s)": 0.01072
+    },
+    {
+      "epoch": 0.6227027027027027,
+      "grad_norm": 4.744035720825195,
+      "learning_rate": 0.0001866555948493699,
+      "logits/chosen": -0.5849882960319519,
+      "logits/rejected": -0.8727071285247803,
+      "logps/chosen": -2.891716718673706,
+      "logps/rejected": -92.83207702636719,
+      "loss": 0.31115370988845825,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.22799423336982727,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19745981693267822,
+      "rewards/margins": 7.553801536560059,
+      "rewards/rejected": -7.35634183883667,
+      "step": 108,
+      "train_speed(iter/s)": 0.01072
+    },
+    {
+      "epoch": 0.6284684684684685,
+      "grad_norm": 2.028069019317627,
+      "learning_rate": 0.00018633580392892648,
+      "logits/chosen": -0.5856168270111084,
+      "logits/rejected": -0.8363710641860962,
+      "logps/chosen": -5.515890121459961,
+      "logps/rejected": -83.518310546875,
+      "loss": 0.40675532817840576,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.33311402797698975,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.19537094235420227,
+      "rewards/margins": 6.882720947265625,
+      "rewards/rejected": -6.687350273132324,
+      "step": 109,
+      "train_speed(iter/s)": 0.01072
+    },
+    {
+      "epoch": 0.6342342342342342,
+      "grad_norm": 1.8457213640213013,
+      "learning_rate": 0.00018601250714198302,
+      "logits/chosen": -0.6694027185440063,
+      "logits/rejected": -0.8495978713035583,
+      "logps/chosen": -6.527522087097168,
+      "logps/rejected": -69.24601745605469,
+      "loss": 0.4539996385574341,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3218171000480652,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.16864001750946045,
+      "rewards/margins": 5.4212541580200195,
+      "rewards/rejected": -5.252613544464111,
+      "step": 110,
+      "train_speed(iter/s)": 0.01072
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 1.4716756343841553,
+      "learning_rate": 0.00018568571761675893,
+      "logits/chosen": -0.5902320742607117,
+      "logits/rejected": -0.8840252757072449,
+      "logps/chosen": -5.05216121673584,
+      "logps/rejected": -96.3882064819336,
+      "loss": 0.3378267288208008,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2499268501996994,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.21043285727500916,
+      "rewards/margins": 7.197650909423828,
+      "rewards/rejected": -6.987217903137207,
+      "step": 111,
+      "train_speed(iter/s)": 0.01072
+    },
+    {
+      "epoch": 0.6457657657657657,
+      "grad_norm": 0.7805799245834351,
+      "learning_rate": 0.00018535544862330436,
+      "logits/chosen": -0.6686023473739624,
+      "logits/rejected": -0.986074686050415,
+      "logps/chosen": -2.231717824935913,
+      "logps/rejected": -96.76434326171875,
+      "loss": 0.16498756408691406,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.14222995936870575,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.14315873384475708,
+      "rewards/margins": 7.930060386657715,
+      "rewards/rejected": -7.786901473999023,
+      "step": 112,
+      "train_speed(iter/s)": 0.01072
+    },
+    {
+      "epoch": 0.6515315315315315,
+      "grad_norm": 2.115647792816162,
+      "learning_rate": 0.00018502171357296144,
+      "logits/chosen": -0.7186330556869507,
+      "logits/rejected": -0.9198458194732666,
+      "logps/chosen": -6.138432502746582,
+      "logps/rejected": -76.40232849121094,
+      "loss": 0.42215695977211,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.34653419256210327,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3074129521846771,
+      "rewards/margins": 6.320471286773682,
+      "rewards/rejected": -6.013057708740234,
+      "step": 113,
+      "train_speed(iter/s)": 0.010721
+    },
+    {
+      "epoch": 0.6572972972972972,
+      "grad_norm": 8.20527458190918,
+      "learning_rate": 0.00018468452601781948,
+      "logits/chosen": -0.6733801364898682,
+      "logits/rejected": -0.9577184319496155,
+      "logps/chosen": -5.523393154144287,
+      "logps/rejected": -94.21669006347656,
+      "loss": 0.4980089068412781,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.37918829917907715,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.07570207118988037,
+      "rewards/margins": 7.661881446838379,
+      "rewards/rejected": -7.586179256439209,
+      "step": 114,
+      "train_speed(iter/s)": 0.010721
+    },
+    {
+      "epoch": 0.6630630630630631,
+      "grad_norm": 1.4824076890945435,
+      "learning_rate": 0.00018434389965016495,
+      "logits/chosen": -0.6685976386070251,
+      "logits/rejected": -0.9235626459121704,
+      "logps/chosen": -4.8032307624816895,
+      "logps/rejected": -89.84734344482422,
+      "loss": 0.3988059163093567,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3272039592266083,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1318884938955307,
+      "rewards/margins": 7.2766876220703125,
+      "rewards/rejected": -7.14479923248291,
+      "step": 115,
+      "train_speed(iter/s)": 0.010721
+    },
+    {
+      "epoch": 0.6688288288288289,
+      "grad_norm": 6.457195281982422,
+      "learning_rate": 0.00018399984830192522,
+      "logits/chosen": -0.6563739776611328,
+      "logits/rejected": -0.9446012377738953,
+      "logps/chosen": -5.333343505859375,
+      "logps/rejected": -96.29754638671875,
+      "loss": 0.6648478507995605,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.5736638307571411,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.1766398400068283,
+      "rewards/margins": 7.466494560241699,
+      "rewards/rejected": -7.289853572845459,
+      "step": 116,
+      "train_speed(iter/s)": 0.010721
+    },
+    {
+      "epoch": 0.6745945945945946,
+      "grad_norm": 0.6062999367713928,
+      "learning_rate": 0.00018365238594410696,
+      "logits/chosen": -0.6826354265213013,
+      "logits/rejected": -0.9526041150093079,
+      "logps/chosen": -2.1560752391815186,
+      "logps/rejected": -97.08880615234375,
+      "loss": 0.15202483534812927,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1097334772348404,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.08674149215221405,
+      "rewards/margins": 8.09538745880127,
+      "rewards/rejected": -8.008646011352539,
+      "step": 117,
+      "train_speed(iter/s)": 0.010721
+    },
+    {
+      "epoch": 0.6803603603603604,
+      "grad_norm": 4.866965293884277,
+      "learning_rate": 0.00018330152668622892,
+      "logits/chosen": -0.6270961761474609,
+      "logits/rejected": -0.8659616708755493,
+      "logps/chosen": -6.8050456047058105,
+      "logps/rejected": -94.24461364746094,
+      "loss": 0.5779068470001221,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.4634403884410858,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08972588181495667,
+      "rewards/margins": 7.540095806121826,
+      "rewards/rejected": -7.450369834899902,
+      "step": 118,
+      "train_speed(iter/s)": 0.010721
+    },
+    {
+      "epoch": 0.6861261261261261,
+      "grad_norm": 1.9984689950942993,
+      "learning_rate": 0.00018294728477574886,
+      "logits/chosen": -0.6620824933052063,
+      "logits/rejected": -0.9279792308807373,
+      "logps/chosen": -3.62056303024292,
+      "logps/rejected": -88.79190826416016,
+      "loss": 0.5508708357810974,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.44049572944641113,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.04914311319589615,
+      "rewards/margins": 7.090651512145996,
+      "rewards/rejected": -7.041508197784424,
+      "step": 119,
+      "train_speed(iter/s)": 0.010721
+    },
+    {
+      "epoch": 0.6918918918918919,
+      "grad_norm": 2.634037971496582,
+      "learning_rate": 0.000182589674597485,
+      "logits/chosen": -0.6129943132400513,
+      "logits/rejected": -0.891259491443634,
+      "logps/chosen": -4.1637163162231445,
+      "logps/rejected": -89.52510833740234,
+      "loss": 0.5133303999900818,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.35086461901664734,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.015482299029827118,
+      "rewards/margins": 7.1094279289245605,
+      "rewards/rejected": -7.0939459800720215,
+      "step": 120,
+      "train_speed(iter/s)": 0.010722
+    },
+    {
+      "epoch": 0.6976576576576576,
+      "grad_norm": 2.7602248191833496,
+      "learning_rate": 0.00018222871067303192,
+      "logits/chosen": -0.644217848777771,
+      "logits/rejected": -0.8804385662078857,
+      "logps/chosen": -4.665724277496338,
+      "logps/rejected": -75.18159484863281,
+      "loss": 0.4733228385448456,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3284577429294586,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.11940830945968628,
+      "rewards/margins": 6.475360870361328,
+      "rewards/rejected": -6.355952262878418,
+      "step": 121,
+      "train_speed(iter/s)": 0.010722
+    },
+    {
+      "epoch": 0.7034234234234235,
+      "grad_norm": 8.460821151733398,
+      "learning_rate": 0.0001818644076601709,
+      "logits/chosen": -0.6995394825935364,
+      "logits/rejected": -0.9071613550186157,
+      "logps/chosen": -8.95744800567627,
+      "logps/rejected": -77.84011840820312,
+      "loss": 0.6963841915130615,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.5519821047782898,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.2606813609600067,
+      "rewards/margins": 6.603857040405273,
+      "rewards/rejected": -6.343176364898682,
+      "step": 122,
+      "train_speed(iter/s)": 0.010722
+    },
+    {
+      "epoch": 0.7091891891891892,
+      "grad_norm": 3.587855100631714,
+      "learning_rate": 0.00018149678035227473,
+      "logits/chosen": -0.6442317962646484,
+      "logits/rejected": -0.8487176895141602,
+      "logps/chosen": -6.872650623321533,
+      "logps/rejected": -82.89649963378906,
+      "loss": 0.4728536605834961,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3608010411262512,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.009775836020708084,
+      "rewards/margins": 6.596733093261719,
+      "rewards/rejected": -6.586956977844238,
+      "step": 123,
+      "train_speed(iter/s)": 0.010723
+    },
+    {
+      "epoch": 0.714954954954955,
+      "grad_norm": 1.7244524955749512,
+      "learning_rate": 0.00018112584367770685,
+      "logits/chosen": -0.6685165762901306,
+      "logits/rejected": -0.8828461766242981,
+      "logps/chosen": -5.397648334503174,
+      "logps/rejected": -85.60098266601562,
+      "loss": 0.389848530292511,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.32140427827835083,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.12320394814014435,
+      "rewards/margins": 7.233897686004639,
+      "rewards/rejected": -7.11069393157959,
+      "step": 124,
+      "train_speed(iter/s)": 0.010723
+    },
+    {
+      "epoch": 0.7207207207207207,
+      "grad_norm": 0.7455915808677673,
+      "learning_rate": 0.00018075161269921527,
+      "logits/chosen": -0.5614687204360962,
+      "logits/rejected": -0.8907942771911621,
+      "logps/chosen": -1.7438545227050781,
+      "logps/rejected": -92.23735809326172,
+      "loss": 0.12226351350545883,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.10527773201465607,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.15749390423297882,
+      "rewards/margins": 7.594954967498779,
+      "rewards/rejected": -7.437460422515869,
+      "step": 125,
+      "train_speed(iter/s)": 0.010723
+    },
+    {
+      "epoch": 0.7264864864864865,
+      "grad_norm": 1.0758198499679565,
+      "learning_rate": 0.000180374102613321,
+      "logits/chosen": -0.6551331877708435,
+      "logits/rejected": -0.8264045119285583,
+      "logps/chosen": -4.308568000793457,
+      "logps/rejected": -80.06883239746094,
+      "loss": 0.36240947246551514,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.24730846285820007,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.10061657428741455,
+      "rewards/margins": 6.429587364196777,
+      "rewards/rejected": -6.328970909118652,
+      "step": 126,
+      "train_speed(iter/s)": 0.010723
+    },
+    {
+      "epoch": 0.7322522522522522,
+      "grad_norm": 1.0857983827590942,
+      "learning_rate": 0.0001799933287497007,
+      "logits/chosen": -0.6380553841590881,
+      "logits/rejected": -0.8699716925621033,
+      "logps/chosen": -1.9937247037887573,
+      "logps/rejected": -87.90292358398438,
+      "loss": 0.18343912065029144,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.12965701520442963,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.09683795273303986,
+      "rewards/margins": 7.11926794052124,
+      "rewards/rejected": -7.022429943084717,
+      "step": 127,
+      "train_speed(iter/s)": 0.010724
+    },
+    {
+      "epoch": 0.738018018018018,
+      "grad_norm": 9.539369583129883,
+      "learning_rate": 0.00017960930657056438,
+      "logits/chosen": -0.6350418925285339,
+      "logits/rejected": -0.8755402565002441,
+      "logps/chosen": -2.645111560821533,
+      "logps/rejected": -82.07209777832031,
+      "loss": 0.23406416177749634,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.17941246926784515,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.07386763393878937,
+      "rewards/margins": 6.727149486541748,
+      "rewards/rejected": -6.653281211853027,
+      "step": 128,
+      "train_speed(iter/s)": 0.010724
+    },
+    {
+      "epoch": 0.7437837837837837,
+      "grad_norm": 10.807381629943848,
+      "learning_rate": 0.00017922205167002754,
+      "logits/chosen": -0.6329447031021118,
+      "logits/rejected": -0.8397620320320129,
+      "logps/chosen": -4.6306023597717285,
+      "logps/rejected": -67.41867065429688,
+      "loss": 0.5849911570549011,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.41021403670310974,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.06239153444766998,
+      "rewards/margins": 5.311315059661865,
+      "rewards/rejected": -5.248923301696777,
+      "step": 129,
+      "train_speed(iter/s)": 0.010724
+    },
+    {
+      "epoch": 0.7495495495495496,
+      "grad_norm": 1.8455349206924438,
+      "learning_rate": 0.00017883157977347774,
+      "logits/chosen": -0.5277632474899292,
+      "logits/rejected": -0.785979151725769,
+      "logps/chosen": -4.069696426391602,
+      "logps/rejected": -82.98876953125,
+      "loss": 0.3647957146167755,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.24460868537425995,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.12994280457496643,
+      "rewards/margins": 6.137853622436523,
+      "rewards/rejected": -6.00791072845459,
+      "step": 130,
+      "train_speed(iter/s)": 0.010724
+    },
+    {
+      "epoch": 0.7553153153153153,
+      "grad_norm": 1.5157482624053955,
+      "learning_rate": 0.00017843790673693625,
+      "logits/chosen": -0.533730685710907,
+      "logits/rejected": -0.8200556635856628,
+      "logps/chosen": -4.038061141967773,
+      "logps/rejected": -82.76172637939453,
+      "loss": 0.2888023555278778,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2086402326822281,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2728549540042877,
+      "rewards/margins": 6.415480613708496,
+      "rewards/rejected": -6.142626762390137,
+      "step": 131,
+      "train_speed(iter/s)": 0.010724
+    },
+    {
+      "epoch": 0.7610810810810811,
+      "grad_norm": 1.936601996421814,
+      "learning_rate": 0.00017804104854641408,
+      "logits/chosen": -0.6249516606330872,
+      "logits/rejected": -0.8721716403961182,
+      "logps/chosen": -3.304114818572998,
+      "logps/rejected": -60.27552795410156,
+      "loss": 0.415643572807312,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.28065580129623413,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2913169264793396,
+      "rewards/margins": 4.785688400268555,
+      "rewards/rejected": -4.49437141418457,
+      "step": 132,
+      "train_speed(iter/s)": 0.010724
+    },
+    {
+      "epoch": 0.7668468468468469,
+      "grad_norm": 3.6359870433807373,
+      "learning_rate": 0.0001776410213172628,
+      "logits/chosen": -0.5627670288085938,
+      "logits/rejected": -0.8085115551948547,
+      "logps/chosen": -1.7719781398773193,
+      "logps/rejected": -72.61830139160156,
+      "loss": 0.2871711552143097,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1900225728750229,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.089139424264431,
+      "rewards/margins": 5.8200178146362305,
+      "rewards/rejected": -5.730878829956055,
+      "step": 133,
+      "train_speed(iter/s)": 0.010725
+    },
+    {
+      "epoch": 0.7726126126126126,
+      "grad_norm": 1.4323631525039673,
+      "learning_rate": 0.00017723784129352018,
+      "logits/chosen": -0.6231422424316406,
+      "logits/rejected": -0.8504309058189392,
+      "logps/chosen": -1.730849027633667,
+      "logps/rejected": -68.78044128417969,
+      "loss": 0.2242366373538971,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.13950544595718384,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.18260075151920319,
+      "rewards/margins": 5.708943843841553,
+      "rewards/rejected": -5.526342391967773,
+      "step": 134,
+      "train_speed(iter/s)": 0.010725
+    },
+    {
+      "epoch": 0.7783783783783784,
+      "grad_norm": 1.6163887977600098,
+      "learning_rate": 0.00017683152484725066,
+      "logits/chosen": -0.6224597692489624,
+      "logits/rejected": -0.7799749374389648,
+      "logps/chosen": -4.717617988586426,
+      "logps/rejected": -61.31199264526367,
+      "loss": 0.4397742450237274,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3140389323234558,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.25990185141563416,
+      "rewards/margins": 4.902276039123535,
+      "rewards/rejected": -4.642374515533447,
+      "step": 135,
+      "train_speed(iter/s)": 0.010726
+    },
+    {
+      "epoch": 0.7841441441441441,
+      "grad_norm": 1.2988686561584473,
+      "learning_rate": 0.00017642208847788032,
+      "logits/chosen": -0.4455900192260742,
+      "logits/rejected": -0.8272579312324524,
+      "logps/chosen": -3.7604730129241943,
+      "logps/rejected": -91.75283813476562,
+      "loss": 0.3069111108779907,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.257954478263855,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.17708764970302582,
+      "rewards/margins": 7.115184783935547,
+      "rewards/rejected": -6.93809700012207,
+      "step": 136,
+      "train_speed(iter/s)": 0.010725
+    },
+    {
+      "epoch": 0.78990990990991,
+      "grad_norm": 3.960479974746704,
+      "learning_rate": 0.00017600954881152693,
+      "logits/chosen": -0.480897456407547,
+      "logits/rejected": -0.806812584400177,
+      "logps/chosen": -2.339344024658203,
+      "logps/rejected": -88.25721740722656,
+      "loss": 0.25455033779144287,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1620534360408783,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.10610076785087585,
+      "rewards/margins": 6.920160293579102,
+      "rewards/rejected": -6.814059734344482,
+      "step": 137,
+      "train_speed(iter/s)": 0.010725
+    },
+    {
+      "epoch": 0.7956756756756757,
+      "grad_norm": 9.401166915893555,
+      "learning_rate": 0.00017559392260032506,
+      "logits/chosen": -0.4589274525642395,
+      "logits/rejected": -0.7634912729263306,
+      "logps/chosen": -5.8359503746032715,
+      "logps/rejected": -96.87335205078125,
+      "loss": 0.5270398855209351,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.43480122089385986,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.14989905059337616,
+      "rewards/margins": 7.5454607009887695,
+      "rewards/rejected": -7.69536018371582,
+      "step": 138,
+      "train_speed(iter/s)": 0.010725
+    },
+    {
+      "epoch": 0.8014414414414415,
+      "grad_norm": 4.686446666717529,
+      "learning_rate": 0.00017517522672174548,
+      "logits/chosen": -0.5010412931442261,
+      "logits/rejected": -0.8063571453094482,
+      "logps/chosen": -2.5103189945220947,
+      "logps/rejected": -96.94129943847656,
+      "loss": 0.3778824508190155,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.252981573343277,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.10596343874931335,
+      "rewards/margins": 7.605740070343018,
+      "rewards/rejected": -7.499776840209961,
+      "step": 139,
+      "train_speed(iter/s)": 0.010724
+    },
+    {
+      "epoch": 0.8072072072072072,
+      "grad_norm": 1.6708407402038574,
+      "learning_rate": 0.00017475347817790996,
+      "logits/chosen": -0.4751094877719879,
+      "logits/rejected": -0.8238545060157776,
+      "logps/chosen": -3.2878451347351074,
+      "logps/rejected": -86.26036071777344,
+      "loss": 0.24279947578907013,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.16919009387493134,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.07166648656129837,
+      "rewards/margins": 6.587783336639404,
+      "rewards/rejected": -6.516116142272949,
+      "step": 140,
+      "train_speed(iter/s)": 0.010724
+    },
+    {
+      "epoch": 0.812972972972973,
+      "grad_norm": 1.4335037469863892,
+      "learning_rate": 0.00017432869409490093,
+      "logits/chosen": -0.5489400625228882,
+      "logits/rejected": -0.7731753587722778,
+      "logps/chosen": -5.348051071166992,
+      "logps/rejected": -70.64154815673828,
+      "loss": 0.36098673939704895,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2713705897331238,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.23904970288276672,
+      "rewards/margins": 5.921088695526123,
+      "rewards/rejected": -5.682039737701416,
+      "step": 141,
+      "train_speed(iter/s)": 0.010725
+    },
+    {
+      "epoch": 0.8187387387387387,
+      "grad_norm": 2.012854814529419,
+      "learning_rate": 0.00017390089172206592,
+      "logits/chosen": -0.467896968126297,
+      "logits/rejected": -0.793892502784729,
+      "logps/chosen": -1.7580054998397827,
+      "logps/rejected": -85.476318359375,
+      "loss": 0.23942089080810547,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.17797623574733734,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.09021879732608795,
+      "rewards/margins": 6.8085222244262695,
+      "rewards/rejected": -6.718303680419922,
+      "step": 142,
+      "train_speed(iter/s)": 0.010725
+    },
+    {
+      "epoch": 0.8245045045045045,
+      "grad_norm": 5.257446765899658,
+      "learning_rate": 0.00017347008843131712,
+      "logits/chosen": -0.4858510196208954,
+      "logits/rejected": -0.7565414309501648,
+      "logps/chosen": -3.7341561317443848,
+      "logps/rejected": -75.44990539550781,
+      "loss": 0.43588101863861084,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3324872553348541,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1589755266904831,
+      "rewards/margins": 5.903942584991455,
+      "rewards/rejected": -5.744967460632324,
+      "step": 143,
+      "train_speed(iter/s)": 0.010725
+    },
+    {
+      "epoch": 0.8302702702702702,
+      "grad_norm": 1.9966129064559937,
+      "learning_rate": 0.00017303630171642607,
+      "logits/chosen": -0.4013749659061432,
+      "logits/rejected": -0.7098768353462219,
+      "logps/chosen": -2.0640015602111816,
+      "logps/rejected": -80.94207763671875,
+      "loss": 0.2669733166694641,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2289275974035263,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.20508992671966553,
+      "rewards/margins": 5.947912216186523,
+      "rewards/rejected": -5.742822170257568,
+      "step": 144,
+      "train_speed(iter/s)": 0.010725
+    },
+    {
+      "epoch": 0.836036036036036,
+      "grad_norm": 4.144430637359619,
+      "learning_rate": 0.0001725995491923131,
+      "logits/chosen": -0.34412139654159546,
+      "logits/rejected": -0.7268174290657043,
+      "logps/chosen": -2.0844953060150146,
+      "logps/rejected": -86.29147338867188,
+      "loss": 0.2860412001609802,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.21140313148498535,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.1814691722393036,
+      "rewards/margins": 6.325296401977539,
+      "rewards/rejected": -6.143826961517334,
+      "step": 145,
+      "train_speed(iter/s)": 0.010724
+    },
+    {
+      "epoch": 0.8418018018018018,
+      "grad_norm": 6.901271343231201,
+      "learning_rate": 0.0001721598485943322,
+      "logits/chosen": -0.4293047785758972,
+      "logits/rejected": -0.6931589841842651,
+      "logps/chosen": -3.5573177337646484,
+      "logps/rejected": -80.00125885009766,
+      "loss": 0.402279257774353,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.30397528409957886,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.027988389134407043,
+      "rewards/margins": 6.108896732330322,
+      "rewards/rejected": -6.136884689331055,
+      "step": 146,
+      "train_speed(iter/s)": 0.010724
+    },
+    {
+      "epoch": 0.8475675675675676,
+      "grad_norm": 1.3138624429702759,
+      "learning_rate": 0.00017171721777755074,
+      "logits/chosen": -0.441506952047348,
+      "logits/rejected": -0.6183236241340637,
+      "logps/chosen": -5.245753288269043,
+      "logps/rejected": -61.3995246887207,
+      "loss": 0.5000537633895874,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3299512565135956,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1415078043937683,
+      "rewards/margins": 4.831845760345459,
+      "rewards/rejected": -4.690337181091309,
+      "step": 147,
+      "train_speed(iter/s)": 0.010724
+    },
+    {
+      "epoch": 0.8533333333333334,
+      "grad_norm": 2.7394113540649414,
+      "learning_rate": 0.00017127167471602447,
+      "logits/chosen": -0.44564759731292725,
+      "logits/rejected": -0.7324830889701843,
+      "logps/chosen": -2.2436249256134033,
+      "logps/rejected": -73.5162124633789,
+      "loss": 0.20336386561393738,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.11223578453063965,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.17428822815418243,
+      "rewards/margins": 5.678185939788818,
+      "rewards/rejected": -5.503897666931152,
+      "step": 148,
+      "train_speed(iter/s)": 0.010724
+    },
+    {
+      "epoch": 0.8590990990990991,
+      "grad_norm": 1.488072395324707,
+      "learning_rate": 0.00017082323750206761,
+      "logits/chosen": -0.44784775376319885,
+      "logits/rejected": -0.6909451484680176,
+      "logps/chosen": -3.5523109436035156,
+      "logps/rejected": -70.1702651977539,
+      "loss": 0.31512442231178284,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2102314829826355,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.11678414791822433,
+      "rewards/margins": 5.312015533447266,
+      "rewards/rejected": -5.1952314376831055,
+      "step": 149,
+      "train_speed(iter/s)": 0.010724
+    },
+    {
+      "epoch": 0.8648648648648649,
+      "grad_norm": 2.6578073501586914,
+      "learning_rate": 0.00017037192434551823,
+      "logits/chosen": -0.39348986744880676,
+      "logits/rejected": -0.690170407295227,
+      "logps/chosen": -3.4318065643310547,
+      "logps/rejected": -76.70863342285156,
+      "loss": 0.40308576822280884,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2762233316898346,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1851682811975479,
+      "rewards/margins": 5.912222385406494,
+      "rewards/rejected": -5.727054119110107,
+      "step": 150,
+      "train_speed(iter/s)": 0.010724
+    },
+    {
+      "epoch": 0.8706306306306306,
+      "grad_norm": 2.9397995471954346,
+      "learning_rate": 0.00016991775357299866,
+      "logits/chosen": -0.4844151735305786,
+      "logits/rejected": -0.7689380645751953,
+      "logps/chosen": -4.010564804077148,
+      "logps/rejected": -85.45973205566406,
+      "loss": 0.3323879539966583,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2697053551673889,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.17146898806095123,
+      "rewards/margins": 6.537592887878418,
+      "rewards/rejected": -6.366123676300049,
+      "step": 151,
+      "train_speed(iter/s)": 0.010712
+    },
+    {
+      "epoch": 0.8763963963963964,
+      "grad_norm": 2.216646909713745,
+      "learning_rate": 0.00016946074362717147,
+      "logits/chosen": -0.5326839685440063,
+      "logits/rejected": -0.7718309760093689,
+      "logps/chosen": -4.843362808227539,
+      "logps/rejected": -73.73052978515625,
+      "loss": 0.4268698990345001,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.36223095655441284,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.21485474705696106,
+      "rewards/margins": 5.929214000701904,
+      "rewards/rejected": -5.714358329772949,
+      "step": 152,
+      "train_speed(iter/s)": 0.010712
+    },
+    {
+      "epoch": 0.8821621621621621,
+      "grad_norm": 6.382073879241943,
+      "learning_rate": 0.00016900091306599042,
+      "logits/chosen": -0.6236099004745483,
+      "logits/rejected": -0.7957028150558472,
+      "logps/chosen": -6.768365859985352,
+      "logps/rejected": -75.18216705322266,
+      "loss": 0.5070593953132629,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.39859798550605774,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.03324725478887558,
+      "rewards/margins": 6.097735404968262,
+      "rewards/rejected": -6.130982875823975,
+      "step": 153,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.887927927927928,
+      "grad_norm": 5.002453327178955,
+      "learning_rate": 0.00016853828056194697,
+      "logits/chosen": -0.47666168212890625,
+      "logits/rejected": -0.7901638150215149,
+      "logps/chosen": -1.6101099252700806,
+      "logps/rejected": -83.95780944824219,
+      "loss": 0.2384922355413437,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.15521878004074097,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.027309579774737358,
+      "rewards/margins": 6.557631492614746,
+      "rewards/rejected": -6.530322074890137,
+      "step": 154,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.8936936936936937,
+      "grad_norm": 2.2284231185913086,
+      "learning_rate": 0.00016807286490131196,
+      "logits/chosen": -0.4581748843193054,
+      "logits/rejected": -0.7216594815254211,
+      "logps/chosen": -4.746794700622559,
+      "logps/rejected": -65.14288330078125,
+      "loss": 0.4944155812263489,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.38365647196769714,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.09537286311388016,
+      "rewards/margins": 4.88486909866333,
+      "rewards/rejected": -4.789495944976807,
+      "step": 155,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.8994594594594595,
+      "grad_norm": 1.4425420761108398,
+      "learning_rate": 0.00016760468498337283,
+      "logits/chosen": -0.4141704738140106,
+      "logits/rejected": -0.753037691116333,
+      "logps/chosen": -2.9428489208221436,
+      "logps/rejected": -85.63114166259766,
+      "loss": 0.2691604495048523,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.22332294285297394,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.29303914308547974,
+      "rewards/margins": 6.752739906311035,
+      "rewards/rejected": -6.459700584411621,
+      "step": 156,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.9052252252252252,
+      "grad_norm": 3.795686960220337,
+      "learning_rate": 0.00016713375981966612,
+      "logits/chosen": -0.4735071659088135,
+      "logits/rejected": -0.7943378686904907,
+      "logps/chosen": -9.046199798583984,
+      "logps/rejected": -86.67741394042969,
+      "loss": 0.602267324924469,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.5201234817504883,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.15314966440200806,
+      "rewards/margins": 6.5488057136535645,
+      "rewards/rejected": -6.7019548416137695,
+      "step": 157,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.910990990990991,
+      "grad_norm": 4.890588760375977,
+      "learning_rate": 0.00016666010853320543,
+      "logits/chosen": -0.5752784013748169,
+      "logits/rejected": -0.7697316408157349,
+      "logps/chosen": -9.034675598144531,
+      "logps/rejected": -79.48356628417969,
+      "loss": 0.7191926836967468,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.5265187621116638,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.07981552183628082,
+      "rewards/margins": 6.082505226135254,
+      "rewards/rejected": -6.162321090698242,
+      "step": 158,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.9167567567567567,
+      "grad_norm": 1.312125563621521,
+      "learning_rate": 0.00016618375035770498,
+      "logits/chosen": -0.5153398513793945,
+      "logits/rejected": -0.8238294720649719,
+      "logps/chosen": -3.0522966384887695,
+      "logps/rejected": -84.81144714355469,
+      "loss": 0.32918140292167664,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2058570384979248,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.19958336651325226,
+      "rewards/margins": 6.894275665283203,
+      "rewards/rejected": -6.694692611694336,
+      "step": 159,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.9225225225225225,
+      "grad_norm": 1.3166769742965698,
+      "learning_rate": 0.00016570470463679856,
+      "logits/chosen": -0.4472997486591339,
+      "logits/rejected": -0.7798503041267395,
+      "logps/chosen": -2.2529990673065186,
+      "logps/rejected": -94.50987243652344,
+      "loss": 0.2068968117237091,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.13873136043548584,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.15320263803005219,
+      "rewards/margins": 7.273505687713623,
+      "rewards/rejected": -7.120303153991699,
+      "step": 160,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.9282882882882882,
+      "grad_norm": 1.4938724040985107,
+      "learning_rate": 0.00016522299082325382,
+      "logits/chosen": -0.4986165463924408,
+      "logits/rejected": -0.7801789045333862,
+      "logps/chosen": -2.9465794563293457,
+      "logps/rejected": -73.10553741455078,
+      "loss": 0.30796459317207336,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2202310413122177,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.38857752084732056,
+      "rewards/margins": 5.931979656219482,
+      "rewards/rejected": -5.543401718139648,
+      "step": 161,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.9340540540540541,
+      "grad_norm": 1.7879637479782104,
+      "learning_rate": 0.00016473862847818277,
+      "logits/chosen": -0.5118488073348999,
+      "logits/rejected": -0.8153480291366577,
+      "logps/chosen": -4.100103855133057,
+      "logps/rejected": -91.44921875,
+      "loss": 0.35479840636253357,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.30052870512008667,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.14176566898822784,
+      "rewards/margins": 7.297792911529541,
+      "rewards/rejected": -7.156026840209961,
+      "step": 162,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.9398198198198198,
+      "grad_norm": 2.240473985671997,
+      "learning_rate": 0.00016425163727024707,
+      "logits/chosen": -0.578972339630127,
+      "logits/rejected": -0.7973175048828125,
+      "logps/chosen": -4.10087776184082,
+      "logps/rejected": -75.45283508300781,
+      "loss": 0.41070497035980225,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.24780017137527466,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.003609701991081238,
+      "rewards/margins": 6.174129009246826,
+      "rewards/rejected": -6.17051887512207,
+      "step": 163,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.9455855855855856,
+      "grad_norm": 1.820946455001831,
+      "learning_rate": 0.0001637620369748595,
+      "logits/chosen": -0.5063561201095581,
+      "logits/rejected": -0.839270293712616,
+      "logps/chosen": -3.7182536125183105,
+      "logps/rejected": -102.9184341430664,
+      "loss": 0.2363234907388687,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.16097372770309448,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.07240395247936249,
+      "rewards/margins": 8.28852653503418,
+      "rewards/rejected": -8.216121673583984,
+      "step": 164,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.9513513513513514,
+      "grad_norm": 1.6907538175582886,
+      "learning_rate": 0.00016326984747338095,
+      "logits/chosen": -0.5052527785301208,
+      "logits/rejected": -0.7384231686592102,
+      "logps/chosen": -4.822022914886475,
+      "logps/rejected": -71.9358901977539,
+      "loss": 0.4056674838066101,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.30248117446899414,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.24947428703308105,
+      "rewards/margins": 5.717057704925537,
+      "rewards/rejected": -5.467583179473877,
+      "step": 165,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.9571171171171171,
+      "grad_norm": 2.038442850112915,
+      "learning_rate": 0.00016277508875231302,
+      "logits/chosen": -0.5684705972671509,
+      "logits/rejected": -0.7857838273048401,
+      "logps/chosen": -5.808892250061035,
+      "logps/rejected": -83.46935272216797,
+      "loss": 0.41192197799682617,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3203110098838806,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.05355273187160492,
+      "rewards/margins": 6.413900375366211,
+      "rewards/rejected": -6.360347747802734,
+      "step": 166,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.9628828828828829,
+      "grad_norm": 1.4706615209579468,
+      "learning_rate": 0.00016227778090248648,
+      "logits/chosen": -0.48619288206100464,
+      "logits/rejected": -0.7674732208251953,
+      "logps/chosen": -1.154846429824829,
+      "logps/rejected": -72.29942321777344,
+      "loss": 0.18448084592819214,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1356724500656128,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2431672066450119,
+      "rewards/margins": 5.5912065505981445,
+      "rewards/rejected": -5.348039150238037,
+      "step": 167,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.9686486486486486,
+      "grad_norm": 1.3399150371551514,
+      "learning_rate": 0.00016177794411824544,
+      "logits/chosen": -0.5658018589019775,
+      "logits/rejected": -0.8080291748046875,
+      "logps/chosen": -3.5855631828308105,
+      "logps/rejected": -70.36351013183594,
+      "loss": 0.33944055438041687,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2699050009250641,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.18274028599262238,
+      "rewards/margins": 5.833596229553223,
+      "rewards/rejected": -5.650856018066406,
+      "step": 168,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.9744144144144145,
+      "grad_norm": 3.7518484592437744,
+      "learning_rate": 0.00016127559869662722,
+      "logits/chosen": -0.46642014384269714,
+      "logits/rejected": -0.8066117167472839,
+      "logps/chosen": -3.712211847305298,
+      "logps/rejected": -80.26597595214844,
+      "loss": 0.25077763199806213,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.19776861369609833,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.023500367999076843,
+      "rewards/margins": 6.045979022979736,
+      "rewards/rejected": -6.069479942321777,
+      "step": 169,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.9801801801801802,
+      "grad_norm": 1.6079647541046143,
+      "learning_rate": 0.00016077076503653826,
+      "logits/chosen": -0.47919565439224243,
+      "logits/rejected": -0.7426984310150146,
+      "logps/chosen": -3.771472454071045,
+      "logps/rejected": -72.3167495727539,
+      "loss": 0.34788501262664795,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.26689645648002625,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.13779021799564362,
+      "rewards/margins": 5.563295364379883,
+      "rewards/rejected": -5.4255051612854,
+      "step": 170,
+      "train_speed(iter/s)": 0.010714
+    },
+    {
+      "epoch": 0.985945945945946,
+      "grad_norm": 3.0278408527374268,
+      "learning_rate": 0.00016026346363792567,
+      "logits/chosen": -0.431247353553772,
+      "logits/rejected": -0.7434531450271606,
+      "logps/chosen": -3.02232027053833,
+      "logps/rejected": -71.19493103027344,
+      "loss": 0.3027627468109131,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2018585056066513,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.1917632669210434,
+      "rewards/margins": 5.246973991394043,
+      "rewards/rejected": -5.055210113525391,
+      "step": 171,
+      "train_speed(iter/s)": 0.010713
+    },
+    {
+      "epoch": 0.9917117117117117,
+      "grad_norm": 1.8670094013214111,
+      "learning_rate": 0.00015975371510094485,
+      "logits/chosen": -0.5508197546005249,
+      "logits/rejected": -0.7615008354187012,
+      "logps/chosen": -6.864560604095459,
+      "logps/rejected": -65.1205825805664,
+      "loss": 0.4220251142978668,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.38794711232185364,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.46989697217941284,
+      "rewards/margins": 5.622817039489746,
+      "rewards/rejected": -5.152919769287109,
+      "step": 172,
+      "train_speed(iter/s)": 0.010714
+    },
+    {
+      "epoch": 0.9974774774774775,
+      "grad_norm": 1.1435422897338867,
+      "learning_rate": 0.00015924154012512308,
+      "logits/chosen": -0.5378420948982239,
+      "logits/rejected": -0.837982177734375,
+      "logps/chosen": -2.298398971557617,
+      "logps/rejected": -76.451904296875,
+      "loss": 0.18354284763336182,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1457872986793518,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.08272568881511688,
+      "rewards/margins": 5.861605167388916,
+      "rewards/rejected": -5.778879165649414,
+      "step": 173,
+      "train_speed(iter/s)": 0.010714
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 1.1435422897338867,
+      "learning_rate": 0.0001587269595085186,
+      "logits/chosen": -0.46139007806777954,
+      "logits/rejected": -0.787911593914032,
+      "logps/chosen": -2.2199597358703613,
+      "logps/rejected": -95.19585418701172,
+      "loss": 0.09489377588033676,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.14853443205356598,
+      "rewards/accuracies": 0.9285714030265808,
+      "rewards/chosen": 0.0750870555639267,
+      "rewards/margins": 7.632567882537842,
+      "rewards/rejected": -7.557480812072754,
+      "step": 174,
+      "train_speed(iter/s)": 0.010751
+    },
+    {
+      "epoch": 1.0057657657657657,
+      "grad_norm": 1.8974206447601318,
+      "learning_rate": 0.00015820999414687656,
+      "logits/chosen": -0.5884040594100952,
+      "logits/rejected": -0.821685254573822,
+      "logps/chosen": -2.4951252937316895,
+      "logps/rejected": -70.88331604003906,
+      "loss": 0.2344619631767273,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.19315816462039948,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.15108144283294678,
+      "rewards/margins": 5.873671531677246,
+      "rewards/rejected": -5.72258996963501,
+      "step": 175,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.0115315315315316,
+      "grad_norm": 0.6439478397369385,
+      "learning_rate": 0.00015769066503277997,
+      "logits/chosen": -0.52037113904953,
+      "logits/rejected": -0.7941011786460876,
+      "logps/chosen": -2.5466148853302,
+      "logps/rejected": -88.6285171508789,
+      "loss": 0.16956239938735962,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.14753001928329468,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19697943329811096,
+      "rewards/margins": 6.911043167114258,
+      "rewards/rejected": -6.714064121246338,
+      "step": 176,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.0172972972972973,
+      "grad_norm": 0.8241242170333862,
+      "learning_rate": 0.0001571689932547978,
+      "logits/chosen": -0.5486235618591309,
+      "logits/rejected": -0.8218496441841125,
+      "logps/chosen": -4.344364643096924,
+      "logps/rejected": -78.90025329589844,
+      "loss": 0.23550763726234436,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1828317493200302,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.23757538199424744,
+      "rewards/margins": 6.283997058868408,
+      "rewards/rejected": -6.046421527862549,
+      "step": 177,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.023063063063063,
+      "grad_norm": 0.8632805943489075,
+      "learning_rate": 0.00015664499999662815,
+      "logits/chosen": -0.5551503896713257,
+      "logits/rejected": -0.8251141309738159,
+      "logps/chosen": -2.980712890625,
+      "logps/rejected": -83.22628784179688,
+      "loss": 0.15064574778079987,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.13642281293869019,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2624984383583069,
+      "rewards/margins": 6.998701572418213,
+      "rewards/rejected": -6.736203193664551,
+      "step": 178,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.0288288288288288,
+      "grad_norm": 1.1632269620895386,
+      "learning_rate": 0.00015611870653623825,
+      "logits/chosen": -0.4921662211418152,
+      "logits/rejected": -0.8483188152313232,
+      "logps/chosen": -1.3713245391845703,
+      "logps/rejected": -85.42861938476562,
+      "loss": 0.1704276204109192,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1270376294851303,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.12698447704315186,
+      "rewards/margins": 6.958097457885742,
+      "rewards/rejected": -6.831112861633301,
+      "step": 179,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.0345945945945947,
+      "grad_norm": 1.6630468368530273,
+      "learning_rate": 0.00015559013424500047,
+      "logits/chosen": -0.6110640168190002,
+      "logits/rejected": -0.8926798105239868,
+      "logps/chosen": -2.6163129806518555,
+      "logps/rejected": -90.13837432861328,
+      "loss": 0.14455236494541168,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1374558061361313,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3422771096229553,
+      "rewards/margins": 7.565336227416992,
+      "rewards/rejected": -7.223059177398682,
+      "step": 180,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.0403603603603604,
+      "grad_norm": 1.077152132987976,
+      "learning_rate": 0.0001550593045868244,
+      "logits/chosen": -0.603461742401123,
+      "logits/rejected": -0.8620364665985107,
+      "logps/chosen": -4.633229732513428,
+      "logps/rejected": -76.99014282226562,
+      "loss": 0.40596532821655273,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3524565100669861,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.26899972558021545,
+      "rewards/margins": 6.21177864074707,
+      "rewards/rejected": -5.942779064178467,
+      "step": 181,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.046126126126126,
+      "grad_norm": 12.267168045043945,
+      "learning_rate": 0.00015452623911728523,
+      "logits/chosen": -0.6155453324317932,
+      "logits/rejected": -0.906221866607666,
+      "logps/chosen": -4.583263397216797,
+      "logps/rejected": -87.18157196044922,
+      "loss": 0.3373698592185974,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3133515417575836,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.26982182264328003,
+      "rewards/margins": 7.339932441711426,
+      "rewards/rejected": -7.070110321044922,
+      "step": 182,
+      "train_speed(iter/s)": 0.01075
+    },
+    {
+      "epoch": 1.0518918918918918,
+      "grad_norm": 0.7905959486961365,
+      "learning_rate": 0.00015399095948274852,
+      "logits/chosen": -0.5579067468643188,
+      "logits/rejected": -0.8554020524024963,
+      "logps/chosen": -1.4490176439285278,
+      "logps/rejected": -91.41957092285156,
+      "loss": 0.12704932689666748,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.09653271734714508,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.26760008931159973,
+      "rewards/margins": 7.848196029663086,
+      "rewards/rejected": -7.58059549331665,
+      "step": 183,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.0576576576576577,
+      "grad_norm": 1.1468485593795776,
+      "learning_rate": 0.00015345348741949117,
+      "logits/chosen": -0.5733897089958191,
+      "logits/rejected": -0.9108660221099854,
+      "logps/chosen": -3.66715669631958,
+      "logps/rejected": -95.78589630126953,
+      "loss": 0.24044549465179443,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.21640615165233612,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.16394057869911194,
+      "rewards/margins": 7.828693389892578,
+      "rewards/rejected": -7.664752960205078,
+      "step": 184,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.0634234234234234,
+      "grad_norm": 2.031723976135254,
+      "learning_rate": 0.00015291384475281877,
+      "logits/chosen": -0.49676764011383057,
+      "logits/rejected": -0.8489770293235779,
+      "logps/chosen": -1.7427412271499634,
+      "logps/rejected": -96.12220764160156,
+      "loss": 0.12766067683696747,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.11571483314037323,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.14926967024803162,
+      "rewards/margins": 7.909836769104004,
+      "rewards/rejected": -7.760568141937256,
+      "step": 185,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.0691891891891891,
+      "grad_norm": 2.5470285415649414,
+      "learning_rate": 0.00015237205339617917,
+      "logits/chosen": -0.5048971772193909,
+      "logits/rejected": -0.7917367219924927,
+      "logps/chosen": -4.352810382843018,
+      "logps/rejected": -81.40713500976562,
+      "loss": 0.31023305654525757,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2897438108921051,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.14250265061855316,
+      "rewards/margins": 6.402338981628418,
+      "rewards/rejected": -6.259836196899414,
+      "step": 186,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.0749549549549549,
+      "grad_norm": 1.091139316558838,
+      "learning_rate": 0.000151828135350273,
+      "logits/chosen": -0.5620720982551575,
+      "logits/rejected": -0.841740608215332,
+      "logps/chosen": -1.8221709728240967,
+      "logps/rejected": -82.46046447753906,
+      "loss": 0.15480715036392212,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.12393927574157715,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3422683775424957,
+      "rewards/margins": 6.792727470397949,
+      "rewards/rejected": -6.450459003448486,
+      "step": 187,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.0807207207207208,
+      "grad_norm": 0.9704499244689941,
+      "learning_rate": 0.00015128211270215992,
+      "logits/chosen": -0.6280016899108887,
+      "logits/rejected": -0.8943365812301636,
+      "logps/chosen": -2.3877384662628174,
+      "logps/rejected": -81.13067626953125,
+      "loss": 0.18478192389011383,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.16890017688274384,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.34920331835746765,
+      "rewards/margins": 6.807940483093262,
+      "rewards/rejected": -6.458736896514893,
+      "step": 188,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.0864864864864865,
+      "grad_norm": 1.0640044212341309,
+      "learning_rate": 0.00015073400762436197,
+      "logits/chosen": -0.6785641312599182,
+      "logits/rejected": -0.8755528330802917,
+      "logps/chosen": -3.03802490234375,
+      "logps/rejected": -80.31719970703125,
+      "loss": 0.23102031648159027,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.20030677318572998,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.299454927444458,
+      "rewards/margins": 6.583179473876953,
+      "rewards/rejected": -6.283725261688232,
+      "step": 189,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.0922522522522522,
+      "grad_norm": 1.2426904439926147,
+      "learning_rate": 0.00015018384237396292,
+      "logits/chosen": -0.6075233221054077,
+      "logits/rejected": -0.9030817747116089,
+      "logps/chosen": -3.1166131496429443,
+      "logps/rejected": -88.71156311035156,
+      "loss": 0.1606086939573288,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1422414779663086,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.22253718972206116,
+      "rewards/margins": 7.480426788330078,
+      "rewards/rejected": -7.257890224456787,
+      "step": 190,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.098018018018018,
+      "grad_norm": 13.646331787109375,
+      "learning_rate": 0.0001496316392917049,
+      "logits/chosen": -0.5769588351249695,
+      "logits/rejected": -0.9033970236778259,
+      "logps/chosen": -1.5371689796447754,
+      "logps/rejected": -104.17735290527344,
+      "loss": 0.10281458497047424,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.09561118483543396,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2810855209827423,
+      "rewards/margins": 8.708104133605957,
+      "rewards/rejected": -8.427018165588379,
+      "step": 191,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.1037837837837838,
+      "grad_norm": 1.8991550207138062,
+      "learning_rate": 0.00014907742080108073,
+      "logits/chosen": -0.6996522545814514,
+      "logits/rejected": -0.9485704302787781,
+      "logps/chosen": -3.0516557693481445,
+      "logps/rejected": -86.58012390136719,
+      "loss": 0.4684174954891205,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.4487941563129425,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19735433161258698,
+      "rewards/margins": 7.334392547607422,
+      "rewards/rejected": -7.137038230895996,
+      "step": 192,
+      "train_speed(iter/s)": 0.01075
+    },
+    {
+      "epoch": 1.1095495495495495,
+      "grad_norm": 19.20090675354004,
+      "learning_rate": 0.0001485212094074237,
+      "logits/chosen": -0.576482892036438,
+      "logits/rejected": -0.84342360496521,
+      "logps/chosen": -4.030856132507324,
+      "logps/rejected": -86.30321502685547,
+      "loss": 0.6711006760597229,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.6366056799888611,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.26511046290397644,
+      "rewards/margins": 7.336245536804199,
+      "rewards/rejected": -7.071135520935059,
+      "step": 193,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.1153153153153152,
+      "grad_norm": 0.9619101881980896,
+      "learning_rate": 0.0001479630276969936,
+      "logits/chosen": -0.6200534701347351,
+      "logits/rejected": -0.8699415922164917,
+      "logps/chosen": -3.693956136703491,
+      "logps/rejected": -93.85980987548828,
+      "loss": 0.09817313402891159,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.08495140075683594,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.26861289143562317,
+      "rewards/margins": 8.128625869750977,
+      "rewards/rejected": -7.860013008117676,
+      "step": 194,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.1210810810810812,
+      "grad_norm": 0.7348951101303101,
+      "learning_rate": 0.00014740289833605939,
+      "logits/chosen": -0.6436381936073303,
+      "logits/rejected": -0.9239126443862915,
+      "logps/chosen": -3.7549028396606445,
+      "logps/rejected": -93.33834838867188,
+      "loss": 0.26953551173210144,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.24437427520751953,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1677757203578949,
+      "rewards/margins": 7.494346618652344,
+      "rewards/rejected": -7.326571464538574,
+      "step": 195,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.1268468468468469,
+      "grad_norm": 5.2597246170043945,
+      "learning_rate": 0.00014684084406997903,
+      "logits/chosen": -0.6959158778190613,
+      "logits/rejected": -0.9090582132339478,
+      "logps/chosen": -2.6071348190307617,
+      "logps/rejected": -81.92942810058594,
+      "loss": 0.1599394530057907,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.13409104943275452,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.21387647092342377,
+      "rewards/margins": 6.953554630279541,
+      "rewards/rejected": -6.739677906036377,
+      "step": 196,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.1326126126126126,
+      "grad_norm": 0.9787479639053345,
+      "learning_rate": 0.0001462768877222757,
+      "logits/chosen": -0.5895656943321228,
+      "logits/rejected": -0.9225438833236694,
+      "logps/chosen": -1.224118947982788,
+      "logps/rejected": -100.8663558959961,
+      "loss": 0.06828276813030243,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.06501688808202744,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2447878122329712,
+      "rewards/margins": 8.5127534866333,
+      "rewards/rejected": -8.267965316772461,
+      "step": 197,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.1383783783783783,
+      "grad_norm": 0.7639157176017761,
+      "learning_rate": 0.00014571105219371102,
+      "logits/chosen": -0.6577166318893433,
+      "logits/rejected": -0.9819097518920898,
+      "logps/chosen": -0.7252616286277771,
+      "logps/rejected": -81.32587432861328,
+      "loss": 0.1278780996799469,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.10438857972621918,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.18735525012016296,
+      "rewards/margins": 6.4731879234313965,
+      "rewards/rejected": -6.28583288192749,
+      "step": 198,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.1441441441441442,
+      "grad_norm": 0.956177830696106,
+      "learning_rate": 0.00014514336046135518,
+      "logits/chosen": -0.7244867086410522,
+      "logits/rejected": -0.9211379289627075,
+      "logps/chosen": -3.929692268371582,
+      "logps/rejected": -73.1805419921875,
+      "loss": 0.2801938056945801,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.22287607192993164,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.23973776400089264,
+      "rewards/margins": 6.041521072387695,
+      "rewards/rejected": -5.801782608032227,
+      "step": 199,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.14990990990991,
+      "grad_norm": 0.9859761595726013,
+      "learning_rate": 0.00014457383557765386,
+      "logits/chosen": -0.6032741069793701,
+      "logits/rejected": -0.8709255456924438,
+      "logps/chosen": -2.3330612182617188,
+      "logps/rejected": -78.04156494140625,
+      "loss": 0.21972186863422394,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.16077226400375366,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6756027936935425,
+      "rewards/margins": 6.16083288192749,
+      "rewards/rejected": -5.485230445861816,
+      "step": 200,
+      "train_speed(iter/s)": 0.010749
+    },
+    {
+      "epoch": 1.1556756756756756,
+      "grad_norm": 2.053891658782959,
+      "learning_rate": 0.00014400250066949196,
+      "logits/chosen": -0.7468363642692566,
+      "logits/rejected": -0.9693096876144409,
+      "logps/chosen": -2.202683210372925,
+      "logps/rejected": -83.35052490234375,
+      "loss": 0.1414460837841034,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1286502480506897,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2432466298341751,
+      "rewards/margins": 7.013325214385986,
+      "rewards/rejected": -6.770079612731934,
+      "step": 201,
+      "train_speed(iter/s)": 0.010738
+    },
+    {
+      "epoch": 1.1614414414414413,
+      "grad_norm": 0.7447286248207092,
+      "learning_rate": 0.00014342937893725488,
+      "logits/chosen": -0.6638346910476685,
+      "logits/rejected": -0.9504916071891785,
+      "logps/chosen": -1.4076967239379883,
+      "logps/rejected": -91.8344955444336,
+      "loss": 0.11076842993497849,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.0927545577287674,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2952735722064972,
+      "rewards/margins": 7.439762592315674,
+      "rewards/rejected": -7.144488334655762,
+      "step": 202,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.1672072072072073,
+      "grad_norm": 1.7305192947387695,
+      "learning_rate": 0.00014285449365388598,
+      "logits/chosen": -0.6913129687309265,
+      "logits/rejected": -1.0035713911056519,
+      "logps/chosen": -2.595938205718994,
+      "logps/rejected": -93.46743774414062,
+      "loss": 0.1786770224571228,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.15392035245895386,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.16032256186008453,
+      "rewards/margins": 7.787504196166992,
+      "rewards/rejected": -7.627182483673096,
+      "step": 203,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.172972972972973,
+      "grad_norm": 0.7990829944610596,
+      "learning_rate": 0.00014227786816394184,
+      "logits/chosen": -0.7262576818466187,
+      "logits/rejected": -1.0095360279083252,
+      "logps/chosen": -4.604730606079102,
+      "logps/rejected": -94.75875091552734,
+      "loss": 0.1926305592060089,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.18425899744033813,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.40172773599624634,
+      "rewards/margins": 7.936948776245117,
+      "rewards/rejected": -7.535220623016357,
+      "step": 204,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.1787387387387387,
+      "grad_norm": 0.7804811596870422,
+      "learning_rate": 0.00014169952588264417,
+      "logits/chosen": -0.7492111921310425,
+      "logits/rejected": -1.052622675895691,
+      "logps/chosen": -2.9088637828826904,
+      "logps/rejected": -99.19158935546875,
+      "loss": 0.13855330646038055,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.12824688851833344,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.23516345024108887,
+      "rewards/margins": 8.24970531463623,
+      "rewards/rejected": -8.014541625976562,
+      "step": 205,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.1845045045045044,
+      "grad_norm": 1.3688875436782837,
+      "learning_rate": 0.0001411194902949289,
+      "logits/chosen": -0.7413324117660522,
+      "logits/rejected": -1.0183649063110352,
+      "logps/chosen": -3.5715129375457764,
+      "logps/rejected": -105.68714141845703,
+      "loss": 0.2724800109863281,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.25694820284843445,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.313816636800766,
+      "rewards/margins": 8.691929817199707,
+      "rewards/rejected": -8.37811279296875,
+      "step": 206,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.1902702702702703,
+      "grad_norm": 2.0642895698547363,
+      "learning_rate": 0.0001405377849544927,
+      "logits/chosen": -0.6753020286560059,
+      "logits/rejected": -1.0321030616760254,
+      "logps/chosen": -2.917701005935669,
+      "logps/rejected": -96.5132827758789,
+      "loss": 0.17677046358585358,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.15049855411052704,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.27416104078292847,
+      "rewards/margins": 7.89042854309082,
+      "rewards/rejected": -7.616267681121826,
+      "step": 207,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.196036036036036,
+      "grad_norm": 1.8087902069091797,
+      "learning_rate": 0.00013995443348283645,
+      "logits/chosen": -0.6556892395019531,
+      "logits/rejected": -0.991775631904602,
+      "logps/chosen": -2.722226858139038,
+      "logps/rejected": -102.24663543701172,
+      "loss": 0.24504928290843964,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.20121188461780548,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2066883146762848,
+      "rewards/margins": 8.376320838928223,
+      "rewards/rejected": -8.169632911682129,
+      "step": 208,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.2018018018018017,
+      "grad_norm": 1.0671130418777466,
+      "learning_rate": 0.00013936945956830602,
+      "logits/chosen": -0.7256425023078918,
+      "logits/rejected": -1.002614140510559,
+      "logps/chosen": -3.5971474647521973,
+      "logps/rejected": -90.5836181640625,
+      "loss": 0.22293436527252197,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.18081282079219818,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19337910413742065,
+      "rewards/margins": 7.3919477462768555,
+      "rewards/rejected": -7.198568344116211,
+      "step": 209,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.2075675675675677,
+      "grad_norm": 1.3437083959579468,
+      "learning_rate": 0.00013878288696513022,
+      "logits/chosen": -0.7431738376617432,
+      "logits/rejected": -1.0425618886947632,
+      "logps/chosen": -2.174896240234375,
+      "logps/rejected": -93.76087951660156,
+      "loss": 0.15365180373191833,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.14746138453483582,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2575794756412506,
+      "rewards/margins": 7.875104904174805,
+      "rewards/rejected": -7.617526054382324,
+      "step": 210,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.2133333333333334,
+      "grad_norm": 0.76457679271698,
+      "learning_rate": 0.00013819473949245654,
+      "logits/chosen": -0.6461102962493896,
+      "logits/rejected": -1.0543663501739502,
+      "logps/chosen": -0.8342275619506836,
+      "logps/rejected": -104.50704193115234,
+      "loss": 0.09112594276666641,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.08857303112745285,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.23656663298606873,
+      "rewards/margins": 8.866386413574219,
+      "rewards/rejected": -8.629819869995117,
+      "step": 211,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.219099099099099,
+      "grad_norm": 0.6871019005775452,
+      "learning_rate": 0.0001376050410333836,
+      "logits/chosen": -0.6719679236412048,
+      "logits/rejected": -0.9832717776298523,
+      "logps/chosen": -3.1014583110809326,
+      "logps/rejected": -98.14430236816406,
+      "loss": 0.3030906915664673,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.21772870421409607,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.02887733280658722,
+      "rewards/margins": 7.6625895500183105,
+      "rewards/rejected": -7.633711814880371,
+      "step": 212,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.2248648648648648,
+      "grad_norm": 2.0604000091552734,
+      "learning_rate": 0.00013701381553399145,
+      "logits/chosen": -0.6421749591827393,
+      "logits/rejected": -1.0211272239685059,
+      "logps/chosen": -2.438856840133667,
+      "logps/rejected": -86.31884002685547,
+      "loss": 0.20499145984649658,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1656690537929535,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1947031468153,
+      "rewards/margins": 7.004463195800781,
+      "rewards/rejected": -6.809760093688965,
+      "step": 213,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.2306306306306307,
+      "grad_norm": 2.321046829223633,
+      "learning_rate": 0.00013642108700236916,
+      "logits/chosen": -0.7420238852500916,
+      "logits/rejected": -1.0413094758987427,
+      "logps/chosen": -0.9865455031394958,
+      "logps/rejected": -72.17105102539062,
+      "loss": 0.13595286011695862,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.10351557284593582,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.11004337668418884,
+      "rewards/margins": 5.79013204574585,
+      "rewards/rejected": -5.680088520050049,
+      "step": 214,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.2363963963963964,
+      "grad_norm": 4.301558494567871,
+      "learning_rate": 0.00013582687950764,
+      "logits/chosen": -0.6356464624404907,
+      "logits/rejected": -0.986944854259491,
+      "logps/chosen": -1.5492937564849854,
+      "logps/rejected": -79.20093536376953,
+      "loss": 0.19731514155864716,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.15034356713294983,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.15903238952159882,
+      "rewards/margins": 6.342228412628174,
+      "rewards/rejected": -6.1831955909729,
+      "step": 215,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.2421621621621621,
+      "grad_norm": 1.1075836420059204,
+      "learning_rate": 0.00013523121717898387,
+      "logits/chosen": -0.6029863953590393,
+      "logits/rejected": -0.9658622741699219,
+      "logps/chosen": -3.6208183765411377,
+      "logps/rejected": -97.2186508178711,
+      "loss": 0.4900016784667969,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.44314447045326233,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.04553697258234024,
+      "rewards/margins": 7.420025825500488,
+      "rewards/rejected": -7.374488830566406,
+      "step": 216,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.2479279279279278,
+      "grad_norm": 2.592005968093872,
+      "learning_rate": 0.00013463412420465767,
+      "logits/chosen": -0.6447511911392212,
+      "logits/rejected": -0.9609813094139099,
+      "logps/chosen": -2.2061100006103516,
+      "logps/rejected": -88.9801254272461,
+      "loss": 0.21441426873207092,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.20333540439605713,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2582980692386627,
+      "rewards/margins": 6.966536521911621,
+      "rewards/rejected": -6.70823860168457,
+      "step": 217,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.2536936936936938,
+      "grad_norm": 1.5074249505996704,
+      "learning_rate": 0.00013403562483101298,
+      "logits/chosen": -0.66890949010849,
+      "logits/rejected": -0.8881216049194336,
+      "logps/chosen": -4.03311014175415,
+      "logps/rejected": -70.53189086914062,
+      "loss": 0.306526780128479,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.22115619480609894,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.29509076476097107,
+      "rewards/margins": 5.955531120300293,
+      "rewards/rejected": -5.660440444946289,
+      "step": 218,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.2594594594594595,
+      "grad_norm": 0.588188886642456,
+      "learning_rate": 0.00013343574336151153,
+      "logits/chosen": -0.6300919055938721,
+      "logits/rejected": -0.9817159175872803,
+      "logps/chosen": -2.001368761062622,
+      "logps/rejected": -94.01361083984375,
+      "loss": 0.3623042702674866,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2725469172000885,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.15840183198451996,
+      "rewards/margins": 7.825695991516113,
+      "rewards/rejected": -7.667293548583984,
+      "step": 219,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.2652252252252252,
+      "grad_norm": 14.042351722717285,
+      "learning_rate": 0.0001328345041557382,
+      "logits/chosen": -0.759201169013977,
+      "logits/rejected": -1.016777515411377,
+      "logps/chosen": -3.4606502056121826,
+      "logps/rejected": -93.0523910522461,
+      "loss": 0.20867016911506653,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.15421104431152344,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.11695647984743118,
+      "rewards/margins": 7.954014301300049,
+      "rewards/rejected": -7.8370585441589355,
+      "step": 220,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.2709909909909909,
+      "grad_norm": 1.2167177200317383,
+      "learning_rate": 0.000132231931628412,
+      "logits/chosen": -0.5667409896850586,
+      "logits/rejected": -0.8965315222740173,
+      "logps/chosen": -3.431696891784668,
+      "logps/rejected": -105.54657745361328,
+      "loss": 0.1707317978143692,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1522923707962036,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.24732309579849243,
+      "rewards/margins": 8.84961223602295,
+      "rewards/rejected": -8.602288246154785,
+      "step": 221,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.2767567567567568,
+      "grad_norm": 0.9449612498283386,
+      "learning_rate": 0.00013162805024839448,
+      "logits/chosen": -0.6432092189788818,
+      "logits/rejected": -0.8541032075881958,
+      "logps/chosen": -3.5057826042175293,
+      "logps/rejected": -85.99465942382812,
+      "loss": 0.2346688210964203,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.19423571228981018,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5884712338447571,
+      "rewards/margins": 7.24459981918335,
+      "rewards/rejected": -6.656128883361816,
+      "step": 222,
+      "train_speed(iter/s)": 0.010736
+    },
+    {
+      "epoch": 1.2825225225225225,
+      "grad_norm": 1.486234426498413,
+      "learning_rate": 0.00013102288453769632,
+      "logits/chosen": -0.6213188171386719,
+      "logits/rejected": -0.9732288718223572,
+      "logps/chosen": -2.9368274211883545,
+      "logps/rejected": -108.50143432617188,
+      "loss": 0.19141153991222382,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1635173112154007,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.09779652208089828,
+      "rewards/margins": 8.9135103225708,
+      "rewards/rejected": -8.815712928771973,
+      "step": 223,
+      "train_speed(iter/s)": 0.010736
+    },
+    {
+      "epoch": 1.2882882882882882,
+      "grad_norm": 1.301498532295227,
+      "learning_rate": 0.0001304164590704813,
+      "logits/chosen": -0.6701276302337646,
+      "logits/rejected": -0.9731598496437073,
+      "logps/chosen": -3.8457467555999756,
+      "logps/rejected": -93.66019439697266,
+      "loss": 0.28035908937454224,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.24942338466644287,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.4500879943370819,
+      "rewards/margins": 8.23204517364502,
+      "rewards/rejected": -7.781956672668457,
+      "step": 224,
+      "train_speed(iter/s)": 0.010736
+    },
+    {
+      "epoch": 1.2940540540540542,
+      "grad_norm": 1.0099408626556396,
+      "learning_rate": 0.0001298087984720687,
+      "logits/chosen": -0.7134125232696533,
+      "logits/rejected": -0.9936617612838745,
+      "logps/chosen": -2.324589967727661,
+      "logps/rejected": -90.27128601074219,
+      "loss": 0.39343133568763733,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2453940212726593,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.25489863753318787,
+      "rewards/margins": 7.6775102615356445,
+      "rewards/rejected": -7.422612190246582,
+      "step": 225,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.2998198198198199,
+      "grad_norm": 2.37233829498291,
+      "learning_rate": 0.00012919992741793307,
+      "logits/chosen": -0.6083082556724548,
+      "logits/rejected": -0.9392067193984985,
+      "logps/chosen": -2.33506441116333,
+      "logps/rejected": -91.54042053222656,
+      "loss": 0.18458783626556396,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.17687150835990906,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.32263070344924927,
+      "rewards/margins": 7.811326503753662,
+      "rewards/rejected": -7.488696098327637,
+      "step": 226,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.3055855855855856,
+      "grad_norm": 0.9204962253570557,
+      "learning_rate": 0.0001285898706327023,
+      "logits/chosen": -0.6583359837532043,
+      "logits/rejected": -0.9856835007667542,
+      "logps/chosen": -2.480973720550537,
+      "logps/rejected": -98.34219360351562,
+      "loss": 0.22872069478034973,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2118297517299652,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.37609827518463135,
+      "rewards/margins": 8.75016975402832,
+      "rewards/rejected": -8.37407112121582,
+      "step": 227,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.3113513513513513,
+      "grad_norm": 1.0065797567367554,
+      "learning_rate": 0.0001279786528891538,
+      "logits/chosen": -0.564249575138092,
+      "logits/rejected": -0.8596559762954712,
+      "logps/chosen": -2.2762789726257324,
+      "logps/rejected": -94.75420379638672,
+      "loss": 0.09863093495368958,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.09180156141519547,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.30846285820007324,
+      "rewards/margins": 8.076614379882812,
+      "rewards/rejected": -7.76815128326416,
+      "step": 228,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.317117117117117,
+      "grad_norm": 15.840254783630371,
+      "learning_rate": 0.0001273662990072083,
+      "logits/chosen": -0.6420232057571411,
+      "logits/rejected": -0.9717917442321777,
+      "logps/chosen": -4.723894119262695,
+      "logps/rejected": -91.68743896484375,
+      "loss": 0.5773552656173706,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.4086422324180603,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.09256955981254578,
+      "rewards/margins": 7.509539604187012,
+      "rewards/rejected": -7.416969299316406,
+      "step": 229,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.322882882882883,
+      "grad_norm": 0.9773613810539246,
+      "learning_rate": 0.00012675283385292212,
+      "logits/chosen": -0.5419533252716064,
+      "logits/rejected": -0.908346951007843,
+      "logps/chosen": -2.8323910236358643,
+      "logps/rejected": -92.77593994140625,
+      "loss": 0.18030104041099548,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1703605055809021,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3452431559562683,
+      "rewards/margins": 7.826876163482666,
+      "rewards/rejected": -7.48163366317749,
+      "step": 230,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.3286486486486486,
+      "grad_norm": 0.5861178636550903,
+      "learning_rate": 0.00012613828233747727,
+      "logits/chosen": -0.5227465629577637,
+      "logits/rejected": -0.867357611656189,
+      "logps/chosen": -2.319690704345703,
+      "logps/rejected": -90.09024810791016,
+      "loss": 0.17545996606349945,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.14297524094581604,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.15114043653011322,
+      "rewards/margins": 7.286042213439941,
+      "rewards/rejected": -7.134902477264404,
+      "step": 231,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.3344144144144146,
+      "grad_norm": 1.1118627786636353,
+      "learning_rate": 0.00012552266941617018,
+      "logits/chosen": -0.6113250851631165,
+      "logits/rejected": -0.7813447713851929,
+      "logps/chosen": -4.297638893127441,
+      "logps/rejected": -73.71047973632812,
+      "loss": 0.2730986773967743,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2158653289079666,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.24944612383842468,
+      "rewards/margins": 6.196266174316406,
+      "rewards/rejected": -5.946820259094238,
+      "step": 232,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.3401801801801803,
+      "grad_norm": 0.8542975783348083,
+      "learning_rate": 0.000124906020087398,
+      "logits/chosen": -0.45566728711128235,
+      "logits/rejected": -0.8274158239364624,
+      "logps/chosen": -1.3975021839141846,
+      "logps/rejected": -92.9134750366211,
+      "loss": 0.10510716587305069,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.08654335141181946,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.292056143283844,
+      "rewards/margins": 7.4350056648254395,
+      "rewards/rejected": -7.142949104309082,
+      "step": 233,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.345945945945946,
+      "grad_norm": 0.6710683703422546,
+      "learning_rate": 0.00012428835939164363,
+      "logits/chosen": -0.48423632979393005,
+      "logits/rejected": -0.931830883026123,
+      "logps/chosen": -0.9655717611312866,
+      "logps/rejected": -103.34365844726562,
+      "loss": 0.10704699903726578,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.10177168250083923,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.27281564474105835,
+      "rewards/margins": 8.59388256072998,
+      "rewards/rejected": -8.321066856384277,
+      "step": 234,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.3517117117117117,
+      "grad_norm": 1.0295995473861694,
+      "learning_rate": 0.00012366971241045894,
+      "logits/chosen": -0.5070984959602356,
+      "logits/rejected": -0.8951232433319092,
+      "logps/chosen": -1.928631067276001,
+      "logps/rejected": -88.40235900878906,
+      "loss": 0.21813911199569702,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.16451862454414368,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.064000703394413,
+      "rewards/margins": 7.2806854248046875,
+      "rewards/rejected": -7.216684341430664,
+      "step": 235,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.3574774774774774,
+      "grad_norm": 0.8053532242774963,
+      "learning_rate": 0.00012305010426544614,
+      "logits/chosen": -0.5998705625534058,
+      "logits/rejected": -0.9042835831642151,
+      "logps/chosen": -4.446467876434326,
+      "logps/rejected": -86.84009552001953,
+      "loss": 0.25722736120224,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.24677737057209015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.43149375915527344,
+      "rewards/margins": 7.540010929107666,
+      "rewards/rejected": -7.108516693115234,
+      "step": 236,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.3632432432432433,
+      "grad_norm": 1.9170420169830322,
+      "learning_rate": 0.00012242956011723782,
+      "logits/chosen": -0.5368235111236572,
+      "logits/rejected": -0.8908076286315918,
+      "logps/chosen": -1.4597830772399902,
+      "logps/rejected": -100.62641906738281,
+      "loss": 0.1386258900165558,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.13322040438652039,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2073771357536316,
+      "rewards/margins": 8.687051773071289,
+      "rewards/rejected": -8.479674339294434,
+      "step": 237,
+      "train_speed(iter/s)": 0.010736
+    },
+    {
+      "epoch": 1.369009009009009,
+      "grad_norm": 1.0038492679595947,
+      "learning_rate": 0.00012180810516447512,
+      "logits/chosen": -0.5559844970703125,
+      "logits/rejected": -0.8453220725059509,
+      "logps/chosen": -3.5673227310180664,
+      "logps/rejected": -92.9088134765625,
+      "loss": 0.23567795753479004,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.20739604532718658,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.27056020498275757,
+      "rewards/margins": 7.564149379730225,
+      "rewards/rejected": -7.293589115142822,
+      "step": 238,
+      "train_speed(iter/s)": 0.010736
+    },
+    {
+      "epoch": 1.3747747747747747,
+      "grad_norm": 0.7483298182487488,
+      "learning_rate": 0.0001211857646427845,
+      "logits/chosen": -0.5241140723228455,
+      "logits/rejected": -0.8691558837890625,
+      "logps/chosen": -1.9950472116470337,
+      "logps/rejected": -82.87139892578125,
+      "loss": 0.15872085094451904,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.12550276517868042,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3112674951553345,
+      "rewards/margins": 6.984847545623779,
+      "rewards/rejected": -6.673579216003418,
+      "step": 239,
+      "train_speed(iter/s)": 0.010736
+    },
+    {
+      "epoch": 1.3805405405405407,
+      "grad_norm": 0.9307670593261719,
+      "learning_rate": 0.00012056256382375308,
+      "logits/chosen": -0.5428832769393921,
+      "logits/rejected": -0.9613853096961975,
+      "logps/chosen": -1.8748772144317627,
+      "logps/rejected": -97.14224243164062,
+      "loss": 0.12880003452301025,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.12042552977800369,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.31744644045829773,
+      "rewards/margins": 8.378560066223145,
+      "rewards/rejected": -8.061113357543945,
+      "step": 240,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.3863063063063064,
+      "grad_norm": 2.9521400928497314,
+      "learning_rate": 0.00011993852801390226,
+      "logits/chosen": -0.4635058045387268,
+      "logits/rejected": -0.8949097394943237,
+      "logps/chosen": -0.5122127532958984,
+      "logps/rejected": -96.43069458007812,
+      "loss": 0.10679192841053009,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.053764842450618744,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.165335014462471,
+      "rewards/margins": 8.006298065185547,
+      "rewards/rejected": -7.840963840484619,
+      "step": 241,
+      "train_speed(iter/s)": 0.010737
+    },
+    {
+      "epoch": 1.392072072072072,
+      "grad_norm": 1.0468616485595703,
+      "learning_rate": 0.00011931368255366027,
+      "logits/chosen": -0.45876914262771606,
+      "logits/rejected": -0.831932544708252,
+      "logps/chosen": -2.4560790061950684,
+      "logps/rejected": -97.22357177734375,
+      "loss": 0.2074245661497116,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.17626309394836426,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.24911226332187653,
+      "rewards/margins": 8.137984275817871,
+      "rewards/rejected": -7.888872146606445,
+      "step": 242,
+      "train_speed(iter/s)": 0.010736
+    },
+    {
+      "epoch": 1.3978378378378378,
+      "grad_norm": 0.7075209021568298,
+      "learning_rate": 0.00011868805281633304,
+      "logits/chosen": -0.58433997631073,
+      "logits/rejected": -0.9207597970962524,
+      "logps/chosen": -2.2052416801452637,
+      "logps/rejected": -101.76717376708984,
+      "loss": 0.18332861363887787,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1732126772403717,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4298679530620575,
+      "rewards/margins": 8.984416961669922,
+      "rewards/rejected": -8.554548263549805,
+      "step": 243,
+      "train_speed(iter/s)": 0.010736
+    },
+    {
+      "epoch": 1.4036036036036035,
+      "grad_norm": 0.9815582036972046,
+      "learning_rate": 0.00011806166420707392,
+      "logits/chosen": -0.48037832975387573,
+      "logits/rejected": -0.9194313883781433,
+      "logps/chosen": -1.8615732192993164,
+      "logps/rejected": -102.38887023925781,
+      "loss": 0.21181951463222504,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.15694458782672882,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.29181793332099915,
+      "rewards/margins": 8.388344764709473,
+      "rewards/rejected": -8.096527099609375,
+      "step": 244,
+      "train_speed(iter/s)": 0.010736
+    },
+    {
+      "epoch": 1.4093693693693694,
+      "grad_norm": 2.933603525161743,
+      "learning_rate": 0.00011743454216185201,
+      "logits/chosen": -0.640617311000824,
+      "logits/rejected": -0.9676884412765503,
+      "logps/chosen": -3.7124361991882324,
+      "logps/rejected": -97.49540710449219,
+      "loss": 0.2786301076412201,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.26465094089508057,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.17746347188949585,
+      "rewards/margins": 8.408309936523438,
+      "rewards/rejected": -8.23084545135498,
+      "step": 245,
+      "train_speed(iter/s)": 0.010736
+    },
+    {
+      "epoch": 1.4151351351351351,
+      "grad_norm": 0.6771543025970459,
+      "learning_rate": 0.00011680671214641927,
+      "logits/chosen": -0.4917660057544708,
+      "logits/rejected": -0.9493937492370605,
+      "logps/chosen": -1.4353026151657104,
+      "logps/rejected": -133.36654663085938,
+      "loss": 0.10904774069786072,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.10567378252744675,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.24828633666038513,
+      "rewards/margins": 11.476335525512695,
+      "rewards/rejected": -11.228048324584961,
+      "step": 246,
+      "train_speed(iter/s)": 0.010736
+    },
+    {
+      "epoch": 1.4209009009009008,
+      "grad_norm": 0.5642514824867249,
+      "learning_rate": 0.0001161781996552765,
+      "logits/chosen": -0.5594913363456726,
+      "logits/rejected": -1.0035367012023926,
+      "logps/chosen": -2.4414238929748535,
+      "logps/rejected": -129.03379821777344,
+      "loss": 0.1480759233236313,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1389404833316803,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.01771201565861702,
+      "rewards/margins": 11.155719757080078,
+      "rewards/rejected": -11.138009071350098,
+      "step": 247,
+      "train_speed(iter/s)": 0.010736
+    },
+    {
+      "epoch": 1.4266666666666667,
+      "grad_norm": 2.8613696098327637,
+      "learning_rate": 0.0001155490302106379,
+      "logits/chosen": -0.5504012703895569,
+      "logits/rejected": -0.993097722530365,
+      "logps/chosen": -1.1884583234786987,
+      "logps/rejected": -113.57362365722656,
+      "loss": 0.11941150575876236,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.10435314476490021,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.20970577001571655,
+      "rewards/margins": 9.547948837280273,
+      "rewards/rejected": -9.338242530822754,
+      "step": 248,
+      "train_speed(iter/s)": 0.010736
+    },
+    {
+      "epoch": 1.4324324324324325,
+      "grad_norm": 0.8018273711204529,
+      "learning_rate": 0.00011491922936139499,
+      "logits/chosen": -0.5908212661743164,
+      "logits/rejected": -0.9416083693504333,
+      "logps/chosen": -2.546377182006836,
+      "logps/rejected": -98.2177963256836,
+      "loss": 0.20611125230789185,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.19133996963500977,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.17961813509464264,
+      "rewards/margins": 8.596184730529785,
+      "rewards/rejected": -8.416565895080566,
+      "step": 249,
+      "train_speed(iter/s)": 0.010736
+    },
+    {
+      "epoch": 1.4381981981981982,
+      "grad_norm": 1.2365566492080688,
+      "learning_rate": 0.00011428882268207872,
+      "logits/chosen": -0.6211739778518677,
+      "logits/rejected": -0.8576729893684387,
+      "logps/chosen": -6.602561950683594,
+      "logps/rejected": -92.08356475830078,
+      "loss": 0.3254232406616211,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2950092554092407,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.45778965950012207,
+      "rewards/margins": 8.364259719848633,
+      "rewards/rejected": -7.90647029876709,
+      "step": 250,
+      "train_speed(iter/s)": 0.010736
+    },
+    {
+      "epoch": 1.4439639639639639,
+      "grad_norm": 0.8326837420463562,
+      "learning_rate": 0.00011365783577182132,
+      "logits/chosen": -0.5177187919616699,
+      "logits/rejected": -0.9024335145950317,
+      "logps/chosen": -2.21883225440979,
+      "logps/rejected": -105.33366394042969,
+      "loss": 0.1589798927307129,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.13007135689258575,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3507290780544281,
+      "rewards/margins": 9.073866844177246,
+      "rewards/rejected": -8.723137855529785,
+      "step": 251,
+      "train_speed(iter/s)": 0.010728
+    },
+    {
+      "epoch": 1.4497297297297298,
+      "grad_norm": 1.8344132900238037,
+      "learning_rate": 0.00011302629425331666,
+      "logits/chosen": -0.603495180606842,
+      "logits/rejected": -0.801387369632721,
+      "logps/chosen": -5.291406154632568,
+      "logps/rejected": -84.78588104248047,
+      "loss": 0.3241059482097626,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2689138948917389,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2446574866771698,
+      "rewards/margins": 7.256671905517578,
+      "rewards/rejected": -7.012013912200928,
+      "step": 252,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 1.4554954954954955,
+      "grad_norm": 0.6599681377410889,
+      "learning_rate": 0.00011239422377177973,
+      "logits/chosen": -0.5089681148529053,
+      "logits/rejected": -0.8658185005187988,
+      "logps/chosen": -3.079047679901123,
+      "logps/rejected": -127.54818725585938,
+      "loss": 0.17893767356872559,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.17471365630626678,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5883893370628357,
+      "rewards/margins": 11.444487571716309,
+      "rewards/rejected": -10.856100082397461,
+      "step": 253,
+      "train_speed(iter/s)": 0.010728
+    },
+    {
+      "epoch": 1.4612612612612612,
+      "grad_norm": 0.6386237144470215,
+      "learning_rate": 0.00011176164999390522,
+      "logits/chosen": -0.5488162040710449,
+      "logits/rejected": -0.9546905755996704,
+      "logps/chosen": -3.4519522190093994,
+      "logps/rejected": -109.98165893554688,
+      "loss": 0.22669866681098938,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.20005784928798676,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3646531105041504,
+      "rewards/margins": 9.544320106506348,
+      "rewards/rejected": -9.179666519165039,
+      "step": 254,
+      "train_speed(iter/s)": 0.010728
+    },
+    {
+      "epoch": 1.4670270270270271,
+      "grad_norm": 17.654590606689453,
+      "learning_rate": 0.00011112859860682547,
+      "logits/chosen": -0.5480679869651794,
+      "logits/rejected": -0.9488638639450073,
+      "logps/chosen": -3.5623536109924316,
+      "logps/rejected": -115.0560073852539,
+      "loss": 0.4454716145992279,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3780556321144104,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.10135876387357712,
+      "rewards/margins": 9.828811645507812,
+      "rewards/rejected": -9.727452278137207,
+      "step": 255,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 1.4727927927927928,
+      "grad_norm": 0.6790575385093689,
+      "learning_rate": 0.0001104950953170672,
+      "logits/chosen": -0.5322855710983276,
+      "logits/rejected": -0.8911909461021423,
+      "logps/chosen": -2.056039571762085,
+      "logps/rejected": -107.23054504394531,
+      "loss": 0.2234971523284912,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.20333045721054077,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2641701102256775,
+      "rewards/margins": 9.115086555480957,
+      "rewards/rejected": -8.850915908813477,
+      "step": 256,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 1.4785585585585586,
+      "grad_norm": 0.8362141847610474,
+      "learning_rate": 0.00010986116584950774,
+      "logits/chosen": -0.46502387523651123,
+      "logits/rejected": -0.8917255401611328,
+      "logps/chosen": -1.898627758026123,
+      "logps/rejected": -115.86322021484375,
+      "loss": 0.12826178967952728,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.11593392491340637,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.29406896233558655,
+      "rewards/margins": 9.646780014038086,
+      "rewards/rejected": -9.352710723876953,
+      "step": 257,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 1.4843243243243243,
+      "grad_norm": 0.681954026222229,
+      "learning_rate": 0.00010922683594633021,
+      "logits/chosen": -0.5689491033554077,
+      "logits/rejected": -0.8625025153160095,
+      "logps/chosen": -1.7617994546890259,
+      "logps/rejected": -99.32975769042969,
+      "loss": 0.1266973614692688,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.11154472082853317,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.283980131149292,
+      "rewards/margins": 8.681015014648438,
+      "rewards/rejected": -8.397034645080566,
+      "step": 258,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 1.49009009009009,
+      "grad_norm": 0.6331071853637695,
+      "learning_rate": 0.00010859213136597853,
+      "logits/chosen": -0.5191855430603027,
+      "logits/rejected": -0.8886591196060181,
+      "logps/chosen": -1.3958189487457275,
+      "logps/rejected": -97.13671112060547,
+      "loss": 0.13129664957523346,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.10424058139324188,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.33293408155441284,
+      "rewards/margins": 8.300026893615723,
+      "rewards/rejected": -7.967093467712402,
+      "step": 259,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 1.495855855855856,
+      "grad_norm": 0.7260443568229675,
+      "learning_rate": 0.00010795707788211118,
+      "logits/chosen": -0.4627129137516022,
+      "logits/rejected": -0.824970543384552,
+      "logps/chosen": -3.0662460327148438,
+      "logps/rejected": -102.18391418457031,
+      "loss": 0.20565594732761383,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.18343128263950348,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.37244486808776855,
+      "rewards/margins": 8.65424919128418,
+      "rewards/rejected": -8.281805038452148,
+      "step": 260,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 1.5016216216216216,
+      "grad_norm": 0.8002501726150513,
+      "learning_rate": 0.00010732170128255468,
+      "logits/chosen": -0.4538012444972992,
+      "logits/rejected": -0.7942509651184082,
+      "logps/chosen": -3.3011088371276855,
+      "logps/rejected": -107.73184204101562,
+      "loss": 0.23947054147720337,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.18957434594631195,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.30125465989112854,
+      "rewards/margins": 8.832905769348145,
+      "rewards/rejected": -8.531651496887207,
+      "step": 261,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 1.5073873873873875,
+      "grad_norm": 0.789520800113678,
+      "learning_rate": 0.00010668602736825641,
+      "logits/chosen": -0.5109374523162842,
+      "logits/rejected": -0.8500669002532959,
+      "logps/chosen": -1.8806393146514893,
+      "logps/rejected": -104.79667663574219,
+      "loss": 0.20148895680904388,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.14439257979393005,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4746500253677368,
+      "rewards/margins": 9.070365905761719,
+      "rewards/rejected": -8.595715522766113,
+      "step": 262,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 1.5131531531531532,
+      "grad_norm": 0.6022177338600159,
+      "learning_rate": 0.00010605008195223694,
+      "logits/chosen": -0.5372226238250732,
+      "logits/rejected": -0.8244591951370239,
+      "logps/chosen": -1.747965693473816,
+      "logps/rejected": -95.3003158569336,
+      "loss": 0.14818598330020905,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.14205510914325714,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.252836138010025,
+      "rewards/margins": 8.310218811035156,
+      "rewards/rejected": -8.057382583618164,
+      "step": 263,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 1.518918918918919,
+      "grad_norm": 0.893842339515686,
+      "learning_rate": 0.00010541389085854176,
+      "logits/chosen": -0.5556601285934448,
+      "logits/rejected": -0.8188039660453796,
+      "logps/chosen": -3.6805174350738525,
+      "logps/rejected": -88.07212829589844,
+      "loss": 0.3132077157497406,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.28640279173851013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.16909292340278625,
+      "rewards/margins": 7.427695274353027,
+      "rewards/rejected": -7.258602619171143,
+      "step": 264,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 1.5246846846846847,
+      "grad_norm": 2.960935592651367,
+      "learning_rate": 0.00010477747992119273,
+      "logits/chosen": -0.47233614325523376,
+      "logits/rejected": -0.8489798903465271,
+      "logps/chosen": -2.1952314376831055,
+      "logps/rejected": -109.46432495117188,
+      "loss": 0.1649601310491562,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.15151429176330566,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3918912410736084,
+      "rewards/margins": 9.63163948059082,
+      "rewards/rejected": -9.23974895477295,
+      "step": 265,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 1.5304504504504504,
+      "grad_norm": 1.122962474822998,
+      "learning_rate": 0.0001041408749831389,
+      "logits/chosen": -0.3698769807815552,
+      "logits/rejected": -0.8481926321983337,
+      "logps/chosen": -3.5086236000061035,
+      "logps/rejected": -113.31342315673828,
+      "loss": 0.5645697116851807,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.42728477716445923,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.08735576272010803,
+      "rewards/margins": 9.291422843933105,
+      "rewards/rejected": -9.20406723022461,
+      "step": 266,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 1.536216216216216,
+      "grad_norm": 8.757622718811035,
+      "learning_rate": 0.00010350410189520723,
+      "logits/chosen": -0.48632851243019104,
+      "logits/rejected": -0.8631247282028198,
+      "logps/chosen": -2.381420850753784,
+      "logps/rejected": -115.440673828125,
+      "loss": 0.17880821228027344,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1437128782272339,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3020068407058716,
+      "rewards/margins": 9.818366050720215,
+      "rewards/rejected": -9.516358375549316,
+      "step": 267,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 1.541981981981982,
+      "grad_norm": 1.859763264656067,
+      "learning_rate": 0.00010286718651505275,
+      "logits/chosen": -0.6271436214447021,
+      "logits/rejected": -0.9017850756645203,
+      "logps/chosen": -3.4276139736175537,
+      "logps/rejected": -94.77530670166016,
+      "loss": 0.23817601799964905,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.23029769957065582,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2481326013803482,
+      "rewards/margins": 8.214079856872559,
+      "rewards/rejected": -7.965947151184082,
+      "step": 268,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 1.5477477477477477,
+      "grad_norm": 3.0863282680511475,
+      "learning_rate": 0.00010223015470610871,
+      "logits/chosen": -0.5537242889404297,
+      "logits/rejected": -0.8814469575881958,
+      "logps/chosen": -4.503087520599365,
+      "logps/rejected": -90.9027328491211,
+      "loss": 0.3578626811504364,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.3030712604522705,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.2642664313316345,
+      "rewards/margins": 7.56298828125,
+      "rewards/rejected": -7.298722267150879,
+      "step": 269,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.5535135135135136,
+      "grad_norm": 1.3857908248901367,
+      "learning_rate": 0.00010159303233653604,
+      "logits/chosen": -0.570176899433136,
+      "logits/rejected": -0.8510745763778687,
+      "logps/chosen": -2.5669748783111572,
+      "logps/rejected": -91.87413024902344,
+      "loss": 0.24940097332000732,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2209518700838089,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4336439371109009,
+      "rewards/margins": 8.137587547302246,
+      "rewards/rejected": -7.703943252563477,
+      "step": 270,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.5592792792792793,
+      "grad_norm": 3.1311755180358887,
+      "learning_rate": 0.00010095584527817319,
+      "logits/chosen": -0.584690272808075,
+      "logits/rejected": -0.9461975693702698,
+      "logps/chosen": -1.4432144165039062,
+      "logps/rejected": -90.90803527832031,
+      "loss": 0.17027394473552704,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.15755578875541687,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1908416450023651,
+      "rewards/margins": 7.701169013977051,
+      "rewards/rejected": -7.510327339172363,
+      "step": 271,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.565045045045045,
+      "grad_norm": 0.8555260300636292,
+      "learning_rate": 0.00010031861940548555,
+      "logits/chosen": -0.5870640873908997,
+      "logits/rejected": -0.8846918940544128,
+      "logps/chosen": -3.1194140911102295,
+      "logps/rejected": -84.05816650390625,
+      "loss": 0.2235853672027588,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.19342783093452454,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.32051804661750793,
+      "rewards/margins": 7.1265106201171875,
+      "rewards/rejected": -6.805992126464844,
+      "step": 272,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.5708108108108108,
+      "grad_norm": 0.4469135105609894,
+      "learning_rate": 9.968138059451446e-05,
+      "logits/chosen": -0.561904788017273,
+      "logits/rejected": -0.9012641310691833,
+      "logps/chosen": -3.200127363204956,
+      "logps/rejected": -87.4157943725586,
+      "loss": 0.2398141324520111,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.21223551034927368,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.25336456298828125,
+      "rewards/margins": 7.232346534729004,
+      "rewards/rejected": -6.978981971740723,
+      "step": 273,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.5765765765765765,
+      "grad_norm": 1.172140121459961,
+      "learning_rate": 9.904415472182682e-05,
+      "logits/chosen": -0.4839547276496887,
+      "logits/rejected": -0.8654844760894775,
+      "logps/chosen": -1.2689763307571411,
+      "logps/rejected": -100.70805358886719,
+      "loss": 0.13727417588233948,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.08727098256349564,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.45773637294769287,
+      "rewards/margins": 8.578876495361328,
+      "rewards/rejected": -8.121140480041504,
+      "step": 274,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.5823423423423424,
+      "grad_norm": 1.1815749406814575,
+      "learning_rate": 9.840696766346401e-05,
+      "logits/chosen": -0.6512535810470581,
+      "logits/rejected": -0.8718971610069275,
+      "logps/chosen": -2.4228830337524414,
+      "logps/rejected": -83.79597473144531,
+      "loss": 0.2599056363105774,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.17275774478912354,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.30599865317344666,
+      "rewards/margins": 7.236001968383789,
+      "rewards/rejected": -6.930003643035889,
+      "step": 275,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.588108108108108,
+      "grad_norm": 1.087708830833435,
+      "learning_rate": 9.776984529389131e-05,
+      "logits/chosen": -0.5125508904457092,
+      "logits/rejected": -0.8501510620117188,
+      "logps/chosen": -1.924040675163269,
+      "logps/rejected": -95.89266967773438,
+      "loss": 0.13055917620658875,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.11521827429533005,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2806589901447296,
+      "rewards/margins": 7.590930938720703,
+      "rewards/rejected": -7.310272216796875,
+      "step": 276,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.593873873873874,
+      "grad_norm": 0.926349401473999,
+      "learning_rate": 9.713281348494726e-05,
+      "logits/chosen": -0.5143563747406006,
+      "logits/rejected": -0.9174257516860962,
+      "logps/chosen": -0.9051769971847534,
+      "logps/rejected": -98.711669921875,
+      "loss": 0.11217982321977615,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.057560034096241,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.14916111528873444,
+      "rewards/margins": 7.899416446685791,
+      "rewards/rejected": -7.750256538391113,
+      "step": 277,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.5996396396396397,
+      "grad_norm": 1.0033612251281738,
+      "learning_rate": 9.64958981047928e-05,
+      "logits/chosen": -0.5166518688201904,
+      "logits/rejected": -0.8793089389801025,
+      "logps/chosen": -4.466638565063477,
+      "logps/rejected": -103.69050598144531,
+      "loss": 0.3073112368583679,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2843497693538666,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.32026752829551697,
+      "rewards/margins": 8.899625778198242,
+      "rewards/rejected": -8.579357147216797,
+      "step": 278,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.6054054054054054,
+      "grad_norm": 1.239680528640747,
+      "learning_rate": 9.585912501686111e-05,
+      "logits/chosen": -0.4977273643016815,
+      "logits/rejected": -0.8297019004821777,
+      "logps/chosen": -2.0252184867858887,
+      "logps/rejected": -93.17738342285156,
+      "loss": 0.2024317979812622,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1512202024459839,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.35597655177116394,
+      "rewards/margins": 7.539138317108154,
+      "rewards/rejected": -7.183161735534668,
+      "step": 279,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 1.6111711711711711,
+      "grad_norm": 1.532649040222168,
+      "learning_rate": 9.522252007880732e-05,
+      "logits/chosen": -0.6540271639823914,
+      "logits/rejected": -0.9306864738464355,
+      "logps/chosen": -1.1256556510925293,
+      "logps/rejected": -96.81440734863281,
+      "loss": 0.13118696212768555,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.06020975112915039,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.30623379349708557,
+      "rewards/margins": 8.51535415649414,
+      "rewards/rejected": -8.20911979675293,
+      "step": 280,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.6169369369369369,
+      "grad_norm": 1.0006412267684937,
+      "learning_rate": 9.458610914145826e-05,
+      "logits/chosen": -0.5862869024276733,
+      "logits/rejected": -0.9326885938644409,
+      "logps/chosen": -3.7132813930511475,
+      "logps/rejected": -95.93994903564453,
+      "loss": 0.2073487639427185,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.18085618317127228,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3810235559940338,
+      "rewards/margins": 8.14185619354248,
+      "rewards/rejected": -7.760832786560059,
+      "step": 281,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.6227027027027026,
+      "grad_norm": 0.9711495637893677,
+      "learning_rate": 9.394991804776309e-05,
+      "logits/chosen": -0.552708089351654,
+      "logits/rejected": -0.8055617809295654,
+      "logps/chosen": -2.3333301544189453,
+      "logps/rejected": -84.41504669189453,
+      "loss": 0.13116180896759033,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.10828734934329987,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.40816813707351685,
+      "rewards/margins": 7.26495361328125,
+      "rewards/rejected": -6.856784820556641,
+      "step": 282,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.6284684684684685,
+      "grad_norm": 0.6350001692771912,
+      "learning_rate": 9.331397263174364e-05,
+      "logits/chosen": -0.4492097496986389,
+      "logits/rejected": -0.8619819283485413,
+      "logps/chosen": -1.9441540241241455,
+      "logps/rejected": -109.03350067138672,
+      "loss": 0.1653510183095932,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.14039570093154907,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.42938756942749023,
+      "rewards/margins": 8.919137954711914,
+      "rewards/rejected": -8.489750862121582,
+      "step": 283,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 1.6342342342342342,
+      "grad_norm": 0.6960185170173645,
+      "learning_rate": 9.267829871744536e-05,
+      "logits/chosen": -0.618625283241272,
+      "logits/rejected": -0.9121880531311035,
+      "logps/chosen": -1.6954020261764526,
+      "logps/rejected": -94.96092224121094,
+      "loss": 0.07639533281326294,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.05519339069724083,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.28837311267852783,
+      "rewards/margins": 8.253684997558594,
+      "rewards/rejected": -7.965311527252197,
+      "step": 284,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 1.6400000000000001,
+      "grad_norm": 1.3940598964691162,
+      "learning_rate": 9.204292211788884e-05,
+      "logits/chosen": -0.578076183795929,
+      "logits/rejected": -0.9528270959854126,
+      "logps/chosen": -1.8992458581924438,
+      "logps/rejected": -92.11747741699219,
+      "loss": 0.16904497146606445,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.13233791291713715,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.26542994379997253,
+      "rewards/margins": 7.916376113891602,
+      "rewards/rejected": -7.650946140289307,
+      "step": 285,
+      "train_speed(iter/s)": 0.010729
+    },
+    {
+      "epoch": 1.6457657657657658,
+      "grad_norm": 0.9293711185455322,
+      "learning_rate": 9.140786863402147e-05,
+      "logits/chosen": -0.6649171113967896,
+      "logits/rejected": -0.9644867181777954,
+      "logps/chosen": -2.1775617599487305,
+      "logps/rejected": -78.7835693359375,
+      "loss": 0.19159023463726044,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1725030243396759,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2781451642513275,
+      "rewards/margins": 6.788741111755371,
+      "rewards/rejected": -6.51059627532959,
+      "step": 286,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.6515315315315315,
+      "grad_norm": 1.1537113189697266,
+      "learning_rate": 9.077316405366981e-05,
+      "logits/chosen": -0.6535583138465881,
+      "logits/rejected": -0.9935054183006287,
+      "logps/chosen": -3.1851658821105957,
+      "logps/rejected": -92.34557342529297,
+      "loss": 0.24425984919071198,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.23288863897323608,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.31673336029052734,
+      "rewards/margins": 8.090534210205078,
+      "rewards/rejected": -7.773800849914551,
+      "step": 287,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.6572972972972972,
+      "grad_norm": 2.202179193496704,
+      "learning_rate": 9.01388341504923e-05,
+      "logits/chosen": -0.5745314955711365,
+      "logits/rejected": -0.9523424506187439,
+      "logps/chosen": -4.3446502685546875,
+      "logps/rejected": -119.23814392089844,
+      "loss": 0.2483672946691513,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.20222075283527374,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.33365774154663086,
+      "rewards/margins": 10.17822265625,
+      "rewards/rejected": -9.844566345214844,
+      "step": 288,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.663063063063063,
+      "grad_norm": 0.6017576456069946,
+      "learning_rate": 8.950490468293279e-05,
+      "logits/chosen": -0.6120154857635498,
+      "logits/rejected": -0.9212927222251892,
+      "logps/chosen": -2.7989425659179688,
+      "logps/rejected": -86.53015899658203,
+      "loss": 0.1996924877166748,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.17490829527378082,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.286940336227417,
+      "rewards/margins": 7.214242458343506,
+      "rewards/rejected": -6.927302837371826,
+      "step": 289,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.6688288288288289,
+      "grad_norm": 0.8434000611305237,
+      "learning_rate": 8.887140139317454e-05,
+      "logits/chosen": -0.6056364178657532,
+      "logits/rejected": -0.9164049029350281,
+      "logps/chosen": -4.816060543060303,
+      "logps/rejected": -95.74642944335938,
+      "loss": 0.2582835555076599,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2520279884338379,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7545455694198608,
+      "rewards/margins": 8.554335594177246,
+      "rewards/rejected": -7.799790382385254,
+      "step": 290,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.6745945945945946,
+      "grad_norm": 4.302709102630615,
+      "learning_rate": 8.823835000609482e-05,
+      "logits/chosen": -0.5834745168685913,
+      "logits/rejected": -1.0115468502044678,
+      "logps/chosen": -2.5484602451324463,
+      "logps/rejected": -108.22164916992188,
+      "loss": 0.3237055838108063,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2526852786540985,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.12026344239711761,
+      "rewards/margins": 9.108240127563477,
+      "rewards/rejected": -8.98797607421875,
+      "step": 291,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.6803603603603605,
+      "grad_norm": 1.4632378816604614,
+      "learning_rate": 8.76057762282203e-05,
+      "logits/chosen": -0.5660637617111206,
+      "logits/rejected": -1.061202049255371,
+      "logps/chosen": -2.468435764312744,
+      "logps/rejected": -112.27369689941406,
+      "loss": 0.24110177159309387,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.19869710505008698,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.07396674156188965,
+      "rewards/margins": 9.049920082092285,
+      "rewards/rejected": -8.975954055786133,
+      "step": 292,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.6861261261261262,
+      "grad_norm": 1.314245343208313,
+      "learning_rate": 8.697370574668335e-05,
+      "logits/chosen": -0.6602072715759277,
+      "logits/rejected": -0.9728937149047852,
+      "logps/chosen": -2.3553061485290527,
+      "logps/rejected": -101.07364654541016,
+      "loss": 0.20281532406806946,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.17743095755577087,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5697655081748962,
+      "rewards/margins": 8.947297096252441,
+      "rewards/rejected": -8.377532005310059,
+      "step": 293,
+      "train_speed(iter/s)": 0.01073
+    },
+    {
+      "epoch": 1.691891891891892,
+      "grad_norm": 1.3260091543197632,
+      "learning_rate": 8.634216422817867e-05,
+      "logits/chosen": -0.7734063267707825,
+      "logits/rejected": -0.987154483795166,
+      "logps/chosen": -4.309966564178467,
+      "logps/rejected": -78.10920715332031,
+      "loss": 0.3564646542072296,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2989146113395691,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2878839373588562,
+      "rewards/margins": 6.840456008911133,
+      "rewards/rejected": -6.552572727203369,
+      "step": 294,
+      "train_speed(iter/s)": 0.010731
+    },
+    {
+      "epoch": 1.6976576576576576,
+      "grad_norm": 1.6897776126861572,
+      "learning_rate": 8.57111773179213e-05,
+      "logits/chosen": -0.6336952447891235,
+      "logits/rejected": -1.0445548295974731,
+      "logps/chosen": -1.379799723625183,
+      "logps/rejected": -93.2787857055664,
+      "loss": 0.14356985688209534,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.09856417775154114,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.23449544608592987,
+      "rewards/margins": 7.955099582672119,
+      "rewards/rejected": -7.720603942871094,
+      "step": 295,
+      "train_speed(iter/s)": 0.010731
+    },
+    {
+      "epoch": 1.7034234234234233,
+      "grad_norm": 0.9327660799026489,
+      "learning_rate": 8.508077063860506e-05,
+      "logits/chosen": -0.5344192981719971,
+      "logits/rejected": -1.0205243825912476,
+      "logps/chosen": -0.8863539695739746,
+      "logps/rejected": -122.19490051269531,
+      "loss": 0.20819178223609924,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.1362844705581665,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.10618732124567032,
+      "rewards/margins": 9.871743202209473,
+      "rewards/rejected": -9.765556335449219,
+      "step": 296,
+      "train_speed(iter/s)": 0.010731
+    },
+    {
+      "epoch": 1.709189189189189,
+      "grad_norm": 7.680380821228027,
+      "learning_rate": 8.44509697893621e-05,
+      "logits/chosen": -0.6720832586288452,
+      "logits/rejected": -0.9245396852493286,
+      "logps/chosen": -4.568170070648193,
+      "logps/rejected": -91.47016906738281,
+      "loss": 0.3026784062385559,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.2759172320365906,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.28561854362487793,
+      "rewards/margins": 7.839561939239502,
+      "rewards/rejected": -7.553943634033203,
+      "step": 297,
+      "train_speed(iter/s)": 0.010731
+    },
+    {
+      "epoch": 1.714954954954955,
+      "grad_norm": 1.1304014921188354,
+      "learning_rate": 8.382180034472353e-05,
+      "logits/chosen": -0.5577611327171326,
+      "logits/rejected": -0.9694806337356567,
+      "logps/chosen": -2.482516288757324,
+      "logps/rejected": -124.2740478515625,
+      "loss": 0.17773966491222382,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.16795828938484192,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.18939006328582764,
+      "rewards/margins": 10.547513961791992,
+      "rewards/rejected": -10.358122825622559,
+      "step": 298,
+      "train_speed(iter/s)": 0.010731
+    },
+    {
+      "epoch": 1.7207207207207207,
+      "grad_norm": 3.0718917846679688,
+      "learning_rate": 8.319328785358078e-05,
+      "logits/chosen": -0.6541886329650879,
+      "logits/rejected": -0.9405132532119751,
+      "logps/chosen": -4.128439903259277,
+      "logps/rejected": -86.01166534423828,
+      "loss": 0.2915968596935272,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.28397953510284424,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6267665028572083,
+      "rewards/margins": 7.876523971557617,
+      "rewards/rejected": -7.249757289886475,
+      "step": 299,
+      "train_speed(iter/s)": 0.010731
+    },
+    {
+      "epoch": 1.7264864864864866,
+      "grad_norm": 1.0582029819488525,
+      "learning_rate": 8.256545783814802e-05,
+      "logits/chosen": -0.6806445717811584,
+      "logits/rejected": -0.9849521517753601,
+      "logps/chosen": -1.6571637392044067,
+      "logps/rejected": -89.55918884277344,
+      "loss": 0.15995512902736664,
+      "memory(GiB)": 41.24,
+      "nll_loss": 0.12567465007305145,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3345286250114441,
+      "rewards/margins": 7.534932613372803,
+      "rewards/rejected": -7.200402736663818,
+      "step": 300,
+      "train_speed(iter/s)": 0.010731
+    },
+    {
+      "epoch": 1.7264864864864866,
+      "eval_logits/chosen": -0.6343019604682922,
+      "eval_logits/rejected": -0.9546191096305847,
+      "eval_logps/chosen": -4.193188190460205,
+      "eval_logps/rejected": -85.89741516113281,
+      "eval_loss": 0.4250725209712982,
+      "eval_nll_loss": 0.36724570393562317,
+      "eval_rewards/accuracies": 0.9642857313156128,
+      "eval_rewards/chosen": 0.2654857933521271,
+      "eval_rewards/margins": 7.137094020843506,
+      "eval_rewards/rejected": -6.871608734130859,
+      "eval_runtime": 64.4253,
+      "eval_samples_per_second": 0.869,
+      "eval_steps_per_second": 0.435,
+      "step": 300
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 519,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.6381078727661978e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}