Training in progress, step 200, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:268aa3d2814a792a1ce12fc0ee5a43e0bc3f4dfbe66bca24ad57492c892f8b91
+oid sha256:065322e97e075055ae2c6bcbf10fdfffbac7dd29ef45906fca7a9bacc7abec43
 size 204500912

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d14bdbb174576769aa6486b61934c2015edc41a72d409074143c0b546c4f989b
-size 104062923
+oid sha256:c0e97a95edb102b68426558e6f6306ffd55d9a0ba3fc011ab3b76edaea3a99e3
+size 104062731

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3d6d8fafcd1ee268414be5acf0366296af5b03d60871978712eac1979cb42d65
+oid sha256:7c800b778fa7e115e4c34de8529902de8b61c9a1b4bab3eb8295d06dafff030e
 size 14645

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bf3f9c1ea54f8f95e6812b6b4e99596105233cd3e123554db760e4aba93f83e4
+oid sha256:fb9a36b5dff54d4dc680b7c932dff5afaac16543707cbf68dd86d83d274f369f
 size 1465

last-checkpoint/trainer_state.json

@@ -2,1518 +2,318 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.6540378863409773,
+  "epoch": 0.5317381189764041,
   "eval_steps": 500,
-  "global_step": 1000,
+  "global_step": 200,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.026586905948820207,
-      "grad_norm": 92.03909301757812,
-      "learning_rate": 9e-08,
-      "logits/chosen": 1.8763988018035889,
-      "logits/rejected": 2.256129264831543,
-      "logps/chosen": -180.8492431640625,
-      "logps/rejected": -294.6668395996094,
-      "loss": 16.764971923828124,
+      "grad_norm": 229.74172973632812,
+      "learning_rate": 1.8000000000000001e-06,
+      "logits/chosen": 1.892960548400879,
+      "logits/rejected": 2.2739109992980957,
+      "logps/chosen": -180.7786102294922,
+      "logps/rejected": -296.7843017578125,
+      "loss": 41.754217529296874,
       "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 50.876712799072266,
-      "rewards/margins": 20.225709915161133,
-      "rewards/rejected": 30.651004791259766,
+      "rewards/chosen": 127.22708892822266,
+      "rewards/margins": 51.65830612182617,
+      "rewards/rejected": 75.56879425048828,
       "step": 10
     },
     {
       "epoch": 0.053173811897640415,
-      "grad_norm": 72.75655364990234,
-      "learning_rate": 1.8999999999999998e-07,
-      "logits/chosen": 2.2195005416870117,
-      "logits/rejected": 2.3702588081359863,
-      "logps/chosen": -199.29591369628906,
-      "logps/rejected": -293.90887451171875,
-      "loss": 14.240003967285157,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 53.17363357543945,
-      "rewards/margins": 24.12602996826172,
-      "rewards/rejected": 29.0476016998291,
+      "grad_norm": 127.54531860351562,
+      "learning_rate": 3.8000000000000005e-06,
+      "logits/chosen": 2.360779285430908,
+      "logits/rejected": 2.5129952430725098,
+      "logps/chosen": -198.55577087402344,
+      "logps/rejected": -311.8116149902344,
+      "loss": 34.334066772460936,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 133.30413818359375,
+      "rewards/margins": 69.63650512695312,
+      "rewards/rejected": 63.667640686035156,
       "step": 20
     },
     {
       "epoch": 0.07976071784646062,
-      "grad_norm": 101.80017852783203,
-      "learning_rate": 2.9e-07,
-      "logits/chosen": 2.360567569732666,
-      "logits/rejected": 2.484600305557251,
-      "logps/chosen": -201.53787231445312,
-      "logps/rejected": -286.9433898925781,
-      "loss": 13.708811950683593,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 59.95117950439453,
-      "rewards/margins": 25.14548110961914,
-      "rewards/rejected": 34.805702209472656,
+      "grad_norm": 195.1651153564453,
+      "learning_rate": 5.8e-06,
+      "logits/chosen": 2.7846579551696777,
+      "logits/rejected": 2.912212610244751,
+      "logps/chosen": -201.8841094970703,
+      "logps/rejected": -343.28619384765625,
+      "loss": 31.000405883789064,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 149.70481872558594,
+      "rewards/margins": 90.86196899414062,
+      "rewards/rejected": 58.842857360839844,
       "step": 30
     },
     {
       "epoch": 0.10634762379528083,
-      "grad_norm": 85.12960052490234,
-      "learning_rate": 3.8999999999999997e-07,
-      "logits/chosen": 1.8842649459838867,
-      "logits/rejected": 2.0478363037109375,
-      "logps/chosen": -178.1483917236328,
-      "logps/rejected": -285.0755920410156,
-      "loss": 17.000025939941406,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 50.454071044921875,
-      "rewards/margins": 17.00231170654297,
-      "rewards/rejected": 33.451759338378906,
+      "grad_norm": 217.8875274658203,
+      "learning_rate": 7.800000000000002e-06,
+      "logits/chosen": 2.687530040740967,
+      "logits/rejected": 2.89595365524292,
+      "logps/chosen": -180.60574340820312,
+      "logps/rejected": -395.2937927246094,
+      "loss": 33.74924926757812,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 124.906494140625,
+      "rewards/margins": 96.38619995117188,
+      "rewards/rejected": 28.52029037475586,
       "step": 40
     },
     {
       "epoch": 0.13293452974410103,
-      "grad_norm": 33.85184097290039,
-      "learning_rate": 4.9e-07,
-      "logits/chosen": 2.229463577270508,
-      "logits/rejected": 2.204373836517334,
-      "logps/chosen": -212.55178833007812,
-      "logps/rejected": -280.9806213378906,
-      "loss": 18.418368530273437,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 57.14277267456055,
-      "rewards/margins": 21.43265151977539,
-      "rewards/rejected": 35.710121154785156,
+      "grad_norm": 123.57787322998047,
+      "learning_rate": 9.800000000000001e-06,
+      "logits/chosen": 3.6891541481018066,
+      "logits/rejected": 3.7287964820861816,
+      "logps/chosen": -233.40304565429688,
+      "logps/rejected": -501.949951171875,
+      "loss": 33.165060424804686,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 132.43130493164062,
+      "rewards/margins": 153.64065551757812,
+      "rewards/rejected": -21.209354400634766,
       "step": 50
     },
     {
       "epoch": 0.15952143569292124,
-      "grad_norm": 61.26063537597656,
-      "learning_rate": 5.9e-07,
-      "logits/chosen": 2.045487403869629,
-      "logits/rejected": 2.2564761638641357,
-      "logps/chosen": -183.6549072265625,
-      "logps/rejected": -311.9967956542969,
-      "loss": 12.530684661865234,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 40.03901672363281,
-      "rewards/margins": 22.57087516784668,
-      "rewards/rejected": 17.468143463134766,
+      "grad_norm": 240.26422119140625,
+      "learning_rate": 9.742857142857143e-06,
+      "logits/chosen": 4.325669288635254,
+      "logits/rejected": 4.669508934020996,
+      "logps/chosen": -227.3274688720703,
+      "logps/rejected": -761.2488403320312,
+      "loss": 9.919349670410156,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 78.26126861572266,
+      "rewards/margins": 259.2169494628906,
+      "rewards/rejected": -180.95565795898438,
       "step": 60
     },
     {
       "epoch": 0.18610834164174145,
-      "grad_norm": 65.56060791015625,
-      "learning_rate": 6.9e-07,
-      "logits/chosen": 2.3583855628967285,
-      "logits/rejected": 2.518134593963623,
-      "logps/chosen": -215.484375,
-      "logps/rejected": -292.7709045410156,
-      "loss": 17.06499786376953,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": 54.328125,
-      "rewards/margins": 18.9035587310791,
-      "rewards/rejected": 35.42456817626953,
+      "grad_norm": 18.67397117614746,
+      "learning_rate": 9.457142857142858e-06,
+      "logits/chosen": 6.029001235961914,
+      "logits/rejected": 6.398881912231445,
+      "logps/chosen": -361.8104248046875,
+      "logps/rejected": -936.2852783203125,
+      "loss": 8.505984497070312,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 62.65727615356445,
+      "rewards/margins": 295.85296630859375,
+      "rewards/rejected": -233.1957550048828,
       "step": 70
     },
     {
       "epoch": 0.21269524759056166,
-      "grad_norm": 78.31404876708984,
-      "learning_rate": 7.9e-07,
-      "logits/chosen": 2.389976978302002,
-      "logits/rejected": 2.5061419010162354,
-      "logps/chosen": -199.54867553710938,
-      "logps/rejected": -313.9349670410156,
-      "loss": 14.476513671875,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": 58.9052619934082,
-      "rewards/margins": 28.68975830078125,
-      "rewards/rejected": 30.215505599975586,
+      "grad_norm": 424.631591796875,
+      "learning_rate": 9.171428571428572e-06,
+      "logits/chosen": 7.3346147537231445,
+      "logits/rejected": 7.6371870040893555,
+      "logps/chosen": -487.5819396972656,
+      "logps/rejected": -1231.750244140625,
+      "loss": 5.0393016815185545,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 3.2465362548828125,
+      "rewards/margins": 386.61541748046875,
+      "rewards/rejected": -383.368896484375,
       "step": 80
     },
     {
       "epoch": 0.23928215353938184,
-      "grad_norm": 55.09129333496094,
-      "learning_rate": 8.9e-07,
-      "logits/chosen": 2.343313694000244,
-      "logits/rejected": 2.381267547607422,
-      "logps/chosen": -195.31007385253906,
-      "logps/rejected": -315.2503356933594,
-      "loss": 13.177040100097656,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 62.564231872558594,
-      "rewards/margins": 32.514305114746094,
-      "rewards/rejected": 30.049936294555664,
+      "grad_norm": 58.2872200012207,
+      "learning_rate": 8.885714285714286e-06,
+      "logits/chosen": 7.502760410308838,
+      "logits/rejected": 7.7781524658203125,
+      "logps/chosen": -515.661376953125,
+      "logps/rejected": -1255.3602294921875,
+      "loss": 2.7182350158691406,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -3.7650718688964844,
+      "rewards/margins": 391.1650390625,
+      "rewards/rejected": -394.9300842285156,
       "step": 90
     },
     {
       "epoch": 0.26586905948820205,
-      "grad_norm": 94.35275268554688,
-      "learning_rate": 9.9e-07,
-      "logits/chosen": 2.3925650119781494,
-      "logits/rejected": 2.607084274291992,
-      "logps/chosen": -189.29811096191406,
-      "logps/rejected": -319.96844482421875,
-      "loss": 18.26203155517578,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 50.321632385253906,
-      "rewards/margins": 22.894689559936523,
-      "rewards/rejected": 27.426937103271484,
+      "grad_norm": 0.0009401601273566484,
+      "learning_rate": 8.6e-06,
+      "logits/chosen": 6.9696502685546875,
+      "logits/rejected": 7.490464687347412,
+      "logps/chosen": -407.62591552734375,
+      "logps/rejected": -1201.7757568359375,
+      "loss": 1.852958869934082,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 16.64017105102539,
+      "rewards/margins": 388.97650146484375,
+      "rewards/rejected": -372.3362731933594,
       "step": 100
     },
     {
       "epoch": 0.2924559654370223,
-      "grad_norm": 93.89908599853516,
-      "learning_rate": 9.9e-07,
-      "logits/chosen": 2.4763197898864746,
-      "logits/rejected": 2.6758036613464355,
-      "logps/chosen": -187.5391082763672,
-      "logps/rejected": -340.25250244140625,
-      "loss": 10.57765884399414,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 58.184059143066406,
-      "rewards/margins": 36.878929138183594,
-      "rewards/rejected": 21.305124282836914,
+      "grad_norm": 776.8017578125,
+      "learning_rate": 8.314285714285715e-06,
+      "logits/chosen": 7.605508327484131,
+      "logits/rejected": 8.056883811950684,
+      "logps/chosen": -500.75714111328125,
+      "logps/rejected": -1358.50732421875,
+      "loss": 0.4602807998657227,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -11.148886680603027,
+      "rewards/margins": 444.7156677246094,
+      "rewards/rejected": -455.8646545410156,
       "step": 110
     },
     {
       "epoch": 0.3190428713858425,
-      "grad_norm": 91.25633239746094,
-      "learning_rate": 9.788888888888889e-07,
-      "logits/chosen": 2.5278210639953613,
-      "logits/rejected": 2.6886465549468994,
-      "logps/chosen": -205.1584014892578,
-      "logps/rejected": -349.90093994140625,
-      "loss": 13.945356750488282,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 46.65082550048828,
-      "rewards/margins": 27.151325225830078,
-      "rewards/rejected": 19.499500274658203,
+      "grad_norm": 9.229455307652179e-14,
+      "learning_rate": 8.02857142857143e-06,
+      "logits/chosen": 7.707437992095947,
+      "logits/rejected": 8.208626747131348,
+      "logps/chosen": -493.72503662109375,
+      "logps/rejected": -1418.4669189453125,
+      "loss": 0.4660654544830322,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -27.65629005432129,
+      "rewards/margins": 457.8779296875,
+      "rewards/rejected": -485.53424072265625,
       "step": 120
     },
     {
       "epoch": 0.34562977733466266,
-      "grad_norm": 96.81977844238281,
-      "learning_rate": 9.677777777777777e-07,
-      "logits/chosen": 3.0266711711883545,
-      "logits/rejected": 3.194408416748047,
-      "logps/chosen": -198.1504669189453,
-      "logps/rejected": -356.78485107421875,
-      "loss": 15.321591186523438,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 58.57115936279297,
-      "rewards/margins": 34.659385681152344,
-      "rewards/rejected": 23.911775588989258,
+      "grad_norm": 3.0277444440507395e-10,
+      "learning_rate": 7.742857142857144e-06,
+      "logits/chosen": 7.820192813873291,
+      "logits/rejected": 8.277512550354004,
+      "logps/chosen": -490.735595703125,
+      "logps/rejected": -1337.3505859375,
+      "loss": 1.2780420303344726,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": 0.13531294465065002,
+      "rewards/margins": 430.638671875,
+      "rewards/rejected": -430.50335693359375,
       "step": 130
     },
     {
       "epoch": 0.3722166832834829,
-      "grad_norm": 93.63339233398438,
-      "learning_rate": 9.566666666666667e-07,
-      "logits/chosen": 3.055471181869507,
-      "logits/rejected": 3.145911455154419,
-      "logps/chosen": -219.0845184326172,
-      "logps/rejected": -345.9827880859375,
-      "loss": 13.172528076171876,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 55.67122268676758,
-      "rewards/margins": 32.06965637207031,
-      "rewards/rejected": 23.6015682220459,
+      "grad_norm": 1.2154150397236663e-07,
+      "learning_rate": 7.457142857142857e-06,
+      "logits/chosen": 7.428654670715332,
+      "logits/rejected": 7.884097099304199,
+      "logps/chosen": -490.9019470214844,
+      "logps/rejected": -1213.6722412109375,
+      "loss": 0.03439792990684509,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": 3.2693276405334473,
+      "rewards/margins": 378.11016845703125,
+      "rewards/rejected": -374.8408203125,
       "step": 140
     },
     {
       "epoch": 0.3988035892323031,
-      "grad_norm": 73.2032699584961,
-      "learning_rate": 9.455555555555556e-07,
-      "logits/chosen": 2.777052640914917,
-      "logits/rejected": 2.8150634765625,
-      "logps/chosen": -197.19174194335938,
-      "logps/rejected": -374.73822021484375,
-      "loss": 15.409014892578124,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 48.60415267944336,
-      "rewards/margins": 28.52492332458496,
-      "rewards/rejected": 20.079227447509766,
+      "grad_norm": 0.0008078943355940282,
+      "learning_rate": 7.1714285714285725e-06,
+      "logits/chosen": 7.338967800140381,
+      "logits/rejected": 7.722776889801025,
+      "logps/chosen": -444.8753356933594,
+      "logps/rejected": -1266.625244140625,
+      "loss": 1.42781343460083,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -2.331421375274658,
+      "rewards/margins": 393.41400146484375,
+      "rewards/rejected": -395.74542236328125,
       "step": 150
     },
     {
       "epoch": 0.4253904951811233,
-      "grad_norm": 66.92320251464844,
-      "learning_rate": 9.344444444444444e-07,
-      "logits/chosen": 2.996166467666626,
-      "logits/rejected": 3.1385650634765625,
-      "logps/chosen": -212.75375366210938,
-      "logps/rejected": -371.66522216796875,
-      "loss": 10.577291870117188,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 63.399871826171875,
-      "rewards/margins": 42.74666213989258,
-      "rewards/rejected": 20.653209686279297,
+      "grad_norm": 0.0,
+      "learning_rate": 6.885714285714287e-06,
+      "logits/chosen": 7.96518087387085,
+      "logits/rejected": 8.39413833618164,
+      "logps/chosen": -586.011962890625,
+      "logps/rejected": -1365.374267578125,
+      "loss": 1.9711128234863282,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -28.12943458557129,
+      "rewards/margins": 417.09210205078125,
+      "rewards/rejected": -445.22149658203125,
       "step": 160
     },
     {
       "epoch": 0.4519774011299435,
-      "grad_norm": 64.92620849609375,
-      "learning_rate": 9.233333333333333e-07,
-      "logits/chosen": 2.832219362258911,
-      "logits/rejected": 3.1428098678588867,
-      "logps/chosen": -196.91094970703125,
-      "logps/rejected": -397.47064208984375,
-      "loss": 12.637787628173829,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 54.22844314575195,
-      "rewards/margins": 42.87944793701172,
-      "rewards/rejected": 11.348990440368652,
+      "grad_norm": 5.4764127260797935e-12,
+      "learning_rate": 6.600000000000001e-06,
+      "logits/chosen": 7.3422675132751465,
+      "logits/rejected": 7.857165336608887,
+      "logps/chosen": -465.5393981933594,
+      "logps/rejected": -1300.1103515625,
+      "loss": 0.004332171380519867,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": 1.256854772567749,
+      "rewards/margins": 424.2041931152344,
+      "rewards/rejected": -422.9473571777344,
       "step": 170
     },
     {
       "epoch": 0.4785643070787637,
-      "grad_norm": 88.73342895507812,
-      "learning_rate": 9.122222222222222e-07,
-      "logits/chosen": 3.001598358154297,
-      "logits/rejected": 3.18257737159729,
-      "logps/chosen": -204.97628784179688,
-      "logps/rejected": -451.9921875,
-      "loss": 10.512740325927734,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 49.384117126464844,
-      "rewards/margins": 53.00005340576172,
-      "rewards/rejected": -3.6159355640411377,
+      "grad_norm": 422.58892822265625,
+      "learning_rate": 6.314285714285715e-06,
+      "logits/chosen": 7.199074745178223,
+      "logits/rejected": 7.617570400238037,
+      "logps/chosen": -446.2982482910156,
+      "logps/rejected": -1317.12890625,
+      "loss": 0.9593421936035156,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 2.7993197441101074,
+      "rewards/margins": 444.40753173828125,
+      "rewards/rejected": -441.60821533203125,
       "step": 180
     },
     {
       "epoch": 0.5051512130275839,
-      "grad_norm": 97.91619110107422,
-      "learning_rate": 9.01111111111111e-07,
-      "logits/chosen": 2.735273599624634,
-      "logits/rejected": 2.9921531677246094,
-      "logps/chosen": -185.69210815429688,
-      "logps/rejected": -439.780029296875,
-      "loss": 7.097893524169922,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 51.99469757080078,
-      "rewards/margins": 52.051963806152344,
-      "rewards/rejected": -0.057262420654296875,
+      "grad_norm": 0.0,
+      "learning_rate": 6.028571428571429e-06,
+      "logits/chosen": 7.070580959320068,
+      "logits/rejected": 7.494720458984375,
+      "logps/chosen": -430.97760009765625,
+      "logps/rejected": -1291.35205078125,
+      "loss": 0.09595458507537842,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": 7.344033241271973,
+      "rewards/margins": 433.2731018066406,
+      "rewards/rejected": -425.9291076660156,
       "step": 190
     },
     {
       "epoch": 0.5317381189764041,
-      "grad_norm": 75.27459716796875,
-      "learning_rate": 8.9e-07,
-      "logits/chosen": 3.015864610671997,
-      "logits/rejected": 3.321819305419922,
-      "logps/chosen": -192.43789672851562,
-      "logps/rejected": -471.51055908203125,
-      "loss": 11.397718048095703,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 49.085548400878906,
-      "rewards/margins": 58.6345329284668,
-      "rewards/rejected": -9.54898452758789,
-      "step": 200
-    },
-    {
-      "epoch": 0.5583250249252243,
-      "grad_norm": 67.41759490966797,
-      "learning_rate": 8.788888888888889e-07,
-      "logits/chosen": 3.120459794998169,
-      "logits/rejected": 3.3150908946990967,
-      "logps/chosen": -201.68368530273438,
-      "logps/rejected": -479.9122619628906,
-      "loss": 9.525629425048828,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 49.86951446533203,
-      "rewards/margins": 59.1157112121582,
-      "rewards/rejected": -9.246195793151855,
-      "step": 210
-    },
-    {
-      "epoch": 0.5849119308740446,
-      "grad_norm": 89.28022003173828,
-      "learning_rate": 8.677777777777777e-07,
-      "logits/chosen": 3.2067043781280518,
-      "logits/rejected": 3.2518234252929688,
-      "logps/chosen": -198.66351318359375,
-      "logps/rejected": -459.51287841796875,
-      "loss": 11.018878936767578,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 52.5185432434082,
-      "rewards/margins": 51.389251708984375,
-      "rewards/rejected": 1.129294991493225,
-      "step": 220
-    },
-    {
-      "epoch": 0.6114988368228648,
-      "grad_norm": 57.3789176940918,
-      "learning_rate": 8.566666666666667e-07,
-      "logits/chosen": 3.428664445877075,
-      "logits/rejected": 3.6689727306365967,
-      "logps/chosen": -188.14273071289062,
-      "logps/rejected": -484.1524963378906,
-      "loss": 8.581022644042969,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 49.485206604003906,
-      "rewards/margins": 54.331947326660156,
-      "rewards/rejected": -4.846745491027832,
-      "step": 230
-    },
-    {
-      "epoch": 0.638085742771685,
-      "grad_norm": 78.1611099243164,
-      "learning_rate": 8.455555555555555e-07,
-      "logits/chosen": 3.273719072341919,
-      "logits/rejected": 3.5895423889160156,
-      "logps/chosen": -198.40890502929688,
-      "logps/rejected": -568.0107421875,
-      "loss": 8.444003295898437,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 55.26072311401367,
-      "rewards/margins": 84.45598602294922,
-      "rewards/rejected": -29.195270538330078,
-      "step": 240
-    },
-    {
-      "epoch": 0.6646726487205051,
-      "grad_norm": 87.57330322265625,
-      "learning_rate": 8.344444444444444e-07,
-      "logits/chosen": 3.921356201171875,
-      "logits/rejected": 4.107032775878906,
-      "logps/chosen": -219.56887817382812,
-      "logps/rejected": -529.9613647460938,
-      "loss": 11.188172912597656,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 63.129974365234375,
-      "rewards/margins": 69.01350402832031,
-      "rewards/rejected": -5.883524417877197,
-      "step": 250
-    },
-    {
-      "epoch": 0.6912595546693253,
-      "grad_norm": 78.89329528808594,
-      "learning_rate": 8.233333333333333e-07,
-      "logits/chosen": 4.225001335144043,
-      "logits/rejected": 4.386289119720459,
-      "logps/chosen": -241.30734252929688,
-      "logps/rejected": -510.9679260253906,
-      "loss": 10.638973236083984,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 55.27728271484375,
-      "rewards/margins": 60.547119140625,
-      "rewards/rejected": -5.269834041595459,
-      "step": 260
-    },
-    {
-      "epoch": 0.7178464606181456,
-      "grad_norm": 88.03643798828125,
-      "learning_rate": 8.122222222222221e-07,
-      "logits/chosen": 3.850262403488159,
-      "logits/rejected": 4.041484832763672,
-      "logps/chosen": -241.4730987548828,
-      "logps/rejected": -527.6182250976562,
-      "loss": 11.718121337890626,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 48.15225601196289,
-      "rewards/margins": 54.50339889526367,
-      "rewards/rejected": -6.351143836975098,
-      "step": 270
-    },
-    {
-      "epoch": 0.7444333665669658,
-      "grad_norm": 161.95907592773438,
-      "learning_rate": 8.01111111111111e-07,
-      "logits/chosen": 3.8429579734802246,
-      "logits/rejected": 4.090743064880371,
-      "logps/chosen": -197.0509033203125,
-      "logps/rejected": -592.664794921875,
-      "loss": 8.627317810058594,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 44.076698303222656,
-      "rewards/margins": 71.58090209960938,
-      "rewards/rejected": -27.504215240478516,
-      "step": 280
-    },
-    {
-      "epoch": 0.771020272515786,
-      "grad_norm": 59.20638656616211,
-      "learning_rate": 7.9e-07,
-      "logits/chosen": 3.9963154792785645,
-      "logits/rejected": 4.247437477111816,
-      "logps/chosen": -198.98069763183594,
-      "logps/rejected": -636.8574829101562,
-      "loss": 7.916163635253906,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": 45.44010543823242,
-      "rewards/margins": 82.26910400390625,
-      "rewards/rejected": -36.82899475097656,
-      "step": 290
-    },
-    {
-      "epoch": 0.7976071784646062,
-      "grad_norm": 3.873155947076157e-05,
-      "learning_rate": 7.788888888888889e-07,
-      "logits/chosen": 3.5579922199249268,
-      "logits/rejected": 3.9878501892089844,
-      "logps/chosen": -208.97335815429688,
-      "logps/rejected": -655.0528564453125,
-      "loss": 6.305292129516602,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 36.67095184326172,
-      "rewards/margins": 88.25028991699219,
-      "rewards/rejected": -51.57932662963867,
-      "step": 300
-    },
-    {
-      "epoch": 0.8241940844134263,
-      "grad_norm": 101.79195404052734,
-      "learning_rate": 7.677777777777778e-07,
-      "logits/chosen": 4.391497611999512,
-      "logits/rejected": 4.607339382171631,
-      "logps/chosen": -249.1584930419922,
-      "logps/rejected": -639.23974609375,
-      "loss": 8.43834228515625,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 43.91454315185547,
-      "rewards/margins": 78.60186004638672,
-      "rewards/rejected": -34.68731689453125,
-      "step": 310
-    },
-    {
-      "epoch": 0.8507809903622466,
-      "grad_norm": 91.69438171386719,
-      "learning_rate": 7.566666666666667e-07,
-      "logits/chosen": 4.2728271484375,
-      "logits/rejected": 4.45902156829834,
-      "logps/chosen": -242.8008575439453,
-      "logps/rejected": -614.6475830078125,
-      "loss": 8.622640228271484,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 51.20795440673828,
-      "rewards/margins": 79.56620788574219,
-      "rewards/rejected": -28.358264923095703,
-      "step": 320
-    },
-    {
-      "epoch": 0.8773678963110668,
-      "grad_norm": 64.15619659423828,
-      "learning_rate": 7.455555555555555e-07,
-      "logits/chosen": 4.106622219085693,
-      "logits/rejected": 4.50801944732666,
-      "logps/chosen": -230.63919067382812,
-      "logps/rejected": -676.0430908203125,
-      "loss": 5.273190307617187,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 40.392784118652344,
-      "rewards/margins": 91.14765930175781,
-      "rewards/rejected": -50.75487518310547,
-      "step": 330
-    },
-    {
-      "epoch": 0.903954802259887,
-      "grad_norm": 97.1626205444336,
-      "learning_rate": 7.344444444444444e-07,
-      "logits/chosen": 4.293347358703613,
-      "logits/rejected": 4.595992088317871,
-      "logps/chosen": -215.70938110351562,
-      "logps/rejected": -666.0777587890625,
-      "loss": 5.9850719451904295,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 44.4474983215332,
-      "rewards/margins": 83.35242462158203,
-      "rewards/rejected": -38.90492630004883,
-      "step": 340
-    },
-    {
-      "epoch": 0.9305417082087072,
-      "grad_norm": 85.94694519042969,
-      "learning_rate": 7.233333333333333e-07,
-      "logits/chosen": 4.838589191436768,
-      "logits/rejected": 5.159350872039795,
-      "logps/chosen": -264.10711669921875,
-      "logps/rejected": -796.7295532226562,
-      "loss": 3.4507820129394533,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 42.16107940673828,
-      "rewards/margins": 113.5744857788086,
-      "rewards/rejected": -71.41340637207031,
-      "step": 350
-    },
-    {
-      "epoch": 0.9571286141575274,
-      "grad_norm": 236.06187438964844,
-      "learning_rate": 7.122222222222221e-07,
-      "logits/chosen": 4.589522361755371,
-      "logits/rejected": 4.999955177307129,
-      "logps/chosen": -258.4150695800781,
-      "logps/rejected": -727.7996826171875,
-      "loss": 6.599867248535157,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": 35.76854705810547,
-      "rewards/margins": 92.19223022460938,
-      "rewards/rejected": -56.42368698120117,
-      "step": 360
-    },
-    {
-      "epoch": 0.9837155201063477,
-      "grad_norm": 100.6878662109375,
-      "learning_rate": 7.01111111111111e-07,
-      "logits/chosen": 5.191050052642822,
-      "logits/rejected": 5.309014320373535,
-      "logps/chosen": -296.0009460449219,
-      "logps/rejected": -759.0372924804688,
-      "loss": 3.534566116333008,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 38.79043960571289,
-      "rewards/margins": 100.0891342163086,
-      "rewards/rejected": -61.29869842529297,
-      "step": 370
-    },
-    {
-      "epoch": 1.007976071784646,
-      "grad_norm": 71.57064819335938,
-      "learning_rate": 6.9e-07,
-      "logits/chosen": 5.157181262969971,
-      "logits/rejected": 5.5708699226379395,
-      "logps/chosen": -288.7073059082031,
-      "logps/rejected": -775.53173828125,
-      "loss": 2.5931621551513673,
-      "rewards/accuracies": 0.8698630332946777,
-      "rewards/chosen": 35.64141845703125,
-      "rewards/margins": 96.54639434814453,
-      "rewards/rejected": -60.904972076416016,
-      "step": 380
-    },
-    {
-      "epoch": 1.0345629777334662,
-      "grad_norm": 80.57012939453125,
-      "learning_rate": 6.788888888888889e-07,
-      "logits/chosen": 4.908313274383545,
-      "logits/rejected": 5.281552791595459,
-      "logps/chosen": -255.53994750976562,
-      "logps/rejected": -876.7796630859375,
-      "loss": 3.100166130065918,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 29.122013092041016,
-      "rewards/margins": 121.50981140136719,
-      "rewards/rejected": -92.38780212402344,
-      "step": 390
-    },
-    {
-      "epoch": 1.0611498836822866,
-      "grad_norm": 62.662662506103516,
-      "learning_rate": 6.677777777777778e-07,
-      "logits/chosen": 5.439974308013916,
-      "logits/rejected": 5.852138519287109,
-      "logps/chosen": -291.69049072265625,
-      "logps/rejected": -910.0514526367188,
-      "loss": 3.5051338195800783,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 25.592533111572266,
-      "rewards/margins": 119.5553207397461,
-      "rewards/rejected": -93.96280670166016,
-      "step": 400
-    },
-    {
-      "epoch": 1.0877367896311068,
-      "grad_norm": 178.87042236328125,
-      "learning_rate": 6.566666666666666e-07,
-      "logits/chosen": 5.411637783050537,
-      "logits/rejected": 5.973018646240234,
-      "logps/chosen": -308.68487548828125,
-      "logps/rejected": -894.93359375,
-      "loss": 1.9858436584472656,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 25.565839767456055,
-      "rewards/margins": 120.69632720947266,
-      "rewards/rejected": -95.13047790527344,
-      "step": 410
-    },
-    {
-      "epoch": 1.114323695579927,
-      "grad_norm": 5.069334747531684e-07,
-      "learning_rate": 6.455555555555555e-07,
-      "logits/chosen": 5.404343605041504,
-      "logits/rejected": 5.771250247955322,
-      "logps/chosen": -301.6144104003906,
-      "logps/rejected": -933.3482666015625,
-      "loss": 2.1804153442382814,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 25.12398910522461,
-      "rewards/margins": 123.10489654541016,
-      "rewards/rejected": -97.98091125488281,
-      "step": 420
-    },
-    {
-      "epoch": 1.1409106015287471,
-      "grad_norm": 42.221588134765625,
-      "learning_rate": 6.344444444444444e-07,
-      "logits/chosen": 5.612006187438965,
-      "logits/rejected": 6.014307022094727,
-      "logps/chosen": -324.22314453125,
-      "logps/rejected": -896.99658203125,
-      "loss": 2.2177616119384767,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 22.134746551513672,
-      "rewards/margins": 109.2309341430664,
-      "rewards/rejected": -87.09618377685547,
-      "step": 430
-    },
-    {
-      "epoch": 1.1674975074775673,
-      "grad_norm": 26.945816040039062,
-      "learning_rate": 6.233333333333332e-07,
-      "logits/chosen": 5.816843509674072,
-      "logits/rejected": 6.254372596740723,
-      "logps/chosen": -350.730712890625,
-      "logps/rejected": -915.1998291015625,
-      "loss": 1.6761651992797852,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 27.9744873046875,
-      "rewards/margins": 122.52757263183594,
-      "rewards/rejected": -94.55308532714844,
-      "step": 440
-    },
-    {
-      "epoch": 1.1940844134263875,
-      "grad_norm": 201.8599853515625,
-      "learning_rate": 6.122222222222222e-07,
-      "logits/chosen": 5.693143844604492,
-      "logits/rejected": 6.162411689758301,
-      "logps/chosen": -299.4298095703125,
-      "logps/rejected": -959.1011962890625,
-      "loss": 2.2222429275512696,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 22.612167358398438,
-      "rewards/margins": 126.55992126464844,
-      "rewards/rejected": -103.94776916503906,
-      "step": 450
-    },
-    {
-      "epoch": 1.2206713193752077,
-      "grad_norm": 139.24583435058594,
-      "learning_rate": 6.011111111111112e-07,
-      "logits/chosen": 5.940896034240723,
-      "logits/rejected": 6.417025566101074,
-      "logps/chosen": -349.1092834472656,
-      "logps/rejected": -947.8054809570312,
-      "loss": 1.6525358200073241,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 20.343765258789062,
-      "rewards/margins": 119.97611999511719,
-      "rewards/rejected": -99.63237762451172,
-      "step": 460
-    },
-    {
-      "epoch": 1.2472582253240279,
-      "grad_norm": 207.5171661376953,
-      "learning_rate": 5.9e-07,
-      "logits/chosen": 6.329422950744629,
-      "logits/rejected": 6.6328911781311035,
-      "logps/chosen": -353.35308837890625,
-      "logps/rejected": -962.7057495117188,
-      "loss": 2.509499740600586,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 18.188751220703125,
-      "rewards/margins": 119.3648910522461,
-      "rewards/rejected": -101.1761474609375,
-      "step": 470
-    },
-    {
-      "epoch": 1.273845131272848,
-      "grad_norm": 88.39582824707031,
-      "learning_rate": 5.788888888888889e-07,
-      "logits/chosen": 6.743104457855225,
-      "logits/rejected": 7.0454840660095215,
-      "logps/chosen": -417.47314453125,
-      "logps/rejected": -1049.904052734375,
-      "loss": 1.0497099876403808,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 14.974113464355469,
-      "rewards/margins": 132.9074249267578,
-      "rewards/rejected": -117.93331146240234,
-      "step": 480
-    },
-    {
-      "epoch": 1.3004320372216682,
-      "grad_norm": 22.960351943969727,
-      "learning_rate": 5.677777777777778e-07,
-      "logits/chosen": 6.46866512298584,
-      "logits/rejected": 6.957917213439941,
-      "logps/chosen": -405.8673400878906,
-      "logps/rejected": -1068.978759765625,
-      "loss": 1.990153694152832,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 11.08216381072998,
-      "rewards/margins": 135.7135772705078,
-      "rewards/rejected": -124.63139343261719,
-      "step": 490
-    },
-    {
-      "epoch": 1.3270189431704886,
-      "grad_norm": 2.325967418670416e-09,
-      "learning_rate": 5.566666666666666e-07,
-      "logits/chosen": 7.048731327056885,
-      "logits/rejected": 7.304018497467041,
-      "logps/chosen": -447.47161865234375,
-      "logps/rejected": -1018.6476440429688,
-      "loss": 1.1221290588378907,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 12.060667037963867,
-      "rewards/margins": 117.6533203125,
-      "rewards/rejected": -105.5926513671875,
-      "step": 500
-    },
-    {
-      "epoch": 1.3536058491193088,
-      "grad_norm": 0.007949860766530037,
-      "learning_rate": 5.455555555555555e-07,
-      "logits/chosen": 6.571198463439941,
-      "logits/rejected": 6.9949541091918945,
-      "logps/chosen": -425.16229248046875,
-      "logps/rejected": -1075.010009765625,
-      "loss": 1.1256651878356934,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 6.658470153808594,
-      "rewards/margins": 131.23922729492188,
-      "rewards/rejected": -124.58077239990234,
-      "step": 510
-    },
-    {
-      "epoch": 1.380192755068129,
-      "grad_norm": 162.58592224121094,
-      "learning_rate": 5.344444444444445e-07,
-      "logits/chosen": 6.9003005027771,
-      "logits/rejected": 7.386146545410156,
-      "logps/chosen": -386.6710510253906,
-      "logps/rejected": -1033.316650390625,
-      "loss": 2.134552001953125,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 8.298912048339844,
-      "rewards/margins": 122.3122329711914,
-      "rewards/rejected": -114.0133056640625,
-      "step": 520
-    },
-    {
-      "epoch": 1.4067796610169492,
-      "grad_norm": 2.28546106484373e-08,
-      "learning_rate": 5.233333333333333e-07,
-      "logits/chosen": 6.5765581130981445,
-      "logits/rejected": 7.117588996887207,
-      "logps/chosen": -364.48895263671875,
-      "logps/rejected": -1099.998291015625,
-      "loss": 1.2829697608947754,
+      "grad_norm": 1.3654603958129883,
+      "learning_rate": 5.742857142857143e-06,
+      "logits/chosen": 7.252472877502441,
+      "logits/rejected": 7.820960998535156,
+      "logps/chosen": -435.6075134277344,
+      "logps/rejected": -1333.8228759765625,
+      "loss": 0.7474074840545655,
       "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 8.331583976745605,
-      "rewards/margins": 137.1312713623047,
-      "rewards/rejected": -128.7996826171875,
-      "step": 530
-    },
-    {
-      "epoch": 1.4333665669657694,
-      "grad_norm": 0.8024188280105591,
-      "learning_rate": 5.122222222222222e-07,
-      "logits/chosen": 6.9486517906188965,
-      "logits/rejected": 7.226126194000244,
-      "logps/chosen": -452.52410888671875,
-      "logps/rejected": -1101.6812744140625,
-      "loss": 0.855518913269043,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 3.314232587814331,
-      "rewards/margins": 140.82302856445312,
-      "rewards/rejected": -137.50875854492188,
-      "step": 540
-    },
-    {
-      "epoch": 1.4599534729145895,
-      "grad_norm": 283.451416015625,
-      "learning_rate": 5.011111111111111e-07,
-      "logits/chosen": 6.747658729553223,
-      "logits/rejected": 7.270951271057129,
-      "logps/chosen": -445.3887634277344,
-      "logps/rejected": -1164.9342041015625,
-      "loss": 0.7842754364013672,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 1.7988097667694092,
-      "rewards/margins": 148.59938049316406,
-      "rewards/rejected": -146.80056762695312,
-      "step": 550
-    },
-    {
-      "epoch": 1.4865403788634097,
-      "grad_norm": 6.495264507836457e-20,
-      "learning_rate": 4.9e-07,
-      "logits/chosen": 7.210297584533691,
-      "logits/rejected": 7.638421535491943,
-      "logps/chosen": -495.406494140625,
-      "logps/rejected": -1289.328125,
-      "loss": 1.998776626586914,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 1.3115170001983643,
-      "rewards/margins": 165.7671661376953,
-      "rewards/rejected": -164.4556427001953,
-      "step": 560
-    },
-    {
-      "epoch": 1.5131272848122301,
-      "grad_norm": 162.8050994873047,
-      "learning_rate": 4.788888888888889e-07,
-      "logits/chosen": 7.229719638824463,
-      "logits/rejected": 7.55483341217041,
-      "logps/chosen": -467.35791015625,
-      "logps/rejected": -1125.8382568359375,
-      "loss": 0.433735990524292,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 1.6890428066253662,
-      "rewards/margins": 135.47544860839844,
-      "rewards/rejected": -133.78640747070312,
-      "step": 570
-    },
-    {
-      "epoch": 1.53971419076105,
-      "grad_norm": 198.3883514404297,
-      "learning_rate": 4.677777777777778e-07,
-      "logits/chosen": 6.802654266357422,
-      "logits/rejected": 7.2844390869140625,
-      "logps/chosen": -412.0586853027344,
-      "logps/rejected": -1095.768798828125,
-      "loss": 0.20694947242736816,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 3.9473280906677246,
-      "rewards/margins": 135.24732971191406,
-      "rewards/rejected": -131.3000030517578,
-      "step": 580
-    },
-    {
-      "epoch": 1.5663010967098705,
-      "grad_norm": 2.5432399297642405e-08,
-      "learning_rate": 4.5666666666666665e-07,
-      "logits/chosen": 7.081494331359863,
-      "logits/rejected": 7.492499351501465,
-      "logps/chosen": -423.6611328125,
-      "logps/rejected": -1167.4932861328125,
-      "loss": 1.276815414428711,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": -0.2825419008731842,
-      "rewards/margins": 142.3505096435547,
-      "rewards/rejected": -142.63302612304688,
-      "step": 590
-    },
-    {
-      "epoch": 1.5928880026586905,
-      "grad_norm": 0.6176006197929382,
-      "learning_rate": 4.455555555555555e-07,
-      "logits/chosen": 7.086031913757324,
-      "logits/rejected": 7.627284049987793,
-      "logps/chosen": -448.5660705566406,
-      "logps/rejected": -1207.191650390625,
-      "loss": 0.29495222568511964,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.9667795300483704,
-      "rewards/margins": 149.69699096679688,
-      "rewards/rejected": -148.73023986816406,
-      "step": 600
-    },
-    {
-      "epoch": 1.6194749086075109,
-      "grad_norm": 66.45056915283203,
-      "learning_rate": 4.344444444444444e-07,
-      "logits/chosen": 7.270118713378906,
-      "logits/rejected": 7.5960588455200195,
-      "logps/chosen": -503.7193298339844,
-      "logps/rejected": -1094.6021728515625,
-      "loss": 1.5946972846984864,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": -3.655120372772217,
-      "rewards/margins": 125.68217468261719,
-      "rewards/rejected": -129.33731079101562,
-      "step": 610
-    },
-    {
-      "epoch": 1.646061814556331,
-      "grad_norm": 280.2427978515625,
-      "learning_rate": 4.2333333333333334e-07,
-      "logits/chosen": 7.251768589019775,
-      "logits/rejected": 7.520864009857178,
-      "logps/chosen": -517.1514892578125,
-      "logps/rejected": -1172.587158203125,
-      "loss": 2.4477691650390625,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.9268826246261597,
-      "rewards/margins": 142.18368530273438,
-      "rewards/rejected": -141.2567901611328,
-      "step": 620
-    },
-    {
-      "epoch": 1.6726487205051512,
-      "grad_norm": 1.0393255949020386,
-      "learning_rate": 4.122222222222222e-07,
-      "logits/chosen": 7.011075019836426,
-      "logits/rejected": 7.46621561050415,
-      "logps/chosen": -447.34124755859375,
-      "logps/rejected": -1143.3458251953125,
-      "loss": 1.0738434791564941,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 1.6229969263076782,
-      "rewards/margins": 142.9796600341797,
-      "rewards/rejected": -141.35665893554688,
-      "step": 630
-    },
-    {
-      "epoch": 1.6992356264539714,
-      "grad_norm": 119.75847625732422,
-      "learning_rate": 4.0111111111111106e-07,
-      "logits/chosen": 6.9999237060546875,
-      "logits/rejected": 7.578449249267578,
-      "logps/chosen": -469.19012451171875,
-      "logps/rejected": -1200.680419921875,
-      "loss": 0.9937694549560547,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.2244775295257568,
-      "rewards/margins": 147.14993286132812,
-      "rewards/rejected": -145.92544555664062,
-      "step": 640
-    },
-    {
-      "epoch": 1.7258225324027916,
-      "grad_norm": 27.75540542602539,
-      "learning_rate": 3.8999999999999997e-07,
-      "logits/chosen": 6.937554359436035,
-      "logits/rejected": 7.374237060546875,
-      "logps/chosen": -463.05438232421875,
-      "logps/rejected": -1168.0521240234375,
-      "loss": 0.39649856090545654,
-      "rewards/accuracies": 0.981249988079071,
-      "rewards/chosen": 3.329749345779419,
-      "rewards/margins": 145.75735473632812,
-      "rewards/rejected": -142.4276123046875,
-      "step": 650
-    },
-    {
-      "epoch": 1.7524094383516118,
-      "grad_norm": 31.218721389770508,
-      "learning_rate": 3.788888888888889e-07,
-      "logits/chosen": 7.072316646575928,
-      "logits/rejected": 7.550895690917969,
-      "logps/chosen": -483.40234375,
-      "logps/rejected": -1184.9073486328125,
-      "loss": 0.25033409595489503,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": -2.29609751701355,
-      "rewards/margins": 147.64785766601562,
-      "rewards/rejected": -149.94395446777344,
-      "step": 660
-    },
-    {
-      "epoch": 1.778996344300432,
-      "grad_norm": 6.414053359549143e-07,
-      "learning_rate": 3.6777777777777774e-07,
-      "logits/chosen": 7.303959846496582,
-      "logits/rejected": 7.623525142669678,
-      "logps/chosen": -508.6453552246094,
-      "logps/rejected": -1223.940673828125,
-      "loss": 0.31205618381500244,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": -2.5321922302246094,
-      "rewards/margins": 150.60842895507812,
-      "rewards/rejected": -153.14060974121094,
-      "step": 670
-    },
-    {
-      "epoch": 1.8055832502492524,
-      "grad_norm": 1.0985974499902462e-12,
-      "learning_rate": 3.5666666666666666e-07,
-      "logits/chosen": 7.344334602355957,
-      "logits/rejected": 7.8254547119140625,
-      "logps/chosen": -532.2833251953125,
-      "logps/rejected": -1228.1844482421875,
-      "loss": 1.0204992294311523,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 3.5795791149139404,
-      "rewards/margins": 159.29647827148438,
-      "rewards/rejected": -155.71688842773438,
-      "step": 680
-    },
-    {
-      "epoch": 1.8321701561980723,
-      "grad_norm": 66.1689453125,
-      "learning_rate": 3.4555555555555557e-07,
-      "logits/chosen": 7.0121636390686035,
-      "logits/rejected": 7.367627143859863,
-      "logps/chosen": -453.792236328125,
-      "logps/rejected": -1141.1865234375,
-      "loss": 0.38547022342681886,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": -0.3728172183036804,
-      "rewards/margins": 139.9238739013672,
-      "rewards/rejected": -140.29669189453125,
-      "step": 690
-    },
-    {
-      "epoch": 1.8587570621468927,
-      "grad_norm": 1.7826409438004044e-20,
-      "learning_rate": 3.3444444444444443e-07,
-      "logits/chosen": 6.914497375488281,
-      "logits/rejected": 7.344313144683838,
-      "logps/chosen": -456.8873596191406,
-      "logps/rejected": -1159.482666015625,
-      "loss": 0.2864746332168579,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": -0.41872739791870117,
-      "rewards/margins": 139.0840606689453,
-      "rewards/rejected": -139.50277709960938,
-      "step": 700
-    },
-    {
-      "epoch": 1.8853439680957127,
-      "grad_norm": 0.6577161550521851,
-      "learning_rate": 3.233333333333333e-07,
-      "logits/chosen": 7.24100399017334,
-      "logits/rejected": 7.729971408843994,
-      "logps/chosen": -453.352783203125,
-      "logps/rejected": -1139.1920166015625,
-      "loss": 0.40453357696533204,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": -0.5374351739883423,
-      "rewards/margins": 134.17910766601562,
-      "rewards/rejected": -134.71653747558594,
-      "step": 710
-    },
-    {
-      "epoch": 1.911930874044533,
-      "grad_norm": 58.065155029296875,
-      "learning_rate": 3.122222222222222e-07,
-      "logits/chosen": 7.2779541015625,
-      "logits/rejected": 7.5862250328063965,
-      "logps/chosen": -502.33489990234375,
-      "logps/rejected": -1174.3145751953125,
-      "loss": 0.2982128143310547,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.528018593788147,
-      "rewards/margins": 134.43031311035156,
-      "rewards/rejected": -133.9022979736328,
-      "step": 720
-    },
-    {
-      "epoch": 1.9385177799933533,
-      "grad_norm": 3.057793140411377,
-      "learning_rate": 3.011111111111111e-07,
-      "logits/chosen": 7.26782751083374,
-      "logits/rejected": 7.741539001464844,
-      "logps/chosen": -496.8504943847656,
-      "logps/rejected": -1235.9169921875,
-      "loss": 0.8299455642700195,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": -0.2519731819629669,
-      "rewards/margins": 152.95582580566406,
-      "rewards/rejected": -153.20779418945312,
-      "step": 730
-    },
-    {
-      "epoch": 1.9651046859421735,
-      "grad_norm": 185.18174743652344,
-      "learning_rate": 2.9e-07,
-      "logits/chosen": 7.201784610748291,
-      "logits/rejected": 7.585198402404785,
-      "logps/chosen": -474.599853515625,
-      "logps/rejected": -1184.7464599609375,
-      "loss": 1.5328912734985352,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": -3.7899742126464844,
-      "rewards/margins": 142.3949432373047,
-      "rewards/rejected": -146.1849365234375,
-      "step": 740
-    },
-    {
-      "epoch": 1.9916915918909937,
-      "grad_norm": 173.19436645507812,
-      "learning_rate": 2.788888888888889e-07,
-      "logits/chosen": 7.347403526306152,
-      "logits/rejected": 7.8731865882873535,
-      "logps/chosen": -484.5741271972656,
-      "logps/rejected": -1241.9910888671875,
-      "loss": 0.9414227485656739,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": -0.26702070236206055,
-      "rewards/margins": 149.2693328857422,
-      "rewards/rejected": -149.53634643554688,
-      "step": 750
-    },
-    {
-      "epoch": 2.015952143569292,
-      "grad_norm": 277.53521728515625,
-      "learning_rate": 2.6777777777777775e-07,
-      "logits/chosen": 7.591332912445068,
-      "logits/rejected": 7.759430408477783,
-      "logps/chosen": -578.1312866210938,
-      "logps/rejected": -1104.3414306640625,
-      "loss": 0.3479891538619995,
-      "rewards/accuracies": 0.9726027250289917,
-      "rewards/chosen": -3.462564468383789,
-      "rewards/margins": 117.48858642578125,
-      "rewards/rejected": -120.95115661621094,
-      "step": 760
-    },
-    {
-      "epoch": 2.0425390495181124,
-      "grad_norm": 81.4224624633789,
-      "learning_rate": 2.5666666666666666e-07,
-      "logits/chosen": 6.820937156677246,
-      "logits/rejected": 7.438076972961426,
-      "logps/chosen": -445.38592529296875,
-      "logps/rejected": -1255.2547607421875,
-      "loss": 0.7632743835449218,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": -1.3056232929229736,
-      "rewards/margins": 162.7215576171875,
-      "rewards/rejected": -164.02719116210938,
-      "step": 770
-    },
-    {
-      "epoch": 2.0691259554669323,
-      "grad_norm": 80.839111328125,
-      "learning_rate": 2.455555555555555e-07,
-      "logits/chosen": 7.089077949523926,
-      "logits/rejected": 7.562623500823975,
-      "logps/chosen": -479.9771423339844,
-      "logps/rejected": -1212.7470703125,
-      "loss": 0.37755522727966306,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -0.4610620439052582,
-      "rewards/margins": 147.49462890625,
-      "rewards/rejected": -147.9556884765625,
-      "step": 780
-    },
-    {
-      "epoch": 2.0957128614157527,
-      "grad_norm": 102.21258544921875,
-      "learning_rate": 2.3444444444444444e-07,
-      "logits/chosen": 7.348860740661621,
-      "logits/rejected": 7.812272548675537,
-      "logps/chosen": -545.9133911132812,
-      "logps/rejected": -1209.3829345703125,
-      "loss": 0.8432134628295899,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": -4.405800819396973,
-      "rewards/margins": 136.8076629638672,
-      "rewards/rejected": -141.2134552001953,
-      "step": 790
-    },
-    {
-      "epoch": 2.122299767364573,
-      "grad_norm": 7.490438461303711,
-      "learning_rate": 2.2333333333333332e-07,
-      "logits/chosen": 6.928166389465332,
-      "logits/rejected": 7.589695930480957,
-      "logps/chosen": -411.7215881347656,
-      "logps/rejected": -1234.5572509765625,
-      "loss": 0.08373026251792907,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": -1.1743253469467163,
-      "rewards/margins": 159.31773376464844,
-      "rewards/rejected": -160.49208068847656,
-      "step": 800
-    },
-    {
-      "epoch": 2.148886673313393,
-      "grad_norm": 0.0003583618381526321,
-      "learning_rate": 2.122222222222222e-07,
-      "logits/chosen": 7.2483720779418945,
-      "logits/rejected": 7.788289546966553,
-      "logps/chosen": -497.52069091796875,
-      "logps/rejected": -1158.642822265625,
-      "loss": 0.1160581350326538,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": 2.365230083465576,
-      "rewards/margins": 146.15255737304688,
-      "rewards/rejected": -143.78732299804688,
-      "step": 810
-    },
-    {
-      "epoch": 2.1754735792622135,
-      "grad_norm": 6.3310980796813965,
-      "learning_rate": 2.011111111111111e-07,
-      "logits/chosen": 6.909984588623047,
-      "logits/rejected": 7.344359397888184,
-      "logps/chosen": -432.39764404296875,
-      "logps/rejected": -1212.4569091796875,
-      "loss": 0.9685474395751953,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": -2.124849319458008,
-      "rewards/margins": 155.05654907226562,
-      "rewards/rejected": -157.18141174316406,
-      "step": 820
-    },
-    {
-      "epoch": 2.2020604852110335,
-      "grad_norm": 5.151050697094206e-09,
-      "learning_rate": 1.8999999999999998e-07,
-      "logits/chosen": 7.006634712219238,
-      "logits/rejected": 7.5766754150390625,
-      "logps/chosen": -431.0802307128906,
-      "logps/rejected": -1220.452392578125,
-      "loss": 1.1500192642211915,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": -2.019794464111328,
-      "rewards/margins": 152.16506958007812,
-      "rewards/rejected": -154.18484497070312,
-      "step": 830
-    },
-    {
-      "epoch": 2.228647391159854,
-      "grad_norm": 1.951496702049138e-18,
-      "learning_rate": 1.7888888888888887e-07,
-      "logits/chosen": 6.816000938415527,
-      "logits/rejected": 7.375506401062012,
-      "logps/chosen": -439.57891845703125,
-      "logps/rejected": -1222.27001953125,
-      "loss": 0.3972776889801025,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.9101593494415283,
-      "rewards/margins": 158.9185333251953,
-      "rewards/rejected": -157.0083770751953,
-      "step": 840
-    },
-    {
-      "epoch": 2.255234297108674,
-      "grad_norm": 2.449645117964328e-15,
-      "learning_rate": 1.6777777777777778e-07,
-      "logits/chosen": 7.166296482086182,
-      "logits/rejected": 7.5857744216918945,
-      "logps/chosen": -484.2479553222656,
-      "logps/rejected": -1235.645263671875,
-      "loss": 0.15833470821380616,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": -0.6545869708061218,
-      "rewards/margins": 155.0919952392578,
-      "rewards/rejected": -155.74655151367188,
-      "step": 850
-    },
-    {
-      "epoch": 2.2818212030574943,
-      "grad_norm": 67.49964141845703,
-      "learning_rate": 1.5666666666666667e-07,
-      "logits/chosen": 6.9471001625061035,
-      "logits/rejected": 7.408398628234863,
-      "logps/chosen": -406.9446105957031,
-      "logps/rejected": -1206.536376953125,
-      "loss": 0.3223508358001709,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 4.314828395843506,
-      "rewards/margins": 160.91775512695312,
-      "rewards/rejected": -156.60293579101562,
-      "step": 860
-    },
-    {
-      "epoch": 2.308408109006314,
-      "grad_norm": 3.4588420021464117e-06,
-      "learning_rate": 1.4555555555555555e-07,
-      "logits/chosen": 6.990222930908203,
-      "logits/rejected": 7.685202598571777,
-      "logps/chosen": -426.66973876953125,
-      "logps/rejected": -1176.889404296875,
-      "loss": 0.8611475944519043,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.3860576152801514,
-      "rewards/margins": 144.15267944335938,
-      "rewards/rejected": -142.76663208007812,
-      "step": 870
-    },
-    {
-      "epoch": 2.3349950149551346,
-      "grad_norm": 4.1328581182331625e-12,
-      "learning_rate": 1.3444444444444444e-07,
-      "logits/chosen": 7.256162166595459,
-      "logits/rejected": 7.685450553894043,
-      "logps/chosen": -462.0904846191406,
-      "logps/rejected": -1166.178466796875,
-      "loss": 0.024902737140655516,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.212837815284729,
-      "rewards/margins": 140.08041381835938,
-      "rewards/rejected": -138.8675537109375,
-      "step": 880
-    },
-    {
-      "epoch": 2.361581920903955,
-      "grad_norm": 22.725154876708984,
-      "learning_rate": 1.2333333333333333e-07,
-      "logits/chosen": 7.431256294250488,
-      "logits/rejected": 7.865132808685303,
-      "logps/chosen": -456.8827209472656,
-      "logps/rejected": -1153.871337890625,
-      "loss": 0.13207526206970216,
-      "rewards/accuracies": 0.981249988079071,
-      "rewards/chosen": 1.6622031927108765,
-      "rewards/margins": 136.24082946777344,
-      "rewards/rejected": -134.57862854003906,
-      "step": 890
-    },
-    {
-      "epoch": 2.388168826852775,
-      "grad_norm": 132.83956909179688,
-      "learning_rate": 1.1222222222222221e-07,
-      "logits/chosen": 7.010849952697754,
-      "logits/rejected": 7.441749572753906,
-      "logps/chosen": -502.49371337890625,
-      "logps/rejected": -1215.2733154296875,
-      "loss": 0.5922121524810791,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": -2.6824889183044434,
-      "rewards/margins": 148.62466430664062,
-      "rewards/rejected": -151.30715942382812,
-      "step": 900
-    },
-    {
-      "epoch": 2.4147557328015954,
-      "grad_norm": 0.005245895590633154,
-      "learning_rate": 1.011111111111111e-07,
-      "logits/chosen": 6.980523109436035,
-      "logits/rejected": 7.430232048034668,
-      "logps/chosen": -470.41253662109375,
-      "logps/rejected": -1160.4951171875,
-      "loss": 1.195225143432617,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": -3.2118802070617676,
-      "rewards/margins": 144.29278564453125,
-      "rewards/rejected": -147.50466918945312,
-      "step": 910
-    },
-    {
-      "epoch": 2.4413426387504154,
-      "grad_norm": 194.52578735351562,
-      "learning_rate": 9e-08,
-      "logits/chosen": 6.884810447692871,
-      "logits/rejected": 7.503731727600098,
-      "logps/chosen": -440.31976318359375,
-      "logps/rejected": -1206.906494140625,
-      "loss": 0.44526066780090334,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 1.156048059463501,
-      "rewards/margins": 149.27732849121094,
-      "rewards/rejected": -148.1212921142578,
-      "step": 920
-    },
-    {
-      "epoch": 2.4679295446992358,
-      "grad_norm": 1.5737574004387467e-14,
-      "learning_rate": 7.888888888888889e-08,
-      "logits/chosen": 7.322862148284912,
-      "logits/rejected": 7.748003959655762,
-      "logps/chosen": -508.33245849609375,
-      "logps/rejected": -1189.603759765625,
-      "loss": 0.18692436218261718,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.148808479309082,
-      "rewards/margins": 146.20956420898438,
-      "rewards/rejected": -145.06076049804688,
-      "step": 930
-    },
-    {
-      "epoch": 2.4945164506480557,
-      "grad_norm": 0.22959347069263458,
-      "learning_rate": 6.777777777777778e-08,
-      "logits/chosen": 7.375940799713135,
-      "logits/rejected": 7.710402011871338,
-      "logps/chosen": -490.12384033203125,
-      "logps/rejected": -1171.1483154296875,
-      "loss": 0.27915282249450685,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.145845651626587,
-      "rewards/margins": 140.29800415039062,
-      "rewards/rejected": -139.1521453857422,
-      "step": 940
-    },
-    {
-      "epoch": 2.521103356596876,
-      "grad_norm": 22.964818954467773,
-      "learning_rate": 5.666666666666666e-08,
-      "logits/chosen": 7.258917331695557,
-      "logits/rejected": 7.766401767730713,
-      "logps/chosen": -467.205322265625,
-      "logps/rejected": -1157.4315185546875,
-      "loss": 1.6006925582885743,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 0.41268739104270935,
-      "rewards/margins": 135.27273559570312,
-      "rewards/rejected": -134.86004638671875,
-      "step": 950
-    },
-    {
-      "epoch": 2.547690262545696,
-      "grad_norm": 3.8648969441501535e-11,
-      "learning_rate": 4.555555555555556e-08,
-      "logits/chosen": 7.018073081970215,
-      "logits/rejected": 7.558196067810059,
-      "logps/chosen": -449.532958984375,
-      "logps/rejected": -1138.4356689453125,
-      "loss": 0.28522279262542727,
-      "rewards/accuracies": 0.981249988079071,
-      "rewards/chosen": -0.8609614372253418,
-      "rewards/margins": 139.2249298095703,
-      "rewards/rejected": -140.0858917236328,
-      "step": 960
-    },
-    {
-      "epoch": 2.5742771684945165,
-      "grad_norm": 84.71375274658203,
-      "learning_rate": 3.4444444444444444e-08,
-      "logits/chosen": 7.130776405334473,
-      "logits/rejected": 7.609295845031738,
-      "logps/chosen": -438.42694091796875,
-      "logps/rejected": -1249.6336669921875,
-      "loss": 0.4750792980194092,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -1.273829460144043,
-      "rewards/margins": 160.90731811523438,
-      "rewards/rejected": -162.18113708496094,
-      "step": 970
-    },
-    {
-      "epoch": 2.6008640744433364,
-      "grad_norm": 85.9113540649414,
-      "learning_rate": 2.3333333333333334e-08,
-      "logits/chosen": 7.113263130187988,
-      "logits/rejected": 7.686596870422363,
-      "logps/chosen": -434.325439453125,
-      "logps/rejected": -1194.6849365234375,
-      "loss": 0.33106160163879395,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": -1.2038366794586182,
-      "rewards/margins": 146.78440856933594,
-      "rewards/rejected": -147.98825073242188,
-      "step": 980
-    },
-    {
-      "epoch": 2.627450980392157,
-      "grad_norm": 2.4605165866986043e-20,
-      "learning_rate": 1.2222222222222222e-08,
-      "logits/chosen": 7.00030517578125,
-      "logits/rejected": 7.477368354797363,
-      "logps/chosen": -450.455078125,
-      "logps/rejected": -1269.2520751953125,
-      "loss": 0.2776132583618164,
-      "rewards/accuracies": 0.981249988079071,
-      "rewards/chosen": -3.280397891998291,
-      "rewards/margins": 163.57626342773438,
-      "rewards/rejected": -166.85665893554688,
-      "step": 990
-    },
-    {
-      "epoch": 2.6540378863409773,
-      "grad_norm": 80.78559112548828,
-      "learning_rate": 1.111111111111111e-09,
-      "logits/chosen": 7.087013244628906,
-      "logits/rejected": 7.507058620452881,
-      "logps/chosen": -490.28857421875,
-      "logps/rejected": -1223.248046875,
-      "loss": 0.2815593719482422,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": -2.167093276977539,
-      "rewards/margins": 151.61813354492188,
-      "rewards/rejected": -153.78524780273438,
-      "step": 1000
+      "rewards/chosen": 1.129046082496643,
+      "rewards/margins": 456.1576232910156,
+      "rewards/rejected": -455.028564453125,
+      "step": 200
     }
   ],
   "logging_steps": 10,
-  "max_steps": 1000,
+  "max_steps": 400,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 3,
+  "num_train_epochs": 2,
   "save_steps": 200,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -1522,7 +322,7 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:07d1084fbcea73eed4529408d2dd186b09d81c71318b95b1f0d3c71ddb884015
+oid sha256:fa5979d784b3be5f03398730b0db9a0aaad24ae1fdea10accf8ecc4f7c831b44
 size 6289