Training in progress, step 100, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6578cecfe50e7d3a20d040c9b1cdda32c58f9cfcb54fdc5334ee95fe69c2eddf
+oid sha256:96db5154f98f00e2835e99ed5a8fbdf293d8f63243a6f707c81db39c6c06f0a2
 size 204500912

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9d1cf02cb795b4d866231697a267728073ceac5b926c4192928980f2ac671e58
-size 104062923
+oid sha256:cfdbeaba6f405ddd835994d4498a70666b22a02f42c00b97d4334a7f1e1e3bb2
+size 104062731

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:181c5f0270cf39930062ddfa3767a2481d0c360f120b11f8e25dbf533a1cdaba
+oid sha256:7c800b778fa7e115e4c34de8529902de8b61c9a1b4bab3eb8295d06dafff030e
 size 14645

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b147ab74f32279b151eadea15baf5f3d04f7bd6e1b4f97bb054a7a56cf5a70d9
+oid sha256:89c7e585bf25f929251599f24890acceb440d06b049a496384748f1eed552840
 size 1465

last-checkpoint/trainer_state.json

@@ -2,618 +2,168 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0611498836822866,
+  "epoch": 0.26586905948820205,
   "eval_steps": 500,
-  "global_step": 400,
+  "global_step": 100,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.026586905948820207,
-      "grad_norm": 229.74172973632812,
-      "learning_rate": 1.8000000000000001e-06,
-      "logits/chosen": 1.892960548400879,
-      "logits/rejected": 2.2739109992980957,
-      "logps/chosen": -180.7786102294922,
-      "logps/rejected": -296.7843017578125,
-      "loss": 41.754217529296874,
+      "grad_norm": 235.8455810546875,
+      "learning_rate": 9e-06,
+      "logits/chosen": 1.961168646812439,
+      "logits/rejected": 2.3471131324768066,
+      "logps/chosen": -180.52096557617188,
+      "logps/rejected": -306.41729736328125,
+      "loss": 41.09808349609375,
       "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 127.22708892822266,
-      "rewards/margins": 51.65830612182617,
-      "rewards/rejected": 75.56879425048828,
+      "rewards/chosen": 127.35591888427734,
+      "rewards/margins": 56.6036376953125,
+      "rewards/rejected": 70.75228118896484,
       "step": 10
     },
     {
       "epoch": 0.053173811897640415,
-      "grad_norm": 127.54531860351562,
-      "learning_rate": 3.8000000000000005e-06,
-      "logits/chosen": 2.360779285430908,
-      "logits/rejected": 2.5129952430725098,
-      "logps/chosen": -198.55577087402344,
-      "logps/rejected": -311.8116149902344,
-      "loss": 34.334066772460936,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 133.30413818359375,
-      "rewards/margins": 69.63650512695312,
-      "rewards/rejected": 63.667640686035156,
+      "grad_norm": 132.22105407714844,
+      "learning_rate": 9e-06,
+      "logits/chosen": 2.862368106842041,
+      "logits/rejected": 3.0271706581115723,
+      "logps/chosen": -199.9718017578125,
+      "logps/rejected": -383.2444763183594,
+      "loss": 30.093466186523436,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 132.59613037109375,
+      "rewards/margins": 104.64494323730469,
+      "rewards/rejected": 27.951187133789062,
       "step": 20
     },
     {
       "epoch": 0.07976071784646062,
-      "grad_norm": 195.1651153564453,
-      "learning_rate": 5.8e-06,
-      "logits/chosen": 2.7846579551696777,
-      "logits/rejected": 2.912212610244751,
-      "logps/chosen": -201.8841094970703,
-      "logps/rejected": -343.28619384765625,
-      "loss": 31.000405883789064,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 149.70481872558594,
-      "rewards/margins": 90.86196899414062,
-      "rewards/rejected": 58.842857360839844,
+      "grad_norm": 234.4345703125,
+      "learning_rate": 7.88888888888889e-06,
+      "logits/chosen": 3.7322630882263184,
+      "logits/rejected": 3.8905537128448486,
+      "logps/chosen": -219.1993865966797,
+      "logps/rejected": -487.88043212890625,
+      "loss": 23.500108337402345,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 141.04718017578125,
+      "rewards/margins": 154.50144958496094,
+      "rewards/rejected": -13.454266548156738,
       "step": 30
     },
     {
       "epoch": 0.10634762379528083,
-      "grad_norm": 217.8875274658203,
-      "learning_rate": 7.800000000000002e-06,
-      "logits/chosen": 2.687530040740967,
-      "logits/rejected": 2.89595365524292,
-      "logps/chosen": -180.60574340820312,
-      "logps/rejected": -395.2937927246094,
-      "loss": 33.74924926757812,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 124.906494140625,
-      "rewards/margins": 96.38619995117188,
-      "rewards/rejected": 28.52029037475586,
+      "grad_norm": 110.76012420654297,
+      "learning_rate": 6.777777777777779e-06,
+      "logits/chosen": 3.839090347290039,
+      "logits/rejected": 4.133347988128662,
+      "logps/chosen": -210.9350128173828,
+      "logps/rejected": -591.01513671875,
+      "loss": 23.24909210205078,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 109.74186706542969,
+      "rewards/margins": 179.082275390625,
+      "rewards/rejected": -69.34040832519531,
       "step": 40
     },
     {
       "epoch": 0.13293452974410103,
-      "grad_norm": 123.57787322998047,
-      "learning_rate": 9.800000000000001e-06,
-      "logits/chosen": 3.6891541481018066,
-      "logits/rejected": 3.7287964820861816,
-      "logps/chosen": -233.40304565429688,
-      "logps/rejected": -501.949951171875,
-      "loss": 33.165060424804686,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 132.43130493164062,
-      "rewards/margins": 153.64065551757812,
-      "rewards/rejected": -21.209354400634766,
+      "grad_norm": 221.89686584472656,
+      "learning_rate": 5.666666666666667e-06,
+      "logits/chosen": 5.0413312911987305,
+      "logits/rejected": 5.170095920562744,
+      "logps/chosen": -311.14190673828125,
+      "logps/rejected": -735.1285400390625,
+      "loss": 20.513507080078124,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 93.5618667602539,
+      "rewards/margins": 231.3604736328125,
+      "rewards/rejected": -137.79861450195312,
       "step": 50
     },
     {
       "epoch": 0.15952143569292124,
-      "grad_norm": 240.26422119140625,
-      "learning_rate": 9.742857142857143e-06,
-      "logits/chosen": 4.325669288635254,
-      "logits/rejected": 4.669508934020996,
-      "logps/chosen": -227.3274688720703,
-      "logps/rejected": -761.2488403320312,
-      "loss": 9.919349670410156,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 78.26126861572266,
-      "rewards/margins": 259.2169494628906,
-      "rewards/rejected": -180.95565795898438,
+      "grad_norm": 190.1571044921875,
+      "learning_rate": 4.555555555555556e-06,
+      "logits/chosen": 5.5310869216918945,
+      "logits/rejected": 5.918412208557129,
+      "logps/chosen": -291.5699157714844,
+      "logps/rejected": -943.5242919921875,
+      "loss": 4.122843551635742,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 46.140045166015625,
+      "rewards/margins": 318.23345947265625,
+      "rewards/rejected": -272.0934143066406,
       "step": 60
     },
     {
       "epoch": 0.18610834164174145,
-      "grad_norm": 18.67397117614746,
-      "learning_rate": 9.457142857142858e-06,
-      "logits/chosen": 6.029001235961914,
-      "logits/rejected": 6.398881912231445,
-      "logps/chosen": -361.8104248046875,
-      "logps/rejected": -936.2852783203125,
-      "loss": 8.505984497070312,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 62.65727615356445,
-      "rewards/margins": 295.85296630859375,
-      "rewards/rejected": -233.1957550048828,
+      "grad_norm": 20.02957534790039,
+      "learning_rate": 3.444444444444445e-06,
+      "logits/chosen": 6.543715476989746,
+      "logits/rejected": 6.924801826477051,
+      "logps/chosen": -418.068359375,
+      "logps/rejected": -989.2503662109375,
+      "loss": 5.07203254699707,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 34.52830123901367,
+      "rewards/margins": 294.2066345214844,
+      "rewards/rejected": -259.6783447265625,
       "step": 70
     },
     {
       "epoch": 0.21269524759056166,
-      "grad_norm": 424.631591796875,
-      "learning_rate": 9.171428571428572e-06,
-      "logits/chosen": 7.3346147537231445,
-      "logits/rejected": 7.6371870040893555,
-      "logps/chosen": -487.5819396972656,
-      "logps/rejected": -1231.750244140625,
-      "loss": 5.0393016815185545,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 3.2465362548828125,
-      "rewards/margins": 386.61541748046875,
-      "rewards/rejected": -383.368896484375,
+      "grad_norm": 365.81671142578125,
+      "learning_rate": 2.3333333333333336e-06,
+      "logits/chosen": 7.004462242126465,
+      "logits/rejected": 7.306379795074463,
+      "logps/chosen": -456.623291015625,
+      "logps/rejected": -1127.0947265625,
+      "loss": 4.6394092559814455,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 18.725841522216797,
+      "rewards/margins": 349.7669372558594,
+      "rewards/rejected": -331.0411071777344,
       "step": 80
     },
     {
       "epoch": 0.23928215353938184,
-      "grad_norm": 58.2872200012207,
-      "learning_rate": 8.885714285714286e-06,
-      "logits/chosen": 7.502760410308838,
-      "logits/rejected": 7.7781524658203125,
-      "logps/chosen": -515.661376953125,
-      "logps/rejected": -1255.3602294921875,
-      "loss": 2.7182350158691406,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": -3.7650718688964844,
-      "rewards/margins": 391.1650390625,
-      "rewards/rejected": -394.9300842285156,
+      "grad_norm": 442.5915222167969,
+      "learning_rate": 1.2222222222222223e-06,
+      "logits/chosen": 7.063316345214844,
+      "logits/rejected": 7.30740213394165,
+      "logps/chosen": -479.52117919921875,
+      "logps/rejected": -1115.75146484375,
+      "loss": 3.6795909881591795,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 14.30505084991455,
+      "rewards/margins": 339.4307556152344,
+      "rewards/rejected": -325.1257019042969,
       "step": 90
     },
     {
       "epoch": 0.26586905948820205,
-      "grad_norm": 0.0009401601273566484,
-      "learning_rate": 8.6e-06,
-      "logits/chosen": 6.9696502685546875,
-      "logits/rejected": 7.490464687347412,
-      "logps/chosen": -407.62591552734375,
-      "logps/rejected": -1201.7757568359375,
-      "loss": 1.852958869934082,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 16.64017105102539,
-      "rewards/margins": 388.97650146484375,
-      "rewards/rejected": -372.3362731933594,
+      "grad_norm": 170.1773223876953,
+      "learning_rate": 1.1111111111111112e-07,
+      "logits/chosen": 6.971957206726074,
+      "logits/rejected": 7.46600866317749,
+      "logps/chosen": -429.0472106933594,
+      "logps/rejected": -1123.8218994140625,
+      "loss": 3.1557437896728517,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 5.929503440856934,
+      "rewards/margins": 339.28887939453125,
+      "rewards/rejected": -333.359375,
       "step": 100
-    },
-    {
-      "epoch": 0.2924559654370223,
-      "grad_norm": 776.8017578125,
-      "learning_rate": 8.314285714285715e-06,
-      "logits/chosen": 7.605508327484131,
-      "logits/rejected": 8.056883811950684,
-      "logps/chosen": -500.75714111328125,
-      "logps/rejected": -1358.50732421875,
-      "loss": 0.4602807998657227,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": -11.148886680603027,
-      "rewards/margins": 444.7156677246094,
-      "rewards/rejected": -455.8646545410156,
-      "step": 110
-    },
-    {
-      "epoch": 0.3190428713858425,
-      "grad_norm": 9.229455307652179e-14,
-      "learning_rate": 8.02857142857143e-06,
-      "logits/chosen": 7.707437992095947,
-      "logits/rejected": 8.208626747131348,
-      "logps/chosen": -493.72503662109375,
-      "logps/rejected": -1418.4669189453125,
-      "loss": 0.4660654544830322,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": -27.65629005432129,
-      "rewards/margins": 457.8779296875,
-      "rewards/rejected": -485.53424072265625,
-      "step": 120
-    },
-    {
-      "epoch": 0.34562977733466266,
-      "grad_norm": 3.0277444440507395e-10,
-      "learning_rate": 7.742857142857144e-06,
-      "logits/chosen": 7.820192813873291,
-      "logits/rejected": 8.277512550354004,
-      "logps/chosen": -490.735595703125,
-      "logps/rejected": -1337.3505859375,
-      "loss": 1.2780420303344726,
-      "rewards/accuracies": 0.981249988079071,
-      "rewards/chosen": 0.13531294465065002,
-      "rewards/margins": 430.638671875,
-      "rewards/rejected": -430.50335693359375,
-      "step": 130
-    },
-    {
-      "epoch": 0.3722166832834829,
-      "grad_norm": 1.2154150397236663e-07,
-      "learning_rate": 7.457142857142857e-06,
-      "logits/chosen": 7.428654670715332,
-      "logits/rejected": 7.884097099304199,
-      "logps/chosen": -490.9019470214844,
-      "logps/rejected": -1213.6722412109375,
-      "loss": 0.03439792990684509,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": 3.2693276405334473,
-      "rewards/margins": 378.11016845703125,
-      "rewards/rejected": -374.8408203125,
-      "step": 140
-    },
-    {
-      "epoch": 0.3988035892323031,
-      "grad_norm": 0.0008078943355940282,
-      "learning_rate": 7.1714285714285725e-06,
-      "logits/chosen": 7.338967800140381,
-      "logits/rejected": 7.722776889801025,
-      "logps/chosen": -444.8753356933594,
-      "logps/rejected": -1266.625244140625,
-      "loss": 1.42781343460083,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -2.331421375274658,
-      "rewards/margins": 393.41400146484375,
-      "rewards/rejected": -395.74542236328125,
-      "step": 150
-    },
-    {
-      "epoch": 0.4253904951811233,
-      "grad_norm": 0.0,
-      "learning_rate": 6.885714285714287e-06,
-      "logits/chosen": 7.96518087387085,
-      "logits/rejected": 8.39413833618164,
-      "logps/chosen": -586.011962890625,
-      "logps/rejected": -1365.374267578125,
-      "loss": 1.9711128234863282,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -28.12943458557129,
-      "rewards/margins": 417.09210205078125,
-      "rewards/rejected": -445.22149658203125,
-      "step": 160
-    },
-    {
-      "epoch": 0.4519774011299435,
-      "grad_norm": 5.4764127260797935e-12,
-      "learning_rate": 6.600000000000001e-06,
-      "logits/chosen": 7.3422675132751465,
-      "logits/rejected": 7.857165336608887,
-      "logps/chosen": -465.5393981933594,
-      "logps/rejected": -1300.1103515625,
-      "loss": 0.004332171380519867,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": 1.256854772567749,
-      "rewards/margins": 424.2041931152344,
-      "rewards/rejected": -422.9473571777344,
-      "step": 170
-    },
-    {
-      "epoch": 0.4785643070787637,
-      "grad_norm": 422.58892822265625,
-      "learning_rate": 6.314285714285715e-06,
-      "logits/chosen": 7.199074745178223,
-      "logits/rejected": 7.617570400238037,
-      "logps/chosen": -446.2982482910156,
-      "logps/rejected": -1317.12890625,
-      "loss": 0.9593421936035156,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 2.7993197441101074,
-      "rewards/margins": 444.40753173828125,
-      "rewards/rejected": -441.60821533203125,
-      "step": 180
-    },
-    {
-      "epoch": 0.5051512130275839,
-      "grad_norm": 0.0,
-      "learning_rate": 6.028571428571429e-06,
-      "logits/chosen": 7.070580959320068,
-      "logits/rejected": 7.494720458984375,
-      "logps/chosen": -430.97760009765625,
-      "logps/rejected": -1291.35205078125,
-      "loss": 0.09595458507537842,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": 7.344033241271973,
-      "rewards/margins": 433.2731018066406,
-      "rewards/rejected": -425.9291076660156,
-      "step": 190
-    },
-    {
-      "epoch": 0.5317381189764041,
-      "grad_norm": 1.3654603958129883,
-      "learning_rate": 5.742857142857143e-06,
-      "logits/chosen": 7.252472877502441,
-      "logits/rejected": 7.820960998535156,
-      "logps/chosen": -435.6075134277344,
-      "logps/rejected": -1333.8228759765625,
-      "loss": 0.7474074840545655,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 1.129046082496643,
-      "rewards/margins": 456.1576232910156,
-      "rewards/rejected": -455.028564453125,
-      "step": 200
-    },
-    {
-      "epoch": 0.5583250249252243,
-      "grad_norm": 32.07512283325195,
-      "learning_rate": 5.457142857142858e-06,
-      "logits/chosen": 7.640280723571777,
-      "logits/rejected": 8.078705787658691,
-      "logps/chosen": -507.60870361328125,
-      "logps/rejected": -1413.518798828125,
-      "loss": 0.023566444218158723,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -28.288721084594727,
-      "rewards/margins": 461.63006591796875,
-      "rewards/rejected": -489.9187927246094,
-      "step": 210
-    },
-    {
-      "epoch": 0.5849119308740446,
-      "grad_norm": 0.0,
-      "learning_rate": 5.171428571428571e-06,
-      "logits/chosen": 7.952935218811035,
-      "logits/rejected": 8.260855674743652,
-      "logps/chosen": -547.0781860351562,
-      "logps/rejected": -1421.848388671875,
-      "loss": 0.036107948422431944,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -42.91097640991211,
-      "rewards/margins": 435.43353271484375,
-      "rewards/rejected": -478.34454345703125,
-      "step": 220
-    },
-    {
-      "epoch": 0.6114988368228648,
-      "grad_norm": 0.0,
-      "learning_rate": 4.885714285714286e-06,
-      "logits/chosen": 8.114786148071289,
-      "logits/rejected": 8.457345962524414,
-      "logps/chosen": -507.6402893066406,
-      "logps/rejected": -1458.050537109375,
-      "loss": 0.014430716633796692,
-      "rewards/accuracies": 0.981249988079071,
-      "rewards/chosen": -36.03577423095703,
-      "rewards/margins": 463.03021240234375,
-      "rewards/rejected": -499.06591796875,
-      "step": 230
-    },
-    {
-      "epoch": 0.638085742771685,
-      "grad_norm": 0.0,
-      "learning_rate": 4.600000000000001e-06,
-      "logits/chosen": 7.714877128601074,
-      "logits/rejected": 8.194944381713867,
-      "logps/chosen": -527.2571411132812,
-      "logps/rejected": -1552.201171875,
-      "loss": 0.07465354204177857,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -26.272327423095703,
-      "rewards/margins": 538.8110961914062,
-      "rewards/rejected": -565.0833740234375,
-      "step": 240
-    },
-    {
-      "epoch": 0.6646726487205051,
-      "grad_norm": 0.0,
-      "learning_rate": 4.314285714285714e-06,
-      "logits/chosen": 8.05278205871582,
-      "logits/rejected": 8.397611618041992,
-      "logps/chosen": -562.1492919921875,
-      "logps/rejected": -1432.1495361328125,
-      "loss": 1.750390303248341e-16,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -13.465299606323242,
-      "rewards/margins": 452.33758544921875,
-      "rewards/rejected": -465.80291748046875,
-      "step": 250
-    },
-    {
-      "epoch": 0.6912595546693253,
-      "grad_norm": 0.0,
-      "learning_rate": 4.028571428571429e-06,
-      "logits/chosen": 7.9839582443237305,
-      "logits/rejected": 8.325165748596191,
-      "logps/chosen": -531.9362182617188,
-      "logps/rejected": -1311.642578125,
-      "loss": 0.5010682582855225,
-      "rewards/accuracies": 0.981249988079071,
-      "rewards/chosen": -7.1212358474731445,
-      "rewards/margins": 406.3907165527344,
-      "rewards/rejected": -413.511962890625,
-      "step": 260
-    },
-    {
-      "epoch": 0.7178464606181456,
-      "grad_norm": 72.22783660888672,
-      "learning_rate": 3.742857142857143e-06,
-      "logits/chosen": 7.493854522705078,
-      "logits/rejected": 7.991011142730713,
-      "logps/chosen": -503.574462890625,
-      "logps/rejected": -1281.570068359375,
-      "loss": 0.3572836875915527,
-      "rewards/accuracies": 0.981249988079071,
-      "rewards/chosen": -10.670049667358398,
-      "rewards/margins": 382.18377685546875,
-      "rewards/rejected": -392.85382080078125,
-      "step": 270
-    },
-    {
-      "epoch": 0.7444333665669658,
-      "grad_norm": 209.91946411132812,
-      "learning_rate": 3.4571428571428574e-06,
-      "logits/chosen": 7.536074161529541,
-      "logits/rejected": 7.9550018310546875,
-      "logps/chosen": -423.2442321777344,
-      "logps/rejected": -1381.945068359375,
-      "loss": 0.241387939453125,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": -2.9049324989318848,
-      "rewards/margins": 460.4957580566406,
-      "rewards/rejected": -463.40069580078125,
-      "step": 280
-    },
-    {
-      "epoch": 0.771020272515786,
-      "grad_norm": 0.0,
-      "learning_rate": 3.1714285714285714e-06,
-      "logits/chosen": 7.7217535972595215,
-      "logits/rejected": 8.095100402832031,
-      "logps/chosen": -452.80010986328125,
-      "logps/rejected": -1453.189208984375,
-      "loss": 7.635563088115305e-06,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -13.30943489074707,
-      "rewards/margins": 486.92889404296875,
-      "rewards/rejected": -500.23834228515625,
-      "step": 290
-    },
-    {
-      "epoch": 0.7976071784646062,
-      "grad_norm": 0.0,
-      "learning_rate": 2.885714285714286e-06,
-      "logits/chosen": 7.356575012207031,
-      "logits/rejected": 7.875399589538574,
-      "logps/chosen": -446.17254638671875,
-      "logps/rejected": -1462.5728759765625,
-      "loss": 0.026995140314102172,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": -26.922204971313477,
-      "rewards/margins": 505.7860412597656,
-      "rewards/rejected": -532.708251953125,
-      "step": 300
-    },
-    {
-      "epoch": 0.8241940844134263,
-      "grad_norm": 0.0,
-      "learning_rate": 2.6e-06,
-      "logits/chosen": 7.945906162261963,
-      "logits/rejected": 8.316286087036133,
-      "logps/chosen": -521.7435302734375,
-      "logps/rejected": -1435.0,
-      "loss": 2.198499298095703,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": -26.506107330322266,
-      "rewards/margins": 458.09234619140625,
-      "rewards/rejected": -484.59844970703125,
-      "step": 310
-    },
-    {
-      "epoch": 0.8507809903622466,
-      "grad_norm": 0.0,
-      "learning_rate": 2.3142857142857145e-06,
-      "logits/chosen": 7.83941650390625,
-      "logits/rejected": 8.193696975708008,
-      "logps/chosen": -551.3535766601562,
-      "logps/rejected": -1372.135498046875,
-      "loss": 0.004357181861996651,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -26.256458282470703,
-      "rewards/margins": 423.38323974609375,
-      "rewards/rejected": -449.63970947265625,
-      "step": 320
-    },
-    {
-      "epoch": 0.8773678963110668,
-      "grad_norm": 0.0,
-      "learning_rate": 2.028571428571429e-06,
-      "logits/chosen": 7.511332035064697,
-      "logits/rejected": 8.009842872619629,
-      "logps/chosen": -482.1061096191406,
-      "logps/rejected": -1416.7646484375,
-      "loss": 0.0004897473379969596,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -24.751493453979492,
-      "rewards/margins": 472.49639892578125,
-      "rewards/rejected": -497.2478942871094,
-      "step": 330
-    },
-    {
-      "epoch": 0.903954802259887,
-      "grad_norm": 4.885124487524457e-19,
-      "learning_rate": 1.7428571428571432e-06,
-      "logits/chosen": 7.67673397064209,
-      "logits/rejected": 8.061668395996094,
-      "logps/chosen": -477.88983154296875,
-      "logps/rejected": -1371.306396484375,
-      "loss": 0.02846769690513611,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": -19.97150230407715,
-      "rewards/margins": 429.90509033203125,
-      "rewards/rejected": -449.87664794921875,
-      "step": 340
-    },
-    {
-      "epoch": 0.9305417082087072,
-      "grad_norm": 0.0,
-      "learning_rate": 1.4571428571428573e-06,
-      "logits/chosen": 7.819458961486816,
-      "logits/rejected": 8.190754890441895,
-      "logps/chosen": -501.7386169433594,
-      "logps/rejected": -1499.830322265625,
-      "loss": 0.0005395257845520973,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -13.413067817687988,
-      "rewards/margins": 516.6709594726562,
-      "rewards/rejected": -530.083984375,
-      "step": 350
-    },
-    {
-      "epoch": 0.9571286141575274,
-      "grad_norm": 0.0,
-      "learning_rate": 1.1714285714285715e-06,
-      "logits/chosen": 7.6596269607543945,
-      "logits/rejected": 8.10220718383789,
-      "logps/chosen": -496.450439453125,
-      "logps/rejected": -1371.864501953125,
-      "loss": 0.00866500437259674,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": -29.596303939819336,
-      "rewards/margins": 433.4952697753906,
-      "rewards/rejected": -463.09149169921875,
-      "step": 360
-    },
-    {
-      "epoch": 0.9837155201063477,
-      "grad_norm": 1.5750356366836245e-15,
-      "learning_rate": 8.857142857142857e-07,
-      "logits/chosen": 7.884603977203369,
-      "logits/rejected": 8.13658618927002,
-      "logps/chosen": -544.7962646484375,
-      "logps/rejected": -1370.7030029296875,
-      "loss": 0.030383533239364623,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": -27.421581268310547,
-      "rewards/margins": 431.65802001953125,
-      "rewards/rejected": -459.07965087890625,
-      "step": 370
-    },
-    {
-      "epoch": 1.007976071784646,
-      "grad_norm": 0.0,
-      "learning_rate": 6.000000000000001e-07,
-      "logits/chosen": 7.871831893920898,
-      "logits/rejected": 8.307570457458496,
-      "logps/chosen": -519.7576904296875,
-      "logps/rejected": -1396.640869140625,
-      "loss": 0.06974923014640808,
-      "rewards/accuracies": 0.9931507110595703,
-      "rewards/chosen": -26.421640396118164,
-      "rewards/margins": 436.3954162597656,
-      "rewards/rejected": -462.8170471191406,
-      "step": 380
-    },
-    {
-      "epoch": 1.0345629777334662,
-      "grad_norm": 3.976285881214031e-21,
-      "learning_rate": 3.1428571428571433e-07,
-      "logits/chosen": 7.635078430175781,
-      "logits/rejected": 8.017423629760742,
-      "logps/chosen": -461.38946533203125,
-      "logps/rejected": -1499.245361328125,
-      "loss": 0.03160061240196228,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": -30.11972427368164,
-      "rewards/margins": 512.0826416015625,
-      "rewards/rejected": -542.2023315429688,
-      "step": 390
-    },
-    {
-      "epoch": 1.0611498836822866,
-      "grad_norm": 0.0,
-      "learning_rate": 2.8571428571428575e-08,
-      "logits/chosen": 7.7975664138793945,
-      "logits/rejected": 8.254159927368164,
-      "logps/chosen": -477.83282470703125,
-      "logps/rejected": -1497.345703125,
-      "loss": 0.0043321698904037476,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -29.0898380279541,
-      "rewards/margins": 499.4642639160156,
-      "rewards/rejected": -528.5540771484375,
-      "step": 400
     }
   ],
   "logging_steps": 10,
-  "max_steps": 400,
+  "max_steps": 100,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 2,
+  "num_train_epochs": 1,
   "save_steps": 200,
   "stateful_callbacks": {
     "TrainerControl": {

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa5979d784b3be5f03398730b0db9a0aaad24ae1fdea10accf8ecc4f7c831b44
+oid sha256:d0e9844d94b2ddeacda52d988f72cd6b4206ea325ad209511ab311437d2b42ef
 size 6289