Upload task output abc123

loss.txt

@@ -1 +1 @@
-202,0.00023559275723528117
+202,0.00028668390586972237

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b316ceca0fd69a88ca78e6263ebb4eed5c0bc75c3b1d2a7b77cb662432a40310
+oid sha256:ba1541b31a6b2a77f67aa4a5bd2369f4898ef815907a3883221836f9a7856474
 size 3087467144

trainer_state.json

@@ -11,634 +11,634 @@
   "log_history": [
     {
       "epoch": 0.04926108374384237,
-      "grad_norm": 109.0,
+      "grad_norm": 108.0,
       "learning_rate": 2.52589e-06,
-      "logits/chosen": 0.05971701070666313,
-      "logits/rejected": -0.009489008225500584,
-      "logps/chosen": -715.102294921875,
-      "logps/rejected": -992.0411376953125,
-      "loss": 0.6912,
-      "rewards/accuracies": 0.3333333432674408,
-      "rewards/chosen": -0.0028032560367137194,
-      "rewards/margins": 0.006950830575078726,
-      "rewards/rejected": -0.009754086844623089,
+      "logits/chosen": 0.06021759659051895,
+      "logits/rejected": -0.009350213222205639,
+      "logps/chosen": -714.9625244140625,
+      "logps/rejected": -992.0533447265625,
+      "loss": 0.6838,
+      "rewards/accuracies": 0.366666704416275,
+      "rewards/chosen": 0.011166826821863651,
+      "rewards/margins": 0.022149886935949326,
+      "rewards/rejected": -0.010983060114085674,
       "step": 5
     },
     {
       "epoch": 0.09852216748768473,
-      "grad_norm": 75.0,
+      "grad_norm": 73.5,
       "learning_rate": 5.6832524999999995e-06,
-      "logits/chosen": 0.04588876664638519,
-      "logits/rejected": 0.003985819406807423,
-      "logps/chosen": -671.5519409179688,
-      "logps/rejected": -1003.0562744140625,
+      "logits/chosen": 0.045873817056417465,
+      "logits/rejected": 0.003844240214675665,
+      "logps/chosen": -671.487060546875,
+      "logps/rejected": -1002.9957885742188,
       "loss": 0.5469,
-      "rewards/accuracies": 0.9083333015441895,
-      "rewards/chosen": 0.09333531558513641,
-      "rewards/margins": 0.3377665579319,
-      "rewards/rejected": -0.24443121254444122,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.0998205691576004,
+      "rewards/margins": 0.3382015824317932,
+      "rewards/rejected": -0.2383810579776764,
       "step": 10
     },
     {
       "epoch": 0.1477832512315271,
-      "grad_norm": 29.125,
+      "grad_norm": 30.625,
       "learning_rate": 8.840615e-06,
-      "logits/chosen": -0.02368784323334694,
-      "logits/rejected": 0.04294583946466446,
-      "logps/chosen": -677.3331909179688,
-      "logps/rejected": -989.4844970703125,
-      "loss": 0.2525,
+      "logits/chosen": -0.023920465260744095,
+      "logits/rejected": 0.042549438774585724,
+      "logps/chosen": -677.3431396484375,
+      "logps/rejected": -989.5335693359375,
+      "loss": 0.2497,
       "rewards/accuracies": 0.98333340883255,
-      "rewards/chosen": 0.3687068819999695,
-      "rewards/margins": 1.4707039594650269,
-      "rewards/rejected": -1.101996898651123,
+      "rewards/chosen": 0.3677126467227936,
+      "rewards/margins": 1.4746158123016357,
+      "rewards/rejected": -1.106903076171875,
       "step": 15
     },
     {
       "epoch": 0.19704433497536947,
-      "grad_norm": 3.703125,
+      "grad_norm": 3.71875,
       "learning_rate": 1.19979775e-05,
-      "logits/chosen": 0.008908278308808804,
-      "logits/rejected": 0.017966633662581444,
-      "logps/chosen": -703.7459716796875,
-      "logps/rejected": -1041.655029296875,
-      "loss": 0.0458,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.7780574560165405,
-      "rewards/margins": 4.076639652252197,
-      "rewards/rejected": -3.298582077026367,
+      "logits/chosen": 0.008176160976290703,
+      "logits/rejected": 0.018394723534584045,
+      "logps/chosen": -703.6475830078125,
+      "logps/rejected": -1041.759765625,
+      "loss": 0.0449,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.787887454032898,
+      "rewards/margins": 4.096963405609131,
+      "rewards/rejected": -3.3090755939483643,
       "step": 20
     },
     {
       "epoch": 0.24630541871921183,
-      "grad_norm": 12.0,
+      "grad_norm": 12.5,
       "learning_rate": 1.5155340000000002e-05,
-      "logits/chosen": -0.024580026045441628,
-      "logits/rejected": 0.03568558394908905,
-      "logps/chosen": -717.9066162109375,
-      "logps/rejected": -1072.755126953125,
-      "loss": 0.016,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.2561551332473755,
-      "rewards/margins": 7.924638271331787,
-      "rewards/rejected": -6.668482780456543,
+      "logits/chosen": -0.024350730702280998,
+      "logits/rejected": 0.03716661036014557,
+      "logps/chosen": -717.7275390625,
+      "logps/rejected": -1072.546630859375,
+      "loss": 0.0161,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.2740657329559326,
+      "rewards/margins": 7.921713829040527,
+      "rewards/rejected": -6.647648811340332,
       "step": 25
     },
     {
       "epoch": 0.2955665024630542,
-      "grad_norm": 0.2138671875,
+      "grad_norm": 0.2109375,
       "learning_rate": 1.8312702500000002e-05,
-      "logits/chosen": 0.021068373695015907,
-      "logits/rejected": -0.008218185044825077,
-      "logps/chosen": -672.0543212890625,
-      "logps/rejected": -1118.3712158203125,
+      "logits/chosen": 0.02022477611899376,
+      "logits/rejected": -0.005769997835159302,
+      "logps/chosen": -672.1201171875,
+      "logps/rejected": -1118.390380859375,
       "loss": 0.0009,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.809730887413025,
-      "rewards/margins": 12.14515495300293,
-      "rewards/rejected": -10.335424423217773,
+      "rewards/chosen": 1.8031457662582397,
+      "rewards/margins": 12.140481948852539,
+      "rewards/rejected": -10.337335586547852,
       "step": 30
     },
     {
       "epoch": 0.3448275862068966,
-      "grad_norm": 7.1875,
+      "grad_norm": 8.6875,
       "learning_rate": 2.1470065e-05,
-      "logits/chosen": 0.0051301405765116215,
-      "logits/rejected": -0.02955557033419609,
-      "logps/chosen": -665.8123779296875,
-      "logps/rejected": -1053.738037109375,
-      "loss": 0.0086,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.6763460636138916,
-      "rewards/margins": 13.757688522338867,
-      "rewards/rejected": -12.081342697143555,
+      "logits/chosen": 0.0037163645029067993,
+      "logits/rejected": -0.027010012418031693,
+      "logps/chosen": -665.9528198242188,
+      "logps/rejected": -1054.0537109375,
+      "loss": 0.0097,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.6622908115386963,
+      "rewards/margins": 13.775215148925781,
+      "rewards/rejected": -12.112924575805664,
       "step": 35
     },
     {
       "epoch": 0.39408866995073893,
-      "grad_norm": 4.1875,
+      "grad_norm": 4.53125,
       "learning_rate": 2.2092428004483083e-05,
-      "logits/chosen": 0.07094338536262512,
-      "logits/rejected": 0.05427859351038933,
-      "logps/chosen": -715.7424926757812,
-      "logps/rejected": -1175.1407470703125,
-      "loss": 0.002,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.44687819480896,
-      "rewards/margins": 15.842519760131836,
-      "rewards/rejected": -13.39564037322998,
+      "logits/chosen": 0.0688682422041893,
+      "logits/rejected": 0.05512089654803276,
+      "logps/chosen": -715.6795654296875,
+      "logps/rejected": -1175.619140625,
+      "loss": 0.0022,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.4531707763671875,
+      "rewards/margins": 15.8966703414917,
+      "rewards/rejected": -13.443501472473145,
       "step": 40
     },
     {
       "epoch": 0.4433497536945813,
-      "grad_norm": 0.00194549560546875,
+      "grad_norm": 0.0022735595703125,
       "learning_rate": 2.205545499586072e-05,
-      "logits/chosen": -0.023238930851221085,
-      "logits/rejected": -0.05826393887400627,
-      "logps/chosen": -682.96142578125,
-      "logps/rejected": -1163.7904052734375,
+      "logits/chosen": -0.025104057043790817,
+      "logits/rejected": -0.059289705008268356,
+      "logps/chosen": -683.1431884765625,
+      "logps/rejected": -1164.0970458984375,
       "loss": 0.0002,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.871985912322998,
-      "rewards/margins": 16.598369598388672,
-      "rewards/rejected": -13.7263822555542,
+      "rewards/chosen": 2.8538169860839844,
+      "rewards/margins": 16.61087989807129,
+      "rewards/rejected": -13.757061958312988,
       "step": 45
     },
     {
       "epoch": 0.49261083743842365,
-      "grad_norm": 0.5390625,
+      "grad_norm": 0.5078125,
       "learning_rate": 2.1990176011484834e-05,
-      "logits/chosen": 0.07197682559490204,
-      "logits/rejected": -0.03699414059519768,
-      "logps/chosen": -663.2843627929688,
-      "logps/rejected": -1101.3746337890625,
+      "logits/chosen": 0.06939034163951874,
+      "logits/rejected": -0.03862878680229187,
+      "logps/chosen": -663.315185546875,
+      "logps/rejected": -1101.8394775390625,
       "loss": 0.0012,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.7080256938934326,
-      "rewards/margins": 16.22707176208496,
-      "rewards/rejected": -13.519048690795898,
+      "rewards/chosen": 2.704948663711548,
+      "rewards/margins": 16.27048110961914,
+      "rewards/rejected": -13.565534591674805,
       "step": 50
     },
     {
       "epoch": 0.541871921182266,
-      "grad_norm": 1.1484375,
+      "grad_norm": 1.09375,
       "learning_rate": 2.1896815242896443e-05,
-      "logits/chosen": -0.07335661351680756,
-      "logits/rejected": -0.07419757544994354,
-      "logps/chosen": -613.652099609375,
-      "logps/rejected": -1092.04052734375,
-      "loss": 0.0016,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 2.8754172325134277,
-      "rewards/margins": 16.448772430419922,
-      "rewards/rejected": -13.573356628417969,
+      "logits/chosen": -0.07729315012693405,
+      "logits/rejected": -0.07702396810054779,
+      "logps/chosen": -613.7337036132812,
+      "logps/rejected": -1092.618408203125,
+      "loss": 0.0014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.867260694503784,
+      "rewards/margins": 16.498395919799805,
+      "rewards/rejected": -13.631135940551758,
       "step": 55
     },
     {
       "epoch": 0.5911330049261084,
-      "grad_norm": 0.00213623046875,
+      "grad_norm": 0.002410888671875,
       "learning_rate": 2.1775693324596527e-05,
-      "logits/chosen": -0.03252274543046951,
-      "logits/rejected": -0.03407047688961029,
-      "logps/chosen": -665.9483642578125,
-      "logps/rejected": -1142.9947509765625,
+      "logits/chosen": -0.03670992702245712,
+      "logits/rejected": -0.035895608365535736,
+      "logps/chosen": -666.19970703125,
+      "logps/rejected": -1143.6500244140625,
       "loss": 0.0005,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.2369766235351562,
-      "rewards/margins": 17.267452239990234,
-      "rewards/rejected": -14.030476570129395,
+      "rewards/chosen": 3.211841583251953,
+      "rewards/margins": 17.307846069335938,
+      "rewards/rejected": -14.0960054397583,
       "step": 60
     },
     {
       "epoch": 0.6403940886699507,
-      "grad_norm": 0.00213623046875,
+      "grad_norm": 0.001922607421875,
       "learning_rate": 2.1627226232871688e-05,
-      "logits/chosen": -0.02839934267103672,
-      "logits/rejected": -0.16694848239421844,
-      "logps/chosen": -728.3703002929688,
-      "logps/rejected": -1130.7135009765625,
+      "logits/chosen": -0.031517110764980316,
+      "logits/rejected": -0.16813872754573822,
+      "logps/chosen": -728.2982177734375,
+      "logps/rejected": -1131.3756103515625,
       "loss": 0.0001,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.4549667835235596,
-      "rewards/margins": 17.37836265563965,
-      "rewards/rejected": -13.9233980178833,
+      "rewards/chosen": 3.4621803760528564,
+      "rewards/margins": 17.45178985595703,
+      "rewards/rejected": -13.98961067199707,
       "step": 65
     },
     {
       "epoch": 0.6896551724137931,
-      "grad_norm": 0.1435546875,
+      "grad_norm": 0.11474609375,
       "learning_rate": 2.1451923857181784e-05,
-      "logits/chosen": -0.0006940827006474137,
-      "logits/rejected": -0.10031332820653915,
-      "logps/chosen": -674.8314819335938,
-      "logps/rejected": -1093.62890625,
+      "logits/chosen": -0.003007475985214114,
+      "logits/rejected": -0.1008603423833847,
+      "logps/chosen": -675.1759033203125,
+      "logps/rejected": -1094.3150634765625,
       "loss": 0.0007,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.454847812652588,
-      "rewards/margins": 16.464736938476562,
-      "rewards/rejected": -13.0098876953125,
+      "rewards/chosen": 3.4204094409942627,
+      "rewards/margins": 16.498920440673828,
+      "rewards/rejected": -13.078511238098145,
       "step": 70
     },
     {
       "epoch": 0.7389162561576355,
-      "grad_norm": 0.27734375,
+      "grad_norm": 0.302734375,
       "learning_rate": 2.12503882490159e-05,
-      "logits/chosen": -0.08651714771986008,
-      "logits/rejected": -0.1013587936758995,
-      "logps/chosen": -670.3421020507812,
-      "logps/rejected": -1148.192626953125,
+      "logits/chosen": -0.08852293342351913,
+      "logits/rejected": -0.10156140476465225,
+      "logps/chosen": -670.6781005859375,
+      "logps/rejected": -1148.835693359375,
       "loss": 0.0003,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.3947932720184326,
-      "rewards/margins": 16.52945899963379,
-      "rewards/rejected": -13.134666442871094,
+      "rewards/chosen": 3.3611931800842285,
+      "rewards/margins": 16.560178756713867,
+      "rewards/rejected": -13.198987007141113,
       "step": 75
     },
     {
       "epoch": 0.7881773399014779,
-      "grad_norm": 0.2890625,
+      "grad_norm": 0.29296875,
       "learning_rate": 2.1023311554230692e-05,
-      "logits/chosen": 0.04961782321333885,
-      "logits/rejected": -0.06455739587545395,
-      "logps/chosen": -684.1675415039062,
-      "logps/rejected": -1145.384033203125,
+      "logits/chosen": 0.0469479113817215,
+      "logits/rejected": -0.06342597305774689,
+      "logps/chosen": -684.2789916992188,
+      "logps/rejected": -1146.1676025390625,
       "loss": 0.0002,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.4156594276428223,
-      "rewards/margins": 16.826187133789062,
-      "rewards/rejected": -13.410530090332031,
+      "rewards/chosen": 3.4045166969299316,
+      "rewards/margins": 16.89340591430664,
+      "rewards/rejected": -13.488889694213867,
       "step": 80
     },
     {
       "epoch": 0.8374384236453202,
-      "grad_norm": 0.11962890625,
+      "grad_norm": 0.1240234375,
       "learning_rate": 2.0771473635972312e-05,
-      "logits/chosen": -0.07051565498113632,
-      "logits/rejected": -0.06034456938505173,
-      "logps/chosen": -641.9849853515625,
-      "logps/rejected": -1189.5823974609375,
+      "logits/chosen": -0.07357874512672424,
+      "logits/rejected": -0.06026272848248482,
+      "logps/chosen": -642.2686767578125,
+      "logps/rejected": -1190.205810546875,
       "loss": 0.0001,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.3825697898864746,
-      "rewards/margins": 16.967540740966797,
-      "rewards/rejected": -13.584970474243164,
+      "rewards/chosen": 3.354203701019287,
+      "rewards/margins": 17.001527786254883,
+      "rewards/rejected": -13.647323608398438,
       "step": 85
     },
     {
       "epoch": 0.8866995073891626,
-      "grad_norm": 0.051025390625,
+      "grad_norm": 0.060302734375,
       "learning_rate": 2.0495739396345457e-05,
-      "logits/chosen": -0.08308638632297516,
-      "logits/rejected": -0.15096285939216614,
-      "logps/chosen": -629.1686401367188,
-      "logps/rejected": -1102.910888671875,
+      "logits/chosen": -0.08571887761354446,
+      "logits/rejected": -0.1507987231016159,
+      "logps/chosen": -629.473388671875,
+      "logps/rejected": -1103.9293212890625,
       "loss": 0.0003,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.448202133178711,
-      "rewards/margins": 17.18029022216797,
-      "rewards/rejected": -13.732088088989258,
+      "rewards/chosen": 3.417731761932373,
+      "rewards/margins": 17.251663208007812,
+      "rewards/rejected": -13.833930969238281,
       "step": 90
     },
     {
       "epoch": 0.9359605911330049,
-      "grad_norm": 0.0172119140625,
+      "grad_norm": 0.0128173828125,
       "learning_rate": 2.01970558060281e-05,
-      "logits/chosen": -0.025356780737638474,
-      "logits/rejected": -0.04741012677550316,
-      "logps/chosen": -725.537353515625,
-      "logps/rejected": -1180.159912109375,
+      "logits/chosen": -0.027310481294989586,
+      "logits/rejected": -0.04605848342180252,
+      "logps/chosen": -725.526611328125,
+      "logps/rejected": -1180.865478515625,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.787024974822998,
-      "rewards/margins": 17.374534606933594,
-      "rewards/rejected": -13.587509155273438,
+      "rewards/chosen": 3.788097858428955,
+      "rewards/margins": 17.4461727142334,
+      "rewards/rejected": -13.658073425292969,
       "step": 95
     },
     {
       "epoch": 0.9852216748768473,
       "grad_norm": 0.004241943359375,
       "learning_rate": 1.9876448652033145e-05,
-      "logits/chosen": -0.12176716327667236,
-      "logits/rejected": -0.1069510206580162,
-      "logps/chosen": -615.0023193359375,
-      "logps/rejected": -1111.8590087890625,
+      "logits/chosen": -0.12386312335729599,
+      "logits/rejected": -0.10513818264007568,
+      "logps/chosen": -615.34033203125,
+      "logps/rejected": -1112.7601318359375,
       "loss": 0.001,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.566725492477417,
-      "rewards/margins": 16.64693832397461,
-      "rewards/rejected": -13.080212593078613,
+      "rewards/chosen": 3.5329208374023438,
+      "rewards/margins": 16.703248977661133,
+      "rewards/rejected": -13.170324325561523,
       "step": 100
     },
     {
       "epoch": 0.9950738916256158,
-      "eval_logits/chosen": -0.10971267521381378,
-      "eval_logits/rejected": -0.11195576936006546,
-      "eval_logps/chosen": -621.429443359375,
-      "eval_logps/rejected": -1077.3065185546875,
-      "eval_loss": 0.00026110102771781385,
+      "eval_logits/chosen": -0.11170963197946548,
+      "eval_logits/rejected": -0.11342550069093704,
+      "eval_logps/chosen": -621.7440795898438,
+      "eval_logps/rejected": -1078.06591796875,
+      "eval_loss": 0.00033413898199796677,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": 3.707453727722168,
-      "eval_rewards/margins": 16.420101165771484,
-      "eval_rewards/rejected": -12.71264934539795,
-      "eval_runtime": 16.3811,
-      "eval_samples_per_second": 12.209,
-      "eval_steps_per_second": 12.209,
+      "eval_rewards/chosen": 3.6759870052337646,
+      "eval_rewards/margins": 16.46457862854004,
+      "eval_rewards/rejected": -12.788591384887695,
+      "eval_runtime": 16.5627,
+      "eval_samples_per_second": 12.075,
+      "eval_steps_per_second": 12.075,
       "step": 101
     },
     {
       "epoch": 1.0295566502463054,
-      "grad_norm": 0.00640869140625,
+      "grad_norm": 0.006683349609375,
       "learning_rate": 1.9535019014786414e-05,
-      "logits/chosen": -0.061172205954790115,
-      "logits/rejected": -0.10132347047328949,
-      "logps/chosen": -672.3265380859375,
-      "logps/rejected": -1173.51171875,
+      "logits/chosen": -0.06293856352567673,
+      "logits/rejected": -0.09947305917739868,
+      "logps/chosen": -672.5732421875,
+      "logps/rejected": -1174.2159423828125,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.708890199661255,
-      "rewards/margins": 17.31882667541504,
-      "rewards/rejected": -13.60993766784668,
+      "rewards/chosen": 3.6842219829559326,
+      "rewards/margins": 17.364566802978516,
+      "rewards/rejected": -13.680344581604004,
       "step": 105
     },
     {
       "epoch": 1.0788177339901477,
-      "grad_norm": 0.001983642578125,
+      "grad_norm": 0.0015106201171875,
       "learning_rate": 1.9173939486619933e-05,
-      "logits/chosen": -0.05034123733639717,
-      "logits/rejected": -0.10526905208826065,
-      "logps/chosen": -647.2199096679688,
-      "logps/rejected": -1188.275634765625,
+      "logits/chosen": -0.05223611742258072,
+      "logits/rejected": -0.10244528949260712,
+      "logps/chosen": -647.3909912109375,
+      "logps/rejected": -1189.0113525390625,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 4.165084362030029,
-      "rewards/margins": 17.745838165283203,
-      "rewards/rejected": -13.5807523727417,
+      "rewards/chosen": 4.147972583770752,
+      "rewards/margins": 17.80228614807129,
+      "rewards/rejected": -13.654312133789062,
       "step": 110
     },
     {
       "epoch": 1.1280788177339902,
-      "grad_norm": 0.0123291015625,
+      "grad_norm": 0.01220703125,
       "learning_rate": 1.8794450144667584e-05,
-      "logits/chosen": -0.037617627531290054,
-      "logits/rejected": -0.12743313610553741,
-      "logps/chosen": -771.9480590820312,
-      "logps/rejected": -1097.7689208984375,
+      "logits/chosen": -0.03947216272354126,
+      "logits/rejected": -0.12559063732624054,
+      "logps/chosen": -772.1909790039062,
+      "logps/rejected": -1098.6795654296875,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 4.783888339996338,
-      "rewards/margins": 18.24011993408203,
-      "rewards/rejected": -13.456230163574219,
+      "rewards/chosen": 4.7596025466918945,
+      "rewards/margins": 18.306901931762695,
+      "rewards/rejected": -13.547297477722168,
       "step": 115
     },
     {
       "epoch": 1.1773399014778325,
-      "grad_norm": 0.000865936279296875,
+      "grad_norm": 0.000743865966796875,
       "learning_rate": 1.839785429199364e-05,
-      "logits/chosen": -0.00499363336712122,
-      "logits/rejected": -0.14917989075183868,
-      "logps/chosen": -711.33447265625,
-      "logps/rejected": -1193.215576171875,
+      "logits/chosen": -0.0065459804609417915,
+      "logits/rejected": -0.1465989649295807,
+      "logps/chosen": -711.794189453125,
+      "logps/rejected": -1193.7640380859375,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 4.206370830535889,
-      "rewards/margins": 18.47035026550293,
-      "rewards/rejected": -14.263978958129883,
+      "rewards/chosen": 4.160403251647949,
+      "rewards/margins": 18.479259490966797,
+      "rewards/rejected": -14.318857192993164,
       "step": 120
     },
     {
       "epoch": 1.2266009852216748,
-      "grad_norm": 0.004608154296875,
+      "grad_norm": 0.003936767578125,
       "learning_rate": 1.7985513981580707e-05,
-      "logits/chosen": -0.07984370738267899,
-      "logits/rejected": -0.14964917302131653,
-      "logps/chosen": -656.4620361328125,
-      "logps/rejected": -1094.466064453125,
+      "logits/chosen": -0.0810680016875267,
+      "logits/rejected": -0.14764997363090515,
+      "logps/chosen": -656.9730224609375,
+      "logps/rejected": -1095.385986328125,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.7183139324188232,
-      "rewards/margins": 17.3912353515625,
-      "rewards/rejected": -13.67292308807373,
+      "rewards/chosen": 3.6672158241271973,
+      "rewards/margins": 17.432126998901367,
+      "rewards/rejected": -13.764910697937012,
       "step": 125
     },
     {
       "epoch": 1.2758620689655173,
-      "grad_norm": 0.01708984375,
+      "grad_norm": 0.0162353515625,
       "learning_rate": 1.7558845338549242e-05,
-      "logits/chosen": -0.07335531711578369,
-      "logits/rejected": -0.25236526131629944,
-      "logps/chosen": -668.4403076171875,
-      "logps/rejected": -1086.022216796875,
+      "logits/chosen": -0.07523246854543686,
+      "logits/rejected": -0.25009721517562866,
+      "logps/chosen": -668.7523193359375,
+      "logps/rejected": -1086.5867919921875,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 4.3127336502075195,
-      "rewards/margins": 17.868593215942383,
-      "rewards/rejected": -13.555859565734863,
+      "rewards/chosen": 4.281540393829346,
+      "rewards/margins": 17.89385986328125,
+      "rewards/rejected": -13.61231803894043,
       "step": 130
     },
     {
       "epoch": 1.3251231527093597,
-      "grad_norm": 0.003265380859375,
+      "grad_norm": 0.002838134765625,
       "learning_rate": 1.711931369667393e-05,
-      "logits/chosen": -0.10842283070087433,
-      "logits/rejected": -0.20305411517620087,
-      "logps/chosen": -665.7677001953125,
-      "logps/rejected": -1082.3480224609375,
+      "logits/chosen": -0.1100362166762352,
+      "logits/rejected": -0.200296089053154,
+      "logps/chosen": -666.0404052734375,
+      "logps/rejected": -1082.836669921875,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.790182590484619,
-      "rewards/margins": 17.2746639251709,
-      "rewards/rejected": -13.484481811523438,
+      "rewards/chosen": 3.762917995452881,
+      "rewards/margins": 17.2962703704834,
+      "rewards/rejected": -13.533352851867676,
       "step": 135
     },
     {
       "epoch": 1.374384236453202,
-      "grad_norm": 0.004364013671875,
+      "grad_norm": 0.0068359375,
       "learning_rate": 1.666842856589978e-05,
-      "logits/chosen": -0.10986592620611191,
-      "logits/rejected": -0.11519701778888702,
-      "logps/chosen": -634.3655395507812,
-      "logps/rejected": -1161.351806640625,
+      "logits/chosen": -0.11170603334903717,
+      "logits/rejected": -0.11424056440591812,
+      "logps/chosen": -634.7129516601562,
+      "logps/rejected": -1162.6055908203125,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.9017367362976074,
-      "rewards/margins": 17.374921798706055,
-      "rewards/rejected": -13.473180770874023,
+      "rewards/chosen": 3.8669981956481934,
+      "rewards/margins": 17.46556854248047,
+      "rewards/rejected": -13.59857177734375,
       "step": 140
     },
     {
       "epoch": 1.4236453201970443,
-      "grad_norm": 0.000965118408203125,
+      "grad_norm": 0.0010833740234375,
       "learning_rate": 1.6207738448141366e-05,
-      "logits/chosen": -0.07416743785142899,
-      "logits/rejected": -0.16824878752231598,
-      "logps/chosen": -629.0369262695312,
-      "logps/rejected": -1140.7796630859375,
+      "logits/chosen": -0.07618793845176697,
+      "logits/rejected": -0.1659926474094391,
+      "logps/chosen": -629.3489379882812,
+      "logps/rejected": -1141.2314453125,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.62548828125,
-      "rewards/margins": 17.625173568725586,
-      "rewards/rejected": -13.999687194824219,
+      "rewards/chosen": 3.594289779663086,
+      "rewards/margins": 17.63915252685547,
+      "rewards/rejected": -14.044862747192383,
       "step": 145
     },
     {
       "epoch": 1.4729064039408866,
-      "grad_norm": 0.0030364990234375,
+      "grad_norm": 0.0028839111328125,
       "learning_rate": 1.573882551916961e-05,
-      "logits/chosen": -0.12653779983520508,
-      "logits/rejected": -0.12018134444952011,
-      "logps/chosen": -682.4252319335938,
-      "logps/rejected": -1164.7567138671875,
+      "logits/chosen": -0.1288968324661255,
+      "logits/rejected": -0.11787639558315277,
+      "logps/chosen": -682.9420166015625,
+      "logps/rejected": -1165.747802734375,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.8734822273254395,
-      "rewards/margins": 17.484914779663086,
-      "rewards/rejected": -13.611432075500488,
+      "rewards/chosen": 3.8218026161193848,
+      "rewards/margins": 17.532339096069336,
+      "rewards/rejected": -13.710535049438477,
       "step": 150
     },
     {
       "epoch": 1.522167487684729,
-      "grad_norm": 0.06494140625,
+      "grad_norm": 0.0712890625,
       "learning_rate": 1.5263300194850375e-05,
-      "logits/chosen": -0.13870279490947723,
-      "logits/rejected": -0.17405085265636444,
-      "logps/chosen": -562.0906372070312,
-      "logps/rejected": -1064.6165771484375,
+      "logits/chosen": -0.14020918309688568,
+      "logits/rejected": -0.17282378673553467,
+      "logps/chosen": -562.5584716796875,
+      "logps/rejected": -1065.2425537109375,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.8661842346191406,
-      "rewards/margins": 16.854862213134766,
-      "rewards/rejected": -12.988679885864258,
+      "rewards/chosen": 3.819398880004883,
+      "rewards/margins": 16.870691299438477,
+      "rewards/rejected": -13.051290512084961,
       "step": 155
     },
     {
       "epoch": 1.5714285714285714,
-      "grad_norm": 0.0101318359375,
+      "grad_norm": 0.01177978515625,
       "learning_rate": 1.478279560039646e-05,
-      "logits/chosen": 0.00353141943924129,
-      "logits/rejected": -0.052297573536634445,
-      "logps/chosen": -631.2164916992188,
-      "logps/rejected": -1117.577392578125,
+      "logits/chosen": 0.0017956334631890059,
+      "logits/rejected": -0.04993182793259621,
+      "logps/chosen": -631.7723388671875,
+      "logps/rejected": -1118.0230712890625,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.469149112701416,
-      "rewards/margins": 16.642498016357422,
-      "rewards/rejected": -13.17335033416748,
+      "rewards/chosen": 3.4135658740997314,
+      "rewards/margins": 16.631460189819336,
+      "rewards/rejected": -13.217893600463867,
       "step": 160
     },
     {
       "epoch": 1.6206896551724137,
-      "grad_norm": 0.003143310546875,
+      "grad_norm": 0.0032806396484375,
       "learning_rate": 1.429896196162745e-05,
-      "logits/chosen": -0.0989263653755188,
-      "logits/rejected": -0.08890175819396973,
-      "logps/chosen": -593.7097778320312,
-      "logps/rejected": -1195.1968994140625,
+      "logits/chosen": -0.10030888020992279,
+      "logits/rejected": -0.0873989462852478,
+      "logps/chosen": -593.56884765625,
+      "logps/rejected": -1196.1925048828125,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 4.126656532287598,
-      "rewards/margins": 17.980117797851562,
-      "rewards/rejected": -13.853459358215332,
+      "rewards/chosen": 4.140743255615234,
+      "rewards/margins": 18.093753814697266,
+      "rewards/rejected": -13.953012466430664,
       "step": 165
     },
     {
       "epoch": 1.6699507389162562,
-      "grad_norm": 0.0030975341796875,
+      "grad_norm": 0.004180908203125,
       "learning_rate": 1.3813460937500001e-05,
-      "logits/chosen": -0.020123030990362167,
-      "logits/rejected": -0.1141321063041687,
-      "logps/chosen": -704.115478515625,
-      "logps/rejected": -1211.7939453125,
+      "logits/chosen": -0.022314613685011864,
+      "logits/rejected": -0.1113683357834816,
+      "logps/chosen": -704.1790771484375,
+      "logps/rejected": -1212.56103515625,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.987809419631958,
-      "rewards/margins": 18.1489315032959,
-      "rewards/rejected": -14.161120414733887,
+      "rewards/chosen": 3.9814553260803223,
+      "rewards/margins": 18.219303131103516,
+      "rewards/rejected": -14.237849235534668,
       "step": 170
     },
     {
       "epoch": 1.7192118226600985,
-      "grad_norm": 0.0244140625,
+      "grad_norm": 0.0185546875,
       "learning_rate": 1.332795991337255e-05,
-      "logits/chosen": -0.07781299948692322,
-      "logits/rejected": -0.1914922446012497,
-      "logps/chosen": -714.4182739257812,
-      "logps/rejected": -1111.824951171875,
+      "logits/chosen": -0.0795547217130661,
+      "logits/rejected": -0.18906092643737793,
+      "logps/chosen": -714.6734619140625,
+      "logps/rejected": -1112.668212890625,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 4.549325942993164,
-      "rewards/margins": 18.17251205444336,
-      "rewards/rejected": -13.623186111450195,
+      "rewards/chosen": 4.523798942565918,
+      "rewards/margins": 18.23130989074707,
+      "rewards/rejected": -13.70750904083252,
       "step": 175
     },
     {
       "epoch": 1.7684729064039408,
-      "grad_norm": 0.0003814697265625,
+      "grad_norm": 0.000331878662109375,
       "learning_rate": 1.2844126274603544e-05,
-      "logits/chosen": -0.04165253788232803,
-      "logits/rejected": -0.11223433166742325,
-      "logps/chosen": -728.790771484375,
-      "logps/rejected": -1157.66552734375,
+      "logits/chosen": -0.04283156991004944,
+      "logits/rejected": -0.11096321046352386,
+      "logps/chosen": -729.1672973632812,
+      "logps/rejected": -1158.585205078125,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 4.202837944030762,
-      "rewards/margins": 17.818559646606445,
-      "rewards/rejected": -13.615720748901367,
+      "rewards/chosen": 4.16518497467041,
+      "rewards/margins": 17.872875213623047,
+      "rewards/rejected": -13.707687377929688,
       "step": 180
     },
     {
       "epoch": 1.8177339901477834,
-      "grad_norm": 0.005523681640625,
+      "grad_norm": 0.005584716796875,
       "learning_rate": 1.2363621680149627e-05,
-      "logits/chosen": 0.018531261011958122,
-      "logits/rejected": -0.08929164707660675,
-      "logps/chosen": -655.3833618164062,
-      "logps/rejected": -1119.193115234375,
+      "logits/chosen": 0.017648298293352127,
+      "logits/rejected": -0.08744337409734726,
+      "logps/chosen": -655.9172973632812,
+      "logps/rejected": -1120.0281982421875,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.5976531505584717,
-      "rewards/margins": 17.0244197845459,
-      "rewards/rejected": -13.426763534545898,
+      "rewards/chosen": 3.544255495071411,
+      "rewards/margins": 17.05453109741211,
+      "rewards/rejected": -13.510274887084961,
       "step": 185
     },
     {
       "epoch": 1.8669950738916257,
-      "grad_norm": 0.00177764892578125,
+      "grad_norm": 0.00147247314453125,
       "learning_rate": 1.1888096355830394e-05,
-      "logits/chosen": -0.008384434506297112,
-      "logits/rejected": -0.078025683760643,
-      "logps/chosen": -641.1484985351562,
-      "logps/rejected": -1152.9144287109375,
+      "logits/chosen": -0.009853709489107132,
+      "logits/rejected": -0.07630355656147003,
+      "logps/chosen": -641.5875244140625,
+      "logps/rejected": -1153.7431640625,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 3.942249298095703,
-      "rewards/margins": 17.213258743286133,
-      "rewards/rejected": -13.271008491516113,
+      "rewards/chosen": 3.8983428478240967,
+      "rewards/margins": 17.25222396850586,
+      "rewards/rejected": -13.353883743286133,
       "step": 190
     },
     {
       "epoch": 1.916256157635468,
-      "grad_norm": 0.0306396484375,
+      "grad_norm": 0.040771484375,
       "learning_rate": 1.1419183426858638e-05,
-      "logits/chosen": -0.11397171020507812,
-      "logits/rejected": -0.14867517352104187,
-      "logps/chosen": -606.4990844726562,
-      "logps/rejected": -1104.8248291015625,
+      "logits/chosen": -0.11551934480667114,
+      "logits/rejected": -0.1467764675617218,
+      "logps/chosen": -606.7501220703125,
+      "logps/rejected": -1105.7303466796875,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 4.0419487953186035,
-      "rewards/margins": 17.517881393432617,
-      "rewards/rejected": -13.475932121276855,
+      "rewards/chosen": 4.016844749450684,
+      "rewards/margins": 17.583324432373047,
+      "rewards/rejected": -13.56648063659668,
       "step": 195
     },
     {
       "epoch": 1.9655172413793105,
-      "grad_norm": 0.0322265625,
+      "grad_norm": 0.029541015625,
       "learning_rate": 1.0958493309100225e-05,
-      "logits/chosen": -0.04851642996072769,
-      "logits/rejected": -0.09745767712593079,
-      "logps/chosen": -662.3472900390625,
-      "logps/rejected": -1182.0906982421875,
+      "logits/chosen": -0.050420112907886505,
+      "logits/rejected": -0.09584550559520721,
+      "logps/chosen": -662.5157470703125,
+      "logps/rejected": -1182.98486328125,
       "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 4.098772048950195,
-      "rewards/margins": 18.182476043701172,
-      "rewards/rejected": -14.083703994750977,
+      "rewards/chosen": 4.08192253112793,
+      "rewards/margins": 18.255043029785156,
+      "rewards/rejected": -14.173120498657227,
       "step": 200
     },
     {
       "epoch": 1.9852216748768474,
-      "eval_logits/chosen": -0.11506563425064087,
-      "eval_logits/rejected": -0.11980016529560089,
-      "eval_logps/chosen": -620.9696655273438,
-      "eval_logps/rejected": -1077.2542724609375,
-      "eval_loss": 0.00023559275723528117,
+      "eval_logits/chosen": -0.11707988381385803,
+      "eval_logits/rejected": -0.12201124429702759,
+      "eval_logps/chosen": -621.3673706054688,
+      "eval_logps/rejected": -1078.1207275390625,
+      "eval_loss": 0.00028668390586972237,
       "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": 3.7534308433532715,
-      "eval_rewards/margins": 16.460861206054688,
-      "eval_rewards/rejected": -12.70743179321289,
-      "eval_runtime": 17.0739,
-      "eval_samples_per_second": 11.714,
-      "eval_steps_per_second": 11.714,
+      "eval_rewards/chosen": 3.7136542797088623,
+      "eval_rewards/margins": 16.507728576660156,
+      "eval_rewards/rejected": -12.794075012207031,
+      "eval_runtime": 17.6249,
+      "eval_samples_per_second": 11.348,
+      "eval_steps_per_second": 11.348,
       "step": 202
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1ca5fc5ec268283fdc7c45c09fa4a36b3a361610f08042358654cebbf1cb9274
+oid sha256:5292107477bd9a079b607dd783099e2b018d5b0a17a19c5c05ea03394a245fa8
 size 6392