Model save

README.md

@@ -3,8 +3,8 @@ library_name: transformers
 model_name: Phi-4-Argunaut-1-SPIN-dev1
 tags:
 - generated_from_trainer
-- dpo
 - trl
+- dpo
 licence: license
 ---
 
@@ -26,7 +26,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/ggbetz/argunauts-training/runs/lp42hqok) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/ggbetz/argunauts-training/runs/1z2x9t5q) 
 
 
 This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 2.0,
     "total_flos": 0.0,
-    "train_loss": 0.2171997334594014,
-    "train_runtime": 3612.0435,
-    "train_samples": 5539,
-    "train_samples_per_second": 3.067,
-    "train_steps_per_second": 0.096
+    "train_loss": 0.2867361557407257,
+    "train_runtime": 4280.6612,
+    "train_samples": 4989,
+    "train_samples_per_second": 2.331,
+    "train_steps_per_second": 0.073
 }

model-00001-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:65af75d35af07125e0b3bf017ec4b398b2c9067b98bc203ee5bd5a5530d92dde
+oid sha256:9b0ea56085ffd706522d837bfb07f83c14f5c8f44b6a9d0b7bd8a8e3415ecd23
 size 4933658528

model-00002-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a57a273238f9c42f1e4337b634935fa7ae3333eac99365e9421f99aa39f669cb
+oid sha256:01b353ed4c7422d6295216f7e1051e831472098cfb6791449260aafef46f81be
 size 4954693112

model-00003-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9ab8312ba59f2e7ab3e2f2f7c5d016179601e7860f6886555ec50eb95c548e3c
+oid sha256:f2aae183aabcb2eb1314dc5d0edad88afe52180500c5e6dd25fa6724f9d652ec
 size 4902243992

model-00004-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa6495e2b7b3faabfb2521d489d3e1fed352d9d566a514e89bb9ae96b8cd14c7
+oid sha256:2d3f14dc985389b4cc36d84d5ace751de00be19b2f1797b535d3aa423b1918cc
 size 4954672440

model-00005-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c295a408d94561912d38721aaf824b7d5691438239174cd46cb106894ae64447
+oid sha256:b43e6befb4e7258089513d8d10dd095913be27ab8012a0e7b5fafdffebf0f8f2
 size 4954672432

model-00006-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e270bbcc9a2da497d6b4a6e7c2c2fd1f1fd2c370e229639cf240ddc36df6491
+oid sha256:c20917ca384f86d677f242d644c35343daa7effe81cdb5cda39ad5262a22e1f7
 size 4619116224

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 2.0,
     "total_flos": 0.0,
-    "train_loss": 0.2171997334594014,
-    "train_runtime": 3612.0435,
-    "train_samples": 5539,
-    "train_samples_per_second": 3.067,
-    "train_steps_per_second": 0.096
+    "train_loss": 0.2867361557407257,
+    "train_runtime": 4280.6612,
+    "train_samples": 4989,
+    "train_samples_per_second": 2.331,
+    "train_steps_per_second": 0.073
 }

trainer_state.json

@@ -4,1058 +4,953 @@
   "best_model_checkpoint": null,
   "epoch": 2.0,
   "eval_steps": 500,
-  "global_step": 348,
+  "global_step": 312,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.02888086642599278,
-      "grad_norm": 11.422099862509954,
-      "learning_rate": 1.111111111111111e-07,
-      "logits/chosen": -2.3837890625,
-      "logits/rejected": -2.0541014671325684,
-      "logps/chosen": -307.1343688964844,
-      "logps/rejected": -332.64373779296875,
-      "loss": 0.3475,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 2.523876905441284,
-      "rewards/margins": 4.429638862609863,
-      "rewards/rejected": -1.906103491783142,
+      "epoch": 0.03205128205128205,
+      "grad_norm": 17.482923590326937,
+      "learning_rate": 1.25e-07,
+      "logits/chosen": -2.0185546875,
+      "logits/rejected": -1.881250023841858,
+      "logps/chosen": -235.671875,
+      "logps/rejected": -302.26873779296875,
+      "loss": 0.38,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 1.7926757335662842,
+      "rewards/margins": 3.30548095703125,
+      "rewards/rejected": -1.5131103992462158,
       "step": 5
     },
     {
-      "epoch": 0.05776173285198556,
-      "grad_norm": 9.346708877153914,
-      "learning_rate": 2.5e-07,
-      "logits/chosen": -2.3408203125,
-      "logits/rejected": -2.0601563453674316,
-      "logps/chosen": -318.1000061035156,
-      "logps/rejected": -406.9624938964844,
-      "loss": 0.3921,
+      "epoch": 0.0641025641025641,
+      "grad_norm": 10.731624225914716,
+      "learning_rate": 2.8125e-07,
+      "logits/chosen": -2.0074219703674316,
+      "logits/rejected": -1.779687523841858,
+      "logps/chosen": -220.52969360351562,
+      "logps/rejected": -415.6499938964844,
+      "loss": 0.4128,
       "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 2.3999266624450684,
-      "rewards/margins": 3.9242920875549316,
-      "rewards/rejected": -1.5220947265625,
+      "rewards/chosen": 1.885888695716858,
+      "rewards/margins": 3.5706787109375,
+      "rewards/rejected": -1.6814696788787842,
       "step": 10
     },
     {
-      "epoch": 0.08664259927797834,
-      "grad_norm": 21.1645613752464,
-      "learning_rate": 3.888888888888889e-07,
-      "logits/chosen": -2.418164014816284,
-      "logits/rejected": -2.087695360183716,
-      "logps/chosen": -357.48748779296875,
-      "logps/rejected": -464.01873779296875,
-      "loss": 0.3932,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 1.938085913658142,
-      "rewards/margins": 3.9061522483825684,
-      "rewards/rejected": -1.965234398841858,
+      "epoch": 0.09615384615384616,
+      "grad_norm": 12.984171783098084,
+      "learning_rate": 4.375e-07,
+      "logits/chosen": -2.097851514816284,
+      "logits/rejected": -1.916015625,
+      "logps/chosen": -212.4031219482422,
+      "logps/rejected": -295.92657470703125,
+      "loss": 0.4187,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 1.880639672279358,
+      "rewards/margins": 3.52978515625,
+      "rewards/rejected": -1.648584008216858,
       "step": 15
     },
     {
-      "epoch": 0.11552346570397112,
-      "grad_norm": 12.894608491427922,
-      "learning_rate": 4.984848484848485e-07,
-      "logits/chosen": -2.3189454078674316,
-      "logits/rejected": -2.0580077171325684,
-      "logps/chosen": -341.4125061035156,
-      "logps/rejected": -354.140625,
-      "loss": 0.3856,
+      "epoch": 0.1282051282051282,
+      "grad_norm": 39.351112208545295,
+      "learning_rate": 4.949324324324325e-07,
+      "logits/chosen": -2.023632764816284,
+      "logits/rejected": -1.8396484851837158,
+      "logps/chosen": -228.0656280517578,
+      "logps/rejected": -375.046875,
+      "loss": 0.5019,
       "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 2.5682129859924316,
-      "rewards/margins": 4.517675876617432,
-      "rewards/rejected": -1.949121117591858,
+      "rewards/chosen": 1.77685546875,
+      "rewards/margins": 3.207226514816284,
+      "rewards/rejected": -1.429071068763733,
       "step": 20
     },
     {
-      "epoch": 0.1444043321299639,
-      "grad_norm": 12.760881593467248,
-      "learning_rate": 4.909090909090909e-07,
-      "logits/chosen": -2.3501954078674316,
-      "logits/rejected": -2.049023389816284,
-      "logps/chosen": -316.30938720703125,
-      "logps/rejected": -418.25,
-      "loss": 0.2819,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": 2.112988233566284,
-      "rewards/margins": 4.386523246765137,
-      "rewards/rejected": -2.271484375,
+      "epoch": 0.16025641025641027,
+      "grad_norm": 8.14663574686874,
+      "learning_rate": 4.864864864864865e-07,
+      "logits/chosen": -1.9912109375,
+      "logits/rejected": -1.8039062023162842,
+      "logps/chosen": -231.5593719482422,
+      "logps/rejected": -515.8937377929688,
+      "loss": 0.4089,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 1.800073266029358,
+      "rewards/margins": 3.571337938308716,
+      "rewards/rejected": -1.771520972251892,
       "step": 25
     },
     {
-      "epoch": 0.17328519855595667,
-      "grad_norm": 12.368541660564347,
-      "learning_rate": 4.833333333333333e-07,
-      "logits/chosen": -2.4140625,
-      "logits/rejected": -2.116992235183716,
-      "logps/chosen": -292.2124938964844,
-      "logps/rejected": -485.0062561035156,
-      "loss": 0.3626,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 2.282482862472534,
-      "rewards/margins": 4.206860542297363,
-      "rewards/rejected": -1.926367163658142,
+      "epoch": 0.19230769230769232,
+      "grad_norm": 16.42433782374998,
+      "learning_rate": 4.780405405405405e-07,
+      "logits/chosen": -1.983789086341858,
+      "logits/rejected": -1.7492187023162842,
+      "logps/chosen": -250.9375,
+      "logps/rejected": -518.8781127929688,
+      "loss": 0.3187,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 1.736535668373108,
+      "rewards/margins": 3.2618165016174316,
+      "rewards/rejected": -1.525964379310608,
       "step": 30
     },
     {
-      "epoch": 0.20216606498194944,
-      "grad_norm": 17.755478161153214,
-      "learning_rate": 4.7575757575757574e-07,
-      "logits/chosen": -2.3724608421325684,
-      "logits/rejected": -2.0888671875,
-      "logps/chosen": -338.65313720703125,
-      "logps/rejected": -413.77813720703125,
-      "loss": 0.3419,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 2.4981446266174316,
-      "rewards/margins": 4.247143745422363,
-      "rewards/rejected": -1.750341773033142,
+      "epoch": 0.22435897435897437,
+      "grad_norm": 16.266393432601383,
+      "learning_rate": 4.695945945945946e-07,
+      "logits/chosen": -2.0166015625,
+      "logits/rejected": -1.91015625,
+      "logps/chosen": -209.7781219482422,
+      "logps/rejected": -289.6625061035156,
+      "loss": 0.3953,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.050332546234131,
+      "rewards/margins": 3.6033051013946533,
+      "rewards/rejected": -1.550323486328125,
       "step": 35
     },
     {
-      "epoch": 0.23104693140794225,
-      "grad_norm": 26.014183301393516,
-      "learning_rate": 4.681818181818182e-07,
-      "logits/chosen": -2.3197264671325684,
-      "logits/rejected": -2.044921875,
-      "logps/chosen": -310.12420654296875,
-      "logps/rejected": -373.03436279296875,
-      "loss": 0.3871,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": 2.195751905441284,
-      "rewards/margins": 4.500244140625,
-      "rewards/rejected": -2.305126905441284,
+      "epoch": 0.2564102564102564,
+      "grad_norm": 25.97441760074817,
+      "learning_rate": 4.611486486486486e-07,
+      "logits/chosen": -1.963476538658142,
+      "logits/rejected": -1.8292968273162842,
+      "logps/chosen": -283.6156311035156,
+      "logps/rejected": -328.79766845703125,
+      "loss": 0.4126,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 1.7740600109100342,
+      "rewards/margins": 3.731884717941284,
+      "rewards/rejected": -1.9563720226287842,
       "step": 40
     },
     {
-      "epoch": 0.259927797833935,
-      "grad_norm": 38.57908256706837,
-      "learning_rate": 4.606060606060606e-07,
-      "logits/chosen": -2.3687500953674316,
-      "logits/rejected": -2.128124952316284,
-      "logps/chosen": -303.8999938964844,
-      "logps/rejected": -405.3687438964844,
-      "loss": 0.3248,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 2.470898389816284,
-      "rewards/margins": 4.250390529632568,
-      "rewards/rejected": -1.774694800376892,
+      "epoch": 0.28846153846153844,
+      "grad_norm": 34.9734844548034,
+      "learning_rate": 4.5270270270270264e-07,
+      "logits/chosen": -2.0054688453674316,
+      "logits/rejected": -1.7921874523162842,
+      "logps/chosen": -274.5687561035156,
+      "logps/rejected": -334.1656188964844,
+      "loss": 0.377,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 2.108358860015869,
+      "rewards/margins": 4.065283298492432,
+      "rewards/rejected": -1.953271508216858,
       "step": 45
     },
     {
-      "epoch": 0.2888086642599278,
-      "grad_norm": 32.89523848493287,
-      "learning_rate": 4.53030303030303e-07,
-      "logits/chosen": -2.338085889816284,
-      "logits/rejected": -2.116406202316284,
-      "logps/chosen": -323.234375,
-      "logps/rejected": -559.8218994140625,
-      "loss": 0.2769,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 2.2285523414611816,
-      "rewards/margins": 4.183837890625,
-      "rewards/rejected": -1.955078125,
+      "epoch": 0.32051282051282054,
+      "grad_norm": 11.913243260486984,
+      "learning_rate": 4.442567567567567e-07,
+      "logits/chosen": -2.0777344703674316,
+      "logits/rejected": -1.947656273841858,
+      "logps/chosen": -225.84219360351562,
+      "logps/rejected": -250.46249389648438,
+      "loss": 0.3373,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 2.101611375808716,
+      "rewards/margins": 3.5291504859924316,
+      "rewards/rejected": -1.426367163658142,
       "step": 50
     },
     {
-      "epoch": 0.3176895306859206,
-      "grad_norm": 11.920071723095447,
-      "learning_rate": 4.4545454545454544e-07,
-      "logits/chosen": -2.395312547683716,
-      "logits/rejected": -2.0990233421325684,
-      "logps/chosen": -304.8843688964844,
-      "logps/rejected": -417.2437438964844,
-      "loss": 0.3615,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": 2.8320069313049316,
-      "rewards/margins": 6.729589939117432,
-      "rewards/rejected": -3.8976807594299316,
+      "epoch": 0.3525641025641026,
+      "grad_norm": 13.672519743198338,
+      "learning_rate": 4.3581081081081076e-07,
+      "logits/chosen": -2.114453077316284,
+      "logits/rejected": -1.878320336341858,
+      "logps/chosen": -331.421875,
+      "logps/rejected": -381.27264404296875,
+      "loss": 0.3941,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 1.777099609375,
+      "rewards/margins": 3.0150146484375,
+      "rewards/rejected": -1.236975073814392,
       "step": 55
     },
     {
-      "epoch": 0.34657039711191334,
-      "grad_norm": 10.026225637915674,
-      "learning_rate": 4.3787878787878784e-07,
-      "logits/chosen": -2.390820264816284,
-      "logits/rejected": -2.187304735183716,
-      "logps/chosen": -341.2906188964844,
-      "logps/rejected": -386.2421875,
-      "loss": 0.3895,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 2.069775342941284,
-      "rewards/margins": 4.311621189117432,
-      "rewards/rejected": -2.2381834983825684,
+      "epoch": 0.38461538461538464,
+      "grad_norm": 11.373635937771688,
+      "learning_rate": 4.2736486486486484e-07,
+      "logits/chosen": -2.139843702316284,
+      "logits/rejected": -1.938867211341858,
+      "logps/chosen": -233.5578155517578,
+      "logps/rejected": -377.8140563964844,
+      "loss": 0.3037,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 2.1447510719299316,
+      "rewards/margins": 3.566601514816284,
+      "rewards/rejected": -1.420263648033142,
       "step": 60
     },
     {
-      "epoch": 0.37545126353790614,
-      "grad_norm": 65.80915152647563,
-      "learning_rate": 4.303030303030303e-07,
-      "logits/chosen": -2.3089842796325684,
-      "logits/rejected": -2.0843749046325684,
-      "logps/chosen": -377.59375,
-      "logps/rejected": -469.63751220703125,
-      "loss": 0.2278,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 1.940588355064392,
-      "rewards/margins": 5.038866996765137,
-      "rewards/rejected": -3.0979981422424316,
+      "epoch": 0.4166666666666667,
+      "grad_norm": 14.605720279749862,
+      "learning_rate": 4.189189189189189e-07,
+      "logits/chosen": -1.915624976158142,
+      "logits/rejected": -1.8369140625,
+      "logps/chosen": -196.640625,
+      "logps/rejected": -297.3812561035156,
+      "loss": 0.3993,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 1.6602294445037842,
+      "rewards/margins": 2.8565430641174316,
+      "rewards/rejected": -1.1953613758087158,
       "step": 65
     },
     {
-      "epoch": 0.4043321299638989,
-      "grad_norm": 19.49794529070635,
-      "learning_rate": 4.227272727272727e-07,
-      "logits/chosen": -2.387500047683716,
-      "logits/rejected": -2.1294922828674316,
-      "logps/chosen": -283.0062561035156,
-      "logps/rejected": -691.9249877929688,
-      "loss": 0.3085,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 1.929785132408142,
-      "rewards/margins": 4.166113376617432,
-      "rewards/rejected": -2.23785400390625,
+      "epoch": 0.44871794871794873,
+      "grad_norm": 12.129690582617949,
+      "learning_rate": 4.1047297297297296e-07,
+      "logits/chosen": -2.043750047683716,
+      "logits/rejected": -1.8582031726837158,
+      "logps/chosen": -269.55780029296875,
+      "logps/rejected": -349.8812561035156,
+      "loss": 0.2719,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 2.2075562477111816,
+      "rewards/margins": 3.8485350608825684,
+      "rewards/rejected": -1.6388671398162842,
       "step": 70
     },
     {
-      "epoch": 0.4332129963898917,
-      "grad_norm": 11.28128140051103,
-      "learning_rate": 4.1515151515151513e-07,
-      "logits/chosen": -2.400195360183716,
-      "logits/rejected": -2.0947265625,
-      "logps/chosen": -291.79766845703125,
-      "logps/rejected": -405.3843688964844,
-      "loss": 0.2438,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 2.5198731422424316,
-      "rewards/margins": 4.618750095367432,
-      "rewards/rejected": -2.0975584983825684,
+      "epoch": 0.4807692307692308,
+      "grad_norm": 10.894196057080642,
+      "learning_rate": 4.02027027027027e-07,
+      "logits/chosen": -2.0833983421325684,
+      "logits/rejected": -1.8416016101837158,
+      "logps/chosen": -218.09375,
+      "logps/rejected": -379.48126220703125,
+      "loss": 0.3121,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 2.30419921875,
+      "rewards/margins": 3.564453125,
+      "rewards/rejected": -1.260766625404358,
       "step": 75
     },
     {
-      "epoch": 0.4620938628158845,
-      "grad_norm": 19.355642911850794,
-      "learning_rate": 4.075757575757576e-07,
-      "logits/chosen": -2.400390625,
-      "logits/rejected": -2.083789110183716,
-      "logps/chosen": -331.4624938964844,
-      "logps/rejected": -432.2875061035156,
-      "loss": 0.1975,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 2.271533250808716,
-      "rewards/margins": 5.00537109375,
-      "rewards/rejected": -2.7354493141174316,
+      "epoch": 0.5128205128205128,
+      "grad_norm": 10.28529818518839,
+      "learning_rate": 3.935810810810811e-07,
+      "logits/chosen": -2.027539014816284,
+      "logits/rejected": -1.8759765625,
+      "logps/chosen": -280.046875,
+      "logps/rejected": -328.8125,
+      "loss": 0.2987,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 2.126843214035034,
+      "rewards/margins": 4.263671875,
+      "rewards/rejected": -2.134082078933716,
       "step": 80
     },
     {
-      "epoch": 0.49097472924187724,
-      "grad_norm": 22.848861871157133,
-      "learning_rate": 4e-07,
-      "logits/chosen": -2.3587889671325684,
-      "logits/rejected": -2.143749952316284,
-      "logps/chosen": -339.46563720703125,
-      "logps/rejected": -404.6968688964844,
-      "loss": 0.2871,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 2.1012206077575684,
-      "rewards/margins": 4.184765815734863,
-      "rewards/rejected": -2.082202196121216,
+      "epoch": 0.5448717948717948,
+      "grad_norm": 10.046587710863294,
+      "learning_rate": 3.851351351351351e-07,
+      "logits/chosen": -2.0047850608825684,
+      "logits/rejected": -1.8369140625,
+      "logps/chosen": -249.7734375,
+      "logps/rejected": -267.5843811035156,
+      "loss": 0.2905,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 2.1821961402893066,
+      "rewards/margins": 3.7508788108825684,
+      "rewards/rejected": -1.568945288658142,
       "step": 85
     },
     {
-      "epoch": 0.51985559566787,
-      "grad_norm": 17.73928799440602,
-      "learning_rate": 3.924242424242424e-07,
-      "logits/chosen": -2.3271484375,
-      "logits/rejected": -2.0726561546325684,
-      "logps/chosen": -289.8374938964844,
-      "logps/rejected": -502.09063720703125,
-      "loss": 0.196,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 2.123095750808716,
-      "rewards/margins": 4.687890529632568,
-      "rewards/rejected": -2.5654053688049316,
+      "epoch": 0.5769230769230769,
+      "grad_norm": 15.815810554242594,
+      "learning_rate": 3.766891891891892e-07,
+      "logits/chosen": -1.9933593273162842,
+      "logits/rejected": -1.7882812023162842,
+      "logps/chosen": -285.0328063964844,
+      "logps/rejected": -244.2062530517578,
+      "loss": 0.2709,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 2.258129835128784,
+      "rewards/margins": 3.865039110183716,
+      "rewards/rejected": -1.611413598060608,
       "step": 90
     },
     {
-      "epoch": 0.5487364620938628,
-      "grad_norm": 44.35578228072781,
-      "learning_rate": 3.8484848484848483e-07,
-      "logits/chosen": -2.315624952316284,
-      "logits/rejected": -2.096874952316284,
-      "logps/chosen": -390.10626220703125,
-      "logps/rejected": -399.70001220703125,
-      "loss": 0.3248,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 2.130859375,
-      "rewards/margins": 4.269433498382568,
-      "rewards/rejected": -2.1373534202575684,
+      "epoch": 0.6089743589743589,
+      "grad_norm": 33.74436261855397,
+      "learning_rate": 3.682432432432432e-07,
+      "logits/chosen": -2.1357421875,
+      "logits/rejected": -1.894140601158142,
+      "logps/chosen": -245.30624389648438,
+      "logps/rejected": -478.70001220703125,
+      "loss": 0.2975,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 1.964135766029358,
+      "rewards/margins": 3.7166991233825684,
+      "rewards/rejected": -1.751123070716858,
       "step": 95
     },
     {
-      "epoch": 0.5776173285198556,
-      "grad_norm": 13.134481184446855,
-      "learning_rate": 3.7727272727272723e-07,
-      "logits/chosen": -2.3763670921325684,
-      "logits/rejected": -2.126953125,
-      "logps/chosen": -337.42498779296875,
-      "logps/rejected": -402.796875,
-      "loss": 0.2493,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 2.370715379714966,
-      "rewards/margins": 4.451855659484863,
-      "rewards/rejected": -2.0814452171325684,
+      "epoch": 0.6410256410256411,
+      "grad_norm": 9.910094853594128,
+      "learning_rate": 3.597972972972973e-07,
+      "logits/chosen": -2.024609327316284,
+      "logits/rejected": -1.865625023841858,
+      "logps/chosen": -264.28436279296875,
+      "logps/rejected": -300.07342529296875,
+      "loss": 0.3171,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 2.304931640625,
+      "rewards/margins": 3.8041014671325684,
+      "rewards/rejected": -1.499169945716858,
       "step": 100
     },
     {
-      "epoch": 0.6064981949458483,
-      "grad_norm": 15.148568274887294,
-      "learning_rate": 3.696969696969697e-07,
-      "logits/chosen": -2.375,
-      "logits/rejected": -2.1224608421325684,
-      "logps/chosen": -331.1499938964844,
-      "logps/rejected": -426.40936279296875,
-      "loss": 0.2124,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 2.0746092796325684,
-      "rewards/margins": 4.863574028015137,
-      "rewards/rejected": -2.7891478538513184,
+      "epoch": 0.6730769230769231,
+      "grad_norm": 11.870080752446105,
+      "learning_rate": 3.5135135135135134e-07,
+      "logits/chosen": -2.0267577171325684,
+      "logits/rejected": -1.8447265625,
+      "logps/chosen": -259.40936279296875,
+      "logps/rejected": -498.8890686035156,
+      "loss": 0.2647,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 2.0042967796325684,
+      "rewards/margins": 3.547900438308716,
+      "rewards/rejected": -1.542810082435608,
       "step": 105
     },
     {
-      "epoch": 0.6353790613718412,
-      "grad_norm": 13.710557758939832,
-      "learning_rate": 3.6212121212121213e-07,
-      "logits/chosen": -2.283203125,
-      "logits/rejected": -2.1001954078674316,
-      "logps/chosen": -349.61248779296875,
-      "logps/rejected": -440.45623779296875,
-      "loss": 0.2573,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 1.9659302234649658,
-      "rewards/margins": 4.624316215515137,
-      "rewards/rejected": -2.6556639671325684,
+      "epoch": 0.7051282051282052,
+      "grad_norm": 8.196509121053467,
+      "learning_rate": 3.429054054054054e-07,
+      "logits/chosen": -2.0044922828674316,
+      "logits/rejected": -1.820703148841858,
+      "logps/chosen": -221.078125,
+      "logps/rejected": -435.2406311035156,
+      "loss": 0.3216,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 2.1091065406799316,
+      "rewards/margins": 3.4317383766174316,
+      "rewards/rejected": -1.322973608970642,
       "step": 110
     },
     {
-      "epoch": 0.6642599277978339,
-      "grad_norm": 64.69545993471166,
-      "learning_rate": 3.545454545454545e-07,
-      "logits/chosen": -2.366406202316284,
-      "logits/rejected": -2.0941405296325684,
-      "logps/chosen": -316.7437438964844,
-      "logps/rejected": -610.2750244140625,
-      "loss": 0.2322,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.97076416015625,
-      "rewards/margins": 4.702880859375,
-      "rewards/rejected": -2.729736328125,
+      "epoch": 0.7371794871794872,
+      "grad_norm": 17.275173620118444,
+      "learning_rate": 3.3445945945945946e-07,
+      "logits/chosen": -2.083203077316284,
+      "logits/rejected": -1.875585913658142,
+      "logps/chosen": -243.6531219482422,
+      "logps/rejected": -477.7124938964844,
+      "loss": 0.2858,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 2.115771532058716,
+      "rewards/margins": 3.8758788108825684,
+      "rewards/rejected": -1.757867455482483,
       "step": 115
     },
     {
-      "epoch": 0.6931407942238267,
-      "grad_norm": 28.158605692193273,
-      "learning_rate": 3.46969696969697e-07,
-      "logits/chosen": -2.3326172828674316,
-      "logits/rejected": -2.194140672683716,
-      "logps/chosen": -422.3999938964844,
-      "logps/rejected": -384.75,
-      "loss": 0.3542,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 2.1262450218200684,
-      "rewards/margins": 4.404687404632568,
-      "rewards/rejected": -2.278759717941284,
+      "epoch": 0.7692307692307693,
+      "grad_norm": 6.190361827721572,
+      "learning_rate": 3.260135135135135e-07,
+      "logits/chosen": -2.025390625,
+      "logits/rejected": -1.809960961341858,
+      "logps/chosen": -276.1343688964844,
+      "logps/rejected": -320.890625,
+      "loss": 0.2334,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 2.201000928878784,
+      "rewards/margins": 4.690966606140137,
+      "rewards/rejected": -2.4933104515075684,
       "step": 120
     },
     {
-      "epoch": 0.7220216606498195,
-      "grad_norm": 54.048997407012784,
-      "learning_rate": 3.393939393939394e-07,
-      "logits/chosen": -2.353515625,
-      "logits/rejected": -2.118945360183716,
-      "logps/chosen": -304.53125,
-      "logps/rejected": -315.09375,
-      "loss": 0.2045,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 2.14715576171875,
-      "rewards/margins": 4.737890720367432,
-      "rewards/rejected": -2.5897459983825684,
+      "epoch": 0.8012820512820513,
+      "grad_norm": 17.015787309272795,
+      "learning_rate": 3.175675675675675e-07,
+      "logits/chosen": -1.984960913658142,
+      "logits/rejected": -1.833593726158142,
+      "logps/chosen": -248.43905639648438,
+      "logps/rejected": -294.2093811035156,
+      "loss": 0.3588,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.134960889816284,
+      "rewards/margins": 3.898571729660034,
+      "rewards/rejected": -1.761315941810608,
       "step": 125
     },
     {
-      "epoch": 0.7509025270758123,
-      "grad_norm": 11.276984222127565,
-      "learning_rate": 3.318181818181818e-07,
-      "logits/chosen": -2.3589844703674316,
-      "logits/rejected": -2.185546875,
-      "logps/chosen": -403.38751220703125,
-      "logps/rejected": -515.4000244140625,
-      "loss": 0.3345,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 1.581787109375,
-      "rewards/margins": 4.064501762390137,
-      "rewards/rejected": -2.4795165061950684,
+      "epoch": 0.8333333333333334,
+      "grad_norm": 9.916096150406192,
+      "learning_rate": 3.091216216216216e-07,
+      "logits/chosen": -2.0804686546325684,
+      "logits/rejected": -1.899999976158142,
+      "logps/chosen": -237.94686889648438,
+      "logps/rejected": -357.84063720703125,
+      "loss": 0.2721,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 2.0490965843200684,
+      "rewards/margins": 3.5159668922424316,
+      "rewards/rejected": -1.4671142101287842,
       "step": 130
     },
     {
-      "epoch": 0.779783393501805,
-      "grad_norm": 14.89194313601513,
-      "learning_rate": 3.242424242424242e-07,
-      "logits/chosen": -2.359570264816284,
-      "logits/rejected": -2.1517577171325684,
-      "logps/chosen": -338.0874938964844,
-      "logps/rejected": -509.6312561035156,
-      "loss": 0.2248,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 2.08935546875,
-      "rewards/margins": 4.466210842132568,
-      "rewards/rejected": -2.376904249191284,
+      "epoch": 0.8653846153846154,
+      "grad_norm": 19.43204229304952,
+      "learning_rate": 3.0067567567567564e-07,
+      "logits/chosen": -1.9519531726837158,
+      "logits/rejected": -1.7833983898162842,
+      "logps/chosen": -268.28436279296875,
+      "logps/rejected": -376.12188720703125,
+      "loss": 0.2836,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 2.1514039039611816,
+      "rewards/margins": 4.179858207702637,
+      "rewards/rejected": -2.028857469558716,
       "step": 135
     },
     {
-      "epoch": 0.8086642599277978,
-      "grad_norm": 14.736139465045111,
-      "learning_rate": 3.166666666666666e-07,
-      "logits/chosen": -2.36328125,
-      "logits/rejected": -2.017773389816284,
-      "logps/chosen": -294.9203186035156,
-      "logps/rejected": -433.78125,
-      "loss": 0.166,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 2.3936524391174316,
-      "rewards/margins": 5.378954887390137,
-      "rewards/rejected": -2.98095703125,
+      "epoch": 0.8974358974358975,
+      "grad_norm": 14.35891063544634,
+      "learning_rate": 2.922297297297297e-07,
+      "logits/chosen": -2.075976610183716,
+      "logits/rejected": -1.883203148841858,
+      "logps/chosen": -203.46875,
+      "logps/rejected": -299.484375,
+      "loss": 0.2292,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 2.165087938308716,
+      "rewards/margins": 3.933666944503784,
+      "rewards/rejected": -1.7722899913787842,
       "step": 140
     },
     {
-      "epoch": 0.8375451263537906,
-      "grad_norm": 16.93771836336577,
-      "learning_rate": 3.0909090909090907e-07,
-      "logits/chosen": -2.420117139816284,
-      "logits/rejected": -2.133984327316284,
-      "logps/chosen": -326.1499938964844,
-      "logps/rejected": -514.8312377929688,
-      "loss": 0.1949,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 2.119824171066284,
-      "rewards/margins": 4.917089939117432,
-      "rewards/rejected": -2.796398878097534,
+      "epoch": 0.9294871794871795,
+      "grad_norm": 15.571301633489812,
+      "learning_rate": 2.8378378378378376e-07,
+      "logits/chosen": -2.001757860183716,
+      "logits/rejected": -1.8171875476837158,
+      "logps/chosen": -257.5218811035156,
+      "logps/rejected": -395.0625,
+      "loss": 0.2054,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 2.296875,
+      "rewards/margins": 4.208886623382568,
+      "rewards/rejected": -1.913354516029358,
       "step": 145
     },
     {
-      "epoch": 0.8664259927797834,
-      "grad_norm": 11.49584268590893,
-      "learning_rate": 3.015151515151515e-07,
-      "logits/chosen": -2.2669920921325684,
-      "logits/rejected": -2.0160155296325684,
-      "logps/chosen": -373.2562561035156,
-      "logps/rejected": -530.3125,
-      "loss": 0.1704,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 2.597363233566284,
-      "rewards/margins": 6.016992092132568,
-      "rewards/rejected": -3.421142578125,
+      "epoch": 0.9615384615384616,
+      "grad_norm": 10.241466724079272,
+      "learning_rate": 2.7533783783783784e-07,
+      "logits/chosen": -2.010937452316284,
+      "logits/rejected": -1.7804687023162842,
+      "logps/chosen": -263.1890563964844,
+      "logps/rejected": -553.5797119140625,
+      "loss": 0.2288,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 2.049511671066284,
+      "rewards/margins": 4.341113090515137,
+      "rewards/rejected": -2.291332960128784,
       "step": 150
     },
     {
-      "epoch": 0.8953068592057761,
-      "grad_norm": 13.443716803588206,
-      "learning_rate": 2.939393939393939e-07,
-      "logits/chosen": -2.3427734375,
-      "logits/rejected": -2.0787110328674316,
-      "logps/chosen": -363.0249938964844,
-      "logps/rejected": -542.6749877929688,
-      "loss": 0.1253,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 1.879052758216858,
-      "rewards/margins": 5.240624904632568,
-      "rewards/rejected": -3.363940477371216,
+      "epoch": 0.9935897435897436,
+      "grad_norm": 8.858764442710157,
+      "learning_rate": 2.6689189189189187e-07,
+      "logits/chosen": -2.089062452316284,
+      "logits/rejected": -1.937890648841858,
+      "logps/chosen": -220.54843139648438,
+      "logps/rejected": -315.1640625,
+      "loss": 0.2811,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 2.0038084983825684,
+      "rewards/margins": 3.595703125,
+      "rewards/rejected": -1.5892822742462158,
       "step": 155
     },
     {
-      "epoch": 0.924187725631769,
-      "grad_norm": 34.79241720252881,
-      "learning_rate": 2.8636363636363637e-07,
-      "logits/chosen": -2.349414110183716,
-      "logits/rejected": -2.1187500953674316,
-      "logps/chosen": -329.6875,
-      "logps/rejected": -474.125,
-      "loss": 0.2088,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 2.0616211891174316,
-      "rewards/margins": 4.8037109375,
-      "rewards/rejected": -2.7415528297424316,
+      "epoch": 1.0256410256410255,
+      "grad_norm": 9.684185372221096,
+      "learning_rate": 2.5844594594594596e-07,
+      "logits/chosen": -2.1142578125,
+      "logits/rejected": -1.8634765148162842,
+      "logps/chosen": -242.419921875,
+      "logps/rejected": -744.0179443359375,
+      "loss": 0.2283,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 2.2816405296325684,
+      "rewards/margins": 4.6513671875,
+      "rewards/rejected": -2.3667969703674316,
       "step": 160
     },
     {
-      "epoch": 0.9530685920577617,
-      "grad_norm": 17.4218945141771,
-      "learning_rate": 2.787878787878788e-07,
-      "logits/chosen": -2.4146485328674316,
-      "logits/rejected": -2.116015672683716,
-      "logps/chosen": -353.875,
-      "logps/rejected": -560.4375,
-      "loss": 0.1459,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 2.0111327171325684,
-      "rewards/margins": 5.4267578125,
-      "rewards/rejected": -3.4139647483825684,
+      "epoch": 1.0576923076923077,
+      "grad_norm": 16.39607167381079,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": -2.0179686546325684,
+      "logits/rejected": -1.838281273841858,
+      "logps/chosen": -285.15155029296875,
+      "logps/rejected": -555.5343627929688,
+      "loss": 0.2506,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 2.1651368141174316,
+      "rewards/margins": 3.9991211891174316,
+      "rewards/rejected": -1.832617163658142,
       "step": 165
     },
     {
-      "epoch": 0.9819494584837545,
-      "grad_norm": 7.763570540153691,
-      "learning_rate": 2.712121212121212e-07,
-      "logits/chosen": -2.291796922683716,
-      "logits/rejected": -2.0296874046325684,
-      "logps/chosen": -390.41876220703125,
-      "logps/rejected": -468.45623779296875,
-      "loss": 0.1389,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 2.46728515625,
-      "rewards/margins": 6.066796779632568,
-      "rewards/rejected": -3.6032471656799316,
+      "epoch": 1.0897435897435896,
+      "grad_norm": 12.71417077582037,
+      "learning_rate": 2.41554054054054e-07,
+      "logits/chosen": -1.9617187976837158,
+      "logits/rejected": -1.7705078125,
+      "logps/chosen": -229.18124389648438,
+      "logps/rejected": -388.0687561035156,
+      "loss": 0.2361,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 2.22900390625,
+      "rewards/margins": 4.506933689117432,
+      "rewards/rejected": -2.277844190597534,
       "step": 170
     },
     {
-      "epoch": 1.0057761732851986,
-      "grad_norm": 5.250963625120634,
-      "learning_rate": 2.636363636363636e-07,
-      "logits/chosen": -2.3120265007019043,
-      "logits/rejected": -2.04237699508667,
-      "logps/chosen": -427.3636474609375,
-      "logps/rejected": -674.1060791015625,
-      "loss": 0.1323,
-      "rewards/accuracies": 0.9217172265052795,
-      "rewards/chosen": 2.187159776687622,
-      "rewards/margins": 6.69081449508667,
-      "rewards/rejected": -4.504527568817139,
+      "epoch": 1.1217948717948718,
+      "grad_norm": 12.537778856529926,
+      "learning_rate": 2.331081081081081e-07,
+      "logits/chosen": -2.107226610183716,
+      "logits/rejected": -1.90234375,
+      "logps/chosen": -215.9656219482422,
+      "logps/rejected": -324.7749938964844,
+      "loss": 0.2086,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 2.130053758621216,
+      "rewards/margins": 4.331640720367432,
+      "rewards/rejected": -2.203198194503784,
       "step": 175
     },
     {
-      "epoch": 1.0346570397111914,
-      "grad_norm": 7.717889023853925,
-      "learning_rate": 2.56060606060606e-07,
-      "logits/chosen": -2.347460985183716,
-      "logits/rejected": -2.016796827316284,
-      "logps/chosen": -360.2875061035156,
-      "logps/rejected": -712.875,
-      "loss": 0.1385,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.92120361328125,
-      "rewards/margins": 6.072070121765137,
-      "rewards/rejected": -4.151269435882568,
+      "epoch": 1.1538461538461537,
+      "grad_norm": 13.46928418539436,
+      "learning_rate": 2.2466216216216216e-07,
+      "logits/chosen": -1.991601586341858,
+      "logits/rejected": -1.790429711341858,
+      "logps/chosen": -239.49063110351562,
+      "logps/rejected": -397.62811279296875,
+      "loss": 0.2422,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 2.213427782058716,
+      "rewards/margins": 4.355273246765137,
+      "rewards/rejected": -2.143115282058716,
       "step": 180
     },
     {
-      "epoch": 1.0635379061371841,
-      "grad_norm": 8.349982709565937,
-      "learning_rate": 2.4848484848484846e-07,
-      "logits/chosen": -2.275195360183716,
-      "logits/rejected": -2.0523438453674316,
-      "logps/chosen": -386.3671875,
-      "logps/rejected": -524.875,
-      "loss": 0.2285,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 2.014880418777466,
-      "rewards/margins": 5.687792778015137,
-      "rewards/rejected": -3.669323682785034,
+      "epoch": 1.185897435897436,
+      "grad_norm": 16.721244760339083,
+      "learning_rate": 2.1621621621621622e-07,
+      "logits/chosen": -2.0091795921325684,
+      "logits/rejected": -1.8250000476837158,
+      "logps/chosen": -260.0874938964844,
+      "logps/rejected": -433.359375,
+      "loss": 0.3083,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.1787109375,
+      "rewards/margins": 3.8218750953674316,
+      "rewards/rejected": -1.6440918445587158,
       "step": 185
     },
     {
-      "epoch": 1.0924187725631769,
-      "grad_norm": 32.542684480595675,
-      "learning_rate": 2.409090909090909e-07,
-      "logits/chosen": -2.368945360183716,
-      "logits/rejected": -2.0582032203674316,
-      "logps/chosen": -374.5289001464844,
-      "logps/rejected": -360.23126220703125,
-      "loss": 0.2207,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.8182861804962158,
-      "rewards/margins": 5.199414253234863,
-      "rewards/rejected": -3.3819823265075684,
+      "epoch": 1.217948717948718,
+      "grad_norm": 6.4545177104485845,
+      "learning_rate": 2.0777027027027025e-07,
+      "logits/chosen": -2.0591797828674316,
+      "logits/rejected": -1.8468749523162842,
+      "logps/chosen": -235.1687469482422,
+      "logps/rejected": -273.58123779296875,
+      "loss": 0.2293,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 2.180835008621216,
+      "rewards/margins": 3.9361329078674316,
+      "rewards/rejected": -1.753662109375,
       "step": 190
     },
     {
-      "epoch": 1.1212996389891696,
-      "grad_norm": 5.443641635105693,
-      "learning_rate": 2.3333333333333333e-07,
-      "logits/chosen": -2.3607420921325684,
-      "logits/rejected": -2.0083985328674316,
-      "logps/chosen": -354.98126220703125,
-      "logps/rejected": -578.9874877929688,
-      "loss": 0.1568,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 2.081469774246216,
-      "rewards/margins": 6.055468559265137,
-      "rewards/rejected": -3.973706007003784,
+      "epoch": 1.25,
+      "grad_norm": 13.390863695555577,
+      "learning_rate": 1.993243243243243e-07,
+      "logits/chosen": -2.015429735183716,
+      "logits/rejected": -1.857812523841858,
+      "logps/chosen": -247.94686889648438,
+      "logps/rejected": -330.03436279296875,
+      "loss": 0.2265,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 2.378588914871216,
+      "rewards/margins": 4.5875244140625,
+      "rewards/rejected": -2.206225633621216,
       "step": 195
     },
     {
-      "epoch": 1.1501805054151624,
-      "grad_norm": 29.210296481306564,
-      "learning_rate": 2.2575757575757576e-07,
-      "logits/chosen": -2.4625000953674316,
-      "logits/rejected": -2.1500000953674316,
-      "logps/chosen": -270.30938720703125,
-      "logps/rejected": -522.21875,
-      "loss": 0.1728,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 1.7206542491912842,
-      "rewards/margins": 5.392187595367432,
-      "rewards/rejected": -3.6748046875,
+      "epoch": 1.282051282051282,
+      "grad_norm": 7.467973287614187,
+      "learning_rate": 1.9087837837837837e-07,
+      "logits/chosen": -1.9660155773162842,
+      "logits/rejected": -1.7736327648162842,
+      "logps/chosen": -242.78125,
+      "logps/rejected": -278.73126220703125,
+      "loss": 0.3117,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 2.247753858566284,
+      "rewards/margins": 3.825390577316284,
+      "rewards/rejected": -1.5797607898712158,
       "step": 200
     },
     {
-      "epoch": 1.1790613718411551,
-      "grad_norm": 8.507315599940252,
-      "learning_rate": 2.1818181818181815e-07,
-      "logits/chosen": -2.3212890625,
-      "logits/rejected": -2.050585985183716,
-      "logps/chosen": -347.82501220703125,
-      "logps/rejected": -477.4375,
-      "loss": 0.238,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 2.0087647438049316,
-      "rewards/margins": 5.121484279632568,
-      "rewards/rejected": -3.111254930496216,
+      "epoch": 1.314102564102564,
+      "grad_norm": 17.275598489252985,
+      "learning_rate": 1.8243243243243243e-07,
+      "logits/chosen": -2.0658202171325684,
+      "logits/rejected": -1.8517577648162842,
+      "logps/chosen": -226.86563110351562,
+      "logps/rejected": -353.4937438964844,
+      "loss": 0.254,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 2.29681396484375,
+      "rewards/margins": 4.391015529632568,
+      "rewards/rejected": -2.097705125808716,
       "step": 205
     },
     {
-      "epoch": 1.207942238267148,
-      "grad_norm": 11.12133440056853,
-      "learning_rate": 2.106060606060606e-07,
-      "logits/chosen": -2.299609422683716,
-      "logits/rejected": -2.0796875953674316,
-      "logps/chosen": -352.25469970703125,
-      "logps/rejected": -428.83905029296875,
-      "loss": 0.2224,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 1.560888648033142,
-      "rewards/margins": 4.144775390625,
-      "rewards/rejected": -2.5865235328674316,
+      "epoch": 1.3461538461538463,
+      "grad_norm": 10.44378100815255,
+      "learning_rate": 1.739864864864865e-07,
+      "logits/chosen": -2.025390625,
+      "logits/rejected": -1.800390601158142,
+      "logps/chosen": -193.6984405517578,
+      "logps/rejected": -289.08282470703125,
+      "loss": 0.2395,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 2.071337938308716,
+      "rewards/margins": 3.8559813499450684,
+      "rewards/rejected": -1.7841675281524658,
       "step": 210
     },
     {
-      "epoch": 1.2368231046931408,
-      "grad_norm": 7.196990053018469,
-      "learning_rate": 2.0303030303030303e-07,
-      "logits/chosen": -2.340625047683716,
-      "logits/rejected": -2.0328125953674316,
-      "logps/chosen": -375.5093688964844,
-      "logps/rejected": -503.96563720703125,
-      "loss": 0.1673,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 2.0465941429138184,
-      "rewards/margins": 5.524023532867432,
-      "rewards/rejected": -3.48193359375,
+      "epoch": 1.3782051282051282,
+      "grad_norm": 97.68377543207546,
+      "learning_rate": 1.6554054054054055e-07,
+      "logits/chosen": -1.9826171398162842,
+      "logits/rejected": -1.755859375,
+      "logps/chosen": -355.3062438964844,
+      "logps/rejected": -485.71563720703125,
+      "loss": 0.2301,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 2.012890577316284,
+      "rewards/margins": 4.411523342132568,
+      "rewards/rejected": -2.397656202316284,
       "step": 215
     },
     {
-      "epoch": 1.2657039711191336,
-      "grad_norm": 30.16546271206658,
-      "learning_rate": 1.9545454545454545e-07,
-      "logits/chosen": -2.3626952171325684,
-      "logits/rejected": -2.0751953125,
-      "logps/chosen": -377.1937561035156,
-      "logps/rejected": -472.3374938964844,
-      "loss": 0.1338,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.931860327720642,
-      "rewards/margins": 5.258203029632568,
-      "rewards/rejected": -3.324414014816284,
+      "epoch": 1.4102564102564101,
+      "grad_norm": 10.308990703109377,
+      "learning_rate": 1.570945945945946e-07,
+      "logits/chosen": -2.0074219703674316,
+      "logits/rejected": -1.796484351158142,
+      "logps/chosen": -291.6031188964844,
+      "logps/rejected": -320.95001220703125,
+      "loss": 0.2796,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 1.8628418445587158,
+      "rewards/margins": 3.919921875,
+      "rewards/rejected": -2.0587158203125,
       "step": 220
     },
     {
-      "epoch": 1.2945848375451263,
-      "grad_norm": 16.65751244106967,
-      "learning_rate": 1.8787878787878785e-07,
-      "logits/chosen": -2.429492235183716,
-      "logits/rejected": -2.100390672683716,
-      "logps/chosen": -269.0625,
-      "logps/rejected": -362.60626220703125,
-      "loss": 0.1772,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 2.430957078933716,
-      "rewards/margins": 4.759179592132568,
-      "rewards/rejected": -2.3294434547424316,
+      "epoch": 1.4423076923076923,
+      "grad_norm": 7.8418596148572455,
+      "learning_rate": 1.4864864864864866e-07,
+      "logits/chosen": -2.051953077316284,
+      "logits/rejected": -1.8517577648162842,
+      "logps/chosen": -221.51171875,
+      "logps/rejected": -234.59375,
+      "loss": 0.2821,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 1.9519531726837158,
+      "rewards/margins": 3.46875,
+      "rewards/rejected": -1.521032691001892,
       "step": 225
     },
     {
-      "epoch": 1.323465703971119,
-      "grad_norm": 7.639249240996568,
-      "learning_rate": 1.803030303030303e-07,
-      "logits/chosen": -2.4078125953674316,
-      "logits/rejected": -2.1470704078674316,
-      "logps/chosen": -328.29998779296875,
-      "logps/rejected": -392.125,
-      "loss": 0.147,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 2.2931885719299316,
-      "rewards/margins": 5.653124809265137,
-      "rewards/rejected": -3.358081102371216,
+      "epoch": 1.4743589743589745,
+      "grad_norm": 19.99054898354689,
+      "learning_rate": 1.402027027027027e-07,
+      "logits/chosen": -1.9873046875,
+      "logits/rejected": -1.816015601158142,
+      "logps/chosen": -234.86874389648438,
+      "logps/rejected": -471.40625,
+      "loss": 0.2533,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 1.637182593345642,
+      "rewards/margins": 3.6805663108825684,
+      "rewards/rejected": -2.044677734375,
       "step": 230
     },
     {
-      "epoch": 1.352346570397112,
-      "grad_norm": 11.235853450673325,
-      "learning_rate": 1.7272727272727272e-07,
-      "logits/chosen": -2.305859327316284,
-      "logits/rejected": -2.067187547683716,
-      "logps/chosen": -335.57501220703125,
-      "logps/rejected": -458.25,
-      "loss": 0.2442,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 2.34521484375,
-      "rewards/margins": 5.45166015625,
-      "rewards/rejected": -3.107348680496216,
+      "epoch": 1.5064102564102564,
+      "grad_norm": 6.927398842550873,
+      "learning_rate": 1.3175675675675673e-07,
+      "logits/chosen": -2.116406202316284,
+      "logits/rejected": -1.890234351158142,
+      "logps/chosen": -236.1374969482422,
+      "logps/rejected": -326.1812438964844,
+      "loss": 0.2292,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 1.9124755859375,
+      "rewards/margins": 3.916796922683716,
+      "rewards/rejected": -2.0068116188049316,
       "step": 235
     },
     {
-      "epoch": 1.3812274368231048,
-      "grad_norm": 24.18415545353277,
-      "learning_rate": 1.6515151515151515e-07,
-      "logits/chosen": -2.288281202316284,
-      "logits/rejected": -1.989843726158142,
-      "logps/chosen": -374.20001220703125,
-      "logps/rejected": -513.2125244140625,
-      "loss": 0.1973,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 2.0380616188049316,
-      "rewards/margins": 6.026757717132568,
-      "rewards/rejected": -3.9879393577575684,
+      "epoch": 1.5384615384615383,
+      "grad_norm": 19.26565095128112,
+      "learning_rate": 1.233108108108108e-07,
+      "logits/chosen": -1.9998047351837158,
+      "logits/rejected": -1.8044922351837158,
+      "logps/chosen": -215.8718719482422,
+      "logps/rejected": -273.55780029296875,
+      "loss": 0.2959,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 1.870080590248108,
+      "rewards/margins": 3.464648485183716,
+      "rewards/rejected": -1.59326171875,
       "step": 240
     },
     {
-      "epoch": 1.4101083032490975,
-      "grad_norm": 14.131437193197566,
-      "learning_rate": 1.5757575757575757e-07,
-      "logits/chosen": -2.3443360328674316,
-      "logits/rejected": -2.125781297683716,
-      "logps/chosen": -314.6875,
-      "logps/rejected": -335.6000061035156,
-      "loss": 0.1387,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 2.135449171066284,
-      "rewards/margins": 4.864843845367432,
-      "rewards/rejected": -2.727935791015625,
+      "epoch": 1.5705128205128205,
+      "grad_norm": 23.53473342051176,
+      "learning_rate": 1.1486486486486487e-07,
+      "logits/chosen": -2.0589842796325684,
+      "logits/rejected": -1.871484398841858,
+      "logps/chosen": -288.2890625,
+      "logps/rejected": -367.5843811035156,
+      "loss": 0.2964,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 1.398657202720642,
+      "rewards/margins": 3.667285203933716,
+      "rewards/rejected": -2.267504930496216,
       "step": 245
     },
     {
-      "epoch": 1.4389891696750903,
-      "grad_norm": 56.26784093160422,
-      "learning_rate": 1.5e-07,
-      "logits/chosen": -2.2816405296325684,
-      "logits/rejected": -2.0853514671325684,
-      "logps/chosen": -376.4281311035156,
-      "logps/rejected": -458.8812561035156,
-      "loss": 0.2482,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.89215087890625,
-      "rewards/margins": 4.9853515625,
-      "rewards/rejected": -3.094287157058716,
+      "epoch": 1.6025641025641026,
+      "grad_norm": 9.487545014290102,
+      "learning_rate": 1.0641891891891891e-07,
+      "logits/chosen": -2.122851610183716,
+      "logits/rejected": -1.9580078125,
+      "logps/chosen": -285.9078063964844,
+      "logps/rejected": -379.8843688964844,
+      "loss": 0.3032,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 1.5487792491912842,
+      "rewards/margins": 3.930615186691284,
+      "rewards/rejected": -2.3807129859924316,
       "step": 250
     },
     {
-      "epoch": 1.467870036101083,
-      "grad_norm": 4.313742509893692,
-      "learning_rate": 1.4242424242424242e-07,
-      "logits/chosen": -2.323046922683716,
-      "logits/rejected": -2.084765672683716,
-      "logps/chosen": -301.89453125,
-      "logps/rejected": -363.7875061035156,
-      "loss": 0.1321,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": 2.201855421066284,
-      "rewards/margins": 5.726758003234863,
-      "rewards/rejected": -3.5247559547424316,
+      "epoch": 1.6346153846153846,
+      "grad_norm": 34.335674181311,
+      "learning_rate": 9.797297297297297e-08,
+      "logits/chosen": -1.9865233898162842,
+      "logits/rejected": -1.8146483898162842,
+      "logps/chosen": -303.875,
+      "logps/rejected": -383.5625,
+      "loss": 0.2268,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 2.1684327125549316,
+      "rewards/margins": 4.968652248382568,
+      "rewards/rejected": -2.7986207008361816,
       "step": 255
     },
     {
-      "epoch": 1.4967509025270758,
-      "grad_norm": 5.9087018367088024,
-      "learning_rate": 1.3484848484848484e-07,
-      "logits/chosen": -2.372265577316284,
-      "logits/rejected": -2.153515577316284,
-      "logps/chosen": -309.9937438964844,
-      "logps/rejected": -372.3374938964844,
-      "loss": 0.1357,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 2.295703172683716,
-      "rewards/margins": 5.535937309265137,
-      "rewards/rejected": -3.2354493141174316,
+      "epoch": 1.6666666666666665,
+      "grad_norm": 12.79380674096048,
+      "learning_rate": 8.952702702702702e-08,
+      "logits/chosen": -1.9738280773162842,
+      "logits/rejected": -1.810156226158142,
+      "logps/chosen": -224.6687469482422,
+      "logps/rejected": -420.2437438964844,
+      "loss": 0.2571,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 1.805883765220642,
+      "rewards/margins": 3.4756011962890625,
+      "rewards/rejected": -1.672705054283142,
       "step": 260
     },
     {
-      "epoch": 1.5256317689530685,
-      "grad_norm": 13.209590602028339,
-      "learning_rate": 1.2727272727272726e-07,
-      "logits/chosen": -2.3701171875,
-      "logits/rejected": -2.107226610183716,
-      "logps/chosen": -297.04998779296875,
-      "logps/rejected": -433.0,
-      "loss": 0.1328,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 2.1886353492736816,
-      "rewards/margins": 5.202734470367432,
-      "rewards/rejected": -3.012500047683716,
+      "epoch": 1.6987179487179487,
+      "grad_norm": 5.288061215805156,
+      "learning_rate": 8.108108108108108e-08,
+      "logits/chosen": -2.00390625,
+      "logits/rejected": -1.875585913658142,
+      "logps/chosen": -238.39688110351562,
+      "logps/rejected": -330.8125,
+      "loss": 0.198,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 2.1185302734375,
+      "rewards/margins": 4.246289253234863,
+      "rewards/rejected": -2.1275634765625,
       "step": 265
     },
     {
-      "epoch": 1.5545126353790613,
-      "grad_norm": 4.27775101978357,
-      "learning_rate": 1.196969696969697e-07,
-      "logits/chosen": -2.3277344703674316,
-      "logits/rejected": -2.1044921875,
-      "logps/chosen": -388.7593688964844,
-      "logps/rejected": -445.0531311035156,
-      "loss": 0.1726,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.5277831554412842,
-      "rewards/margins": 5.085351467132568,
-      "rewards/rejected": -3.555957078933716,
+      "epoch": 1.7307692307692308,
+      "grad_norm": 15.41954334878335,
+      "learning_rate": 7.263513513513512e-08,
+      "logits/chosen": -2.037890672683716,
+      "logits/rejected": -1.8224608898162842,
+      "logps/chosen": -284.765625,
+      "logps/rejected": -533.375,
+      "loss": 0.2335,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 1.4149596691131592,
+      "rewards/margins": 4.398095607757568,
+      "rewards/rejected": -2.9843382835388184,
       "step": 270
     },
     {
-      "epoch": 1.583393501805054,
-      "grad_norm": 18.286296338783334,
-      "learning_rate": 1.1212121212121211e-07,
-      "logits/chosen": -2.396484375,
-      "logits/rejected": -2.110156297683716,
-      "logps/chosen": -309.97186279296875,
-      "logps/rejected": -359.92498779296875,
-      "loss": 0.158,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 2.2788939476013184,
-      "rewards/margins": 5.2080078125,
-      "rewards/rejected": -2.934521436691284,
+      "epoch": 1.7628205128205128,
+      "grad_norm": 7.763713271113468,
+      "learning_rate": 6.418918918918918e-08,
+      "logits/chosen": -2.0380859375,
+      "logits/rejected": -1.8134765625,
+      "logps/chosen": -225.25,
+      "logps/rejected": -430.89373779296875,
+      "loss": 0.2135,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 1.960363745689392,
+      "rewards/margins": 4.0166015625,
+      "rewards/rejected": -2.0557618141174316,
       "step": 275
     },
     {
-      "epoch": 1.612274368231047,
-      "grad_norm": 2.669702691885131,
-      "learning_rate": 1.0454545454545454e-07,
-      "logits/chosen": -2.372851610183716,
-      "logits/rejected": -2.131640672683716,
-      "logps/chosen": -348.1578063964844,
-      "logps/rejected": -405.65936279296875,
-      "loss": 0.2322,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 2.2171874046325684,
-      "rewards/margins": 5.343847751617432,
-      "rewards/rejected": -3.1302733421325684,
+      "epoch": 1.7948717948717947,
+      "grad_norm": 4.521033115765149,
+      "learning_rate": 5.574324324324324e-08,
+      "logits/chosen": -2.044140577316284,
+      "logits/rejected": -1.811914086341858,
+      "logps/chosen": -144.3640594482422,
+      "logps/rejected": -276.8843688964844,
+      "loss": 0.2559,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 2.030041456222534,
+      "rewards/margins": 3.295703172683716,
+      "rewards/rejected": -1.2666351795196533,
       "step": 280
     },
     {
-      "epoch": 1.6411552346570397,
-      "grad_norm": 13.710710847207622,
-      "learning_rate": 9.696969696969696e-08,
-      "logits/chosen": -2.2705078125,
-      "logits/rejected": -2.0806641578674316,
-      "logps/chosen": -350.35626220703125,
-      "logps/rejected": -502.07501220703125,
-      "loss": 0.1197,
-      "rewards/accuracies": 0.981249988079071,
-      "rewards/chosen": 1.909570336341858,
-      "rewards/margins": 5.671875,
-      "rewards/rejected": -3.759765625,
+      "epoch": 1.8269230769230769,
+      "grad_norm": 9.52459220706363,
+      "learning_rate": 4.72972972972973e-08,
+      "logits/chosen": -2.127734422683716,
+      "logits/rejected": -1.897070288658142,
+      "logps/chosen": -280.30157470703125,
+      "logps/rejected": -316.5171813964844,
+      "loss": 0.2486,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 1.485009789466858,
+      "rewards/margins": 3.428417921066284,
+      "rewards/rejected": -1.9419434070587158,
       "step": 285
     },
     {
-      "epoch": 1.6700361010830325,
-      "grad_norm": 7.551122285737169,
-      "learning_rate": 8.93939393939394e-08,
-      "logits/chosen": -2.268749952316284,
-      "logits/rejected": -2.033203125,
-      "logps/chosen": -352.2875061035156,
-      "logps/rejected": -430.5687561035156,
-      "loss": 0.1557,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 2.1177978515625,
-      "rewards/margins": 5.108984470367432,
-      "rewards/rejected": -2.992236375808716,
+      "epoch": 1.858974358974359,
+      "grad_norm": 8.906979076376345,
+      "learning_rate": 3.885135135135135e-08,
+      "logits/chosen": -2.0869140625,
+      "logits/rejected": -1.8507812023162842,
+      "logps/chosen": -217.7156219482422,
+      "logps/rejected": -319.4203186035156,
+      "loss": 0.2271,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 2.053515672683716,
+      "rewards/margins": 4.0621337890625,
+      "rewards/rejected": -2.010498046875,
       "step": 290
     },
     {
-      "epoch": 1.6989169675090254,
-      "grad_norm": 6.684658943158315,
-      "learning_rate": 8.181818181818182e-08,
-      "logits/chosen": -2.3121094703674316,
-      "logits/rejected": -2.0816407203674316,
-      "logps/chosen": -346.4624938964844,
-      "logps/rejected": -449.046875,
-      "loss": 0.1549,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 2.4569334983825684,
-      "rewards/margins": 6.220312595367432,
-      "rewards/rejected": -3.7589354515075684,
+      "epoch": 1.891025641025641,
+      "grad_norm": 9.745814847980592,
+      "learning_rate": 3.040540540540541e-08,
+      "logits/chosen": -2.052539110183716,
+      "logits/rejected": -1.8537108898162842,
+      "logps/chosen": -241.68905639648438,
+      "logps/rejected": -401.51251220703125,
+      "loss": 0.2465,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 1.9554443359375,
+      "rewards/margins": 3.714062452316284,
+      "rewards/rejected": -1.7587372064590454,
       "step": 295
     },
     {
-      "epoch": 1.7277978339350182,
-      "grad_norm": 15.106479425607633,
-      "learning_rate": 7.424242424242424e-08,
-      "logits/chosen": -2.363476514816284,
-      "logits/rejected": -2.167773485183716,
-      "logps/chosen": -351.73748779296875,
-      "logps/rejected": -432.375,
-      "loss": 0.1492,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 2.024670362472534,
-      "rewards/margins": 5.391992092132568,
-      "rewards/rejected": -3.364550828933716,
+      "epoch": 1.9230769230769231,
+      "grad_norm": 10.279258989880054,
+      "learning_rate": 2.195945945945946e-08,
+      "logits/chosen": -1.91796875,
+      "logits/rejected": -1.7571289539337158,
+      "logps/chosen": -250.7375030517578,
+      "logps/rejected": -312.2203063964844,
+      "loss": 0.1891,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 2.004687547683716,
+      "rewards/margins": 4.478320121765137,
+      "rewards/rejected": -2.476269483566284,
       "step": 300
     },
     {
-      "epoch": 1.756678700361011,
-      "grad_norm": 17.071782434502218,
-      "learning_rate": 6.666666666666667e-08,
-      "logits/chosen": -2.3306641578674316,
-      "logits/rejected": -2.131640672683716,
-      "logps/chosen": -343.89373779296875,
-      "logps/rejected": -500.7593688964844,
-      "loss": 0.1674,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 2.3846678733825684,
-      "rewards/margins": 5.525000095367432,
-      "rewards/rejected": -3.142895460128784,
+      "epoch": 1.9551282051282053,
+      "grad_norm": 6.84223500138646,
+      "learning_rate": 1.3513513513513514e-08,
+      "logits/chosen": -2.0062499046325684,
+      "logits/rejected": -1.889062523841858,
+      "logps/chosen": -267.1031188964844,
+      "logps/rejected": -407.0718688964844,
+      "loss": 0.2554,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 1.6201751232147217,
+      "rewards/margins": 3.678515672683716,
+      "rewards/rejected": -2.054980516433716,
       "step": 305
     },
     {
-      "epoch": 1.7855595667870037,
-      "grad_norm": 15.209800869884038,
-      "learning_rate": 5.9090909090909085e-08,
-      "logits/chosen": -2.3880858421325684,
-      "logits/rejected": -2.153515577316284,
-      "logps/chosen": -325.6499938964844,
-      "logps/rejected": -414.95623779296875,
-      "loss": 0.1211,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.990502953529358,
-      "rewards/margins": 5.307031154632568,
-      "rewards/rejected": -3.316601514816284,
+      "epoch": 1.9871794871794872,
+      "grad_norm": 6.084443681112896,
+      "learning_rate": 5.067567567567567e-09,
+      "logits/chosen": -2.08203125,
+      "logits/rejected": -1.8772461414337158,
+      "logps/chosen": -248.9656219482422,
+      "logps/rejected": -302.21875,
+      "loss": 0.2517,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 1.859375,
+      "rewards/margins": 3.5653076171875,
+      "rewards/rejected": -1.707067847251892,
       "step": 310
     },
-    {
-      "epoch": 1.8144404332129964,
-      "grad_norm": 18.27572794541311,
-      "learning_rate": 5.151515151515151e-08,
-      "logits/chosen": -2.367968797683716,
-      "logits/rejected": -2.103710889816284,
-      "logps/chosen": -356.52813720703125,
-      "logps/rejected": -500.12188720703125,
-      "loss": 0.163,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 2.038317918777466,
-      "rewards/margins": 5.278710842132568,
-      "rewards/rejected": -3.2340455055236816,
-      "step": 315
-    },
-    {
-      "epoch": 1.8433212996389892,
-      "grad_norm": 7.0037563817409,
-      "learning_rate": 4.393939393939393e-08,
-      "logits/chosen": -2.3832030296325684,
-      "logits/rejected": -2.078125,
-      "logps/chosen": -257.1625061035156,
-      "logps/rejected": -554.7000122070312,
-      "loss": 0.1503,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 2.0913329124450684,
-      "rewards/margins": 5.219140529632568,
-      "rewards/rejected": -3.128173828125,
-      "step": 320
-    },
-    {
-      "epoch": 1.872202166064982,
-      "grad_norm": 6.3925450697454,
-      "learning_rate": 3.636363636363636e-08,
-      "logits/chosen": -2.378710985183716,
-      "logits/rejected": -2.157031297683716,
-      "logps/chosen": -321.14373779296875,
-      "logps/rejected": -385.79998779296875,
-      "loss": 0.1651,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 2.0872559547424316,
-      "rewards/margins": 5.511328220367432,
-      "rewards/rejected": -3.424072265625,
-      "step": 325
-    },
-    {
-      "epoch": 1.9010830324909747,
-      "grad_norm": 8.51178304792889,
-      "learning_rate": 2.8787878787878787e-08,
-      "logits/chosen": -2.3662109375,
-      "logits/rejected": -2.1107420921325684,
-      "logps/chosen": -349.765625,
-      "logps/rejected": -494.3999938964844,
-      "loss": 0.1741,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 1.6572997570037842,
-      "rewards/margins": 5.140038967132568,
-      "rewards/rejected": -3.4827637672424316,
-      "step": 330
-    },
-    {
-      "epoch": 1.9299638989169674,
-      "grad_norm": 16.678414426541924,
-      "learning_rate": 2.1212121212121214e-08,
-      "logits/chosen": -2.3955078125,
-      "logits/rejected": -2.0966796875,
-      "logps/chosen": -327.1000061035156,
-      "logps/rejected": -492.73748779296875,
-      "loss": 0.1567,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 2.02783203125,
-      "rewards/margins": 5.256445407867432,
-      "rewards/rejected": -3.228686571121216,
-      "step": 335
-    },
-    {
-      "epoch": 1.9588447653429601,
-      "grad_norm": 4.987506994026094,
-      "learning_rate": 1.3636363636363635e-08,
-      "logits/chosen": -2.355273485183716,
-      "logits/rejected": -1.9865233898162842,
-      "logps/chosen": -340.0,
-      "logps/rejected": -424.0687561035156,
-      "loss": 0.1081,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 2.090686082839966,
-      "rewards/margins": 5.492578029632568,
-      "rewards/rejected": -3.401123046875,
-      "step": 340
-    },
-    {
-      "epoch": 1.9877256317689531,
-      "grad_norm": 5.558593262711838,
-      "learning_rate": 6.06060606060606e-09,
-      "logits/chosen": -2.360156297683716,
-      "logits/rejected": -2.049023389816284,
-      "logps/chosen": -311.01251220703125,
-      "logps/rejected": -635.7062377929688,
-      "loss": 0.1415,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 1.959832787513733,
-      "rewards/margins": 5.430273532867432,
-      "rewards/rejected": -3.4686522483825684,
-      "step": 345
-    },
     {
       "epoch": 2.0,
-      "step": 348,
+      "step": 312,
       "total_flos": 0.0,
-      "train_loss": 0.2171997334594014,
-      "train_runtime": 3612.0435,
-      "train_samples_per_second": 3.067,
-      "train_steps_per_second": 0.096
+      "train_loss": 0.2867361557407257,
+      "train_runtime": 4280.6612,
+      "train_samples_per_second": 2.331,
+      "train_steps_per_second": 0.073
     }
   ],
   "logging_steps": 5,
-  "max_steps": 348,
+  "max_steps": 312,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 2,
   "save_steps": 50,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f726cd8e3d8c89b2f6e027abd219a0a60b3a1f5446c82bda65ba8183d52ff7e9
+oid sha256:723f7ffc4922b75a0710d0a8e53e8aeebe78b402100ccee8397bbeee25c1df51
 size 7800