Model save

README.md

@@ -1,9 +1,7 @@
 ---
 base_model: Sim4Rec/inter-play-sim-assistant-sft
-datasets:
-- Sim4Rec/dpo_data
 library_name: transformers
-model_name: Sim4Rec/inter-play-sim-assistant-sft
+model_name: inter-play-sim-assistant-dpo
 tags:
 - generated_from_trainer
 - trl
@@ -11,9 +9,9 @@ tags:
 licence: license
 ---
 
-# Model Card for Sim4Rec/inter-play-sim-assistant-sft
+# Model Card for inter-play-sim-assistant-dpo
 
-This model is a fine-tuned version of [Sim4Rec/inter-play-sim-assistant-sft](https://huggingface.co/Sim4Rec/inter-play-sim-assistant-sft) on the [['Sim4Rec/dpo_data']](https://huggingface.co/datasets/['Sim4Rec/dpo_data']) dataset.
+This model is a fine-tuned version of [Sim4Rec/inter-play-sim-assistant-sft](https://huggingface.co/Sim4Rec/inter-play-sim-assistant-sft).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/jerome-ramos-20/huggingface/runs/bw9lk46t) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/jerome-ramos-20/huggingface/runs/q3zfrxdb) 
 
 
 This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).

all_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 1.9938900203665988,
+    "epoch": 2.0,
     "total_flos": 0.0,
-    "train_loss": 0.5645458033827485,
-    "train_runtime": 476.0531,
-    "train_samples": 3921,
-    "train_samples_per_second": 16.473,
-    "train_steps_per_second": 0.256
+    "train_loss": 0.22515048312672067,
+    "train_runtime": 2958.2669,
+    "train_samples": 45561,
+    "train_samples_per_second": 30.802,
+    "train_steps_per_second": 0.241
 }

config.json

@@ -32,6 +32,6 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.48.2",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 128320
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c662bff069e16a55c0ed3957b7c1a7e2e7b6efe4c7dc1b38a09a42e95da5419d
+oid sha256:228fda64e183c9fdc3a7708fb3208210cdadad23990e8923b3fe182ecf5c0cec
 size 4977222960

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e41f89c0b9e9c5a5e8dd10e5c19c527e4726dc9b373296f7a897cbbaa737621
+oid sha256:5f7204be8a0fc4f037c035587cfd68a41793f7158bf6cf5794a16e4fe64cbc8a
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6e57a5558dd5fcdae6c05b45dd4d4a6294dd49f4b1dd9e5a5bba7cf612cffe6
+oid sha256:48b841d9240b04b4dfe85f97b3b566448cfbabe276eac748a7ad65411df8577f
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c466502280c55a6dd02d9a75fc8f3c83841694b9133c35db128ed464184cfa86
+oid sha256:2c785dc0e0713c891a93b159479d9de5baf67a041e01652bf3e46d23a5251c8c
 size 1168663096

runs/Feb05_22-06-55_w-jerom-inter-play-sim-94c6890b9ccf44ea86f033a3db8a5dbd-5csnwcf/events.out.tfevents.1738793347.w-jerom-inter-play-sim-94c6890b9ccf44ea86f033a3db8a5dbd-5csnwcf.16929.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60bd33b78c7b1b2eb67898c2239db968a826c5b936caca18188d3c546ef10b8e
+size 105020

runs/Feb05_23-51-38_w-jerom-inter-play-sim-94c6890b9ccf44ea86f033a3db8a5dbd-5csnwcf/events.out.tfevents.1738799532.w-jerom-inter-play-sim-94c6890b9ccf44ea86f033a3db8a5dbd-5csnwcf.60419.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95c3abbe38290f51d753e412d52ddc5244ed9e9d32021e7c0d13d40fbb8c6015
+size 6438

runs/Feb05_23-52-52_w-jerom-inter-play-sim-94c6890b9ccf44ea86f033a3db8a5dbd-5csnwcf/events.out.tfevents.1738799601.w-jerom-inter-play-sim-94c6890b9ccf44ea86f033a3db8a5dbd-5csnwcf.62482.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0069be6e747818e3901feb93b381cfe03ea449b10d5f618874b380e666cc459f
+size 7790

runs/Feb06_00-02-58_w-jerom-inter-play-sim-94c6890b9ccf44ea86f033a3db8a5dbd-5fcdrjv/events.out.tfevents.1738800332.w-jerom-inter-play-sim-94c6890b9ccf44ea86f033a3db8a5dbd-5fcdrjv.8855.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cc1728ddee65f74857ea0bc72b128753b118fe8fd026ac833ebb3d5af24cd87
+size 56172

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 1.9938900203665988,
+    "epoch": 2.0,
     "total_flos": 0.0,
-    "train_loss": 0.5645458033827485,
-    "train_runtime": 476.0531,
-    "train_samples": 3921,
-    "train_samples_per_second": 16.473,
-    "train_steps_per_second": 0.256
+    "train_loss": 0.22515048312672067,
+    "train_runtime": 2958.2669,
+    "train_samples": 45561,
+    "train_samples_per_second": 30.802,
+    "train_steps_per_second": 0.241
 }

trainer_state.json

@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.9938900203665988,
+  "epoch": 2.0,
   "eval_steps": 500,
-  "global_step": 122,
+  "global_step": 712,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.016293279022403257,
-      "grad_norm": 1.4111297130584717,
-      "learning_rate": 3.846153846153846e-08,
-      "logits/chosen": -3.2578125,
-      "logits/rejected": -3.19140625,
-      "logps/chosen": -46.375,
-      "logps/rejected": -45.75,
+      "epoch": 0.0028089887640449437,
+      "grad_norm": 1.1993484497070312,
+      "learning_rate": 6.9444444444444435e-09,
+      "logits/chosen": -3.220703125,
+      "logits/rejected": -3.1796875,
+      "logps/chosen": -43.75,
+      "logps/rejected": -42.78125,
       "loss": 0.6914,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -24,197 +24,1082 @@
       "step": 1
     },
     {
-      "epoch": 0.1629327902240326,
-      "grad_norm": 0.7594385147094727,
-      "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -3.2265625,
-      "logits/rejected": -3.2200520038604736,
-      "logps/chosen": -42.77604293823242,
-      "logps/rejected": -41.88541793823242,
-      "loss": 0.6924,
-      "rewards/accuracies": 0.2222222238779068,
-      "rewards/chosen": 0.0023810069542378187,
-      "rewards/margins": -0.0003809928894042969,
-      "rewards/rejected": 0.0027594566345214844,
+      "epoch": 0.028089887640449437,
+      "grad_norm": 0.7986037135124207,
+      "learning_rate": 6.944444444444444e-08,
+      "logits/chosen": -3.2486979961395264,
+      "logits/rejected": -3.2052950859069824,
+      "logps/chosen": -42.34375,
+      "logps/rejected": -41.01215362548828,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.2161458283662796,
+      "rewards/chosen": 0.00020620558643713593,
+      "rewards/margins": -0.00015253490710165352,
+      "rewards/rejected": 0.0003587139945011586,
       "step": 10
     },
     {
-      "epoch": 0.3258655804480652,
-      "grad_norm": 0.7135093808174133,
-      "learning_rate": 4.949291683053768e-07,
-      "logits/chosen": -3.2763671875,
-      "logits/rejected": -3.2466797828674316,
-      "logps/chosen": -40.720314025878906,
-      "logps/rejected": -39.4296875,
-      "loss": 0.6872,
-      "rewards/accuracies": 0.421875,
-      "rewards/chosen": 0.033158015459775925,
-      "rewards/margins": 0.012033844366669655,
-      "rewards/rejected": 0.02114267274737358,
+      "epoch": 0.056179775280898875,
+      "grad_norm": 0.6845986247062683,
+      "learning_rate": 1.3888888888888888e-07,
+      "logits/chosen": -3.244140625,
+      "logits/rejected": -3.219531297683716,
+      "logps/chosen": -41.37812423706055,
+      "logps/rejected": -40.42656326293945,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.3460937440395355,
+      "rewards/chosen": 0.006803750991821289,
+      "rewards/margins": 0.003222918603569269,
+      "rewards/rejected": 0.003580451011657715,
       "step": 20
     },
     {
-      "epoch": 0.48879837067209775,
-      "grad_norm": 0.699894368648529,
-      "learning_rate": 4.70586371748506e-07,
-      "logits/chosen": -3.2841796875,
-      "logits/rejected": -3.2529296875,
-      "logps/chosen": -40.8515625,
-      "logps/rejected": -40.571876525878906,
-      "loss": 0.6765,
-      "rewards/accuracies": 0.528124988079071,
-      "rewards/chosen": 0.0396418571472168,
-      "rewards/margins": 0.034914396703243256,
-      "rewards/rejected": 0.004716300871223211,
+      "epoch": 0.08426966292134831,
+      "grad_norm": 0.5183084607124329,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -3.263671875,
+      "logits/rejected": -3.2261719703674316,
+      "logps/chosen": -41.142189025878906,
+      "logps/rejected": -40.00312423706055,
+      "loss": 0.6862,
+      "rewards/accuracies": 0.44140625,
+      "rewards/chosen": 0.02759246900677681,
+      "rewards/margins": 0.013646435923874378,
+      "rewards/rejected": 0.013950538821518421,
       "step": 30
     },
     {
-      "epoch": 0.6517311608961304,
-      "grad_norm": 1.1233805418014526,
-      "learning_rate": 4.280458575653296e-07,
-      "logits/chosen": -3.195117235183716,
-      "logits/rejected": -3.162890672683716,
-      "logps/chosen": -43.59375,
-      "logps/rejected": -44.532814025878906,
-      "loss": 0.6415,
-      "rewards/accuracies": 0.628125011920929,
-      "rewards/chosen": -0.06923361122608185,
-      "rewards/margins": 0.11445312201976776,
-      "rewards/rejected": -0.1837112456560135,
+      "epoch": 0.11235955056179775,
+      "grad_norm": 0.5848017334938049,
+      "learning_rate": 2.7777777777777776e-07,
+      "logits/chosen": -3.286328077316284,
+      "logits/rejected": -3.229296922683716,
+      "logps/chosen": -41.439064025878906,
+      "logps/rejected": -41.0,
+      "loss": 0.6745,
+      "rewards/accuracies": 0.5546875,
+      "rewards/chosen": 0.039928339421749115,
+      "rewards/margins": 0.038716744631528854,
+      "rewards/rejected": 0.001224255538545549,
       "step": 40
     },
     {
-      "epoch": 0.814663951120163,
-      "grad_norm": 1.2118208408355713,
-      "learning_rate": 3.7081709127108767e-07,
-      "logits/chosen": -3.0589842796325684,
+      "epoch": 0.1404494382022472,
+      "grad_norm": 0.7922030091285706,
+      "learning_rate": 3.472222222222222e-07,
+      "logits/chosen": -3.2728514671325684,
       "logits/rejected": NaN,
-      "logps/chosen": -50.13593673706055,
-      "logps/rejected": -52.04999923706055,
-      "loss": 0.6138,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.3750244081020355,
-      "rewards/margins": 0.18903808295726776,
-      "rewards/rejected": -0.5643554925918579,
+      "logps/chosen": -41.71406173706055,
+      "logps/rejected": -41.890625,
+      "loss": 0.6535,
+      "rewards/accuracies": 0.645312488079071,
+      "rewards/chosen": 0.03761863708496094,
+      "rewards/margins": 0.08368835598230362,
+      "rewards/rejected": -0.04610452800989151,
       "step": 50
     },
     {
-      "epoch": 0.9775967413441955,
-      "grad_norm": 1.2914516925811768,
-      "learning_rate": 3.0362127536287636e-07,
-      "logits/chosen": -3.135546922683716,
-      "logits/rejected": -3.056835889816284,
-      "logps/chosen": -50.14531326293945,
-      "logps/rejected": -56.23749923706055,
-      "loss": 0.566,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.4345153868198395,
-      "rewards/margins": 0.36333543062210083,
-      "rewards/rejected": -0.7979736328125,
+      "epoch": 0.16853932584269662,
+      "grad_norm": 1.3905400037765503,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -3.204296827316284,
+      "logits/rejected": -3.154296875,
+      "logps/chosen": -44.493751525878906,
+      "logps/rejected": -48.446876525878906,
+      "loss": 0.5839,
+      "rewards/accuracies": 0.692187488079071,
+      "rewards/chosen": -0.11696071922779083,
+      "rewards/margins": 0.25516968965530396,
+      "rewards/rejected": -0.3720703125,
       "step": 60
     },
     {
-      "epoch": 1.1466395112016294,
-      "grad_norm": 1.4273715019226074,
-      "learning_rate": 2.3200186419770823e-07,
-      "logits/chosen": -3.161651134490967,
-      "logits/rejected": -3.0788965225219727,
-      "logps/chosen": -53.49691390991211,
-      "logps/rejected": -62.85802459716797,
-      "loss": 0.5371,
-      "rewards/accuracies": 0.6095678806304932,
-      "rewards/chosen": -0.5622889995574951,
-      "rewards/margins": 0.5249747037887573,
-      "rewards/rejected": -1.0871431827545166,
+      "epoch": 0.19662921348314608,
+      "grad_norm": 0.8105548024177551,
+      "learning_rate": 4.861111111111111e-07,
+      "logits/chosen": -3.23828125,
+      "logits/rejected": -3.130859375,
+      "logps/chosen": -47.71875,
+      "logps/rejected": -59.140625,
+      "loss": 0.4754,
+      "rewards/accuracies": 0.7093750238418579,
+      "rewards/chosen": -0.2639709413051605,
+      "rewards/margins": 0.6321045160293579,
+      "rewards/rejected": -0.895751953125,
       "step": 70
     },
     {
-      "epoch": 1.309572301425662,
-      "grad_norm": 1.3244953155517578,
-      "learning_rate": 1.6186724554503237e-07,
-      "logits/chosen": -3.133984327316284,
-      "logits/rejected": -3.0191407203674316,
-      "logps/chosen": -55.234375,
-      "logps/rejected": -68.234375,
-      "loss": 0.4905,
-      "rewards/accuracies": 0.660937488079071,
-      "rewards/chosen": -0.6110439300537109,
-      "rewards/margins": 0.7057861089706421,
-      "rewards/rejected": -1.3170897960662842,
+      "epoch": 0.2247191011235955,
+      "grad_norm": 0.6645036339759827,
+      "learning_rate": 4.998072590601808e-07,
+      "logits/chosen": -3.2529296875,
+      "logits/rejected": NaN,
+      "logps/chosen": -43.765625,
+      "logps/rejected": -72.40937805175781,
+      "loss": 0.3394,
+      "rewards/accuracies": 0.73828125,
+      "rewards/chosen": -0.08822021633386612,
+      "rewards/margins": 1.5029785633087158,
+      "rewards/rejected": -1.591284155845642,
       "step": 80
     },
     {
-      "epoch": 1.4725050916496945,
-      "grad_norm": 1.8561619520187378,
-      "learning_rate": 9.900331622138063e-08,
-      "logits/chosen": -3.107617139816284,
-      "logits/rejected": -2.9839844703674316,
-      "logps/chosen": -55.421875,
-      "logps/rejected": -68.80937194824219,
-      "loss": 0.4936,
-      "rewards/accuracies": 0.640625,
-      "rewards/chosen": -0.6593307256698608,
-      "rewards/margins": 0.737597644329071,
-      "rewards/rejected": -1.3976562023162842,
+      "epoch": 0.25280898876404495,
+      "grad_norm": 0.436211496591568,
+      "learning_rate": 4.990247583129217e-07,
+      "logits/chosen": -3.268749952316284,
+      "logits/rejected": -2.9375,
+      "logps/chosen": -46.046875,
+      "logps/rejected": -89.46875,
+      "loss": 0.2745,
+      "rewards/accuracies": 0.7476562261581421,
+      "rewards/chosen": -0.22807636857032776,
+      "rewards/margins": 2.234375,
+      "rewards/rejected": -2.463183641433716,
       "step": 90
     },
     {
-      "epoch": 1.635437881873727,
-      "grad_norm": 0.9447304606437683,
-      "learning_rate": 4.859616286322094e-08,
-      "logits/chosen": -3.1148438453674316,
-      "logits/rejected": -2.9876952171325684,
-      "logps/chosen": -53.092185974121094,
-      "logps/rejected": -68.9312515258789,
-      "loss": 0.468,
-      "rewards/accuracies": 0.6484375,
-      "rewards/chosen": -0.5302764773368835,
-      "rewards/margins": 0.887438952922821,
-      "rewards/rejected": -1.417944312095642,
+      "epoch": 0.2808988764044944,
+      "grad_norm": 0.43823757767677307,
+      "learning_rate": 4.976423351108942e-07,
+      "logits/chosen": -3.2183594703674316,
+      "logits/rejected": NaN,
+      "logps/chosen": -49.96875,
+      "logps/rejected": -105.32499694824219,
+      "loss": 0.2356,
+      "rewards/accuracies": 0.75390625,
+      "rewards/chosen": -0.3900146484375,
+      "rewards/margins": 2.8427734375,
+      "rewards/rejected": -3.232421875,
       "step": 100
     },
     {
-      "epoch": 1.7983706720977597,
-      "grad_norm": 1.593487024307251,
-      "learning_rate": 1.4804225250339281e-08,
-      "logits/chosen": -3.1207032203674316,
-      "logits/rejected": NaN,
-      "logps/chosen": -54.428123474121094,
-      "logps/rejected": -68.76249694824219,
-      "loss": 0.4758,
-      "rewards/accuracies": 0.640625,
-      "rewards/chosen": -0.6090973019599915,
-      "rewards/margins": 0.817614734172821,
-      "rewards/rejected": -1.4268066883087158,
+      "epoch": 0.3089887640449438,
+      "grad_norm": 0.6719679832458496,
+      "learning_rate": 4.95663319832678e-07,
+      "logits/chosen": -3.2197265625,
+      "logits/rejected": -2.762890577316284,
+      "logps/chosen": -46.25312423706055,
+      "logps/rejected": -114.41874694824219,
+      "loss": 0.236,
+      "rewards/accuracies": 0.7523437738418579,
+      "rewards/chosen": -0.22943687438964844,
+      "rewards/margins": 3.4632811546325684,
+      "rewards/rejected": -3.6929688453674316,
       "step": 110
     },
     {
-      "epoch": 1.9613034623217924,
-      "grad_norm": 2.265634059906006,
-      "learning_rate": 4.152374292708538e-10,
-      "logits/chosen": -3.109179735183716,
-      "logits/rejected": -2.985156297683716,
-      "logps/chosen": -54.99687576293945,
-      "logps/rejected": -71.015625,
-      "loss": 0.4654,
-      "rewards/accuracies": 0.667187511920929,
-      "rewards/chosen": -0.6471847295761108,
-      "rewards/margins": 0.863818347454071,
-      "rewards/rejected": -1.510644555091858,
+      "epoch": 0.33707865168539325,
+      "grad_norm": 0.3029685914516449,
+      "learning_rate": 4.930924800994191e-07,
+      "logits/chosen": -3.2417969703674316,
+      "logits/rejected": -2.752734422683716,
+      "logps/chosen": -45.203125,
+      "logps/rejected": -123.83125305175781,
+      "loss": 0.1955,
+      "rewards/accuracies": 0.77734375,
+      "rewards/chosen": -0.130183607339859,
+      "rewards/margins": 4.001757621765137,
+      "rewards/rejected": -4.1328125,
       "step": 120
     },
     {
-      "epoch": 1.9938900203665988,
-      "step": 122,
+      "epoch": 0.3651685393258427,
+      "grad_norm": 0.5936453938484192,
+      "learning_rate": 4.899360092892144e-07,
+      "logits/chosen": -3.2191405296325684,
+      "logits/rejected": -2.7181639671325684,
+      "logps/chosen": -46.69843673706055,
+      "logps/rejected": -128.28750610351562,
+      "loss": 0.2039,
+      "rewards/accuracies": 0.7679687738418579,
+      "rewards/chosen": -0.2604345381259918,
+      "rewards/margins": 4.139843940734863,
+      "rewards/rejected": -4.399609565734863,
+      "step": 130
+    },
+    {
+      "epoch": 0.39325842696629215,
+      "grad_norm": 0.4349266588687897,
+      "learning_rate": 4.862015116167195e-07,
+      "logits/chosen": -3.243945360183716,
+      "logits/rejected": NaN,
+      "logps/chosen": -45.373435974121094,
+      "logps/rejected": -133.71249389648438,
+      "loss": 0.1869,
+      "rewards/accuracies": 0.7789062261581421,
+      "rewards/chosen": -0.10051727294921875,
+      "rewards/margins": 4.529296875,
+      "rewards/rejected": -4.631054878234863,
+      "step": 140
+    },
+    {
+      "epoch": 0.42134831460674155,
+      "grad_norm": 0.32329249382019043,
+      "learning_rate": 4.81897983813931e-07,
+      "logits/chosen": -3.2386717796325684,
+      "logits/rejected": -2.702929735183716,
+      "logps/chosen": -46.21562576293945,
+      "logps/rejected": -138.9375,
+      "loss": 0.1833,
+      "rewards/accuracies": 0.780468761920929,
+      "rewards/chosen": -0.20498999953269958,
+      "rewards/margins": 4.690625190734863,
+      "rewards/rejected": -4.895898342132568,
+      "step": 150
+    },
+    {
+      "epoch": 0.449438202247191,
+      "grad_norm": 0.24011513590812683,
+      "learning_rate": 4.770357934562704e-07,
+      "logits/chosen": -3.281445264816284,
+      "logits/rejected": NaN,
+      "logps/chosen": -41.82500076293945,
+      "logps/rejected": -132.2624969482422,
+      "loss": 0.195,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": 0.00346546177752316,
+      "rewards/margins": 4.6083984375,
+      "rewards/rejected": -4.605273246765137,
+      "step": 160
+    },
+    {
+      "epoch": 0.47752808988764045,
+      "grad_norm": 0.5602438449859619,
+      "learning_rate": 4.716266539861866e-07,
+      "logits/chosen": -3.237499952316284,
+      "logits/rejected": -2.6552734375,
+      "logps/chosen": -47.9921875,
+      "logps/rejected": -144.77499389648438,
+      "loss": 0.1834,
+      "rewards/accuracies": 0.7757812738418579,
+      "rewards/chosen": -0.31496095657348633,
+      "rewards/margins": 4.906640529632568,
+      "rewards/rejected": -5.219336032867432,
+      "step": 170
+    },
+    {
+      "epoch": 0.5056179775280899,
+      "grad_norm": 0.5013967156410217,
+      "learning_rate": 4.6568359649444796e-07,
+      "logits/chosen": -3.225781202316284,
+      "logits/rejected": -2.6431641578674316,
+      "logps/chosen": -47.4140625,
+      "logps/rejected": -147.6062469482422,
+      "loss": 0.1777,
+      "rewards/accuracies": 0.7835937738418579,
+      "rewards/chosen": -0.26249465346336365,
+      "rewards/margins": 5.070898532867432,
+      "rewards/rejected": -5.333398342132568,
+      "step": 180
+    },
+    {
+      "epoch": 0.5337078651685393,
+      "grad_norm": 0.3109186887741089,
+      "learning_rate": 4.592209383271023e-07,
+      "logits/chosen": -3.2769532203674316,
+      "logits/rejected": -2.708203077316284,
+      "logps/chosen": -40.32500076293945,
+      "logps/rejected": -146.55624389648438,
+      "loss": 0.1691,
+      "rewards/accuracies": 0.7945312261581421,
+      "rewards/chosen": 0.10864410549402237,
+      "rewards/margins": 5.372851371765137,
+      "rewards/rejected": -5.262499809265137,
+      "step": 190
+    },
+    {
+      "epoch": 0.5617977528089888,
+      "grad_norm": 0.329673707485199,
+      "learning_rate": 4.5225424859373684e-07,
+      "logits/chosen": -3.244921922683716,
+      "logits/rejected": -2.625195264816284,
+      "logps/chosen": -46.1875,
+      "logps/rejected": -152.55624389648438,
+      "loss": 0.1722,
+      "rewards/accuracies": 0.784375011920929,
+      "rewards/chosen": -0.20301513373851776,
+      "rewards/margins": 5.3759765625,
+      "rewards/rejected": -5.578711032867432,
+      "step": 200
+    },
+    {
+      "epoch": 0.5898876404494382,
+      "grad_norm": 0.6182069182395935,
+      "learning_rate": 4.448003106601291e-07,
+      "logits/chosen": -3.2255859375,
+      "logits/rejected": NaN,
+      "logps/chosen": -45.904685974121094,
+      "logps/rejected": -153.14999389648438,
+      "loss": 0.1818,
+      "rewards/accuracies": 0.7789062261581421,
+      "rewards/chosen": -0.2197813093662262,
+      "rewards/margins": 5.441601753234863,
+      "rewards/rejected": -5.661913871765137,
+      "step": 210
+    },
+    {
+      "epoch": 0.6179775280898876,
+      "grad_norm": 0.10240374505519867,
+      "learning_rate": 4.3687708171564917e-07,
+      "logits/chosen": -3.2201170921325684,
+      "logits/rejected": NaN,
+      "logps/chosen": -46.890625,
+      "logps/rejected": -154.24374389648438,
+      "loss": 0.18,
+      "rewards/accuracies": 0.76953125,
+      "rewards/chosen": -0.2859039306640625,
+      "rewards/margins": 5.441992282867432,
+      "rewards/rejected": -5.730078220367432,
+      "step": 220
+    },
+    {
+      "epoch": 0.6460674157303371,
+      "grad_norm": 0.5484092235565186,
+      "learning_rate": 4.2850364951281705e-07,
+      "logits/chosen": -3.2359375953674316,
+      "logits/rejected": -2.595507860183716,
+      "logps/chosen": -44.52656173706055,
+      "logps/rejected": -156.5187530517578,
+      "loss": 0.1763,
+      "rewards/accuracies": 0.785937488079071,
+      "rewards/chosen": -0.10360870510339737,
+      "rewards/margins": 5.668359279632568,
+      "rewards/rejected": -5.7724609375,
+      "step": 230
+    },
+    {
+      "epoch": 0.6741573033707865,
+      "grad_norm": 0.33201783895492554,
+      "learning_rate": 4.1970018638323547e-07,
+      "logits/chosen": -3.2476563453674316,
+      "logits/rejected": -2.6099610328674316,
+      "logps/chosen": -41.240623474121094,
+      "logps/rejected": -152.3249969482422,
+      "loss": 0.1836,
+      "rewards/accuracies": 0.778124988079071,
+      "rewards/chosen": 0.05380706861615181,
+      "rewards/margins": 5.612500190734863,
+      "rewards/rejected": -5.559960842132568,
+      "step": 240
+    },
+    {
+      "epoch": 0.702247191011236,
+      "grad_norm": 0.3225279450416565,
+      "learning_rate": 4.1048790064067573e-07,
+      "logits/chosen": -3.2152342796325684,
+      "logits/rejected": NaN,
+      "logps/chosen": -42.44062423706055,
+      "logps/rejected": -152.5437469482422,
+      "loss": 0.1874,
+      "rewards/accuracies": 0.7679687738418579,
+      "rewards/chosen": -0.019408416002988815,
+      "rewards/margins": 5.590234279632568,
+      "rewards/rejected": -5.608788967132568,
+      "step": 250
+    },
+    {
+      "epoch": 0.7303370786516854,
+      "grad_norm": 0.5013627409934998,
+      "learning_rate": 4.0088898548839285e-07,
+      "logits/chosen": -3.1996092796325684,
+      "logits/rejected": -2.558398485183716,
+      "logps/chosen": -46.775001525878906,
+      "logps/rejected": -157.64999389648438,
+      "loss": 0.1809,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.24601340293884277,
+      "rewards/margins": 5.5927734375,
+      "rewards/rejected": -5.837695121765137,
+      "step": 260
+    },
+    {
+      "epoch": 0.7584269662921348,
+      "grad_norm": 0.2542474567890167,
+      "learning_rate": 3.9092656555375414e-07,
+      "logits/chosen": -3.184765577316284,
+      "logits/rejected": -2.549999952316284,
+      "logps/chosen": -48.16718673706055,
+      "logps/rejected": -155.3937530517578,
+      "loss": 0.1927,
+      "rewards/accuracies": 0.7593749761581421,
+      "rewards/chosen": -0.3481277525424957,
+      "rewards/margins": 5.426171779632568,
+      "rewards/rejected": -5.775000095367432,
+      "step": 270
+    },
+    {
+      "epoch": 0.7865168539325843,
+      "grad_norm": 1.486909031867981,
+      "learning_rate": 3.806246411789872e-07,
+      "logits/chosen": -3.206835985183716,
+      "logits/rejected": -2.568554639816284,
+      "logps/chosen": -46.279685974121094,
+      "logps/rejected": -161.4499969482422,
+      "loss": 0.1695,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.17198029160499573,
+      "rewards/margins": 5.812304496765137,
+      "rewards/rejected": -5.983984470367432,
+      "step": 280
+    },
+    {
+      "epoch": 0.8146067415730337,
+      "grad_norm": 0.24136996269226074,
+      "learning_rate": 3.700080306022528e-07,
+      "logits/chosen": -3.238085985183716,
+      "logits/rejected": -2.606250047683716,
+      "logps/chosen": -42.40312576293945,
+      "logps/rejected": -155.80624389648438,
+      "loss": 0.1873,
+      "rewards/accuracies": 0.76953125,
+      "rewards/chosen": -0.016921233385801315,
+      "rewards/margins": 5.745703220367432,
+      "rewards/rejected": -5.7607421875,
+      "step": 290
+    },
+    {
+      "epoch": 0.8426966292134831,
+      "grad_norm": 0.3395313620567322,
+      "learning_rate": 3.5910231016833546e-07,
+      "logits/chosen": -3.2119140625,
+      "logits/rejected": NaN,
+      "logps/chosen": -45.810935974121094,
+      "logps/rejected": -161.1687469482422,
+      "loss": 0.174,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.18533477187156677,
+      "rewards/margins": 5.834374904632568,
+      "rewards/rejected": -6.019140720367432,
+      "step": 300
+    },
+    {
+      "epoch": 0.8707865168539326,
+      "grad_norm": 0.14939536154270172,
+      "learning_rate": 3.4793375271298895e-07,
+      "logits/chosen": -3.1988282203674316,
+      "logits/rejected": -2.597851514816284,
+      "logps/chosen": -44.76874923706055,
+      "logps/rejected": -163.0437469482422,
+      "loss": 0.1656,
+      "rewards/accuracies": 0.7867187261581421,
+      "rewards/chosen": -0.11381302028894424,
+      "rewards/margins": 5.981835842132568,
+      "rewards/rejected": -6.096875190734863,
+      "step": 310
+    },
+    {
+      "epoch": 0.898876404494382,
+      "grad_norm": 0.6819726228713989,
+      "learning_rate": 3.3652926426937325e-07,
+      "logits/chosen": -3.246875047683716,
+      "logits/rejected": -2.609179735183716,
+      "logps/chosen": -41.875,
+      "logps/rejected": -162.28125,
+      "loss": 0.1824,
+      "rewards/accuracies": 0.7828124761581421,
+      "rewards/chosen": 0.06156463548541069,
+      "rewards/margins": 6.103515625,
+      "rewards/rejected": -6.043359279632568,
+      "step": 320
+    },
+    {
+      "epoch": 0.9269662921348315,
+      "grad_norm": 0.10508494079113007,
+      "learning_rate": 3.249163192490642e-07,
+      "logits/chosen": -3.2544922828674316,
+      "logits/rejected": -2.624804735183716,
+      "logps/chosen": -41.25,
+      "logps/rejected": -159.97500610351562,
+      "loss": 0.1816,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": 0.021668624132871628,
+      "rewards/margins": 5.989648342132568,
+      "rewards/rejected": -5.967968940734863,
+      "step": 330
+    },
+    {
+      "epoch": 0.9550561797752809,
+      "grad_norm": 0.2005680650472641,
+      "learning_rate": 3.1312289425378944e-07,
+      "logits/chosen": -3.2759766578674316,
+      "logits/rejected": -2.6624999046325684,
+      "logps/chosen": -39.45781326293945,
+      "logps/rejected": -157.4375,
+      "loss": 0.1757,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.14164963364601135,
+      "rewards/margins": 5.972851753234863,
+      "rewards/rejected": -5.831250190734863,
+      "step": 340
+    },
+    {
+      "epoch": 0.9831460674157303,
+      "grad_norm": 0.2797287404537201,
+      "learning_rate": 3.011774006773449e-07,
+      "logits/chosen": -3.282031297683716,
+      "logits/rejected": NaN,
+      "logps/chosen": -37.98906326293945,
+      "logps/rejected": -157.4375,
+      "loss": 0.1713,
+      "rewards/accuracies": 0.7757812738418579,
+      "rewards/chosen": 0.19436034560203552,
+      "rewards/margins": 6.061718940734863,
+      "rewards/rejected": -5.866406440734863,
+      "step": 350
+    },
+    {
+      "epoch": 1.0112359550561798,
+      "grad_norm": 0.1386144906282425,
+      "learning_rate": 2.8910861626005773e-07,
+      "logits/chosen": -3.2330079078674316,
+      "logits/rejected": -2.6128907203674316,
+      "logps/chosen": -42.69062423706055,
+      "logps/rejected": -162.75625610351562,
+      "loss": 0.1689,
+      "rewards/accuracies": 0.7942708730697632,
+      "rewards/chosen": -0.019468307495117188,
+      "rewards/margins": 6.076367378234863,
+      "rewards/rejected": -6.09765625,
+      "step": 360
+    },
+    {
+      "epoch": 1.0393258426966292,
+      "grad_norm": 0.31265443563461304,
+      "learning_rate": 2.7694561576068983e-07,
+      "logits/chosen": -3.234375,
+      "logits/rejected": -2.5785155296325684,
+      "logps/chosen": -41.896873474121094,
+      "logps/rejected": -160.8125,
+      "loss": 0.1713,
+      "rewards/accuracies": 0.7867187261581421,
+      "rewards/chosen": -0.01103897113353014,
+      "rewards/margins": 6.00390625,
+      "rewards/rejected": -6.014843940734863,
+      "step": 370
+    },
+    {
+      "epoch": 1.0674157303370786,
+      "grad_norm": 0.3972986042499542,
+      "learning_rate": 2.647177009127972e-07,
+      "logits/chosen": -3.2544922828674316,
+      "logits/rejected": -2.5933594703674316,
+      "logps/chosen": -39.5703125,
+      "logps/rejected": -160.85000610351562,
+      "loss": 0.1793,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.09815521538257599,
+      "rewards/margins": 6.125781059265137,
+      "rewards/rejected": -6.029492378234863,
+      "step": 380
+    },
+    {
+      "epoch": 1.095505617977528,
+      "grad_norm": 0.12721529603004456,
+      "learning_rate": 2.524543298342874e-07,
+      "logits/chosen": -3.262890577316284,
+      "logits/rejected": -2.592578172683716,
+      "logps/chosen": -40.4765625,
+      "logps/rejected": -161.91250610351562,
+      "loss": 0.1743,
+      "rewards/accuracies": 0.77734375,
+      "rewards/chosen": 0.05222644656896591,
+      "rewards/margins": 6.142578125,
+      "rewards/rejected": -6.091406345367432,
+      "step": 390
+    },
+    {
+      "epoch": 1.1235955056179776,
+      "grad_norm": 0.09501045197248459,
+      "learning_rate": 2.401850460602329e-07,
+      "logits/chosen": -3.2220702171325684,
+      "logits/rejected": -2.559765577316284,
+      "logps/chosen": -44.423439025878906,
+      "logps/rejected": -167.08749389648438,
+      "loss": 0.1718,
+      "rewards/accuracies": 0.780468761920929,
+      "rewards/chosen": -0.09567908942699432,
+      "rewards/margins": 6.20703125,
+      "rewards/rejected": -6.302734375,
+      "step": 400
+    },
+    {
+      "epoch": 1.151685393258427,
+      "grad_norm": 0.29580923914909363,
+      "learning_rate": 2.2793940736990766e-07,
+      "logits/chosen": -3.2007813453674316,
+      "logits/rejected": NaN,
+      "logps/chosen": -51.12968826293945,
+      "logps/rejected": -170.96875,
+      "loss": 0.172,
+      "rewards/accuracies": 0.780468761920929,
+      "rewards/chosen": -0.42823487520217896,
+      "rewards/margins": 6.079297065734863,
+      "rewards/rejected": -6.508593559265137,
+      "step": 410
+    },
+    {
+      "epoch": 1.1797752808988764,
+      "grad_norm": 0.2765229046344757,
+      "learning_rate": 2.1574691457950803e-07,
+      "logits/chosen": -3.1839842796325684,
+      "logits/rejected": NaN,
+      "logps/chosen": -50.9609375,
+      "logps/rejected": -170.78125,
+      "loss": 0.1704,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": -0.46327972412109375,
+      "rewards/margins": 6.071484565734863,
+      "rewards/rejected": -6.534375190734863,
+      "step": 420
+    },
+    {
+      "epoch": 1.2078651685393258,
+      "grad_norm": 0.15300235152244568,
+      "learning_rate": 2.036369404721023e-07,
+      "logits/chosen": -3.2113280296325684,
+      "logits/rejected": -2.531054735183716,
+      "logps/chosen": -46.8828125,
+      "logps/rejected": -173.1374969482422,
+      "loss": 0.1599,
+      "rewards/accuracies": 0.79296875,
+      "rewards/chosen": -0.21221771836280823,
+      "rewards/margins": 6.374609470367432,
+      "rewards/rejected": -6.587109565734863,
+      "step": 430
+    },
+    {
+      "epoch": 1.2359550561797752,
+      "grad_norm": 0.6692606806755066,
+      "learning_rate": 1.9163865903602372e-07,
+      "logits/chosen": -3.253124952316284,
+      "logits/rejected": -2.5308594703674316,
+      "logps/chosen": -42.287498474121094,
+      "logps/rejected": -169.625,
+      "loss": 0.1762,
+      "rewards/accuracies": 0.776562511920929,
+      "rewards/chosen": -0.029553985223174095,
+      "rewards/margins": 6.40625,
+      "rewards/rejected": -6.434765815734863,
+      "step": 440
+    },
+    {
+      "epoch": 1.2640449438202248,
+      "grad_norm": 0.2745380103588104,
+      "learning_rate": 1.7978097518217702e-07,
+      "logits/chosen": -3.237499952316284,
+      "logits/rejected": -2.541015625,
+      "logps/chosen": -42.74687576293945,
+      "logps/rejected": -168.78750610351562,
+      "loss": 0.1715,
+      "rewards/accuracies": 0.774218738079071,
+      "rewards/chosen": -0.021244239062070847,
+      "rewards/margins": 6.392187595367432,
+      "rewards/rejected": -6.409375190734863,
+      "step": 450
+    },
+    {
+      "epoch": 1.2921348314606742,
+      "grad_norm": 0.3858237564563751,
+      "learning_rate": 1.6809245510957666e-07,
+      "logits/chosen": -3.235546827316284,
+      "logits/rejected": NaN,
+      "logps/chosen": -44.904685974121094,
+      "logps/rejected": -172.96249389648438,
+      "loss": 0.1663,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.10267486423254013,
+      "rewards/margins": 6.484375,
+      "rewards/rejected": -6.587109565734863,
+      "step": 460
+    },
+    {
+      "epoch": 1.3202247191011236,
+      "grad_norm": 0.25496408343315125,
+      "learning_rate": 1.5660125748687093e-07,
+      "logits/chosen": -3.224414110183716,
+      "logits/rejected": NaN,
+      "logps/chosen": -46.64374923706055,
+      "logps/rejected": -171.1374969482422,
+      "loss": 0.1703,
+      "rewards/accuracies": 0.780468761920929,
+      "rewards/chosen": -0.2242431640625,
+      "rewards/margins": 6.318749904632568,
+      "rewards/rejected": -6.542187690734863,
+      "step": 470
+    },
+    {
+      "epoch": 1.348314606741573,
+      "grad_norm": 0.3984578251838684,
+      "learning_rate": 1.4533506561564305e-07,
+      "logits/chosen": -3.224609375,
+      "logits/rejected": -2.529101610183716,
+      "logps/chosen": -47.29375076293945,
+      "logps/rejected": -173.3625030517578,
+      "loss": 0.1669,
+      "rewards/accuracies": 0.780468761920929,
+      "rewards/chosen": -0.2605232298374176,
+      "rewards/margins": 6.368359565734863,
+      "rewards/rejected": -6.627734184265137,
+      "step": 480
+    },
+    {
+      "epoch": 1.3764044943820224,
+      "grad_norm": 0.39370596408843994,
+      "learning_rate": 1.343210207389125e-07,
+      "logits/chosen": -3.203320264816284,
+      "logits/rejected": -2.5302734375,
+      "logps/chosen": -48.0859375,
+      "logps/rejected": -172.93124389648438,
+      "loss": 0.1747,
+      "rewards/accuracies": 0.7757812738418579,
+      "rewards/chosen": -0.27991026639938354,
+      "rewards/margins": 6.329297065734863,
+      "rewards/rejected": -6.610156059265137,
+      "step": 490
+    },
+    {
+      "epoch": 1.404494382022472,
+      "grad_norm": 0.2013687640428543,
+      "learning_rate": 1.2358565665550387e-07,
+      "logits/chosen": -3.244335889816284,
+      "logits/rejected": NaN,
+      "logps/chosen": -41.96875,
+      "logps/rejected": -176.03750610351562,
+      "loss": 0.1557,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 0.01244263630360365,
+      "rewards/margins": 6.784765720367432,
+      "rewards/rejected": -6.769921779632568,
+      "step": 500
+    },
+    {
+      "epoch": 1.4325842696629214,
+      "grad_norm": 0.24023930728435516,
+      "learning_rate": 1.1315483579780094e-07,
+      "logits/chosen": -3.2484374046325684,
+      "logits/rejected": -2.5347657203674316,
+      "logps/chosen": -42.046875,
+      "logps/rejected": -175.8312530517578,
+      "loss": 0.1643,
+      "rewards/accuracies": 0.788281261920929,
+      "rewards/chosen": 0.040112875401973724,
+      "rewards/margins": 6.779296875,
+      "rewards/rejected": -6.739453315734863,
+      "step": 510
+    },
+    {
+      "epoch": 1.4606741573033708,
+      "grad_norm": 0.5066425800323486,
+      "learning_rate": 1.0305368692688174e-07,
+      "logits/chosen": -3.240429639816284,
+      "logits/rejected": -2.522656202316284,
+      "logps/chosen": -43.00468826293945,
+      "logps/rejected": -173.8625030517578,
+      "loss": 0.1791,
+      "rewards/accuracies": 0.778124988079071,
+      "rewards/chosen": -0.03853149339556694,
+      "rewards/margins": 6.608788967132568,
+      "rewards/rejected": -6.646874904632568,
+      "step": 520
+    },
+    {
+      "epoch": 1.4887640449438202,
+      "grad_norm": 0.10324753075838089,
+      "learning_rate": 9.330654459513266e-08,
+      "logits/chosen": -3.233203172683716,
+      "logits/rejected": -2.5269532203674316,
+      "logps/chosen": -41.953125,
+      "logps/rejected": -172.06875610351562,
+      "loss": 0.1683,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.02672729454934597,
+      "rewards/margins": 6.571875095367432,
+      "rewards/rejected": -6.598437309265137,
+      "step": 530
+    },
+    {
+      "epoch": 1.5168539325842696,
+      "grad_norm": 0.06884948909282684,
+      "learning_rate": 8.393689052217964e-08,
+      "logits/chosen": -3.2310547828674316,
+      "logits/rejected": -2.5240235328674316,
+      "logps/chosen": -41.89531326293945,
+      "logps/rejected": -170.4499969482422,
+      "loss": 0.1774,
+      "rewards/accuracies": 0.770312488079071,
+      "rewards/chosen": -0.03569946438074112,
+      "rewards/margins": 6.483984470367432,
+      "rewards/rejected": -6.521093845367432,
+      "step": 540
+    },
+    {
+      "epoch": 1.5449438202247192,
+      "grad_norm": 0.2964838445186615,
+      "learning_rate": 7.49672970253691e-08,
+      "logits/chosen": -3.2562499046325684,
+      "logits/rejected": -2.5296874046325684,
+      "logps/chosen": -40.365623474121094,
+      "logps/rejected": -173.3625030517578,
+      "loss": 0.1644,
+      "rewards/accuracies": 0.78515625,
+      "rewards/chosen": 0.06430435180664062,
+      "rewards/margins": 6.696484565734863,
+      "rewards/rejected": -6.630078315734863,
+      "step": 550
+    },
+    {
+      "epoch": 1.5730337078651684,
+      "grad_norm": 0.16073837876319885,
+      "learning_rate": 6.641937264107867e-08,
+      "logits/chosen": -3.2535157203674316,
+      "logits/rejected": -2.5425782203674316,
+      "logps/chosen": -39.43281173706055,
+      "logps/rejected": -167.91250610351562,
+      "loss": 0.1788,
+      "rewards/accuracies": 0.7671874761581421,
+      "rewards/chosen": 0.08403320610523224,
+      "rewards/margins": 6.482421875,
+      "rewards/rejected": -6.396874904632568,
+      "step": 560
+    },
+    {
+      "epoch": 1.601123595505618,
+      "grad_norm": 0.27828249335289,
+      "learning_rate": 5.831371006785962e-08,
+      "logits/chosen": -3.2699217796325684,
+      "logits/rejected": -2.5228514671325684,
+      "logps/chosen": -39.87187576293945,
+      "logps/rejected": -171.1437530517578,
+      "loss": 0.175,
+      "rewards/accuracies": 0.772656261920929,
+      "rewards/chosen": 0.07085514068603516,
+      "rewards/margins": 6.617578029632568,
+      "rewards/rejected": -6.548047065734863,
+      "step": 570
+    },
+    {
+      "epoch": 1.6292134831460674,
+      "grad_norm": 0.386394202709198,
+      "learning_rate": 5.066983655682325e-08,
+      "logits/chosen": -3.2455077171325684,
+      "logits/rejected": -2.546093702316284,
+      "logps/chosen": -40.72968673706055,
+      "logps/rejected": -168.77499389648438,
+      "loss": 0.1783,
+      "rewards/accuracies": 0.7679687738418579,
+      "rewards/chosen": 0.032080840319395065,
+      "rewards/margins": 6.479296684265137,
+      "rewards/rejected": -6.448828220367432,
+      "step": 580
+    },
+    {
+      "epoch": 1.6573033707865168,
+      "grad_norm": 0.3948213756084442,
+      "learning_rate": 4.3506166868781755e-08,
+      "logits/chosen": -3.2671875953674316,
+      "logits/rejected": -2.521484375,
+      "logps/chosen": -41.20624923706055,
+      "logps/rejected": -172.6875,
+      "loss": 0.1759,
+      "rewards/accuracies": 0.76953125,
+      "rewards/chosen": 0.012204742059111595,
+      "rewards/margins": 6.639843940734863,
+      "rewards/rejected": -6.629296779632568,
+      "step": 590
+    },
+    {
+      "epoch": 1.6853932584269664,
+      "grad_norm": 0.4554503560066223,
+      "learning_rate": 3.683995891147695e-08,
+      "logits/chosen": -3.246289014816284,
+      "logits/rejected": NaN,
+      "logps/chosen": -42.18281173706055,
+      "logps/rejected": -173.86874389648438,
+      "loss": 0.1625,
+      "rewards/accuracies": 0.784375011920929,
+      "rewards/chosen": 0.0017807006370276213,
+      "rewards/margins": 6.689453125,
+      "rewards/rejected": -6.688672065734863,
+      "step": 600
+    },
+    {
+      "epoch": 1.7134831460674156,
+      "grad_norm": 0.6775197386741638,
+      "learning_rate": 3.0687272163768986e-08,
+      "logits/chosen": -3.259960889816284,
+      "logits/rejected": NaN,
+      "logps/chosen": -41.376564025878906,
+      "logps/rejected": -174.4499969482422,
+      "loss": 0.1619,
+      "rewards/accuracies": 0.788281261920929,
+      "rewards/chosen": 0.07793807983398438,
+      "rewards/margins": 6.741796970367432,
+      "rewards/rejected": -6.6640625,
+      "step": 610
+    },
+    {
+      "epoch": 1.7415730337078652,
+      "grad_norm": 0.11740182340145111,
+      "learning_rate": 2.5062928986944676e-08,
+      "logits/chosen": -3.2728514671325684,
+      "logits/rejected": NaN,
+      "logps/chosen": -40.89374923706055,
+      "logps/rejected": -175.0625,
+      "loss": 0.1611,
+      "rewards/accuracies": 0.7835937738418579,
+      "rewards/chosen": 0.07378844916820526,
+      "rewards/margins": 6.797656059265137,
+      "rewards/rejected": -6.723828315734863,
+      "step": 620
+    },
+    {
+      "epoch": 1.7696629213483146,
+      "grad_norm": 0.23929810523986816,
+      "learning_rate": 1.9980478916351296e-08,
+      "logits/chosen": -3.2544922828674316,
+      "logits/rejected": -2.5478515625,
+      "logps/chosen": -42.296875,
+      "logps/rejected": -174.1374969482422,
+      "loss": 0.1689,
+      "rewards/accuracies": 0.785937488079071,
+      "rewards/chosen": -0.01357345562428236,
+      "rewards/margins": 6.648046970367432,
+      "rewards/rejected": -6.661328315734863,
+      "step": 630
+    },
+    {
+      "epoch": 1.797752808988764,
+      "grad_norm": 0.06252361834049225,
+      "learning_rate": 1.5452166019378987e-08,
+      "logits/chosen": -3.2582030296325684,
+      "logits/rejected": -2.5250000953674316,
+      "logps/chosen": -39.837501525878906,
+      "logps/rejected": -173.9875030517578,
+      "loss": 0.1693,
+      "rewards/accuracies": 0.776562511920929,
+      "rewards/chosen": 0.06287650763988495,
+      "rewards/margins": 6.756249904632568,
+      "rewards/rejected": -6.696875095367432,
+      "step": 640
+    },
+    {
+      "epoch": 1.8258426966292136,
+      "grad_norm": 0.6466670632362366,
+      "learning_rate": 1.1488899398429896e-08,
+      "logits/chosen": -3.2587890625,
+      "logits/rejected": -2.5443358421325684,
+      "logps/chosen": -41.12812423706055,
+      "logps/rejected": -175.3125,
+      "loss": 0.1643,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 0.057323455810546875,
+      "rewards/margins": 6.766015529632568,
+      "rewards/rejected": -6.710156440734863,
+      "step": 650
+    },
+    {
+      "epoch": 1.8539325842696628,
+      "grad_norm": 0.364284485578537,
+      "learning_rate": 8.100226909935059e-09,
+      "logits/chosen": -3.2474608421325684,
+      "logits/rejected": -2.5396485328674316,
+      "logps/chosen": -43.040626525878906,
+      "logps/rejected": -175.02499389648438,
+      "loss": 0.1656,
+      "rewards/accuracies": 0.7906249761581421,
+      "rewards/chosen": -0.026231002062559128,
+      "rewards/margins": 6.670312404632568,
+      "rewards/rejected": -6.696093559265137,
+      "step": 660
+    },
+    {
+      "epoch": 1.8820224719101124,
+      "grad_norm": 0.1942785233259201,
+      "learning_rate": 5.2943121627319346e-09,
+      "logits/chosen": -3.252734422683716,
+      "logits/rejected": -2.544140577316284,
+      "logps/chosen": -41.951560974121094,
+      "logps/rejected": -170.24374389648438,
+      "loss": 0.1726,
+      "rewards/accuracies": 0.770312488079071,
+      "rewards/chosen": -0.00887908972799778,
+      "rewards/margins": 6.493359565734863,
+      "rewards/rejected": -6.500390529632568,
+      "step": 670
+    },
+    {
+      "epoch": 1.9101123595505618,
+      "grad_norm": 0.38263440132141113,
+      "learning_rate": 3.077914851215585e-09,
+      "logits/chosen": -3.257031202316284,
+      "logits/rejected": -2.5390625,
+      "logps/chosen": -41.865623474121094,
+      "logps/rejected": -176.53750610351562,
+      "loss": 0.1582,
+      "rewards/accuracies": 0.796093761920929,
+      "rewards/chosen": 0.024802017956972122,
+      "rewards/margins": 6.796093940734863,
+      "rewards/rejected": -6.771484375,
+      "step": 680
+    },
+    {
+      "epoch": 1.9382022471910112,
+      "grad_norm": 0.4378679096698761,
+      "learning_rate": 1.4563744706429514e-09,
+      "logits/chosen": -3.2578125,
+      "logits/rejected": -2.5419921875,
+      "logps/chosen": -40.009376525878906,
+      "logps/rejected": -171.91250610351562,
+      "loss": 0.1703,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.07464599609375,
+      "rewards/margins": 6.642968654632568,
+      "rewards/rejected": -6.567968845367432,
+      "step": 690
+    },
+    {
+      "epoch": 1.9662921348314608,
+      "grad_norm": 0.19188769161701202,
+      "learning_rate": 4.3359745382104405e-10,
+      "logits/chosen": -3.271679639816284,
+      "logits/rejected": -2.5396485328674316,
+      "logps/chosen": -41.52031326293945,
+      "logps/rejected": -175.375,
+      "loss": 0.1601,
+      "rewards/accuracies": 0.7906249761581421,
+      "rewards/chosen": 0.01084976177662611,
+      "rewards/margins": 6.757421970367432,
+      "rewards/rejected": -6.750390529632568,
+      "step": 700
+    },
+    {
+      "epoch": 1.99438202247191,
+      "grad_norm": 0.2792583107948303,
+      "learning_rate": 1.2047760167999133e-11,
+      "logits/chosen": -3.2544922828674316,
+      "logits/rejected": -2.5517578125,
+      "logps/chosen": -42.29999923706055,
+      "logps/rejected": -175.3625030517578,
+      "loss": 0.1594,
+      "rewards/accuracies": 0.7867187261581421,
+      "rewards/chosen": -0.014897918328642845,
+      "rewards/margins": 6.701171875,
+      "rewards/rejected": -6.717187404632568,
+      "step": 710
+    },
+    {
+      "epoch": 2.0,
+      "step": 712,
       "total_flos": 0.0,
-      "train_loss": 0.5645458033827485,
-      "train_runtime": 476.0531,
-      "train_samples_per_second": 16.473,
-      "train_steps_per_second": 0.256
+      "train_loss": 0.22515048312672067,
+      "train_runtime": 2958.2669,
+      "train_samples_per_second": 30.802,
+      "train_steps_per_second": 0.241
     }
   ],
   "logging_steps": 10,
-  "max_steps": 122,
+  "max_steps": 712,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 2,
   "save_steps": 500,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6be99c816b5db8c5a3acc17c9a53f9de39b2335b62b1dba0929d841f39fa676d
+oid sha256:3e47b8bd96f5ed212913b9d45c5a069e52c690acee03a5d1528a81d6eb90ec43
 size 7800