Model save

README.md

@@ -0,0 +1,73 @@
+---
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: dpo-selective-mixdata
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# dpo-selective-mixdata
+
+This model was trained from scratch on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.5805
+- Rewards/chosen: -4.7292
+- Rewards/rejected: -5.2763
+- Rewards/accuracies: 0.6934
+- Rewards/margins: 0.5471
+- Logps/rejected: -654.5243
+- Logps/chosen: -590.7578
+- Logits/rejected: 6.2956
+- Logits/chosen: 6.4467
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 4
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 64
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.5481        | 0.27  | 500  | 0.6089          | -2.6822        | -3.1236          | 0.6705             | 0.4414          | -439.2521      | -386.0565    | 3.9671          | 4.1604        |
+| 0.5519        | 0.53  | 1000 | 0.5867          | -4.2523        | -4.7597          | 0.6894             | 0.5074          | -602.8671      | -543.0739    | 5.1974          | 5.3486        |
+| 0.5597        | 0.8   | 1500 | 0.5821          | -4.7906        | -5.3218          | 0.6959             | 0.5311          | -659.0733      | -596.9037    | 6.4644          | 6.6294        |
+
+
+### Framework versions
+
+- Transformers 4.36.2
+- Pytorch 2.1.2
+- Datasets 2.14.6
+- Tokenizers 0.15.0

all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": 6.446723937988281,
+    "eval_logits/rejected": 6.295619964599609,
+    "eval_logps/chosen": -590.7577514648438,
+    "eval_logps/rejected": -654.5242919921875,
+    "eval_loss": 0.5805472135543823,
+    "eval_rewards/accuracies": 0.6934306621551514,
+    "eval_rewards/chosen": -4.729165554046631,
+    "eval_rewards/margins": 0.5471083521842957,
+    "eval_rewards/rejected": -5.276274681091309,
+    "eval_runtime": 2831.4775,
+    "eval_samples": 35044,
+    "eval_samples_per_second": 12.377,
+    "eval_steps_per_second": 0.387,
+    "train_loss": 0.5658232939218006,
+    "train_runtime": 45309.3847,
+    "train_samples": 120613,
+    "train_samples_per_second": 2.662,
+    "train_steps_per_second": 0.042
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": 6.446723937988281,
+    "eval_logits/rejected": 6.295619964599609,
+    "eval_logps/chosen": -590.7577514648438,
+    "eval_logps/rejected": -654.5242919921875,
+    "eval_loss": 0.5805472135543823,
+    "eval_rewards/accuracies": 0.6934306621551514,
+    "eval_rewards/chosen": -4.729165554046631,
+    "eval_rewards/margins": 0.5471083521842957,
+    "eval_rewards/rejected": -5.276274681091309,
+    "eval_runtime": 2831.4775,
+    "eval_samples": 35044,
+    "eval_samples_per_second": 12.377,
+    "eval_steps_per_second": 0.387
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.36.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd6f506ba8dcb4cfb8c0882cc3a8f8d66208b1900d110162d03eb9401cf1d6b2
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e3374e5638c392e79254199c91b4801e381cfe30e34a5fedc04a5f3f05353b8
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a01997f2bfbeb801204ce2953dcf7cf0cdb53cdbda344b89740d879ccb8f87b
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.5658232939218006,
+    "train_runtime": 45309.3847,
+    "train_samples": 120613,
+    "train_samples_per_second": 2.662,
+    "train_steps_per_second": 0.042
+}

trainer_state.json

@@ -0,0 +1,2724 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9996020692399522,
+  "eval_steps": 500,
+  "global_step": 1884,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.645502645502645e-09,
+      "logits/chosen": -1.7827545404434204,
+      "logits/rejected": -1.8475866317749023,
+      "logps/chosen": -188.38653564453125,
+      "logps/rejected": -220.2354736328125,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 2.6455026455026453e-08,
+      "logits/chosen": -1.9714936017990112,
+      "logits/rejected": -1.876778483390808,
+      "logps/chosen": -288.3788146972656,
+      "logps/rejected": -199.33457946777344,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.3819444477558136,
+      "rewards/chosen": -0.00010286699398420751,
+      "rewards/margins": 0.00021866995666641742,
+      "rewards/rejected": -0.00032153705251403153,
+      "step": 10
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.2910052910052905e-08,
+      "logits/chosen": -1.9301083087921143,
+      "logits/rejected": -1.9124500751495361,
+      "logps/chosen": -180.6164093017578,
+      "logps/rejected": -186.2301483154297,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -4.3160351197002456e-05,
+      "rewards/margins": -7.290135545190424e-05,
+      "rewards/rejected": 2.974100425490178e-05,
+      "step": 20
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 7.936507936507936e-08,
+      "logits/chosen": -1.9063631296157837,
+      "logits/rejected": -1.937107801437378,
+      "logps/chosen": -190.45803833007812,
+      "logps/rejected": -201.83729553222656,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.0003868662752211094,
+      "rewards/margins": -0.00044071293086744845,
+      "rewards/rejected": 5.3846673836233094e-05,
+      "step": 30
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0582010582010581e-07,
+      "logits/chosen": -1.8966213464736938,
+      "logits/rejected": -1.8653703927993774,
+      "logps/chosen": -212.13247680664062,
+      "logps/rejected": -186.06394958496094,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.00024294569448102266,
+      "rewards/margins": 0.00039846505387686193,
+      "rewards/rejected": -0.00015551943215541542,
+      "step": 40
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.3227513227513225e-07,
+      "logits/chosen": -1.9822158813476562,
+      "logits/rejected": -1.8613579273223877,
+      "logps/chosen": -217.9025421142578,
+      "logps/rejected": -174.90078735351562,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.003075198968872428,
+      "rewards/margins": 0.0013211520854383707,
+      "rewards/rejected": 0.001754046999849379,
+      "step": 50
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5873015873015872e-07,
+      "logits/chosen": -1.9449065923690796,
+      "logits/rejected": -1.9192440509796143,
+      "logps/chosen": -176.40371704101562,
+      "logps/rejected": -172.91717529296875,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.003195743542164564,
+      "rewards/margins": 0.0035919342190027237,
+      "rewards/rejected": -0.0003961907350458205,
+      "step": 60
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.8518518518518516e-07,
+      "logits/chosen": -1.9199409484863281,
+      "logits/rejected": -1.8474018573760986,
+      "logps/chosen": -174.99351501464844,
+      "logps/rejected": -197.31735229492188,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 0.0023753070272505283,
+      "rewards/margins": 0.006421695463359356,
+      "rewards/rejected": -0.004046388436108828,
+      "step": 70
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.1164021164021162e-07,
+      "logits/chosen": -1.8320897817611694,
+      "logits/rejected": -1.844348669052124,
+      "logps/chosen": -195.1573028564453,
+      "logps/rejected": -195.3834228515625,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.001768267946317792,
+      "rewards/margins": 0.007629372179508209,
+      "rewards/rejected": -0.009397639892995358,
+      "step": 80
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.3809523809523806e-07,
+      "logits/chosen": -1.909330129623413,
+      "logits/rejected": -1.9080867767333984,
+      "logps/chosen": -196.3611297607422,
+      "logps/rejected": -187.08120727539062,
+      "loss": 0.6879,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.0239607784897089,
+      "rewards/margins": 0.006291805300861597,
+      "rewards/rejected": -0.030252579599618912,
+      "step": 90
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.645502645502645e-07,
+      "logits/chosen": -1.8753902912139893,
+      "logits/rejected": -1.8588756322860718,
+      "logps/chosen": -197.9770050048828,
+      "logps/rejected": -210.04736328125,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.05865932255983353,
+      "rewards/margins": 0.00952382292598486,
+      "rewards/rejected": -0.06818314641714096,
+      "step": 100
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.9100529100529097e-07,
+      "logits/chosen": -1.942865014076233,
+      "logits/rejected": -1.8896948099136353,
+      "logps/chosen": -222.5757598876953,
+      "logps/rejected": -227.91476440429688,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.07471577823162079,
+      "rewards/margins": 0.0264597050845623,
+      "rewards/rejected": -0.1011754721403122,
+      "step": 110
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1746031746031743e-07,
+      "logits/chosen": -1.8088843822479248,
+      "logits/rejected": -1.7518575191497803,
+      "logps/chosen": -222.64022827148438,
+      "logps/rejected": -209.8594512939453,
+      "loss": 0.6776,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.11961501836776733,
+      "rewards/margins": 0.06059970706701279,
+      "rewards/rejected": -0.18021473288536072,
+      "step": 120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.439153439153439e-07,
+      "logits/chosen": -1.7773793935775757,
+      "logits/rejected": -1.7670618295669556,
+      "logps/chosen": -222.9905548095703,
+      "logps/rejected": -228.2213134765625,
+      "loss": 0.669,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.18275949358940125,
+      "rewards/margins": 0.04632816091179848,
+      "rewards/rejected": -0.22908766567707062,
+      "step": 130
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.703703703703703e-07,
+      "logits/chosen": -1.777341604232788,
+      "logits/rejected": -1.6979541778564453,
+      "logps/chosen": -227.1598358154297,
+      "logps/rejected": -202.8821258544922,
+      "loss": 0.6663,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.19891491532325745,
+      "rewards/margins": 0.05222553759813309,
+      "rewards/rejected": -0.25114041566848755,
+      "step": 140
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.968253968253968e-07,
+      "logits/chosen": -1.639807939529419,
+      "logits/rejected": -1.5844175815582275,
+      "logps/chosen": -248.316650390625,
+      "logps/rejected": -248.49240112304688,
+      "loss": 0.6549,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.45894140005111694,
+      "rewards/margins": 0.053535498678684235,
+      "rewards/rejected": -0.512476921081543,
+      "step": 150
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.2328042328042324e-07,
+      "logits/chosen": -1.4632054567337036,
+      "logits/rejected": -1.4093120098114014,
+      "logps/chosen": -265.42877197265625,
+      "logps/rejected": -252.86727905273438,
+      "loss": 0.6609,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.6529312133789062,
+      "rewards/margins": 0.09157294034957886,
+      "rewards/rejected": -0.7445041537284851,
+      "step": 160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.497354497354497e-07,
+      "logits/chosen": -1.4973636865615845,
+      "logits/rejected": -1.3732246160507202,
+      "logps/chosen": -299.1934509277344,
+      "logps/rejected": -286.1786804199219,
+      "loss": 0.6473,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.6386870741844177,
+      "rewards/margins": 0.15349723398685455,
+      "rewards/rejected": -0.7921843528747559,
+      "step": 170
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.761904761904761e-07,
+      "logits/chosen": -1.419722318649292,
+      "logits/rejected": -1.3405072689056396,
+      "logps/chosen": -262.6790466308594,
+      "logps/rejected": -247.8793487548828,
+      "loss": 0.6447,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.6575330495834351,
+      "rewards/margins": 0.12454842031002045,
+      "rewards/rejected": -0.7820814847946167,
+      "step": 180
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999995705919032e-07,
+      "logits/chosen": -1.4580373764038086,
+      "logits/rejected": -1.3449883460998535,
+      "logps/chosen": -265.0930480957031,
+      "logps/rejected": -266.20880126953125,
+      "loss": 0.649,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.6115543246269226,
+      "rewards/margins": 0.11900641769170761,
+      "rewards/rejected": -0.7305607795715332,
+      "step": 190
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.999480434051858e-07,
+      "logits/chosen": -1.7100050449371338,
+      "logits/rejected": -1.6133902072906494,
+      "logps/chosen": -312.9882507324219,
+      "logps/rejected": -298.0437927246094,
+      "loss": 0.6499,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.9183656573295593,
+      "rewards/margins": 0.07652232050895691,
+      "rewards/rejected": -0.9948879480361938,
+      "step": 200
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.998106548810311e-07,
+      "logits/chosen": -1.6054027080535889,
+      "logits/rejected": -1.5358479022979736,
+      "logps/chosen": -283.46551513671875,
+      "logps/rejected": -306.21820068359375,
+      "loss": 0.6371,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.8442398905754089,
+      "rewards/margins": 0.2072862684726715,
+      "rewards/rejected": -1.0515261888504028,
+      "step": 210
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.995874522146975e-07,
+      "logits/chosen": -1.5969936847686768,
+      "logits/rejected": -1.5442743301391602,
+      "logps/chosen": -294.7565002441406,
+      "logps/rejected": -295.92779541015625,
+      "loss": 0.6223,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.9133985638618469,
+      "rewards/margins": 0.20787055790424347,
+      "rewards/rejected": -1.1212691068649292,
+      "step": 220
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.992785120800375e-07,
+      "logits/chosen": -1.0925428867340088,
+      "logits/rejected": -1.0447893142700195,
+      "logps/chosen": -327.7795104980469,
+      "logps/rejected": -339.69500732421875,
+      "loss": 0.6349,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.108730435371399,
+      "rewards/margins": 0.24315257370471954,
+      "rewards/rejected": -1.3518829345703125,
+      "step": 230
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.988839406031596e-07,
+      "logits/chosen": -0.13352422416210175,
+      "logits/rejected": 0.016487866640090942,
+      "logps/chosen": -324.3548278808594,
+      "logps/rejected": -332.9447326660156,
+      "loss": 0.63,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.3696115016937256,
+      "rewards/margins": 0.30091235041618347,
+      "rewards/rejected": -1.6705236434936523,
+      "step": 240
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.98403873325972e-07,
+      "logits/chosen": -0.25656959414482117,
+      "logits/rejected": -0.22962741553783417,
+      "logps/chosen": -276.41180419921875,
+      "logps/rejected": -297.1820983886719,
+      "loss": 0.6098,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.8686075210571289,
+      "rewards/margins": 0.2754421532154083,
+      "rewards/rejected": -1.1440496444702148,
+      "step": 250
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.978384751596212e-07,
+      "logits/chosen": 0.7076885104179382,
+      "logits/rejected": 0.7922934293746948,
+      "logps/chosen": -315.2431640625,
+      "logps/rejected": -349.3119812011719,
+      "loss": 0.6038,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.1041444540023804,
+      "rewards/margins": 0.35109204053878784,
+      "rewards/rejected": -1.4552366733551025,
+      "step": 260
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.971879403278432e-07,
+      "logits/chosen": 1.446623682975769,
+      "logits/rejected": 1.5377308130264282,
+      "logps/chosen": -370.0694580078125,
+      "logps/rejected": -390.1954650878906,
+      "loss": 0.604,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.6548852920532227,
+      "rewards/margins": 0.3384679853916168,
+      "rewards/rejected": -1.9933536052703857,
+      "step": 270
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.964524923002436e-07,
+      "logits/chosen": 0.9782055020332336,
+      "logits/rejected": 1.0214025974273682,
+      "logps/chosen": -374.83990478515625,
+      "logps/rejected": -407.66644287109375,
+      "loss": 0.5943,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.666853666305542,
+      "rewards/margins": 0.4476628303527832,
+      "rewards/rejected": -2.1145167350769043,
+      "step": 280
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.956323837155325e-07,
+      "logits/chosen": 1.1159193515777588,
+      "logits/rejected": 1.1185142993927002,
+      "logps/chosen": -357.0001220703125,
+      "logps/rejected": -394.30279541015625,
+      "loss": 0.6107,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.6487280130386353,
+      "rewards/margins": 0.3608878254890442,
+      "rewards/rejected": -2.009615898132324,
+      "step": 290
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.947278962947386e-07,
+      "logits/chosen": 1.6991510391235352,
+      "logits/rejected": 1.9209808111190796,
+      "logps/chosen": -396.7066955566406,
+      "logps/rejected": -431.6412658691406,
+      "loss": 0.6001,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.0254971981048584,
+      "rewards/margins": 0.42958149313926697,
+      "rewards/rejected": -2.455078601837158,
+      "step": 300
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.937393407444337e-07,
+      "logits/chosen": 1.100517988204956,
+      "logits/rejected": 1.2365609407424927,
+      "logps/chosen": -356.17681884765625,
+      "logps/rejected": -412.0787048339844,
+      "loss": 0.5925,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7612950801849365,
+      "rewards/margins": 0.4073117673397064,
+      "rewards/rejected": -2.168606758117676,
+      "step": 310
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.926670566499992e-07,
+      "logits/chosen": 1.4866538047790527,
+      "logits/rejected": 1.5181796550750732,
+      "logps/chosen": -370.19793701171875,
+      "logps/rejected": -399.6379089355469,
+      "loss": 0.5865,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.8391116857528687,
+      "rewards/margins": 0.3383946418762207,
+      "rewards/rejected": -2.1775062084198,
+      "step": 320
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.915114123589732e-07,
+      "logits/chosen": 3.2927498817443848,
+      "logits/rejected": 3.4409637451171875,
+      "logps/chosen": -474.81134033203125,
+      "logps/rejected": -486.77264404296875,
+      "loss": 0.5826,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.5577211380004883,
+      "rewards/margins": 0.5131586790084839,
+      "rewards/rejected": -3.0708799362182617,
+      "step": 330
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.90272804854517e-07,
+      "logits/chosen": 4.935120582580566,
+      "logits/rejected": 5.241981029510498,
+      "logps/chosen": -558.8336181640625,
+      "logps/rejected": -587.92578125,
+      "loss": 0.6196,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -3.244816303253174,
+      "rewards/margins": 0.4380508065223694,
+      "rewards/rejected": -3.6828665733337402,
+      "step": 340
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.889516596190448e-07,
+      "logits/chosen": 3.1185314655303955,
+      "logits/rejected": 3.6742959022521973,
+      "logps/chosen": -524.4172973632812,
+      "logps/rejected": -528.1964111328125,
+      "loss": 0.602,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.8538341522216797,
+      "rewards/margins": 0.4846366345882416,
+      "rewards/rejected": -3.338470935821533,
+      "step": 350
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.875484304880629e-07,
+      "logits/chosen": 1.2977662086486816,
+      "logits/rejected": 1.6007190942764282,
+      "logps/chosen": -468.362060546875,
+      "logps/rejected": -483.639892578125,
+      "loss": 0.6246,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.143881320953369,
+      "rewards/margins": 0.36468884348869324,
+      "rewards/rejected": -2.5085701942443848,
+      "step": 360
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.860635994942702e-07,
+      "logits/chosen": 1.5786962509155273,
+      "logits/rejected": 1.9745759963989258,
+      "logps/chosen": -395.7223205566406,
+      "logps/rejected": -411.2752990722656,
+      "loss": 0.5782,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.866349458694458,
+      "rewards/margins": 0.4314461350440979,
+      "rewards/rejected": -2.297795534133911,
+      "step": 370
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.844976767019714e-07,
+      "logits/chosen": 2.636617660522461,
+      "logits/rejected": 2.620858669281006,
+      "logps/chosen": -369.3752136230469,
+      "logps/rejected": -416.45574951171875,
+      "loss": 0.576,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.1540982723236084,
+      "rewards/margins": 0.4857994019985199,
+      "rewards/rejected": -2.6398978233337402,
+      "step": 380
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.828512000318616e-07,
+      "logits/chosen": 2.6557507514953613,
+      "logits/rejected": 3.367990016937256,
+      "logps/chosen": -490.51708984375,
+      "logps/rejected": -534.2076416015625,
+      "loss": 0.5691,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.498900890350342,
+      "rewards/margins": 0.7616626620292664,
+      "rewards/rejected": -3.260563373565674,
+      "step": 390
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.811247350762418e-07,
+      "logits/chosen": 2.237056016921997,
+      "logits/rejected": 2.7285072803497314,
+      "logps/chosen": -381.19000244140625,
+      "logps/rejected": -417.8363342285156,
+      "loss": 0.5659,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.9070937633514404,
+      "rewards/margins": 0.5606156587600708,
+      "rewards/rejected": -2.46770977973938,
+      "step": 400
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.79318874904728e-07,
+      "logits/chosen": 4.107834339141846,
+      "logits/rejected": 4.020180702209473,
+      "logps/chosen": -422.3435974121094,
+      "logps/rejected": -466.9422912597656,
+      "loss": 0.5826,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.08524751663208,
+      "rewards/margins": 0.5145748853683472,
+      "rewards/rejected": -2.599822521209717,
+      "step": 410
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.774342398605221e-07,
+      "logits/chosen": 4.6263508796691895,
+      "logits/rejected": 5.094324588775635,
+      "logps/chosen": -447.66790771484375,
+      "logps/rejected": -472.98541259765625,
+      "loss": 0.5864,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.5026144981384277,
+      "rewards/margins": 0.5022978782653809,
+      "rewards/rejected": -3.0049126148223877,
+      "step": 420
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.754714773473134e-07,
+      "logits/chosen": 3.137293577194214,
+      "logits/rejected": 3.4239907264709473,
+      "logps/chosen": -383.85272216796875,
+      "logps/rejected": -439.3240661621094,
+      "loss": 0.5614,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.8029075860977173,
+      "rewards/margins": 0.6045055389404297,
+      "rewards/rejected": -2.4074130058288574,
+      "step": 430
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.734312616068851e-07,
+      "logits/chosen": 3.090550184249878,
+      "logits/rejected": 3.005370855331421,
+      "logps/chosen": -393.83636474609375,
+      "logps/rejected": -456.08917236328125,
+      "loss": 0.5636,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.2565479278564453,
+      "rewards/margins": 0.5205822587013245,
+      "rewards/rejected": -2.777130365371704,
+      "step": 440
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.713142934875005e-07,
+      "logits/chosen": 3.673936128616333,
+      "logits/rejected": 4.1635026931762695,
+      "logps/chosen": -492.8273010253906,
+      "logps/rejected": -526.05810546875,
+      "loss": 0.5837,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.6836562156677246,
+      "rewards/margins": 0.5476377010345459,
+      "rewards/rejected": -3.2312939167022705,
+      "step": 450
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.6912130020314996e-07,
+      "logits/chosen": 3.0116238594055176,
+      "logits/rejected": 3.2378668785095215,
+      "logps/chosen": -389.37811279296875,
+      "logps/rejected": -445.2620544433594,
+      "loss": 0.5657,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.161726951599121,
+      "rewards/margins": 0.5274554491043091,
+      "rewards/rejected": -2.6891825199127197,
+      "step": 460
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.668530350837408e-07,
+      "logits/chosen": 2.4322941303253174,
+      "logits/rejected": 2.488206148147583,
+      "logps/chosen": -389.4209899902344,
+      "logps/rejected": -435.225341796875,
+      "loss": 0.5843,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.8818559646606445,
+      "rewards/margins": 0.40773385763168335,
+      "rewards/rejected": -2.2895896434783936,
+      "step": 470
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.64510277316316e-07,
+      "logits/chosen": 2.517045736312866,
+      "logits/rejected": 2.917738676071167,
+      "logps/chosen": -429.2723693847656,
+      "logps/rejected": -474.98681640625,
+      "loss": 0.568,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.0465638637542725,
+      "rewards/margins": 0.6840267777442932,
+      "rewards/rejected": -2.730590581893921,
+      "step": 480
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.6209383167739015e-07,
+      "logits/chosen": 2.8990161418914795,
+      "logits/rejected": 3.3626670837402344,
+      "logps/chosen": -449.22235107421875,
+      "logps/rejected": -482.134033203125,
+      "loss": 0.5814,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.392901659011841,
+      "rewards/margins": 0.5466801524162292,
+      "rewards/rejected": -2.939582109451294,
+      "step": 490
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.5960452825649526e-07,
+      "logits/chosen": 2.796527147293091,
+      "logits/rejected": 2.8022494316101074,
+      "logps/chosen": -416.53692626953125,
+      "logps/rejected": -469.519287109375,
+      "loss": 0.5481,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.1346237659454346,
+      "rewards/margins": 0.5421117544174194,
+      "rewards/rejected": -2.6767354011535645,
+      "step": 500
+    },
+    {
+      "epoch": 0.27,
+      "eval_logits/chosen": 4.160411834716797,
+      "eval_logits/rejected": 3.967071294784546,
+      "eval_logps/chosen": -386.0565185546875,
+      "eval_logps/rejected": -439.2520751953125,
+      "eval_loss": 0.6089237928390503,
+      "eval_rewards/accuracies": 0.6705063581466675,
+      "eval_rewards/chosen": -2.6821529865264893,
+      "eval_rewards/margins": 0.44139912724494934,
+      "eval_rewards/rejected": -3.123551845550537,
+      "eval_runtime": 2833.1592,
+      "eval_samples_per_second": 12.369,
+      "eval_steps_per_second": 0.387,
+      "step": 500
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.570432221710314e-07,
+      "logits/chosen": 2.99139666557312,
+      "logits/rejected": 2.9831995964050293,
+      "logps/chosen": -414.53643798828125,
+      "logps/rejected": -447.3922424316406,
+      "loss": 0.5896,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.9070537090301514,
+      "rewards/margins": 0.6989824175834656,
+      "rewards/rejected": -2.6060359477996826,
+      "step": 510
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.5441079327251927e-07,
+      "logits/chosen": 2.8209099769592285,
+      "logits/rejected": 3.297544002532959,
+      "logps/chosen": -415.64990234375,
+      "logps/rejected": -483.9609375,
+      "loss": 0.5711,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.1026933193206787,
+      "rewards/margins": 0.5386781096458435,
+      "rewards/rejected": -2.641371726989746,
+      "step": 520
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.5170814584435644e-07,
+      "logits/chosen": 3.1211986541748047,
+      "logits/rejected": 3.634293794631958,
+      "logps/chosen": -445.901611328125,
+      "logps/rejected": -477.486083984375,
+      "loss": 0.5723,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.098557949066162,
+      "rewards/margins": 0.7294325828552246,
+      "rewards/rejected": -2.8279905319213867,
+      "step": 530
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.4893620829118124e-07,
+      "logits/chosen": 4.876758575439453,
+      "logits/rejected": 4.832733154296875,
+      "logps/chosen": -390.5097351074219,
+      "logps/rejected": -458.13433837890625,
+      "loss": 0.5553,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.3329362869262695,
+      "rewards/margins": 0.6324361562728882,
+      "rewards/rejected": -2.965372323989868,
+      "step": 540
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.460959328199497e-07,
+      "logits/chosen": 5.745593070983887,
+      "logits/rejected": 6.544689178466797,
+      "logps/chosen": -479.26312255859375,
+      "logps/rejected": -547.4561767578125,
+      "loss": 0.5592,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.804473876953125,
+      "rewards/margins": 0.6489624977111816,
+      "rewards/rejected": -3.4534363746643066,
+      "step": 550
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.4318829511283707e-07,
+      "logits/chosen": 5.760529518127441,
+      "logits/rejected": 5.908038139343262,
+      "logps/chosen": -500.1220703125,
+      "logps/rejected": -579.2546997070312,
+      "loss": 0.5787,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -3.121411085128784,
+      "rewards/margins": 0.5803627967834473,
+      "rewards/rejected": -3.7017738819122314,
+      "step": 560
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.40214293992074e-07,
+      "logits/chosen": 4.968967914581299,
+      "logits/rejected": 5.733763694763184,
+      "logps/chosen": -449.43182373046875,
+      "logps/rejected": -481.9095153808594,
+      "loss": 0.5705,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.2817494869232178,
+      "rewards/margins": 0.5884373188018799,
+      "rewards/rejected": -2.8701870441436768,
+      "step": 570
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.3717495107683516e-07,
+      "logits/chosen": 6.087460041046143,
+      "logits/rejected": 6.749306678771973,
+      "logps/chosen": -554.03173828125,
+      "logps/rejected": -600.6712646484375,
+      "loss": 0.5706,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -3.579016923904419,
+      "rewards/margins": 0.5537681579589844,
+      "rewards/rejected": -4.132784843444824,
+      "step": 580
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.340713104322953e-07,
+      "logits/chosen": 5.846838474273682,
+      "logits/rejected": 6.166060447692871,
+      "logps/chosen": -547.8532104492188,
+      "logps/rejected": -635.3438110351562,
+      "loss": 0.5318,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -3.4989750385284424,
+      "rewards/margins": 0.7716847658157349,
+      "rewards/rejected": -4.270659446716309,
+      "step": 590
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.3090443821097566e-07,
+      "logits/chosen": 4.9042229652404785,
+      "logits/rejected": 5.197253704071045,
+      "logps/chosen": -483.651123046875,
+      "logps/rejected": -557.0797119140625,
+      "loss": 0.5436,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.6425604820251465,
+      "rewards/margins": 0.6980206370353699,
+      "rewards/rejected": -3.340581178665161,
+      "step": 600
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.276754222865029e-07,
+      "logits/chosen": 4.094830513000488,
+      "logits/rejected": 4.941472053527832,
+      "logps/chosen": -461.6971130371094,
+      "logps/rejected": -490.43377685546875,
+      "loss": 0.5627,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.5023880004882812,
+      "rewards/margins": 0.6450821757316589,
+      "rewards/rejected": -3.147469997406006,
+      "step": 610
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2438537187990565e-07,
+      "logits/chosen": 4.136041641235352,
+      "logits/rejected": 4.636130332946777,
+      "logps/chosen": -462.34136962890625,
+      "logps/rejected": -513.6154174804688,
+      "loss": 0.5457,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.5274147987365723,
+      "rewards/margins": 0.8207530975341797,
+      "rewards/rejected": -3.348167896270752,
+      "step": 620
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.210354171785795e-07,
+      "logits/chosen": 4.823207855224609,
+      "logits/rejected": 5.325142860412598,
+      "logps/chosen": -488.6162109375,
+      "logps/rejected": -548.1345825195312,
+      "loss": 0.5523,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.699159622192383,
+      "rewards/margins": 0.712811291217804,
+      "rewards/rejected": -3.4119713306427,
+      "step": 630
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.1762670894804775e-07,
+      "logits/chosen": 4.793757915496826,
+      "logits/rejected": 5.079339027404785,
+      "logps/chosen": -517.7249755859375,
+      "logps/rejected": -555.1099243164062,
+      "loss": 0.5609,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -3.0854828357696533,
+      "rewards/margins": 0.5394307374954224,
+      "rewards/rejected": -3.624913454055786,
+      "step": 640
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.1416041813665493e-07,
+      "logits/chosen": 4.005798816680908,
+      "logits/rejected": 4.380357265472412,
+      "logps/chosen": -465.93701171875,
+      "logps/rejected": -518.0279541015625,
+      "loss": 0.5483,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.6373207569122314,
+      "rewards/margins": 0.5303686857223511,
+      "rewards/rejected": -3.167689323425293,
+      "step": 650
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.1063773547332584e-07,
+      "logits/chosen": 4.460536956787109,
+      "logits/rejected": 4.873723030090332,
+      "logps/chosen": -462.9532775878906,
+      "logps/rejected": -519.1263427734375,
+      "loss": 0.5651,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.6806063652038574,
+      "rewards/margins": 0.6171140670776367,
+      "rewards/rejected": -3.2977206707000732,
+      "step": 660
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.0705987105853077e-07,
+      "logits/chosen": 4.531912803649902,
+      "logits/rejected": 4.875602722167969,
+      "logps/chosen": -430.66015625,
+      "logps/rejected": -490.43505859375,
+      "loss": 0.5664,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.494072437286377,
+      "rewards/margins": 0.5989924669265747,
+      "rewards/rejected": -3.0930652618408203,
+      "step": 670
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.034280539485952e-07,
+      "logits/chosen": 3.7882587909698486,
+      "logits/rejected": 4.701764106750488,
+      "logps/chosen": -472.9560546875,
+      "logps/rejected": -545.2316284179688,
+      "loss": 0.5388,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.2864925861358643,
+      "rewards/margins": 0.8441749811172485,
+      "rewards/rejected": -3.130667209625244,
+      "step": 680
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.997435317334988e-07,
+      "logits/chosen": 5.6066789627075195,
+      "logits/rejected": 6.343667507171631,
+      "logps/chosen": -546.0533447265625,
+      "logps/rejected": -598.7782592773438,
+      "loss": 0.5491,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -3.1853551864624023,
+      "rewards/margins": 0.7388944625854492,
+      "rewards/rejected": -3.9242501258850098,
+      "step": 690
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.960075701083074e-07,
+      "logits/chosen": 6.01334285736084,
+      "logits/rejected": 6.114634037017822,
+      "logps/chosen": -553.4805908203125,
+      "logps/rejected": -612.23095703125,
+      "loss": 0.5694,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -3.704514980316162,
+      "rewards/margins": 0.5731467008590698,
+      "rewards/rejected": -4.27766227722168,
+      "step": 700
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.92221452438385e-07,
+      "logits/chosen": 5.397694110870361,
+      "logits/rejected": 5.5348615646362305,
+      "logps/chosen": -530.5993041992188,
+      "logps/rejected": -563.1759033203125,
+      "loss": 0.5445,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -3.2607131004333496,
+      "rewards/margins": 0.5572507977485657,
+      "rewards/rejected": -3.817964553833008,
+      "step": 710
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.8838647931853684e-07,
+      "logits/chosen": 5.427793025970459,
+      "logits/rejected": 5.946757793426514,
+      "logps/chosen": -589.3401489257812,
+      "logps/rejected": -668.6383666992188,
+      "loss": 0.5744,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -3.8846659660339355,
+      "rewards/margins": 0.7481729388237,
+      "rewards/rejected": -4.632839679718018,
+      "step": 720
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.845039681262332e-07,
+      "logits/chosen": 5.047539710998535,
+      "logits/rejected": 5.657217025756836,
+      "logps/chosen": -554.4708862304688,
+      "logps/rejected": -616.8558349609375,
+      "loss": 0.5279,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -3.465214967727661,
+      "rewards/margins": 0.6963861584663391,
+      "rewards/rejected": -4.1616010665893555,
+      "step": 730
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.805752525690681e-07,
+      "logits/chosen": 5.46529483795166,
+      "logits/rejected": 5.889483451843262,
+      "logps/chosen": -564.59228515625,
+      "logps/rejected": -660.7513427734375,
+      "loss": 0.5431,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -3.814227342605591,
+      "rewards/margins": 0.7818762063980103,
+      "rewards/rejected": -4.596103191375732,
+      "step": 740
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7660168222660824e-07,
+      "logits/chosen": 5.152674198150635,
+      "logits/rejected": 5.510188102722168,
+      "logps/chosen": -587.24560546875,
+      "logps/rejected": -626.5244140625,
+      "loss": 0.5788,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -3.4678547382354736,
+      "rewards/margins": 0.5621030926704407,
+      "rewards/rejected": -4.0299577713012695,
+      "step": 750
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.725846220867901e-07,
+      "logits/chosen": 4.9715166091918945,
+      "logits/rejected": 5.927099227905273,
+      "logps/chosen": -540.8013916015625,
+      "logps/rejected": -573.8363037109375,
+      "loss": 0.5665,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -3.4430599212646484,
+      "rewards/margins": 0.5011590123176575,
+      "rewards/rejected": -3.944218873977661,
+      "step": 760
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6852545207702393e-07,
+      "logits/chosen": 5.292831897735596,
+      "logits/rejected": 6.226684093475342,
+      "logps/chosen": -585.9905395507812,
+      "logps/rejected": -621.8675537109375,
+      "loss": 0.5763,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.4966049194335938,
+      "rewards/margins": 0.7289837598800659,
+      "rewards/rejected": -4.225588798522949,
+      "step": 770
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6442556659016475e-07,
+      "logits/chosen": 5.347090244293213,
+      "logits/rejected": 6.540925025939941,
+      "logps/chosen": -568.36572265625,
+      "logps/rejected": -611.6026611328125,
+      "loss": 0.5235,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.558441162109375,
+      "rewards/margins": 0.7847374081611633,
+      "rewards/rejected": -4.343178749084473,
+      "step": 780
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.602863740055161e-07,
+      "logits/chosen": 5.637946128845215,
+      "logits/rejected": 6.208116054534912,
+      "logps/chosen": -585.2955322265625,
+      "logps/rejected": -676.1448974609375,
+      "loss": 0.5476,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -3.8783912658691406,
+      "rewards/margins": 0.8531737327575684,
+      "rewards/rejected": -4.731564998626709,
+      "step": 790
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.5610929620502747e-07,
+      "logits/chosen": 4.267381191253662,
+      "logits/rejected": 5.2271528244018555,
+      "logps/chosen": -578.3743286132812,
+      "logps/rejected": -665.9542846679688,
+      "loss": 0.5381,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -3.779019594192505,
+      "rewards/margins": 0.8331762552261353,
+      "rewards/rejected": -4.61219596862793,
+      "step": 800
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5189576808485404e-07,
+      "logits/chosen": 4.498036861419678,
+      "logits/rejected": 5.546736717224121,
+      "logps/chosen": -622.6543579101562,
+      "logps/rejected": -662.39990234375,
+      "loss": 0.5631,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.9034676551818848,
+      "rewards/margins": 0.7405894994735718,
+      "rewards/rejected": -4.644057273864746,
+      "step": 810
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.476472370624464e-07,
+      "logits/chosen": 5.130145072937012,
+      "logits/rejected": 5.417891502380371,
+      "logps/chosen": -601.1119384765625,
+      "logps/rejected": -665.1697387695312,
+      "loss": 0.5375,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.171017646789551,
+      "rewards/margins": 0.6281455159187317,
+      "rewards/rejected": -4.799162864685059,
+      "step": 820
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.43365162579338e-07,
+      "logits/chosen": 5.479987144470215,
+      "logits/rejected": 5.379258155822754,
+      "logps/chosen": -553.6651611328125,
+      "logps/rejected": -656.02880859375,
+      "loss": 0.5448,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -3.905271053314209,
+      "rewards/margins": 0.8390300869941711,
+      "rewards/rejected": -4.744300842285156,
+      "step": 830
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.390510155998023e-07,
+      "logits/chosen": 5.235553741455078,
+      "logits/rejected": 5.778375148773193,
+      "logps/chosen": -578.1744384765625,
+      "logps/rejected": -612.1620483398438,
+      "loss": 0.5477,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.762741804122925,
+      "rewards/margins": 0.507239043712616,
+      "rewards/rejected": -4.269980430603027,
+      "step": 840
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.347062781055526e-07,
+      "logits/chosen": 5.177485942840576,
+      "logits/rejected": 5.3884077072143555,
+      "logps/chosen": -517.0951538085938,
+      "logps/rejected": -611.8797607421875,
+      "loss": 0.5674,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -3.408946990966797,
+      "rewards/margins": 0.6858997344970703,
+      "rewards/rejected": -4.094846725463867,
+      "step": 850
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.303324425866559e-07,
+      "logits/chosen": 4.779083728790283,
+      "logits/rejected": 4.99451208114624,
+      "logps/chosen": -534.8001708984375,
+      "logps/rejected": -588.9842529296875,
+      "loss": 0.5692,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -3.043729543685913,
+      "rewards/margins": 0.7375171184539795,
+      "rewards/rejected": -3.7812466621398926,
+      "step": 860
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2593101152883795e-07,
+      "logits/chosen": 4.893560409545898,
+      "logits/rejected": 4.912612438201904,
+      "logps/chosen": -477.8765563964844,
+      "logps/rejected": -577.2595825195312,
+      "loss": 0.5559,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.9390878677368164,
+      "rewards/margins": 0.735144853591919,
+      "rewards/rejected": -3.6742329597473145,
+      "step": 870
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.21503496897354e-07,
+      "logits/chosen": 4.999721527099609,
+      "logits/rejected": 5.3239426612854,
+      "logps/chosen": -539.9871215820312,
+      "logps/rejected": -568.0904541015625,
+      "loss": 0.5742,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -3.2623775005340576,
+      "rewards/margins": 0.4343135356903076,
+      "rewards/rejected": -3.6966910362243652,
+      "step": 880
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.170514196176037e-07,
+      "logits/chosen": 4.89313268661499,
+      "logits/rejected": 4.976138114929199,
+      "logps/chosen": -531.0595703125,
+      "logps/rejected": -615.2716064453125,
+      "loss": 0.5382,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -3.470829486846924,
+      "rewards/margins": 0.6645902395248413,
+      "rewards/rejected": -4.1354193687438965,
+      "step": 890
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.125763090526674e-07,
+      "logits/chosen": 4.8970746994018555,
+      "logits/rejected": 5.335020542144775,
+      "logps/chosen": -570.1527709960938,
+      "logps/rejected": -649.9376831054688,
+      "loss": 0.5099,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.5448403358459473,
+      "rewards/margins": 0.861894965171814,
+      "rewards/rejected": -4.406734943389893,
+      "step": 900
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.080797024779447e-07,
+      "logits/chosen": 4.570401191711426,
+      "logits/rejected": 5.245053291320801,
+      "logps/chosen": -538.8987426757812,
+      "logps/rejected": -598.7467651367188,
+      "loss": 0.529,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -3.5106894969940186,
+      "rewards/margins": 0.7811245918273926,
+      "rewards/rejected": -4.29181432723999,
+      "step": 910
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.035631445530743e-07,
+      "logits/chosen": 5.412999629974365,
+      "logits/rejected": 5.755091190338135,
+      "logps/chosen": -584.6710205078125,
+      "logps/rejected": -669.8858642578125,
+      "loss": 0.5345,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -3.7473666667938232,
+      "rewards/margins": 0.8245238065719604,
+      "rewards/rejected": -4.571890830993652,
+      "step": 920
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.9902818679131775e-07,
+      "logits/chosen": 5.077193260192871,
+      "logits/rejected": 5.424740791320801,
+      "logps/chosen": -592.6712646484375,
+      "logps/rejected": -663.4064331054688,
+      "loss": 0.5308,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.113033294677734,
+      "rewards/margins": 0.7320185899734497,
+      "rewards/rejected": -4.845052242279053,
+      "step": 930
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.944763870265886e-07,
+      "logits/chosen": 4.157199859619141,
+      "logits/rejected": 5.055262565612793,
+      "logps/chosen": -593.670166015625,
+      "logps/rejected": -670.92578125,
+      "loss": 0.5311,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.861306667327881,
+      "rewards/margins": 0.8220443725585938,
+      "rewards/rejected": -4.683350563049316,
+      "step": 940
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.899093088783105e-07,
+      "logits/chosen": 4.214892864227295,
+      "logits/rejected": 4.872607707977295,
+      "logps/chosen": -627.0443115234375,
+      "logps/rejected": -696.4767456054688,
+      "loss": 0.5336,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.9733481407165527,
+      "rewards/margins": 0.7376152276992798,
+      "rewards/rejected": -4.710963726043701,
+      "step": 950
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8532852121428733e-07,
+      "logits/chosen": 4.1724419593811035,
+      "logits/rejected": 4.470523834228516,
+      "logps/chosen": -547.5701904296875,
+      "logps/rejected": -630.979736328125,
+      "loss": 0.5351,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -3.5739071369171143,
+      "rewards/margins": 0.9093371629714966,
+      "rewards/rejected": -4.4832444190979,
+      "step": 960
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.807355976117716e-07,
+      "logits/chosen": 3.997878313064575,
+      "logits/rejected": 4.5441575050354,
+      "logps/chosen": -573.764892578125,
+      "logps/rejected": -626.8602294921875,
+      "loss": 0.5637,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -3.4168217182159424,
+      "rewards/margins": 0.7197359800338745,
+      "rewards/rejected": -4.1365580558776855,
+      "step": 970
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.761321158169134e-07,
+      "logits/chosen": 3.821613311767578,
+      "logits/rejected": 4.272428512573242,
+      "logps/chosen": -539.5350341796875,
+      "logps/rejected": -624.6610717773438,
+      "loss": 0.5437,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -3.425632953643799,
+      "rewards/margins": 0.7200672030448914,
+      "rewards/rejected": -4.145700454711914,
+      "step": 980
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.715196572027789e-07,
+      "logits/chosen": 3.830516815185547,
+      "logits/rejected": 3.83209228515625,
+      "logps/chosen": -553.0765380859375,
+      "logps/rejected": -614.8712768554688,
+      "loss": 0.5591,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -3.4271316528320312,
+      "rewards/margins": 0.6863579750061035,
+      "rewards/rejected": -4.113489627838135,
+      "step": 990
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6689980622612204e-07,
+      "logits/chosen": 4.027700424194336,
+      "logits/rejected": 4.4086809158325195,
+      "logps/chosen": -504.0587463378906,
+      "logps/rejected": -589.3375854492188,
+      "loss": 0.5519,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -3.1788105964660645,
+      "rewards/margins": 0.6995212435722351,
+      "rewards/rejected": -3.8783316612243652,
+      "step": 1000
+    },
+    {
+      "epoch": 0.53,
+      "eval_logits/chosen": 5.348609447479248,
+      "eval_logits/rejected": 5.197441101074219,
+      "eval_logps/chosen": -543.0738525390625,
+      "eval_logps/rejected": -602.8671264648438,
+      "eval_loss": 0.5866538882255554,
+      "eval_rewards/accuracies": 0.6894388794898987,
+      "eval_rewards/chosen": -4.252326488494873,
+      "eval_rewards/margins": 0.5073760151863098,
+      "eval_rewards/rejected": -4.759702205657959,
+      "eval_runtime": 2829.1466,
+      "eval_samples_per_second": 12.387,
+      "eval_steps_per_second": 0.387,
+      "step": 1000
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.622741498830969e-07,
+      "logits/chosen": 4.356446266174316,
+      "logits/rejected": 4.390655517578125,
+      "logps/chosen": -531.9536743164062,
+      "logps/rejected": -595.4317626953125,
+      "loss": 0.5685,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -3.2892327308654785,
+      "rewards/margins": 0.6568211913108826,
+      "rewards/rejected": -3.9460537433624268,
+      "step": 1010
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5764427716409815e-07,
+      "logits/chosen": 3.622380495071411,
+      "logits/rejected": 4.40954065322876,
+      "logps/chosen": -515.4381103515625,
+      "logps/rejected": -565.3131103515625,
+      "loss": 0.5428,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -3.0056488513946533,
+      "rewards/margins": 0.683438777923584,
+      "rewards/rejected": -3.6890876293182373,
+      "step": 1020
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.5301177850791616e-07,
+      "logits/chosen": 3.9004855155944824,
+      "logits/rejected": 4.826648712158203,
+      "logps/chosen": -568.6829833984375,
+      "logps/rejected": -630.1458740234375,
+      "loss": 0.5394,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.4316725730895996,
+      "rewards/margins": 0.825964093208313,
+      "rewards/rejected": -4.257637023925781,
+      "step": 1030
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4837824525539477e-07,
+      "logits/chosen": 3.6745972633361816,
+      "logits/rejected": 4.580860614776611,
+      "logps/chosen": -539.4935302734375,
+      "logps/rejected": -591.1002197265625,
+      "loss": 0.5549,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -3.340437650680542,
+      "rewards/margins": 0.6588428020477295,
+      "rewards/rejected": -3.9992804527282715,
+      "step": 1040
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4374526910277886e-07,
+      "logits/chosen": 4.059652805328369,
+      "logits/rejected": 4.569832801818848,
+      "logps/chosen": -524.096435546875,
+      "logps/rejected": -599.4136962890625,
+      "loss": 0.566,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.2275619506835938,
+      "rewards/margins": 0.767623782157898,
+      "rewards/rejected": -3.9951858520507812,
+      "step": 1050
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.391144415549403e-07,
+      "logits/chosen": 4.6053290367126465,
+      "logits/rejected": 4.822667121887207,
+      "logps/chosen": -529.7521362304688,
+      "logps/rejected": -581.6422729492188,
+      "loss": 0.5563,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -3.478646755218506,
+      "rewards/margins": 0.4808022081851959,
+      "rewards/rejected": -3.959449052810669,
+      "step": 1060
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3448735337866919e-07,
+      "logits/chosen": 4.680068016052246,
+      "logits/rejected": 5.09098482131958,
+      "logps/chosen": -486.0177307128906,
+      "logps/rejected": -539.49658203125,
+      "loss": 0.5976,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -3.101942539215088,
+      "rewards/margins": 0.5219632983207703,
+      "rewards/rejected": -3.623905897140503,
+      "step": 1070
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.2986559405621886e-07,
+      "logits/chosen": 4.115077495574951,
+      "logits/rejected": 4.646478652954102,
+      "logps/chosen": -508.98480224609375,
+      "logps/rejected": -547.0084838867188,
+      "loss": 0.5705,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.912672281265259,
+      "rewards/margins": 0.5327907204627991,
+      "rewards/rejected": -3.445463180541992,
+      "step": 1080
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2525075123929213e-07,
+      "logits/chosen": 4.412012100219727,
+      "logits/rejected": 4.801966190338135,
+      "logps/chosen": -494.9298400878906,
+      "logps/rejected": -553.3126220703125,
+      "loss": 0.53,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -3.0377612113952637,
+      "rewards/margins": 0.6897190809249878,
+      "rewards/rejected": -3.72748064994812,
+      "step": 1090
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.206444102036565e-07,
+      "logits/chosen": 4.899998664855957,
+      "logits/rejected": 5.011576175689697,
+      "logps/chosen": -533.6759643554688,
+      "logps/rejected": -604.164306640625,
+      "loss": 0.5622,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -3.346374034881592,
+      "rewards/margins": 0.5992105603218079,
+      "rewards/rejected": -3.945584774017334,
+      "step": 1100
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.160481533045751e-07,
+      "logits/chosen": 4.868978977203369,
+      "logits/rejected": 5.403256416320801,
+      "logps/chosen": -569.0045776367188,
+      "logps/rejected": -625.4342041015625,
+      "loss": 0.547,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -3.556847333908081,
+      "rewards/margins": 0.7111159563064575,
+      "rewards/rejected": -4.267963409423828,
+      "step": 1110
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1146355943324148e-07,
+      "logits/chosen": 5.191876411437988,
+      "logits/rejected": 5.507510185241699,
+      "logps/chosen": -557.5202026367188,
+      "logps/rejected": -637.4208374023438,
+      "loss": 0.5456,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -3.72900652885437,
+      "rewards/margins": 0.7132843732833862,
+      "rewards/rejected": -4.442290782928467,
+      "step": 1120
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0689220347440374e-07,
+      "logits/chosen": 4.4830732345581055,
+      "logits/rejected": 5.013645172119141,
+      "logps/chosen": -587.7266235351562,
+      "logps/rejected": -625.2584228515625,
+      "loss": 0.5546,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.556133985519409,
+      "rewards/margins": 0.6282913684844971,
+      "rewards/rejected": -4.184426307678223,
+      "step": 1130
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0233565576536564e-07,
+      "logits/chosen": 4.331673622131348,
+      "logits/rejected": 4.588160037994385,
+      "logps/chosen": -579.3844604492188,
+      "logps/rejected": -643.15380859375,
+      "loss": 0.5457,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -3.580660581588745,
+      "rewards/margins": 0.58534175157547,
+      "rewards/rejected": -4.16600227355957,
+      "step": 1140
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.97795481556549e-07,
+      "logits/chosen": 3.8659698963165283,
+      "logits/rejected": 4.494038105010986,
+      "logps/chosen": -526.2404174804688,
+      "logps/rejected": -560.0923461914062,
+      "loss": 0.5521,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.2603347301483154,
+      "rewards/margins": 0.6364775896072388,
+      "rewards/rejected": -3.8968119621276855,
+      "step": 1150
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9327324047380422e-07,
+      "logits/chosen": 3.7696309089660645,
+      "logits/rejected": 4.594429016113281,
+      "logps/chosen": -504.9520568847656,
+      "logps/rejected": -586.5645751953125,
+      "loss": 0.5119,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -3.0566916465759277,
+      "rewards/margins": 0.8342288732528687,
+      "rewards/rejected": -3.8909201622009277,
+      "step": 1160
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.887704859826528e-07,
+      "logits/chosen": 4.6678924560546875,
+      "logits/rejected": 5.033539772033691,
+      "logps/chosen": -579.7088012695312,
+      "logps/rejected": -635.5621948242188,
+      "loss": 0.5359,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -3.5427486896514893,
+      "rewards/margins": 0.6539695262908936,
+      "rewards/rejected": -4.196718692779541,
+      "step": 1170
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8428876485464572e-07,
+      "logits/chosen": 4.120070934295654,
+      "logits/rejected": 4.802687644958496,
+      "logps/chosen": -546.7890014648438,
+      "logps/rejected": -595.0975341796875,
+      "loss": 0.5792,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -3.6790192127227783,
+      "rewards/margins": 0.6043803095817566,
+      "rewards/rejected": -4.28339958190918,
+      "step": 1180
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.798296166360216e-07,
+      "logits/chosen": 4.212487697601318,
+      "logits/rejected": 5.026057243347168,
+      "logps/chosen": -570.1341552734375,
+      "logps/rejected": -631.1887817382812,
+      "loss": 0.554,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.4938530921936035,
+      "rewards/margins": 0.8111255764961243,
+      "rewards/rejected": -4.304978370666504,
+      "step": 1190
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7539457311884675e-07,
+      "logits/chosen": 4.184502601623535,
+      "logits/rejected": 4.767962455749512,
+      "logps/chosen": -532.1156005859375,
+      "logps/rejected": -596.5296630859375,
+      "loss": 0.5289,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.360400438308716,
+      "rewards/margins": 0.7973008155822754,
+      "rewards/rejected": -4.157701015472412,
+      "step": 1200
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7098515781481883e-07,
+      "logits/chosen": 5.043318748474121,
+      "logits/rejected": 5.477365970611572,
+      "logps/chosen": -554.041015625,
+      "logps/rejected": -602.6400146484375,
+      "loss": 0.5381,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -3.55816388130188,
+      "rewards/margins": 0.6002733111381531,
+      "rewards/rejected": -4.1584367752075195,
+      "step": 1210
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6660288543191568e-07,
+      "logits/chosen": 4.342132568359375,
+      "logits/rejected": 5.391766548156738,
+      "logps/chosen": -545.3358764648438,
+      "logps/rejected": -593.21044921875,
+      "loss": 0.5273,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -3.2604167461395264,
+      "rewards/margins": 0.7034675478935242,
+      "rewards/rejected": -3.9638848304748535,
+      "step": 1220
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6224926135406693e-07,
+      "logits/chosen": 4.523472785949707,
+      "logits/rejected": 4.9561238288879395,
+      "logps/chosen": -541.8357543945312,
+      "logps/rejected": -617.3576049804688,
+      "loss": 0.5279,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.246079206466675,
+      "rewards/margins": 0.7863640785217285,
+      "rewards/rejected": -4.032443046569824,
+      "step": 1230
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.579257811240298e-07,
+      "logits/chosen": 4.174516201019287,
+      "logits/rejected": 4.948155403137207,
+      "logps/chosen": -566.966796875,
+      "logps/rejected": -629.2576293945312,
+      "loss": 0.5383,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -3.5496108531951904,
+      "rewards/margins": 0.6297346949577332,
+      "rewards/rejected": -4.179345607757568,
+      "step": 1240
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5363392992964523e-07,
+      "logits/chosen": 4.812657833099365,
+      "logits/rejected": 5.1759843826293945,
+      "logps/chosen": -545.2940673828125,
+      "logps/rejected": -610.6871948242188,
+      "loss": 0.5599,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -3.605778217315674,
+      "rewards/margins": 0.5686632394790649,
+      "rewards/rejected": -4.174441337585449,
+      "step": 1250
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4937518209365108e-07,
+      "logits/chosen": 4.399840354919434,
+      "logits/rejected": 5.024414539337158,
+      "logps/chosen": -565.787353515625,
+      "logps/rejected": -621.1328125,
+      "loss": 0.5422,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -3.319702625274658,
+      "rewards/margins": 0.7489235997200012,
+      "rewards/rejected": -4.0686259269714355,
+      "step": 1260
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4515100056722708e-07,
+      "logits/chosen": 4.322859287261963,
+      "logits/rejected": 4.647487163543701,
+      "logps/chosen": -473.44915771484375,
+      "logps/rejected": -556.26806640625,
+      "loss": 0.5335,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.9221551418304443,
+      "rewards/margins": 0.8288917541503906,
+      "rewards/rejected": -3.7510476112365723,
+      "step": 1270
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4096283642744716e-07,
+      "logits/chosen": 4.445618152618408,
+      "logits/rejected": 5.310964107513428,
+      "logps/chosen": -529.9594116210938,
+      "logps/rejected": -583.6397094726562,
+      "loss": 0.5489,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -3.098684310913086,
+      "rewards/margins": 0.7321470975875854,
+      "rewards/rejected": -3.8308310508728027,
+      "step": 1280
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3681212837880977e-07,
+      "logits/chosen": 4.398858070373535,
+      "logits/rejected": 4.8625688552856445,
+      "logps/chosen": -509.6946716308594,
+      "logps/rejected": -594.6348876953125,
+      "loss": 0.5397,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.9974119663238525,
+      "rewards/margins": 0.8787434697151184,
+      "rewards/rejected": -3.876155376434326,
+      "step": 1290
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3270030225901908e-07,
+      "logits/chosen": 4.207759857177734,
+      "logits/rejected": 5.357243537902832,
+      "logps/chosen": -540.0505981445312,
+      "logps/rejected": -591.9456787109375,
+      "loss": 0.5405,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.9670040607452393,
+      "rewards/margins": 0.9191484451293945,
+      "rewards/rejected": -3.8861522674560547,
+      "step": 1300
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2862877054918572e-07,
+      "logits/chosen": 3.810641050338745,
+      "logits/rejected": 4.167244911193848,
+      "logps/chosen": -455.54962158203125,
+      "logps/rejected": -525.954833984375,
+      "loss": 0.558,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.536816358566284,
+      "rewards/margins": 0.7640112638473511,
+      "rewards/rejected": -3.300827741622925,
+      "step": 1310
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2459893188861613e-07,
+      "logits/chosen": 3.5092411041259766,
+      "logits/rejected": 4.095297813415527,
+      "logps/chosen": -398.01702880859375,
+      "logps/rejected": -486.40106201171875,
+      "loss": 0.5194,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.352292060852051,
+      "rewards/margins": 0.8707995414733887,
+      "rewards/rejected": -3.2230911254882812,
+      "step": 1320
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.206121705943558e-07,
+      "logits/chosen": 3.8648457527160645,
+      "logits/rejected": 4.79974365234375,
+      "logps/chosen": -450.3089904785156,
+      "logps/rejected": -489.1319274902344,
+      "loss": 0.5501,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.3793253898620605,
+      "rewards/margins": 0.6881033778190613,
+      "rewards/rejected": -3.0674290657043457,
+      "step": 1330
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1666985618565422e-07,
+      "logits/chosen": 4.9529547691345215,
+      "logits/rejected": 5.19863224029541,
+      "logps/chosen": -469.70855712890625,
+      "logps/rejected": -529.8267822265625,
+      "loss": 0.5422,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.916710615158081,
+      "rewards/margins": 0.6288083791732788,
+      "rewards/rejected": -3.545518398284912,
+      "step": 1340
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1277334291351145e-07,
+      "logits/chosen": 5.23069953918457,
+      "logits/rejected": 5.482142925262451,
+      "logps/chosen": -469.21307373046875,
+      "logps/rejected": -589.8045654296875,
+      "loss": 0.5183,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.9651408195495605,
+      "rewards/margins": 0.8228071331977844,
+      "rewards/rejected": -3.7879481315612793,
+      "step": 1350
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.089239692954701e-07,
+      "logits/chosen": 4.438393592834473,
+      "logits/rejected": 5.2009711265563965,
+      "logps/chosen": -496.31463623046875,
+      "logps/rejected": -561.5128173828125,
+      "loss": 0.5497,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.9592137336730957,
+      "rewards/margins": 0.6417844891548157,
+      "rewards/rejected": -3.6009984016418457,
+      "step": 1360
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.051230576558127e-07,
+      "logits/chosen": 5.329151630401611,
+      "logits/rejected": 5.1115946769714355,
+      "logps/chosen": -510.32379150390625,
+      "logps/rejected": -633.5717163085938,
+      "loss": 0.5281,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.305920124053955,
+      "rewards/margins": 0.6949362754821777,
+      "rewards/rejected": -4.000856399536133,
+      "step": 1370
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0137191367132078e-07,
+      "logits/chosen": 4.635811805725098,
+      "logits/rejected": 5.594052314758301,
+      "logps/chosen": -538.316650390625,
+      "logps/rejected": -593.9783325195312,
+      "loss": 0.5179,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.257719039916992,
+      "rewards/margins": 0.8566193580627441,
+      "rewards/rejected": -4.1143388748168945,
+      "step": 1380
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.76718259227532e-08,
+      "logits/chosen": 5.040810585021973,
+      "logits/rejected": 5.872241973876953,
+      "logps/chosen": -559.94140625,
+      "logps/rejected": -600.4876098632812,
+      "loss": 0.5346,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -3.3652939796447754,
+      "rewards/margins": 0.7677735090255737,
+      "rewards/rejected": -4.1330671310424805,
+      "step": 1390
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.402406545219676e-08,
+      "logits/chosen": 5.209423542022705,
+      "logits/rejected": 5.7783403396606445,
+      "logps/chosen": -570.754150390625,
+      "logps/rejected": -631.2447509765625,
+      "loss": 0.5343,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -3.6044132709503174,
+      "rewards/margins": 0.8434345126152039,
+      "rewards/rejected": -4.447848320007324,
+      "step": 1400
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.042988532644249e-08,
+      "logits/chosen": 5.212072372436523,
+      "logits/rejected": 5.8972063064575195,
+      "logps/chosen": -583.494873046875,
+      "logps/rejected": -651.96630859375,
+      "loss": 0.5139,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.3433117866516113,
+      "rewards/margins": 0.9961609840393066,
+      "rewards/rejected": -4.339472770690918,
+      "step": 1410
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.689052020653592e-08,
+      "logits/chosen": 4.78833532333374,
+      "logits/rejected": 5.89553165435791,
+      "logps/chosen": -561.2659912109375,
+      "logps/rejected": -631.704833984375,
+      "loss": 0.5514,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.425673007965088,
+      "rewards/margins": 0.8838043212890625,
+      "rewards/rejected": -4.309477806091309,
+      "step": 1420
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.340718592365037e-08,
+      "logits/chosen": 5.993227958679199,
+      "logits/rejected": 6.045858860015869,
+      "logps/chosen": -575.0015869140625,
+      "logps/rejected": -656.5886840820312,
+      "loss": 0.5134,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -3.8804218769073486,
+      "rewards/margins": 0.6545805931091309,
+      "rewards/rejected": -4.535001754760742,
+      "step": 1430
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 7.998107906142839e-08,
+      "logits/chosen": 5.676799774169922,
+      "logits/rejected": 6.391809940338135,
+      "logps/chosen": -568.3199462890625,
+      "logps/rejected": -637.5294189453125,
+      "loss": 0.5254,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -3.7702269554138184,
+      "rewards/margins": 0.8325515985488892,
+      "rewards/rejected": -4.602778434753418,
+      "step": 1440
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.661337654493575e-08,
+      "logits/chosen": 5.199474334716797,
+      "logits/rejected": 5.9724626541137695,
+      "logps/chosen": -604.0126953125,
+      "logps/rejected": -679.3408203125,
+      "loss": 0.5511,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.842377185821533,
+      "rewards/margins": 0.8208445310592651,
+      "rewards/rejected": -4.663221836090088,
+      "step": 1450
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.330523523636751e-08,
+      "logits/chosen": 4.888028144836426,
+      "logits/rejected": 5.282634258270264,
+      "logps/chosen": -574.2249755859375,
+      "logps/rejected": -649.12158203125,
+      "loss": 0.5353,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -3.574962615966797,
+      "rewards/margins": 0.7398016452789307,
+      "rewards/rejected": -4.314764499664307,
+      "step": 1460
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.005779153764682e-08,
+      "logits/chosen": 5.4382123947143555,
+      "logits/rejected": 5.483049392700195,
+      "logps/chosen": -550.1482543945312,
+      "logps/rejected": -629.5477294921875,
+      "loss": 0.5274,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.7003333568573,
+      "rewards/margins": 0.6894980669021606,
+      "rewards/rejected": -4.389832019805908,
+      "step": 1470
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.687216100005138e-08,
+      "logits/chosen": 5.554679870605469,
+      "logits/rejected": 6.150310516357422,
+      "logps/chosen": -604.5882568359375,
+      "logps/rejected": -694.1824340820312,
+      "loss": 0.5241,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -3.837186336517334,
+      "rewards/margins": 0.8338931202888489,
+      "rewards/rejected": -4.671080589294434,
+      "step": 1480
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.374943794100349e-08,
+      "logits/chosen": 4.995267868041992,
+      "logits/rejected": 6.089870452880859,
+      "logps/chosen": -540.8165283203125,
+      "logps/rejected": -621.503662109375,
+      "loss": 0.5181,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.447101593017578,
+      "rewards/margins": 0.998428463935852,
+      "rewards/rejected": -4.445529937744141,
+      "step": 1490
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.069069506815325e-08,
+      "logits/chosen": 5.665683269500732,
+      "logits/rejected": 5.919187545776367,
+      "logps/chosen": -557.50341796875,
+      "logps/rejected": -635.5902099609375,
+      "loss": 0.5597,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -3.7898292541503906,
+      "rewards/margins": 0.7044454216957092,
+      "rewards/rejected": -4.494275093078613,
+      "step": 1500
+    },
+    {
+      "epoch": 0.8,
+      "eval_logits/chosen": 6.629395008087158,
+      "eval_logits/rejected": 6.464417934417725,
+      "eval_logps/chosen": -596.9036865234375,
+      "eval_logps/rejected": -659.0733032226562,
+      "eval_loss": 0.5820685625076294,
+      "eval_rewards/accuracies": 0.6959397792816162,
+      "eval_rewards/chosen": -4.79062557220459,
+      "eval_rewards/margins": 0.5311391949653625,
+      "eval_rewards/rejected": -5.321763515472412,
+      "eval_runtime": 2832.2152,
+      "eval_samples_per_second": 12.373,
+      "eval_steps_per_second": 0.387,
+      "step": 1500
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.7696983110885746e-08,
+      "logits/chosen": 6.144343376159668,
+      "logits/rejected": 6.1946892738342285,
+      "logps/chosen": -568.4108276367188,
+      "logps/rejected": -656.3994750976562,
+      "loss": 0.5713,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.8781726360321045,
+      "rewards/margins": 0.7777887582778931,
+      "rewards/rejected": -4.655961036682129,
+      "step": 1510
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.47693304593777e-08,
+      "logits/chosen": 5.610565185546875,
+      "logits/rejected": 6.231629371643066,
+      "logps/chosen": -578.0348510742188,
+      "logps/rejected": -634.0302124023438,
+      "loss": 0.5254,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.640714168548584,
+      "rewards/margins": 0.8447046279907227,
+      "rewards/rejected": -4.485418796539307,
+      "step": 1520
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.190874281132851e-08,
+      "logits/chosen": 5.386014938354492,
+      "logits/rejected": 5.870030403137207,
+      "logps/chosen": -544.43212890625,
+      "logps/rejected": -605.0162963867188,
+      "loss": 0.5313,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -3.533620834350586,
+      "rewards/margins": 0.7047785520553589,
+      "rewards/rejected": -4.238399505615234,
+      "step": 1530
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.9116202826486045e-08,
+      "logits/chosen": 5.818629741668701,
+      "logits/rejected": 6.154656410217285,
+      "logps/chosen": -583.7796630859375,
+      "logps/rejected": -635.4794921875,
+      "loss": 0.5337,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -3.777665615081787,
+      "rewards/margins": 0.6795842051506042,
+      "rewards/rejected": -4.457250118255615,
+      "step": 1540
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.639266978908676e-08,
+      "logits/chosen": 5.711030006408691,
+      "logits/rejected": 6.358799934387207,
+      "logps/chosen": -611.2662353515625,
+      "logps/rejected": -676.8931884765625,
+      "loss": 0.5187,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -3.886317491531372,
+      "rewards/margins": 0.7436650991439819,
+      "rewards/rejected": -4.629981994628906,
+      "step": 1550
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.373907927832513e-08,
+      "logits/chosen": 5.366969108581543,
+      "logits/rejected": 5.813288688659668,
+      "logps/chosen": -553.2315673828125,
+      "logps/rejected": -654.0559692382812,
+      "loss": 0.5274,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -3.5649421215057373,
+      "rewards/margins": 0.8516329526901245,
+      "rewards/rejected": -4.416574954986572,
+      "step": 1560
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.115634284696698e-08,
+      "logits/chosen": 5.473841667175293,
+      "logits/rejected": 6.142959117889404,
+      "logps/chosen": -595.4747314453125,
+      "logps/rejected": -675.5442504882812,
+      "loss": 0.4995,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -3.8835277557373047,
+      "rewards/margins": 0.8822458386421204,
+      "rewards/rejected": -4.765774250030518,
+      "step": 1570
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.864534770821559e-08,
+      "logits/chosen": 5.575949668884277,
+      "logits/rejected": 6.3084797859191895,
+      "logps/chosen": -557.6798706054688,
+      "logps/rejected": -628.6642456054688,
+      "loss": 0.5549,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -3.6573452949523926,
+      "rewards/margins": 0.8320104479789734,
+      "rewards/rejected": -4.489355564117432,
+      "step": 1580
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.620695643093924e-08,
+      "logits/chosen": 5.1766839027404785,
+      "logits/rejected": 6.165072441101074,
+      "logps/chosen": -578.5045166015625,
+      "logps/rejected": -645.0850830078125,
+      "loss": 0.5174,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -3.7886593341827393,
+      "rewards/margins": 0.9527637362480164,
+      "rewards/rejected": -4.7414231300354,
+      "step": 1590
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.384200664336412e-08,
+      "logits/chosen": 5.577385425567627,
+      "logits/rejected": 6.328660011291504,
+      "logps/chosen": -591.4779052734375,
+      "logps/rejected": -644.4967041015625,
+      "loss": 0.5339,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.7442920207977295,
+      "rewards/margins": 0.8128402829170227,
+      "rewards/rejected": -4.557132244110107,
+      "step": 1600
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.155131074533529e-08,
+      "logits/chosen": 4.875297546386719,
+      "logits/rejected": 5.868630886077881,
+      "logps/chosen": -590.7601318359375,
+      "logps/rejected": -644.1740112304688,
+      "loss": 0.5532,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -3.6774280071258545,
+      "rewards/margins": 0.69183349609375,
+      "rewards/rejected": -4.369261741638184,
+      "step": 1610
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.9335655629243645e-08,
+      "logits/chosen": 5.1115217208862305,
+      "logits/rejected": 5.944631099700928,
+      "logps/chosen": -572.9088134765625,
+      "logps/rejected": -635.9785766601562,
+      "loss": 0.5635,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.67173433303833,
+      "rewards/margins": 0.6962157487869263,
+      "rewards/rejected": -4.367949962615967,
+      "step": 1620
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.7195802409715197e-08,
+      "logits/chosen": 5.198755741119385,
+      "logits/rejected": 5.86753511428833,
+      "logps/chosen": -608.7496948242188,
+      "logps/rejected": -661.3677978515625,
+      "loss": 0.5482,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -3.853783130645752,
+      "rewards/margins": 0.7783033847808838,
+      "rewards/rejected": -4.632086753845215,
+      "step": 1630
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.513248616215527e-08,
+      "logits/chosen": 4.92915153503418,
+      "logits/rejected": 5.753048896789551,
+      "logps/chosen": -574.3338623046875,
+      "logps/rejected": -647.4317626953125,
+      "loss": 0.5148,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -3.5955843925476074,
+      "rewards/margins": 0.8324259519577026,
+      "rewards/rejected": -4.428009986877441,
+      "step": 1640
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.31464156702382e-08,
+      "logits/chosen": 4.705880165100098,
+      "logits/rejected": 5.527700901031494,
+      "logps/chosen": -576.8226928710938,
+      "logps/rejected": -628.70849609375,
+      "loss": 0.5499,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -3.532841205596924,
+      "rewards/margins": 0.7848026752471924,
+      "rewards/rejected": -4.317643642425537,
+      "step": 1650
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.1238273182427933e-08,
+      "logits/chosen": 5.681646347045898,
+      "logits/rejected": 6.597903251647949,
+      "logps/chosen": -569.5862426757812,
+      "logps/rejected": -620.6526489257812,
+      "loss": 0.5361,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -3.6919121742248535,
+      "rewards/margins": 0.7138382196426392,
+      "rewards/rejected": -4.405750274658203,
+      "step": 1660
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.9408714177614306e-08,
+      "logits/chosen": 5.2946391105651855,
+      "logits/rejected": 6.044753074645996,
+      "logps/chosen": -564.3060302734375,
+      "logps/rejected": -626.62353515625,
+      "loss": 0.5135,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -3.511505126953125,
+      "rewards/margins": 0.8757253885269165,
+      "rewards/rejected": -4.38723087310791,
+      "step": 1670
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.7658367139945228e-08,
+      "logits/chosen": 5.4068779945373535,
+      "logits/rejected": 5.994559288024902,
+      "logps/chosen": -599.1517333984375,
+      "logps/rejected": -650.2825317382812,
+      "loss": 0.5239,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -3.7978172302246094,
+      "rewards/margins": 0.7726759910583496,
+      "rewards/rejected": -4.570493221282959,
+      "step": 1680
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.5987833342931745e-08,
+      "logits/chosen": 5.3490800857543945,
+      "logits/rejected": 6.137597560882568,
+      "logps/chosen": -583.5009155273438,
+      "logps/rejected": -632.0625,
+      "loss": 0.5467,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -3.8200180530548096,
+      "rewards/margins": 0.7188345193862915,
+      "rewards/rejected": -4.538852691650391,
+      "step": 1690
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.439768664290053e-08,
+      "logits/chosen": 5.457294940948486,
+      "logits/rejected": 5.772470474243164,
+      "logps/chosen": -603.9612426757812,
+      "logps/rejected": -673.6254272460938,
+      "loss": 0.5239,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -3.798058032989502,
+      "rewards/margins": 0.787087619304657,
+      "rewards/rejected": -4.585144996643066,
+      "step": 1700
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2888473281864597e-08,
+      "logits/chosen": 5.0094194412231445,
+      "logits/rejected": 5.467037200927734,
+      "logps/chosen": -566.6424560546875,
+      "logps/rejected": -654.4783935546875,
+      "loss": 0.5305,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -3.774543046951294,
+      "rewards/margins": 0.7515050172805786,
+      "rewards/rejected": -4.526047706604004,
+      "step": 1710
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1460711699880082e-08,
+      "logits/chosen": 4.981472015380859,
+      "logits/rejected": 6.010613918304443,
+      "logps/chosen": -587.7794189453125,
+      "logps/rejected": -658.3820190429688,
+      "loss": 0.5277,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -3.5816383361816406,
+      "rewards/margins": 0.8929538726806641,
+      "rewards/rejected": -4.474592208862305,
+      "step": 1720
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.0114892356953397e-08,
+      "logits/chosen": 4.724175453186035,
+      "logits/rejected": 5.377623558044434,
+      "logps/chosen": -572.1964111328125,
+      "logps/rejected": -616.9322509765625,
+      "loss": 0.5707,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -3.6713428497314453,
+      "rewards/margins": 0.6362916231155396,
+      "rewards/rejected": -4.3076348304748535,
+      "step": 1730
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.851477564560061e-09,
+      "logits/chosen": 5.281374454498291,
+      "logits/rejected": 5.823667526245117,
+      "logps/chosen": -574.9295654296875,
+      "logps/rejected": -682.01611328125,
+      "loss": 0.5272,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -3.782670497894287,
+      "rewards/margins": 1.0396969318389893,
+      "rewards/rejected": -4.8223676681518555,
+      "step": 1740
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.670901326832763e-09,
+      "logits/chosen": 5.813240051269531,
+      "logits/rejected": 6.113523006439209,
+      "logps/chosen": -583.0259399414062,
+      "logps/rejected": -707.0408935546875,
+      "loss": 0.5068,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -3.896685838699341,
+      "rewards/margins": 0.9632736444473267,
+      "rewards/rejected": -4.859958648681641,
+      "step": 1750
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.5735691914738936e-09,
+      "logits/chosen": 4.764095783233643,
+      "logits/rejected": 5.158614158630371,
+      "logps/chosen": -563.710693359375,
+      "logps/rejected": -646.1812744140625,
+      "loss": 0.5635,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -3.6481738090515137,
+      "rewards/margins": 0.7430108785629272,
+      "rewards/rejected": -4.3911848068237305,
+      "step": 1760
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.559858110443016e-09,
+      "logits/chosen": 4.941697120666504,
+      "logits/rejected": 5.857718467712402,
+      "logps/chosen": -589.8416137695312,
+      "logps/rejected": -651.1143798828125,
+      "loss": 0.5218,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.781978130340576,
+      "rewards/margins": 0.8180473446846008,
+      "rewards/rejected": -4.600025177001953,
+      "step": 1770
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.6301163104676685e-09,
+      "logits/chosen": 5.284440517425537,
+      "logits/rejected": 5.553745269775391,
+      "logps/chosen": -551.4322509765625,
+      "logps/rejected": -651.212646484375,
+      "loss": 0.5407,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -3.6018593311309814,
+      "rewards/margins": 0.7678863406181335,
+      "rewards/rejected": -4.36974573135376,
+      "step": 1780
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.784663173421438e-09,
+      "logits/chosen": 5.374969959259033,
+      "logits/rejected": 5.7753376960754395,
+      "logps/chosen": -622.2989501953125,
+      "logps/rejected": -689.6795654296875,
+      "loss": 0.5308,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -3.73246693611145,
+      "rewards/margins": 0.8092583417892456,
+      "rewards/rejected": -4.5417256355285645,
+      "step": 1790
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.023789126611137e-09,
+      "logits/chosen": 5.826882839202881,
+      "logits/rejected": 6.438210964202881,
+      "logps/chosen": -580.4129638671875,
+      "logps/rejected": -642.33740234375,
+      "loss": 0.5188,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -3.765054702758789,
+      "rewards/margins": 0.8300830721855164,
+      "rewards/rejected": -4.595137596130371,
+      "step": 1800
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.3477555430100604e-09,
+      "logits/chosen": 5.330014228820801,
+      "logits/rejected": 6.266722679138184,
+      "logps/chosen": -545.0280151367188,
+      "logps/rejected": -623.4840087890625,
+      "loss": 0.5255,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -3.382579803466797,
+      "rewards/margins": 1.0013315677642822,
+      "rewards/rejected": -4.383911609649658,
+      "step": 1810
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7567946514721322e-09,
+      "logits/chosen": 5.459120273590088,
+      "logits/rejected": 5.671516418457031,
+      "logps/chosen": -565.1712036132812,
+      "logps/rejected": -651.7200927734375,
+      "loss": 0.5631,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -3.7416863441467285,
+      "rewards/margins": 0.7198008894920349,
+      "rewards/rejected": -4.46148681640625,
+      "step": 1820
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.2511094569571668e-09,
+      "logits/chosen": 4.77047061920166,
+      "logits/rejected": 5.680732727050781,
+      "logps/chosen": -540.1213989257812,
+      "logps/rejected": -612.4068603515625,
+      "loss": 0.5294,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -3.508056640625,
+      "rewards/margins": 0.8371505737304688,
+      "rewards/rejected": -4.345207214355469,
+      "step": 1830
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.308736707954289e-10,
+      "logits/chosen": 5.112320899963379,
+      "logits/rejected": 6.371096611022949,
+      "logps/chosen": -591.5335693359375,
+      "logps/rejected": -622.4219360351562,
+      "loss": 0.5351,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -3.7684085369110107,
+      "rewards/margins": 0.8042453527450562,
+      "rewards/rejected": -4.572653770446777,
+      "step": 1840
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 4.962316510149222e-10,
+      "logits/chosen": 4.924439430236816,
+      "logits/rejected": 5.723077774047852,
+      "logps/chosen": -528.1791381835938,
+      "logps/rejected": -613.794189453125,
+      "loss": 0.5155,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.5361733436584473,
+      "rewards/margins": 0.8941107988357544,
+      "rewards/rejected": -4.43028450012207,
+      "step": 1850
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.4729835275189016e-10,
+      "logits/chosen": 5.2165093421936035,
+      "logits/rejected": 5.809523105621338,
+      "logps/chosen": -512.5826416015625,
+      "logps/rejected": -562.5357055664062,
+      "loss": 0.5279,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.3880043029785156,
+      "rewards/margins": 0.6139224767684937,
+      "rewards/rejected": -4.001926898956299,
+      "step": 1860
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 8.415928876176482e-11,
+      "logits/chosen": 5.226788520812988,
+      "logits/rejected": 5.463950157165527,
+      "logps/chosen": -568.6761474609375,
+      "logps/rejected": -642.3060302734375,
+      "loss": 0.5796,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -3.871469020843506,
+      "rewards/margins": 0.6007556319236755,
+      "rewards/rejected": -4.472224235534668,
+      "step": 1870
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 6.870500044303673e-12,
+      "logits/chosen": 5.113773345947266,
+      "logits/rejected": 5.360236167907715,
+      "logps/chosen": -532.0494995117188,
+      "logps/rejected": -629.88818359375,
+      "loss": 0.537,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -3.353315830230713,
+      "rewards/margins": 0.727187991142273,
+      "rewards/rejected": -4.080503940582275,
+      "step": 1880
+    },
+    {
+      "epoch": 1.0,
+      "step": 1884,
+      "total_flos": 0.0,
+      "train_loss": 0.5658232939218006,
+      "train_runtime": 45309.3847,
+      "train_samples_per_second": 2.662,
+      "train_steps_per_second": 0.042
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1884,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}