Model save

README.md

@@ -0,0 +1,60 @@
+---
+license: mit
+base_model: HuggingFaceH4/mistral-7b-sft-beta
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: 0.001_idpo_same_scratch_iter_3
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# 0.001_idpo_same_scratch_iter_3
+
+This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 128
+- total_eval_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.36.2
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.583395419260448,
+    "train_runtime": 12448.4703,
+    "train_samples": 61134,
+    "train_samples_per_second": 4.911,
+    "train_steps_per_second": 0.038
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.36.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:336712b96bfd3c8485558f2bc872f520cee05d88c8d89145e925c937399ea4c9
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebec7709c9b204027ff9be3867896e6e1bc8283444d0702005a26903581410a0
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b99dab387f2eb9abc2d4dd2ccb6a8ce4f9e689af52d5e9e3f16dfcb35edd3ed7
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.583395419260448,
+    "train_runtime": 12448.4703,
+    "train_samples": 61134,
+    "train_samples_per_second": 4.911,
+    "train_steps_per_second": 0.038
+}

trainer_state.json

@@ -0,0 +1,798 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 478,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.0416666666666666e-08,
+      "logits/chosen": -2.771872043609619,
+      "logits/rejected": -2.836427688598633,
+      "logps/chosen": -186.35714721679688,
+      "logps/pi_response": -56.843544006347656,
+      "logps/ref_response": -56.843544006347656,
+      "logps/rejected": -159.2776336669922,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": -2.812751054763794,
+      "logits/rejected": -2.805162191390991,
+      "logps/chosen": -250.5966796875,
+      "logps/pi_response": -84.03253173828125,
+      "logps/ref_response": -84.02335357666016,
+      "logps/rejected": -215.7916717529297,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4444444477558136,
+      "rewards/chosen": 0.0001410771074006334,
+      "rewards/margins": 0.00040674934280104935,
+      "rewards/rejected": -0.00026567222084850073,
+      "step": 10
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -2.818342924118042,
+      "logits/rejected": -2.815077304840088,
+      "logps/chosen": -247.14089965820312,
+      "logps/pi_response": -69.06600189208984,
+      "logps/ref_response": -68.91864013671875,
+      "logps/rejected": -225.91415405273438,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": 0.001155191333964467,
+      "rewards/margins": -0.00012961139145772904,
+      "rewards/rejected": 0.0012848025653511286,
+      "step": 20
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -2.7623016834259033,
+      "logits/rejected": -2.729628324508667,
+      "logps/chosen": -243.77188110351562,
+      "logps/pi_response": -83.35380554199219,
+      "logps/ref_response": -82.44132995605469,
+      "logps/rejected": -221.338134765625,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.006917249411344528,
+      "rewards/margins": 0.005432372447103262,
+      "rewards/rejected": 0.00148487719707191,
+      "step": 30
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -2.799405097961426,
+      "logits/rejected": -2.7605228424072266,
+      "logps/chosen": -235.19711303710938,
+      "logps/pi_response": -72.81827545166016,
+      "logps/ref_response": -70.3433837890625,
+      "logps/rejected": -198.21481323242188,
+      "loss": 0.6836,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.027682146057486534,
+      "rewards/margins": 0.023167867213487625,
+      "rewards/rejected": 0.004514278843998909,
+      "step": 40
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999733114418725e-07,
+      "logits/chosen": -2.7923216819763184,
+      "logits/rejected": -2.780092716217041,
+      "logps/chosen": -244.62802124023438,
+      "logps/pi_response": -87.8843002319336,
+      "logps/ref_response": -81.94856262207031,
+      "logps/rejected": -257.96270751953125,
+      "loss": 0.6748,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.04818359389901161,
+      "rewards/margins": 0.05035162717103958,
+      "rewards/rejected": -0.0021680318750441074,
+      "step": 50
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.990398100856366e-07,
+      "logits/chosen": -2.744097948074341,
+      "logits/rejected": -2.7316837310791016,
+      "logps/chosen": -204.2379608154297,
+      "logps/pi_response": -71.70608520507812,
+      "logps/ref_response": -57.30153274536133,
+      "logps/rejected": -191.46578979492188,
+      "loss": 0.6608,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 0.0015774101484566927,
+      "rewards/margins": 0.05679439380764961,
+      "rewards/rejected": -0.055216990411281586,
+      "step": 60
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.967775735898179e-07,
+      "logits/chosen": -2.6771161556243896,
+      "logits/rejected": -2.670588970184326,
+      "logps/chosen": -212.1810302734375,
+      "logps/pi_response": -96.94953155517578,
+      "logps/ref_response": -65.7808609008789,
+      "logps/rejected": -224.86795043945312,
+      "loss": 0.6418,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.05775865167379379,
+      "rewards/margins": 0.1283641755580902,
+      "rewards/rejected": -0.1861228346824646,
+      "step": 70
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.931986719649298e-07,
+      "logits/chosen": -2.6362526416778564,
+      "logits/rejected": -2.591085910797119,
+      "logps/chosen": -237.90524291992188,
+      "logps/pi_response": -120.31071472167969,
+      "logps/ref_response": -76.52632141113281,
+      "logps/rejected": -243.9879608154297,
+      "loss": 0.6356,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.10214054584503174,
+      "rewards/margins": 0.2090938836336136,
+      "rewards/rejected": -0.31123441457748413,
+      "step": 80
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.883222001996351e-07,
+      "logits/chosen": -2.662489414215088,
+      "logits/rejected": -2.6212925910949707,
+      "logps/chosen": -238.72500610351562,
+      "logps/pi_response": -122.1798095703125,
+      "logps/ref_response": -70.05473327636719,
+      "logps/rejected": -236.9344024658203,
+      "loss": 0.6331,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.16823504865169525,
+      "rewards/margins": 0.17120732367038727,
+      "rewards/rejected": -0.3394424021244049,
+      "step": 90
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.821741763807186e-07,
+      "logits/chosen": -2.6857962608337402,
+      "logits/rejected": -2.6634621620178223,
+      "logps/chosen": -234.5189208984375,
+      "logps/pi_response": -132.98020935058594,
+      "logps/ref_response": -72.16676330566406,
+      "logps/rejected": -242.3022918701172,
+      "loss": 0.6265,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.20184996724128723,
+      "rewards/margins": 0.15526965260505676,
+      "rewards/rejected": -0.357119619846344,
+      "step": 100
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.747874028753375e-07,
+      "logits/chosen": -2.6297690868377686,
+      "logits/rejected": -2.594348430633545,
+      "logps/chosen": -280.73272705078125,
+      "logps/pi_response": -147.92088317871094,
+      "logps/ref_response": -78.85017395019531,
+      "logps/rejected": -264.87420654296875,
+      "loss": 0.6288,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.22727267444133759,
+      "rewards/margins": 0.32433170080184937,
+      "rewards/rejected": -0.5516043901443481,
+      "step": 110
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.662012913161997e-07,
+      "logits/chosen": -2.6496646404266357,
+      "logits/rejected": -2.619614839553833,
+      "logps/chosen": -314.30682373046875,
+      "logps/pi_response": -197.1916046142578,
+      "logps/ref_response": -79.45616149902344,
+      "logps/rejected": -317.90936279296875,
+      "loss": 0.5952,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.5182690024375916,
+      "rewards/margins": 0.3716340661048889,
+      "rewards/rejected": -0.8899030685424805,
+      "step": 120
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.5646165232345103e-07,
+      "logits/chosen": -2.669473648071289,
+      "logits/rejected": -2.6336374282836914,
+      "logps/chosen": -293.299072265625,
+      "logps/pi_response": -204.230224609375,
+      "logps/ref_response": -77.86495208740234,
+      "logps/rejected": -316.03448486328125,
+      "loss": 0.6044,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.616094708442688,
+      "rewards/margins": 0.3894798159599304,
+      "rewards/rejected": -1.0055744647979736,
+      "step": 130
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.456204510851956e-07,
+      "logits/chosen": -2.600029706954956,
+      "logits/rejected": -2.552990198135376,
+      "logps/chosen": -274.93023681640625,
+      "logps/pi_response": -175.21990966796875,
+      "logps/ref_response": -76.53099060058594,
+      "logps/rejected": -297.7197570800781,
+      "loss": 0.6096,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.3966221511363983,
+      "rewards/margins": 0.4644131660461426,
+      "rewards/rejected": -0.8610353469848633,
+      "step": 140
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.337355301007335e-07,
+      "logits/chosen": -2.451113224029541,
+      "logits/rejected": -2.440601348876953,
+      "logps/chosen": -244.29443359375,
+      "logps/pi_response": -189.71490478515625,
+      "logps/ref_response": -59.47999954223633,
+      "logps/rejected": -276.4666442871094,
+      "loss": 0.5883,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.521985650062561,
+      "rewards/margins": 0.40682539343833923,
+      "rewards/rejected": -0.9288110733032227,
+      "step": 150
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2087030056579986e-07,
+      "logits/chosen": -2.392782688140869,
+      "logits/rejected": -2.3590378761291504,
+      "logps/chosen": -293.15496826171875,
+      "logps/pi_response": -246.6798095703125,
+      "logps/ref_response": -73.12691497802734,
+      "logps/rejected": -316.5404968261719,
+      "loss": 0.6101,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.7774425745010376,
+      "rewards/margins": 0.32445091009140015,
+      "rewards/rejected": -1.101893424987793,
+      "step": 160
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.070934040463998e-07,
+      "logits/chosen": -2.3637728691101074,
+      "logits/rejected": -2.3082034587860107,
+      "logps/chosen": -330.51300048828125,
+      "logps/pi_response": -247.8793182373047,
+      "logps/ref_response": -69.19013214111328,
+      "logps/rejected": -340.6532287597656,
+      "loss": 0.5765,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.8771257400512695,
+      "rewards/margins": 0.37836703658103943,
+      "rewards/rejected": -1.255492925643921,
+      "step": 170
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.9247834624635404e-07,
+      "logits/chosen": -2.3222761154174805,
+      "logits/rejected": -2.290357828140259,
+      "logps/chosen": -317.992919921875,
+      "logps/pi_response": -246.22482299804688,
+      "logps/ref_response": -64.53086853027344,
+      "logps/rejected": -345.11138916015625,
+      "loss": 0.5736,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.8496101498603821,
+      "rewards/margins": 0.627398669719696,
+      "rewards/rejected": -1.4770088195800781,
+      "step": 180
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7710310482256523e-07,
+      "logits/chosen": -2.2988460063934326,
+      "logits/rejected": -2.2790465354919434,
+      "logps/chosen": -323.6756286621094,
+      "logps/pi_response": -238.6248016357422,
+      "logps/ref_response": -72.57078552246094,
+      "logps/rejected": -355.2505798339844,
+      "loss": 0.5828,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.0282012224197388,
+      "rewards/margins": 0.3955053389072418,
+      "rewards/rejected": -1.4237064123153687,
+      "step": 190
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.610497133404795e-07,
+      "logits/chosen": -2.335862636566162,
+      "logits/rejected": -2.277442693710327,
+      "logps/chosen": -346.7106628417969,
+      "logps/pi_response": -245.89291381835938,
+      "logps/ref_response": -73.11415100097656,
+      "logps/rejected": -370.8809814453125,
+      "loss": 0.5449,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.9884869456291199,
+      "rewards/margins": 0.5412154793739319,
+      "rewards/rejected": -1.5297023057937622,
+      "step": 200
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4440382358952115e-07,
+      "logits/chosen": -2.2967655658721924,
+      "logits/rejected": -2.263054609298706,
+      "logps/chosen": -343.8060302734375,
+      "logps/pi_response": -267.05694580078125,
+      "logps/ref_response": -70.57121276855469,
+      "logps/rejected": -349.5938415527344,
+      "loss": 0.5706,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.133570671081543,
+      "rewards/margins": 0.5091238617897034,
+      "rewards/rejected": -1.6426944732666016,
+      "step": 210
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.272542485937368e-07,
+      "logits/chosen": -2.383333921432495,
+      "logits/rejected": -2.3383095264434814,
+      "logps/chosen": -331.64178466796875,
+      "logps/pi_response": -235.6920623779297,
+      "logps/ref_response": -68.40555572509766,
+      "logps/rejected": -365.8157043457031,
+      "loss": 0.5675,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.9668046236038208,
+      "rewards/margins": 0.5283567905426025,
+      "rewards/rejected": -1.495161533355713,
+      "step": 220
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.096924887558854e-07,
+      "logits/chosen": -2.343796491622925,
+      "logits/rejected": -2.309424877166748,
+      "logps/chosen": -322.5975036621094,
+      "logps/pi_response": -233.7900390625,
+      "logps/ref_response": -67.48506164550781,
+      "logps/rejected": -376.9325256347656,
+      "loss": 0.5833,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.9696311950683594,
+      "rewards/margins": 0.49308356642723083,
+      "rewards/rejected": -1.462714672088623,
+      "step": 230
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9181224366319943e-07,
+      "logits/chosen": -2.346538782119751,
+      "logits/rejected": -2.2966227531433105,
+      "logps/chosen": -333.65643310546875,
+      "logps/pi_response": -260.61004638671875,
+      "logps/ref_response": -67.38578033447266,
+      "logps/rejected": -375.87469482421875,
+      "loss": 0.5634,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.1766176223754883,
+      "rewards/margins": 0.4966951012611389,
+      "rewards/rejected": -1.673312783241272,
+      "step": 240
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7370891215954565e-07,
+      "logits/chosen": -2.3725361824035645,
+      "logits/rejected": -2.3253109455108643,
+      "logps/chosen": -358.3512268066406,
+      "logps/pi_response": -293.8153381347656,
+      "logps/ref_response": -72.65675354003906,
+      "logps/rejected": -365.43853759765625,
+      "loss": 0.5653,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.2577649354934692,
+      "rewards/margins": 0.4211507737636566,
+      "rewards/rejected": -1.6789157390594482,
+      "step": 250
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.55479083351317e-07,
+      "logits/chosen": -2.37805438041687,
+      "logits/rejected": -2.32930326461792,
+      "logps/chosen": -374.30377197265625,
+      "logps/pi_response": -283.6889343261719,
+      "logps/ref_response": -74.38246154785156,
+      "logps/rejected": -373.2997741699219,
+      "loss": 0.5588,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.15468430519104,
+      "rewards/margins": 0.474844366312027,
+      "rewards/rejected": -1.6295287609100342,
+      "step": 260
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.3722002126275822e-07,
+      "logits/chosen": -2.2448177337646484,
+      "logits/rejected": -2.2177035808563232,
+      "logps/chosen": -346.1255187988281,
+      "logps/pi_response": -286.4845275878906,
+      "logps/ref_response": -76.11927032470703,
+      "logps/rejected": -373.6055603027344,
+      "loss": 0.5786,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.2877910137176514,
+      "rewards/margins": 0.400728315114975,
+      "rewards/rejected": -1.6885192394256592,
+      "step": 270
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.19029145890313e-07,
+      "logits/chosen": -2.1976356506347656,
+      "logits/rejected": -2.1738736629486084,
+      "logps/chosen": -318.11578369140625,
+      "logps/pi_response": -277.13848876953125,
+      "logps/ref_response": -61.588165283203125,
+      "logps/rejected": -371.4759216308594,
+      "loss": 0.5439,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.2585313320159912,
+      "rewards/margins": 0.4716874659061432,
+      "rewards/rejected": -1.7302188873291016,
+      "step": 280
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.0100351342479216e-07,
+      "logits/chosen": -2.267279863357544,
+      "logits/rejected": -2.225353240966797,
+      "logps/chosen": -341.911865234375,
+      "logps/pi_response": -263.16766357421875,
+      "logps/ref_response": -70.3861312866211,
+      "logps/rejected": -358.21624755859375,
+      "loss": 0.565,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.13209068775177,
+      "rewards/margins": 0.480673611164093,
+      "rewards/rejected": -1.6127643585205078,
+      "step": 290
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8323929841460178e-07,
+      "logits/chosen": -2.2511088848114014,
+      "logits/rejected": -2.20157790184021,
+      "logps/chosen": -369.5644226074219,
+      "logps/pi_response": -284.96087646484375,
+      "logps/ref_response": -79.78241729736328,
+      "logps/rejected": -390.3699645996094,
+      "loss": 0.5584,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.189987063407898,
+      "rewards/margins": 0.4863070845603943,
+      "rewards/rejected": -1.6762943267822266,
+      "step": 300
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6583128063291573e-07,
+      "logits/chosen": -2.2550597190856934,
+      "logits/rejected": -2.2291643619537354,
+      "logps/chosen": -391.4834289550781,
+      "logps/pi_response": -303.56292724609375,
+      "logps/ref_response": -84.91761016845703,
+      "logps/rejected": -436.4339904785156,
+      "loss": 0.5727,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.2904925346374512,
+      "rewards/margins": 0.6019302606582642,
+      "rewards/rejected": -1.8924226760864258,
+      "step": 310
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.488723393865766e-07,
+      "logits/chosen": -2.292328357696533,
+      "logits/rejected": -2.205026388168335,
+      "logps/chosen": -375.3182678222656,
+      "logps/pi_response": -294.6897277832031,
+      "logps/ref_response": -75.90422821044922,
+      "logps/rejected": -383.18994140625,
+      "loss": 0.547,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.125930905342102,
+      "rewards/margins": 0.6577755212783813,
+      "rewards/rejected": -1.7837063074111938,
+      "step": 320
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3245295796480788e-07,
+      "logits/chosen": -2.238246202468872,
+      "logits/rejected": -2.1797311305999756,
+      "logps/chosen": -344.31427001953125,
+      "logps/pi_response": -295.9718933105469,
+      "logps/ref_response": -69.94654846191406,
+      "logps/rejected": -386.03436279296875,
+      "loss": 0.5343,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.2373181581497192,
+      "rewards/margins": 0.531391978263855,
+      "rewards/rejected": -1.7687098979949951,
+      "step": 330
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1666074087171627e-07,
+      "logits/chosen": -2.2410218715667725,
+      "logits/rejected": -2.182896137237549,
+      "logps/chosen": -320.0640563964844,
+      "logps/pi_response": -265.65252685546875,
+      "logps/ref_response": -72.93882751464844,
+      "logps/rejected": -349.9189453125,
+      "loss": 0.5352,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.1647237539291382,
+      "rewards/margins": 0.5372185707092285,
+      "rewards/rejected": -1.7019424438476562,
+      "step": 340
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0157994641835734e-07,
+      "logits/chosen": -2.214590549468994,
+      "logits/rejected": -2.144507646560669,
+      "logps/chosen": -331.2563171386719,
+      "logps/pi_response": -289.6974182128906,
+      "logps/ref_response": -74.75759887695312,
+      "logps/rejected": -383.2954406738281,
+      "loss": 0.5407,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.2160807847976685,
+      "rewards/margins": 0.6268461346626282,
+      "rewards/rejected": -1.8429267406463623,
+      "step": 350
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.729103716819111e-08,
+      "logits/chosen": -2.2340056896209717,
+      "logits/rejected": -2.1630749702453613,
+      "logps/chosen": -363.3587341308594,
+      "logps/pi_response": -291.95538330078125,
+      "logps/ref_response": -75.47894287109375,
+      "logps/rejected": -398.1706237792969,
+      "loss": 0.5352,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.259597659111023,
+      "rewards/margins": 0.6464335322380066,
+      "rewards/rejected": -1.9060310125350952,
+      "step": 360
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.387025063449081e-08,
+      "logits/chosen": -2.187648296356201,
+      "logits/rejected": -2.1372499465942383,
+      "logps/chosen": -342.2130432128906,
+      "logps/pi_response": -296.50830078125,
+      "logps/ref_response": -68.33134460449219,
+      "logps/rejected": -397.110595703125,
+      "loss": 0.5447,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.2424293756484985,
+      "rewards/margins": 0.5943651795387268,
+      "rewards/rejected": -1.8367946147918701,
+      "step": 370
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.138919252022435e-08,
+      "logits/chosen": -2.223222255706787,
+      "logits/rejected": -2.1696248054504395,
+      "logps/chosen": -345.4095764160156,
+      "logps/pi_response": -297.59759521484375,
+      "logps/ref_response": -72.3606948852539,
+      "logps/rejected": -421.50531005859375,
+      "loss": 0.5508,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.3772599697113037,
+      "rewards/margins": 0.516662061214447,
+      "rewards/rejected": -1.8939220905303955,
+      "step": 380
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.991445467064689e-08,
+      "logits/chosen": -2.2751657962799072,
+      "logits/rejected": -2.190664529800415,
+      "logps/chosen": -357.4648742675781,
+      "logps/pi_response": -295.1226806640625,
+      "logps/ref_response": -70.87272644042969,
+      "logps/rejected": -415.9715270996094,
+      "loss": 0.5434,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.3301585912704468,
+      "rewards/margins": 0.6158719658851624,
+      "rewards/rejected": -1.9460303783416748,
+      "step": 390
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.9507259776993954e-08,
+      "logits/chosen": -2.254105806350708,
+      "logits/rejected": -2.2081995010375977,
+      "logps/chosen": -366.28436279296875,
+      "logps/pi_response": -304.9342041015625,
+      "logps/ref_response": -77.48291015625,
+      "logps/rejected": -364.3297119140625,
+      "loss": 0.5429,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.3418512344360352,
+      "rewards/margins": 0.46248334646224976,
+      "rewards/rejected": -1.8043344020843506,
+      "step": 400
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.022313472693447e-08,
+      "logits/chosen": -2.232144594192505,
+      "logits/rejected": -2.1603074073791504,
+      "logps/chosen": -369.909423828125,
+      "logps/pi_response": -314.9814147949219,
+      "logps/ref_response": -76.4972915649414,
+      "logps/rejected": -432.63824462890625,
+      "loss": 0.5482,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.456408977508545,
+      "rewards/margins": 0.5390293598175049,
+      "rewards/rejected": -1.9954382181167603,
+      "step": 410
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.2111614344599684e-08,
+      "logits/chosen": -2.268048048019409,
+      "logits/rejected": -2.194788694381714,
+      "logps/chosen": -351.391845703125,
+      "logps/pi_response": -303.7065734863281,
+      "logps/ref_response": -70.95718383789062,
+      "logps/rejected": -420.17315673828125,
+      "loss": 0.5353,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.3950040340423584,
+      "rewards/margins": 0.5390046238899231,
+      "rewards/rejected": -1.9340085983276367,
+      "step": 420
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.521597710086439e-08,
+      "logits/chosen": -2.256483554840088,
+      "logits/rejected": -2.182900905609131,
+      "logps/chosen": -394.15948486328125,
+      "logps/pi_response": -322.7268981933594,
+      "logps/ref_response": -71.76502990722656,
+      "logps/rejected": -406.4312438964844,
+      "loss": 0.5666,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.330580711364746,
+      "rewards/margins": 0.6264187693595886,
+      "rewards/rejected": -1.9569995403289795,
+      "step": 430
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.57301420397924e-09,
+      "logits/chosen": -2.261594772338867,
+      "logits/rejected": -2.219043016433716,
+      "logps/chosen": -363.95025634765625,
+      "logps/pi_response": -315.3779296875,
+      "logps/ref_response": -77.31925201416016,
+      "logps/rejected": -394.43280029296875,
+      "loss": 0.5349,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.391911268234253,
+      "rewards/margins": 0.5445317029953003,
+      "rewards/rejected": -1.9364430904388428,
+      "step": 440
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.212833302556258e-09,
+      "logits/chosen": -2.2221949100494385,
+      "logits/rejected": -2.142899513244629,
+      "logps/chosen": -372.461181640625,
+      "logps/pi_response": -308.6124572753906,
+      "logps/ref_response": -75.69192504882812,
+      "logps/rejected": -399.45892333984375,
+      "loss": 0.5203,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.2634239196777344,
+      "rewards/margins": 0.6645442247390747,
+      "rewards/rejected": -1.9279682636260986,
+      "step": 450
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.158697848236607e-09,
+      "logits/chosen": -2.2366528511047363,
+      "logits/rejected": -2.166773796081543,
+      "logps/chosen": -369.5625915527344,
+      "logps/pi_response": -296.8544006347656,
+      "logps/ref_response": -69.99818420410156,
+      "logps/rejected": -397.776123046875,
+      "loss": 0.5432,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.3708467483520508,
+      "rewards/margins": 0.6627798676490784,
+      "rewards/rejected": -2.0336265563964844,
+      "step": 460
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 4.269029751107489e-10,
+      "logits/chosen": -2.2382636070251465,
+      "logits/rejected": -2.1679019927978516,
+      "logps/chosen": -345.45440673828125,
+      "logps/pi_response": -300.89666748046875,
+      "logps/ref_response": -71.26921081542969,
+      "logps/rejected": -400.05804443359375,
+      "loss": 0.5428,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.2617619037628174,
+      "rewards/margins": 0.6952942609786987,
+      "rewards/rejected": -1.9570560455322266,
+      "step": 470
+    },
+    {
+      "epoch": 1.0,
+      "step": 478,
+      "total_flos": 0.0,
+      "train_loss": 0.583395419260448,
+      "train_runtime": 12448.4703,
+      "train_samples_per_second": 4.911,
+      "train_steps_per_second": 0.038
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 478,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}