TX

README.md

@@ -0,0 +1,75 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- alignment_handbook-handbook
+- generated_from_trainer
+datasets:
+- princeton-nlp/mistral-instruct-ultrafeedback
+model-index:
+- name: Mistral-7B-Base-SimPO2-5e-7
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/tengxiao01/huggingface/runs/jcsdn1jt)
+# Mistral-7B-Base-SimPO2-5e-7
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the princeton-nlp/mistral-instruct-ultrafeedback dataset.
+It achieves the following results on the evaluation set:
+- Logits/chosen: -2.9000
+- Logits/rejected: -2.8928
+- Logps/chosen: -1.1346
+- Logps/rejected: -1.2466
+- Loss: -0.0708
+- Rewards/accuracies: 0.5758
+- Rewards/chosen: -1.1346
+- Rewards/margins: 0.1120
+- Rewards/rejected: -1.2466
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 2
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 128
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch  | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
+|:-------------:|:------:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
+| -0.0905       | 0.8573 | 400  | -2.9000       | -2.8928         | -1.1346      | -1.2466        | -0.0708         | 0.5758             | -1.1346        | 0.1120          | -1.2466          |
+
+
+### Framework versions
+
+- Transformers 4.42.0
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,22 @@
+{
+    "epoch": 0.9987943737441393,
+    "eval_logits/chosen": -2.8956403732299805,
+    "eval_logits/rejected": -2.887930154800415,
+    "eval_logps/chosen": -1.1685596704483032,
+    "eval_logps/rejected": -1.2866594791412354,
+    "eval_loss": -0.07405862212181091,
+    "eval_rewards/accuracies": 0.5797872543334961,
+    "eval_rewards/chosen": -1.1685596704483032,
+    "eval_rewards/margins": 0.11809979379177094,
+    "eval_rewards/rejected": -1.2866594791412354,
+    "eval_runtime": 432.6369,
+    "eval_samples": 2994,
+    "eval_samples_per_second": 6.92,
+    "eval_steps_per_second": 0.435,
+    "total_flos": 0.0,
+    "train_loss": 0.0,
+    "train_runtime": 4.3106,
+    "train_samples": 59720,
+    "train_samples_per_second": 13854.197,
+    "train_steps_per_second": 108.105
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "alignment-handbook/zephyr-7b-sft-full",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.42.0",
+  "use_cache": true,
+  "vocab_size": 32000
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 0.9987943737441393,
+    "eval_logits/chosen": -2.8956403732299805,
+    "eval_logits/rejected": -2.887930154800415,
+    "eval_logps/chosen": -1.1685596704483032,
+    "eval_logps/rejected": -1.2866594791412354,
+    "eval_loss": -0.07405862212181091,
+    "eval_rewards/accuracies": 0.5797872543334961,
+    "eval_rewards/chosen": -1.1685596704483032,
+    "eval_rewards/margins": 0.11809979379177094,
+    "eval_rewards/rejected": -1.2866594791412354,
+    "eval_runtime": 432.6369,
+    "eval_samples": 2994,
+    "eval_samples_per_second": 6.92,
+    "eval_steps_per_second": 0.435
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.42.0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e86ac70d9836b2d4195eee1d3dbc4402a7a0c20acffa8d338364fbcbeedf14c5
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a21eafd141af9dd8067a4a78c38a4f8d7d3e1523345be7af9c02bf5a88d9115
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a86de4c4ce516205d7e88bc37604be949702a33ff400ebb201352d450f703996
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9987943737441393,
+    "total_flos": 0.0,
+    "train_loss": 0.0,
+    "train_runtime": 4.3106,
+    "train_samples": 59720,
+    "train_samples_per_second": 13854.197,
+    "train_steps_per_second": 108.105
+}

trainer_state.json

@@ -0,0 +1,1453 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9987943737441393,
+  "eval_steps": 400,
+  "global_step": 466,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010716677829872739,
+      "grad_norm": 8.776682238147446,
+      "learning_rate": 5.3191489361702123e-08,
+      "logits/chosen": -2.8905415534973145,
+      "logits/rejected": -2.8877615928649902,
+      "logps/chosen": -0.9742914438247681,
+      "logps/rejected": -0.9831219911575317,
+      "loss": 0.0001,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.9742914438247681,
+      "rewards/margins": 0.008830604143440723,
+      "rewards/rejected": -0.9831219911575317,
+      "step": 5
+    },
+    {
+      "epoch": 0.021433355659745478,
+      "grad_norm": 9.810248541108757,
+      "learning_rate": 1.0638297872340425e-07,
+      "logits/chosen": -2.9070327281951904,
+      "logits/rejected": -2.8929152488708496,
+      "logps/chosen": -0.978168785572052,
+      "logps/rejected": -1.001371145248413,
+      "loss": -0.0,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.978168785572052,
+      "rewards/margins": 0.023202406242489815,
+      "rewards/rejected": -1.001371145248413,
+      "step": 10
+    },
+    {
+      "epoch": 0.032150033489618215,
+      "grad_norm": 7.0995253496741135,
+      "learning_rate": 1.5957446808510638e-07,
+      "logits/chosen": -2.895216703414917,
+      "logits/rejected": -2.9180421829223633,
+      "logps/chosen": -0.9596306085586548,
+      "logps/rejected": -0.9847423434257507,
+      "loss": -0.0001,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.9596306085586548,
+      "rewards/margins": 0.02511175535619259,
+      "rewards/rejected": -0.9847423434257507,
+      "step": 15
+    },
+    {
+      "epoch": 0.042866711319490956,
+      "grad_norm": 7.785343608601427,
+      "learning_rate": 2.127659574468085e-07,
+      "logits/chosen": -2.902580976486206,
+      "logits/rejected": -2.9009933471679688,
+      "logps/chosen": -0.9715825319290161,
+      "logps/rejected": -0.9759886860847473,
+      "loss": -0.0,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.9715825319290161,
+      "rewards/margins": 0.00440616998821497,
+      "rewards/rejected": -0.9759886860847473,
+      "step": 20
+    },
+    {
+      "epoch": 0.0535833891493637,
+      "grad_norm": 6.677386701450202,
+      "learning_rate": 2.659574468085106e-07,
+      "logits/chosen": -2.8314852714538574,
+      "logits/rejected": -2.8363277912139893,
+      "logps/chosen": -0.9519669413566589,
+      "logps/rejected": -1.0002533197402954,
+      "loss": -0.0003,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.9519669413566589,
+      "rewards/margins": 0.048286281526088715,
+      "rewards/rejected": -1.0002533197402954,
+      "step": 25
+    },
+    {
+      "epoch": 0.06430006697923643,
+      "grad_norm": 8.4552870442767,
+      "learning_rate": 3.1914893617021275e-07,
+      "logits/chosen": -2.895077705383301,
+      "logits/rejected": -2.9221832752227783,
+      "logps/chosen": -0.983117938041687,
+      "logps/rejected": -0.9829673767089844,
+      "loss": -0.0002,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.983117938041687,
+      "rewards/margins": -0.00015044640167616308,
+      "rewards/rejected": -0.9829673767089844,
+      "step": 30
+    },
+    {
+      "epoch": 0.07501674480910918,
+      "grad_norm": 7.60248558460589,
+      "learning_rate": 3.7234042553191484e-07,
+      "logits/chosen": -2.9559242725372314,
+      "logits/rejected": -2.9249794483184814,
+      "logps/chosen": -0.95380699634552,
+      "logps/rejected": -0.9984177350997925,
+      "loss": -0.0004,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.95380699634552,
+      "rewards/margins": 0.04461073502898216,
+      "rewards/rejected": -0.9984177350997925,
+      "step": 35
+    },
+    {
+      "epoch": 0.08573342263898191,
+      "grad_norm": 8.131750300550957,
+      "learning_rate": 4.25531914893617e-07,
+      "logits/chosen": -2.8956754207611084,
+      "logits/rejected": -2.8939049243927,
+      "logps/chosen": -0.9987373352050781,
+      "logps/rejected": -0.9930660128593445,
+      "loss": -0.0015,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.9987373352050781,
+      "rewards/margins": -0.005671312101185322,
+      "rewards/rejected": -0.9930660128593445,
+      "step": 40
+    },
+    {
+      "epoch": 0.09645010046885466,
+      "grad_norm": 7.488747662199227,
+      "learning_rate": 4.787234042553192e-07,
+      "logits/chosen": -2.8674330711364746,
+      "logits/rejected": -2.856853485107422,
+      "logps/chosen": -0.9764472246170044,
+      "logps/rejected": -0.9902997016906738,
+      "loss": -0.001,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.9764472246170044,
+      "rewards/margins": 0.013852563686668873,
+      "rewards/rejected": -0.9902997016906738,
+      "step": 45
+    },
+    {
+      "epoch": 0.1071667782987274,
+      "grad_norm": 7.447612263721868,
+      "learning_rate": 4.999367579541646e-07,
+      "logits/chosen": -2.875795364379883,
+      "logits/rejected": -2.8871512413024902,
+      "logps/chosen": -0.9369581937789917,
+      "logps/rejected": -0.9167054891586304,
+      "loss": -0.0027,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.9369581937789917,
+      "rewards/margins": -0.020252685993909836,
+      "rewards/rejected": -0.9167054891586304,
+      "step": 50
+    },
+    {
+      "epoch": 0.11788345612860013,
+      "grad_norm": 8.432358893020524,
+      "learning_rate": 4.99550394651012e-07,
+      "logits/chosen": -2.915107011795044,
+      "logits/rejected": -2.925875186920166,
+      "logps/chosen": -0.9737237095832825,
+      "logps/rejected": -1.0020536184310913,
+      "loss": -0.0022,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.9737237095832825,
+      "rewards/margins": 0.028329964727163315,
+      "rewards/rejected": -1.0020536184310913,
+      "step": 55
+    },
+    {
+      "epoch": 0.12860013395847286,
+      "grad_norm": 9.97321343058875,
+      "learning_rate": 4.988133448023071e-07,
+      "logits/chosen": -2.8224921226501465,
+      "logits/rejected": -2.8447964191436768,
+      "logps/chosen": -0.9149552583694458,
+      "logps/rejected": -0.9350795745849609,
+      "loss": -0.0041,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.9149552583694458,
+      "rewards/margins": 0.02012430503964424,
+      "rewards/rejected": -0.9350795745849609,
+      "step": 60
+    },
+    {
+      "epoch": 0.13931681178834562,
+      "grad_norm": 8.412802955896868,
+      "learning_rate": 4.977266441646379e-07,
+      "logits/chosen": -2.900336503982544,
+      "logits/rejected": -2.9115452766418457,
+      "logps/chosen": -0.8882263898849487,
+      "logps/rejected": -0.9012020230293274,
+      "loss": -0.0057,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.8882263898849487,
+      "rewards/margins": 0.012975621037185192,
+      "rewards/rejected": -0.9012020230293274,
+      "step": 65
+    },
+    {
+      "epoch": 0.15003348961821836,
+      "grad_norm": 8.21023142941744,
+      "learning_rate": 4.962918198495654e-07,
+      "logits/chosen": -2.816331386566162,
+      "logits/rejected": -2.839690685272217,
+      "logps/chosen": -0.9164741635322571,
+      "logps/rejected": -0.9583846926689148,
+      "loss": -0.007,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.9164741635322571,
+      "rewards/margins": 0.04191048443317413,
+      "rewards/rejected": -0.9583846926689148,
+      "step": 70
+    },
+    {
+      "epoch": 0.1607501674480911,
+      "grad_norm": 8.354728288045035,
+      "learning_rate": 4.945108881776135e-07,
+      "logits/chosen": -2.8184382915496826,
+      "logits/rejected": -2.81858229637146,
+      "logps/chosen": -0.8914631009101868,
+      "logps/rejected": -0.9294856190681458,
+      "loss": -0.0066,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.8914631009101868,
+      "rewards/margins": 0.038022562861442566,
+      "rewards/rejected": -0.9294856190681458,
+      "step": 75
+    },
+    {
+      "epoch": 0.17146684527796383,
+      "grad_norm": 9.546162491226012,
+      "learning_rate": 4.923863518447879e-07,
+      "logits/chosen": -2.840766191482544,
+      "logits/rejected": -2.8635830879211426,
+      "logps/chosen": -0.887871265411377,
+      "logps/rejected": -0.958912193775177,
+      "loss": -0.006,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.887871265411377,
+      "rewards/margins": 0.07104082405567169,
+      "rewards/rejected": -0.958912193775177,
+      "step": 80
+    },
+    {
+      "epoch": 0.18218352310783656,
+      "grad_norm": 9.354318155678731,
+      "learning_rate": 4.899211964056035e-07,
+      "logits/chosen": -2.854689359664917,
+      "logits/rejected": -2.866008758544922,
+      "logps/chosen": -0.9082525968551636,
+      "logps/rejected": -0.9574602842330933,
+      "loss": -0.0132,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.9082525968551636,
+      "rewards/margins": 0.04920773208141327,
+      "rewards/rejected": -0.9574602842330933,
+      "step": 85
+    },
+    {
+      "epoch": 0.19290020093770932,
+      "grad_norm": 9.38237658589797,
+      "learning_rate": 4.871188860775643e-07,
+      "logits/chosen": -2.830294132232666,
+      "logits/rejected": -2.8332040309906006,
+      "logps/chosen": -0.9233630895614624,
+      "logps/rejected": -0.9633795022964478,
+      "loss": -0.0142,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.9233630895614624,
+      "rewards/margins": 0.040016382932662964,
+      "rewards/rejected": -0.9633795022964478,
+      "step": 90
+    },
+    {
+      "epoch": 0.20361687876758205,
+      "grad_norm": 8.979685614931006,
+      "learning_rate": 4.839833588729897e-07,
+      "logits/chosen": -2.8387644290924072,
+      "logits/rejected": -2.8495821952819824,
+      "logps/chosen": -0.9509484171867371,
+      "logps/rejected": -0.974162220954895,
+      "loss": -0.0085,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.9509484171867371,
+      "rewards/margins": 0.02321377769112587,
+      "rewards/rejected": -0.974162220954895,
+      "step": 95
+    },
+    {
+      "epoch": 0.2143335565974548,
+      "grad_norm": 9.427728619757332,
+      "learning_rate": 4.80519021065031e-07,
+      "logits/chosen": -2.860654354095459,
+      "logits/rejected": -2.8680408000946045,
+      "logps/chosen": -0.9869228601455688,
+      "logps/rejected": -1.0385963916778564,
+      "loss": -0.015,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.9869228601455688,
+      "rewards/margins": 0.05167357996106148,
+      "rewards/rejected": -1.0385963916778564,
+      "step": 100
+    },
+    {
+      "epoch": 0.22505023442732752,
+      "grad_norm": 8.191485445947501,
+      "learning_rate": 4.767307409956528e-07,
+      "logits/chosen": -2.859048843383789,
+      "logits/rejected": -2.8625106811523438,
+      "logps/chosen": -0.8909996151924133,
+      "logps/rejected": -0.9282458424568176,
+      "loss": -0.0154,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.8909996151924133,
+      "rewards/margins": 0.03724620118737221,
+      "rewards/rejected": -0.9282458424568176,
+      "step": 105
+    },
+    {
+      "epoch": 0.23576691225720026,
+      "grad_norm": 9.657244741544229,
+      "learning_rate": 4.726238422342805e-07,
+      "logits/chosen": -2.8523192405700684,
+      "logits/rejected": -2.84688138961792,
+      "logps/chosen": -0.9256642460823059,
+      "logps/rejected": -0.9944097399711609,
+      "loss": -0.0136,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.9256642460823059,
+      "rewards/margins": 0.06874561309814453,
+      "rewards/rejected": -0.9944097399711609,
+      "step": 110
+    },
+    {
+      "epoch": 0.24648359008707302,
+      "grad_norm": 11.45037121550009,
+      "learning_rate": 4.6820409609673027e-07,
+      "logits/chosen": -2.895726203918457,
+      "logits/rejected": -2.900205612182617,
+      "logps/chosen": -0.9234002828598022,
+      "logps/rejected": -0.938769519329071,
+      "loss": -0.0216,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.9234002828598022,
+      "rewards/margins": 0.015369323082268238,
+      "rewards/rejected": -0.938769519329071,
+      "step": 115
+    },
+    {
+      "epoch": 0.2572002679169457,
+      "grad_norm": 9.77719736840442,
+      "learning_rate": 4.6347771353493173e-07,
+      "logits/chosen": -2.9136910438537598,
+      "logits/rejected": -2.930593252182007,
+      "logps/chosen": -0.9225813150405884,
+      "logps/rejected": -0.917222797870636,
+      "loss": -0.0168,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.9225813150405884,
+      "rewards/margins": -0.005358507391065359,
+      "rewards/rejected": -0.917222797870636,
+      "step": 120
+    },
+    {
+      "epoch": 0.2679169457468185,
+      "grad_norm": 10.417139695198527,
+      "learning_rate": 4.5845133640884217e-07,
+      "logits/chosen": -2.898656129837036,
+      "logits/rejected": -2.8996481895446777,
+      "logps/chosen": -0.9208585619926453,
+      "logps/rejected": -0.9479470252990723,
+      "loss": -0.0193,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.9208585619926453,
+      "rewards/margins": 0.0270883496850729,
+      "rewards/rejected": -0.9479470252990723,
+      "step": 125
+    },
+    {
+      "epoch": 0.27863362357669125,
+      "grad_norm": 11.697763554027677,
+      "learning_rate": 4.531320281528169e-07,
+      "logits/chosen": -2.9303739070892334,
+      "logits/rejected": -2.9241251945495605,
+      "logps/chosen": -0.9110285639762878,
+      "logps/rejected": -0.9726667404174805,
+      "loss": -0.0227,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.9110285639762878,
+      "rewards/margins": 0.06163806468248367,
+      "rewards/rejected": -0.9726667404174805,
+      "step": 130
+    },
+    {
+      "epoch": 0.289350301406564,
+      "grad_norm": 11.144902490144164,
+      "learning_rate": 4.4752726384955293e-07,
+      "logits/chosen": -2.9404830932617188,
+      "logits/rejected": -2.947343111038208,
+      "logps/chosen": -0.9363792538642883,
+      "logps/rejected": -0.9744508862495422,
+      "loss": -0.0195,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.9363792538642883,
+      "rewards/margins": 0.03807152435183525,
+      "rewards/rejected": -0.9744508862495422,
+      "step": 135
+    },
+    {
+      "epoch": 0.3000669792364367,
+      "grad_norm": 11.921950508886521,
+      "learning_rate": 4.4164491972555294e-07,
+      "logits/chosen": -2.8976681232452393,
+      "logits/rejected": -2.898315906524658,
+      "logps/chosen": -0.9219269752502441,
+      "logps/rejected": -0.9640507698059082,
+      "loss": -0.0259,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.9219269752502441,
+      "rewards/margins": 0.04212382063269615,
+      "rewards/rejected": -0.9640507698059082,
+      "step": 140
+    },
+    {
+      "epoch": 0.31078365706630945,
+      "grad_norm": 13.471098814591679,
+      "learning_rate": 4.3549326208287393e-07,
+      "logits/chosen": -2.9619617462158203,
+      "logits/rejected": -2.9274888038635254,
+      "logps/chosen": -0.9526675343513489,
+      "logps/rejected": -1.007359266281128,
+      "loss": -0.0355,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.9526675343513489,
+      "rewards/margins": 0.0546916201710701,
+      "rewards/rejected": -1.007359266281128,
+      "step": 145
+    },
+    {
+      "epoch": 0.3215003348961822,
+      "grad_norm": 12.766870563789471,
+      "learning_rate": 4.2908093568271187e-07,
+      "logits/chosen": -2.9606540203094482,
+      "logits/rejected": -2.9726486206054688,
+      "logps/chosen": -0.9194633364677429,
+      "logps/rejected": -0.9709534645080566,
+      "loss": -0.0371,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.9194633364677429,
+      "rewards/margins": 0.05149019509553909,
+      "rewards/rejected": -0.9709534645080566,
+      "step": 150
+    },
+    {
+      "epoch": 0.3322170127260549,
+      "grad_norm": 12.28868938106036,
+      "learning_rate": 4.2241695159714843e-07,
+      "logits/chosen": -3.043233633041382,
+      "logits/rejected": -3.0254199504852295,
+      "logps/chosen": -0.9320791959762573,
+      "logps/rejected": -0.9708281755447388,
+      "loss": -0.0274,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.9320791959762573,
+      "rewards/margins": 0.038748957216739655,
+      "rewards/rejected": -0.9708281755447388,
+      "step": 155
+    },
+    {
+      "epoch": 0.34293369055592765,
+      "grad_norm": 16.212616603609877,
+      "learning_rate": 4.1551067454613074e-07,
+      "logits/chosen": -2.943720579147339,
+      "logits/rejected": -2.928537607192993,
+      "logps/chosen": -0.921017050743103,
+      "logps/rejected": -0.9882138967514038,
+      "loss": -0.0359,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.921017050743103,
+      "rewards/margins": 0.06719694286584854,
+      "rewards/rejected": -0.9882138967514038,
+      "step": 160
+    },
+    {
+      "epoch": 0.3536503683858004,
+      "grad_norm": 15.792664568932544,
+      "learning_rate": 4.0837180973747876e-07,
+      "logits/chosen": -3.0298409461975098,
+      "logits/rejected": -3.0069289207458496,
+      "logps/chosen": -0.9174503087997437,
+      "logps/rejected": -0.9726656079292297,
+      "loss": -0.0342,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.9174503087997437,
+      "rewards/margins": 0.05521535873413086,
+      "rewards/rejected": -0.9726656079292297,
+      "step": 165
+    },
+    {
+      "epoch": 0.3643670462156731,
+      "grad_norm": 10.787104513853434,
+      "learning_rate": 4.0101038922841455e-07,
+      "logits/chosen": -2.968522548675537,
+      "logits/rejected": -2.9739623069763184,
+      "logps/chosen": -0.9771224856376648,
+      "logps/rejected": -1.0373847484588623,
+      "loss": -0.0466,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.9771224856376648,
+      "rewards/margins": 0.06026231497526169,
+      "rewards/rejected": -1.0373847484588623,
+      "step": 170
+    },
+    {
+      "epoch": 0.3750837240455459,
+      "grad_norm": 13.17830711446757,
+      "learning_rate": 3.934367578277783e-07,
+      "logits/chosen": -2.9837470054626465,
+      "logits/rejected": -2.9880218505859375,
+      "logps/chosen": -1.000985026359558,
+      "logps/rejected": -1.0419981479644775,
+      "loss": -0.0381,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.000985026359558,
+      "rewards/margins": 0.041013117879629135,
+      "rewards/rejected": -1.0419981479644775,
+      "step": 175
+    },
+    {
+      "epoch": 0.38580040187541864,
+      "grad_norm": 14.72612824066128,
+      "learning_rate": 3.856615585587434e-07,
+      "logits/chosen": -2.9933667182922363,
+      "logits/rejected": -3.0047719478607178,
+      "logps/chosen": -1.0653940439224243,
+      "logps/rejected": -1.1087322235107422,
+      "loss": -0.0319,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.0653940439224243,
+      "rewards/margins": 0.043338216841220856,
+      "rewards/rejected": -1.1087322235107422,
+      "step": 180
+    },
+    {
+      "epoch": 0.3965170797052914,
+      "grad_norm": 13.397319728202142,
+      "learning_rate": 3.776957177024581e-07,
+      "logits/chosen": -2.9812378883361816,
+      "logits/rejected": -2.9986259937286377,
+      "logps/chosen": -1.0316323041915894,
+      "logps/rejected": -1.0616919994354248,
+      "loss": -0.033,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.0316323041915894,
+      "rewards/margins": 0.030059661716222763,
+      "rewards/rejected": -1.0616919994354248,
+      "step": 185
+    },
+    {
+      "epoch": 0.4072337575351641,
+      "grad_norm": 14.059564629221503,
+      "learning_rate": 3.69550429443633e-07,
+      "logits/chosen": -2.8960227966308594,
+      "logits/rejected": -2.913637161254883,
+      "logps/chosen": -1.012019157409668,
+      "logps/rejected": -1.215237021446228,
+      "loss": -0.0447,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.012019157409668,
+      "rewards/margins": 0.20321793854236603,
+      "rewards/rejected": -1.215237021446228,
+      "step": 190
+    },
+    {
+      "epoch": 0.41795043536503684,
+      "grad_norm": 14.952805148520074,
+      "learning_rate": 3.612371401396502e-07,
+      "logits/chosen": -3.007540225982666,
+      "logits/rejected": -2.9880564212799072,
+      "logps/chosen": -1.0366661548614502,
+      "logps/rejected": -1.0943790674209595,
+      "loss": -0.0368,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.0366661548614502,
+      "rewards/margins": 0.05771297961473465,
+      "rewards/rejected": -1.0943790674209595,
+      "step": 195
+    },
+    {
+      "epoch": 0.4286671131949096,
+      "grad_norm": 14.170450035952996,
+      "learning_rate": 3.527675322353011e-07,
+      "logits/chosen": -2.922602415084839,
+      "logits/rejected": -2.9457390308380127,
+      "logps/chosen": -0.970561146736145,
+      "logps/rejected": -1.0033313035964966,
+      "loss": -0.0473,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.970561146736145,
+      "rewards/margins": 0.032770149409770966,
+      "rewards/rejected": -1.0033313035964966,
+      "step": 200
+    },
+    {
+      "epoch": 0.4393837910247823,
+      "grad_norm": 15.488305590600266,
+      "learning_rate": 3.4415350784575693e-07,
+      "logits/chosen": -3.004404067993164,
+      "logits/rejected": -2.9882898330688477,
+      "logps/chosen": -0.9613070487976074,
+      "logps/rejected": -1.0151447057724,
+      "loss": -0.044,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.9613070487976074,
+      "rewards/margins": 0.053837649524211884,
+      "rewards/rejected": -1.0151447057724,
+      "step": 205
+    },
+    {
+      "epoch": 0.45010046885465504,
+      "grad_norm": 13.543647630514888,
+      "learning_rate": 3.3540717203084224e-07,
+      "logits/chosen": -2.956808090209961,
+      "logits/rejected": -2.9622817039489746,
+      "logps/chosen": -0.9170505404472351,
+      "logps/rejected": -0.9884069561958313,
+      "loss": -0.0514,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.9170505404472351,
+      "rewards/margins": 0.07135634124279022,
+      "rewards/rejected": -0.9884069561958313,
+      "step": 210
+    },
+    {
+      "epoch": 0.4608171466845278,
+      "grad_norm": 14.183913951293327,
+      "learning_rate": 3.265408157841153e-07,
+      "logits/chosen": -2.9880080223083496,
+      "logits/rejected": -2.986445903778076,
+      "logps/chosen": -0.9918737411499023,
+      "logps/rejected": -1.0286133289337158,
+      "loss": -0.0446,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.9918737411499023,
+      "rewards/margins": 0.036739740520715714,
+      "rewards/rejected": -1.0286133289337158,
+      "step": 215
+    },
+    {
+      "epoch": 0.4715338245144005,
+      "grad_norm": 16.687524624276737,
+      "learning_rate": 3.1756689876066187e-07,
+      "logits/chosen": -2.940664291381836,
+      "logits/rejected": -2.940556049346924,
+      "logps/chosen": -0.9756967425346375,
+      "logps/rejected": -1.0494619607925415,
+      "loss": -0.0496,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.9756967425346375,
+      "rewards/margins": 0.07376515865325928,
+      "rewards/rejected": -1.0494619607925415,
+      "step": 220
+    },
+    {
+      "epoch": 0.4822505023442733,
+      "grad_norm": 15.106449736541158,
+      "learning_rate": 3.0849803176787184e-07,
+      "logits/chosen": -2.978764057159424,
+      "logits/rejected": -3.02057147026062,
+      "logps/chosen": -0.9675830602645874,
+      "logps/rejected": -1.0782067775726318,
+      "loss": -0.0528,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.9675830602645874,
+      "rewards/margins": 0.11062371730804443,
+      "rewards/rejected": -1.0782067775726318,
+      "step": 225
+    },
+    {
+      "epoch": 0.49296718017414604,
+      "grad_norm": 19.732642999044504,
+      "learning_rate": 2.993469590438065e-07,
+      "logits/chosen": -2.91894793510437,
+      "logits/rejected": -2.92021107673645,
+      "logps/chosen": -0.9777423143386841,
+      "logps/rejected": -1.0478038787841797,
+      "loss": -0.0487,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.9777423143386841,
+      "rewards/margins": 0.07006163895130157,
+      "rewards/rejected": -1.0478038787841797,
+      "step": 230
+    },
+    {
+      "epoch": 0.5036838580040187,
+      "grad_norm": 15.470655040763342,
+      "learning_rate": 2.901265403480597e-07,
+      "logits/chosen": -2.868373394012451,
+      "logits/rejected": -2.8822197914123535,
+      "logps/chosen": -0.9746626615524292,
+      "logps/rejected": -1.109392523765564,
+      "loss": -0.0532,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.9746626615524292,
+      "rewards/margins": 0.13472989201545715,
+      "rewards/rejected": -1.109392523765564,
+      "step": 235
+    },
+    {
+      "epoch": 0.5144005358338914,
+      "grad_norm": 18.117444556863273,
+      "learning_rate": 2.808497328902782e-07,
+      "logits/chosen": -2.829807758331299,
+      "logits/rejected": -2.837395191192627,
+      "logps/chosen": -1.0404951572418213,
+      "logps/rejected": -1.1007030010223389,
+      "loss": -0.0453,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.0404951572418213,
+      "rewards/margins": 0.06020789220929146,
+      "rewards/rejected": -1.1007030010223389,
+      "step": 240
+    },
+    {
+      "epoch": 0.5251172136637642,
+      "grad_norm": 18.628678925066545,
+      "learning_rate": 2.715295731217396e-07,
+      "logits/chosen": -2.81119966506958,
+      "logits/rejected": -2.815497636795044,
+      "logps/chosen": -1.0369569063186646,
+      "logps/rejected": -1.112683892250061,
+      "loss": -0.0596,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.0369569063186646,
+      "rewards/margins": 0.07572712004184723,
+      "rewards/rejected": -1.112683892250061,
+      "step": 245
+    },
+    {
+      "epoch": 0.535833891493637,
+      "grad_norm": 17.866763466807367,
+      "learning_rate": 2.621791584155725e-07,
+      "logits/chosen": -2.83900785446167,
+      "logits/rejected": -2.859978199005127,
+      "logps/chosen": -1.1023390293121338,
+      "logps/rejected": -1.1515963077545166,
+      "loss": -0.0617,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.1023390293121338,
+      "rewards/margins": 0.04925721138715744,
+      "rewards/rejected": -1.1515963077545166,
+      "step": 250
+    },
+    {
+      "epoch": 0.5465505693235098,
+      "grad_norm": 17.478329638335467,
+      "learning_rate": 2.52811628661367e-07,
+      "logits/chosen": -2.8983101844787598,
+      "logits/rejected": -2.908754348754883,
+      "logps/chosen": -1.0288032293319702,
+      "logps/rejected": -1.134108066558838,
+      "loss": -0.0631,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.0288032293319702,
+      "rewards/margins": 0.10530481487512589,
+      "rewards/rejected": -1.134108066558838,
+      "step": 255
+    },
+    {
+      "epoch": 0.5572672471533825,
+      "grad_norm": 15.049601923886751,
+      "learning_rate": 2.434401478000353e-07,
+      "logits/chosen": -2.8707144260406494,
+      "logits/rejected": -2.876131057739258,
+      "logps/chosen": -1.0516908168792725,
+      "logps/rejected": -1.1409704685211182,
+      "loss": -0.0405,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.0516908168792725,
+      "rewards/margins": 0.08927960693836212,
+      "rewards/rejected": -1.1409704685211182,
+      "step": 260
+    },
+    {
+      "epoch": 0.5679839249832552,
+      "grad_norm": 14.714810625194913,
+      "learning_rate": 2.340778853248759e-07,
+      "logits/chosen": -2.86578369140625,
+      "logits/rejected": -2.8567006587982178,
+      "logps/chosen": -1.0564695596694946,
+      "logps/rejected": -1.199512004852295,
+      "loss": -0.0695,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.0564695596694946,
+      "rewards/margins": 0.143042653799057,
+      "rewards/rejected": -1.199512004852295,
+      "step": 265
+    },
+    {
+      "epoch": 0.578700602813128,
+      "grad_norm": 18.053080062016626,
+      "learning_rate": 2.247379977748339e-07,
+      "logits/chosen": -2.9022586345672607,
+      "logits/rejected": -2.9201228618621826,
+      "logps/chosen": -1.0403499603271484,
+      "logps/rejected": -1.1265124082565308,
+      "loss": -0.0673,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.0403499603271484,
+      "rewards/margins": 0.08616244792938232,
+      "rewards/rejected": -1.1265124082565308,
+      "step": 270
+    },
+    {
+      "epoch": 0.5894172806430007,
+      "grad_norm": 14.15566071309189,
+      "learning_rate": 2.1543361024596573e-07,
+      "logits/chosen": -2.9802463054656982,
+      "logits/rejected": -2.988959550857544,
+      "logps/chosen": -0.9800962209701538,
+      "logps/rejected": -1.0803120136260986,
+      "loss": -0.0525,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.9800962209701538,
+      "rewards/margins": 0.1002158373594284,
+      "rewards/rejected": -1.0803120136260986,
+      "step": 275
+    },
+    {
+      "epoch": 0.6001339584728734,
+      "grad_norm": 17.56629547418189,
+      "learning_rate": 2.0617779794709082e-07,
+      "logits/chosen": -2.8620638847351074,
+      "logits/rejected": -2.8629186153411865,
+      "logps/chosen": -1.103073000907898,
+      "logps/rejected": -1.196608304977417,
+      "loss": -0.0632,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.103073000907898,
+      "rewards/margins": 0.09353531897068024,
+      "rewards/rejected": -1.196608304977417,
+      "step": 280
+    },
+    {
+      "epoch": 0.6108506363027462,
+      "grad_norm": 15.46761530330997,
+      "learning_rate": 1.9698356782554687e-07,
+      "logits/chosen": -2.9556548595428467,
+      "logits/rejected": -2.9529995918273926,
+      "logps/chosen": -1.109263300895691,
+      "logps/rejected": -1.1731650829315186,
+      "loss": -0.0564,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.109263300895691,
+      "rewards/margins": 0.06390184909105301,
+      "rewards/rejected": -1.1731650829315186,
+      "step": 285
+    },
+    {
+      "epoch": 0.6215673141326189,
+      "grad_norm": 16.8037567850014,
+      "learning_rate": 1.878638402888727e-07,
+      "logits/chosen": -2.9940178394317627,
+      "logits/rejected": -2.9970099925994873,
+      "logps/chosen": -0.9795197248458862,
+      "logps/rejected": -1.0418453216552734,
+      "loss": -0.0668,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.9795197248458862,
+      "rewards/margins": 0.06232555955648422,
+      "rewards/rejected": -1.0418453216552734,
+      "step": 290
+    },
+    {
+      "epoch": 0.6322839919624916,
+      "grad_norm": 16.60747046570722,
+      "learning_rate": 1.7883143104810223e-07,
+      "logits/chosen": -2.9415085315704346,
+      "logits/rejected": -2.9544739723205566,
+      "logps/chosen": -1.069349765777588,
+      "logps/rejected": -1.2494323253631592,
+      "loss": -0.0739,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.069349765777588,
+      "rewards/margins": 0.1800827533006668,
+      "rewards/rejected": -1.2494323253631592,
+      "step": 295
+    },
+    {
+      "epoch": 0.6430006697923644,
+      "grad_norm": 15.422934564351497,
+      "learning_rate": 1.6989903310818548e-07,
+      "logits/chosen": -3.0317275524139404,
+      "logits/rejected": -3.049278736114502,
+      "logps/chosen": -1.0510863065719604,
+      "logps/rejected": -1.1395753622055054,
+      "loss": -0.0517,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.0510863065719604,
+      "rewards/margins": 0.08848897367715836,
+      "rewards/rejected": -1.1395753622055054,
+      "step": 300
+    },
+    {
+      "epoch": 0.6537173476222371,
+      "grad_norm": 17.56071364411128,
+      "learning_rate": 1.610791989308466e-07,
+      "logits/chosen": -2.9625232219696045,
+      "logits/rejected": -2.9480719566345215,
+      "logps/chosen": -1.0810691118240356,
+      "logps/rejected": -1.1474703550338745,
+      "loss": -0.0711,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.0810691118240356,
+      "rewards/margins": 0.06640125066041946,
+      "rewards/rejected": -1.1474703550338745,
+      "step": 305
+    },
+    {
+      "epoch": 0.6644340254521098,
+      "grad_norm": 19.400175535223255,
+      "learning_rate": 1.523843227949418e-07,
+      "logits/chosen": -3.0158638954162598,
+      "logits/rejected": -3.014029026031494,
+      "logps/chosen": -1.054434061050415,
+      "logps/rejected": -1.1792142391204834,
+      "loss": -0.0711,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.054434061050415,
+      "rewards/margins": 0.12478007376194,
+      "rewards/rejected": -1.1792142391204834,
+      "step": 310
+    },
+    {
+      "epoch": 0.6751507032819826,
+      "grad_norm": 18.063066961906372,
+      "learning_rate": 1.4382662337910867e-07,
+      "logits/chosen": -2.9173102378845215,
+      "logits/rejected": -2.9165103435516357,
+      "logps/chosen": -1.1207153797149658,
+      "logps/rejected": -1.2241191864013672,
+      "loss": -0.0582,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.1207153797149658,
+      "rewards/margins": 0.10340368747711182,
+      "rewards/rejected": -1.2241191864013672,
+      "step": 315
+    },
+    {
+      "epoch": 0.6858673811118553,
+      "grad_norm": 17.656998709658712,
+      "learning_rate": 1.3541812659118104e-07,
+      "logits/chosen": -2.924173593521118,
+      "logits/rejected": -2.901674747467041,
+      "logps/chosen": -1.1284511089324951,
+      "logps/rejected": -1.2347378730773926,
+      "loss": -0.0605,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.1284511089324951,
+      "rewards/margins": 0.10628668963909149,
+      "rewards/rejected": -1.2347378730773926,
+      "step": 320
+    },
+    {
+      "epoch": 0.696584058941728,
+      "grad_norm": 17.674353062779463,
+      "learning_rate": 1.2717064866850047e-07,
+      "logits/chosen": -2.9069221019744873,
+      "logits/rejected": -2.9266703128814697,
+      "logps/chosen": -1.0635308027267456,
+      "logps/rejected": -1.2443645000457764,
+      "loss": -0.076,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.0635308027267456,
+      "rewards/margins": 0.1808338761329651,
+      "rewards/rejected": -1.2443645000457764,
+      "step": 325
+    },
+    {
+      "epoch": 0.7073007367716008,
+      "grad_norm": 16.692003983984833,
+      "learning_rate": 1.1909577957287117e-07,
+      "logits/chosen": -2.9482550621032715,
+      "logits/rejected": -2.9383444786071777,
+      "logps/chosen": -1.1115998029708862,
+      "logps/rejected": -1.269691824913025,
+      "loss": -0.0875,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.1115998029708862,
+      "rewards/margins": 0.15809203684329987,
+      "rewards/rejected": -1.269691824913025,
+      "step": 330
+    },
+    {
+      "epoch": 0.7180174146014735,
+      "grad_norm": 16.832132437998734,
+      "learning_rate": 1.1120486670349441e-07,
+      "logits/chosen": -3.0022974014282227,
+      "logits/rejected": -3.0084285736083984,
+      "logps/chosen": -1.1076433658599854,
+      "logps/rejected": -1.2158708572387695,
+      "loss": -0.0631,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.1076433658599854,
+      "rewards/margins": 0.10822737216949463,
+      "rewards/rejected": -1.2158708572387695,
+      "step": 335
+    },
+    {
+      "epoch": 0.7287340924313462,
+      "grad_norm": 16.746834458412387,
+      "learning_rate": 1.0350899895076948e-07,
+      "logits/chosen": -2.960242509841919,
+      "logits/rejected": -2.9590137004852295,
+      "logps/chosen": -1.1618969440460205,
+      "logps/rejected": -1.3054715394973755,
+      "loss": -0.063,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.1618969440460205,
+      "rewards/margins": 0.14357443153858185,
+      "rewards/rejected": -1.3054715394973755,
+      "step": 340
+    },
+    {
+      "epoch": 0.739450770261219,
+      "grad_norm": 16.472011449839883,
+      "learning_rate": 9.601899111337069e-08,
+      "logits/chosen": -2.9834091663360596,
+      "logits/rejected": -2.9975457191467285,
+      "logps/chosen": -1.126527190208435,
+      "logps/rejected": -1.3144327402114868,
+      "loss": -0.0777,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.126527190208435,
+      "rewards/margins": 0.18790560960769653,
+      "rewards/rejected": -1.3144327402114868,
+      "step": 345
+    },
+    {
+      "epoch": 0.7501674480910918,
+      "grad_norm": 17.215572295903478,
+      "learning_rate": 8.874536870049765e-08,
+      "logits/chosen": -2.969423294067383,
+      "logits/rejected": -2.969230890274048,
+      "logps/chosen": -1.127826452255249,
+      "logps/rejected": -1.271470069885254,
+      "loss": -0.0752,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.127826452255249,
+      "rewards/margins": 0.1436435580253601,
+      "rewards/rejected": -1.271470069885254,
+      "step": 350
+    },
+    {
+      "epoch": 0.7608841259209645,
+      "grad_norm": 19.033752258043222,
+      "learning_rate": 8.169835314065663e-08,
+      "logits/chosen": -2.9632205963134766,
+      "logits/rejected": -2.9548847675323486,
+      "logps/chosen": -1.1820392608642578,
+      "logps/rejected": -1.2698676586151123,
+      "loss": -0.0668,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.1820392608642578,
+      "rewards/margins": 0.08782847970724106,
+      "rewards/rejected": -1.2698676586151123,
+      "step": 355
+    },
+    {
+      "epoch": 0.7716008037508373,
+      "grad_norm": 17.064795493163807,
+      "learning_rate": 7.488784741775816e-08,
+      "logits/chosen": -2.976466178894043,
+      "logits/rejected": -2.9713473320007324,
+      "logps/chosen": -1.054131269454956,
+      "logps/rejected": -1.1566834449768066,
+      "loss": -0.0663,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.054131269454956,
+      "rewards/margins": 0.10255217552185059,
+      "rewards/rejected": -1.1566834449768066,
+      "step": 360
+    },
+    {
+      "epoch": 0.78231748158071,
+      "grad_norm": 17.868156711335107,
+      "learning_rate": 6.832342215471715e-08,
+      "logits/chosen": -2.98659086227417,
+      "logits/rejected": -2.986912488937378,
+      "logps/chosen": -1.1025737524032593,
+      "logps/rejected": -1.242219090461731,
+      "loss": -0.0823,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.1025737524032593,
+      "rewards/margins": 0.13964524865150452,
+      "rewards/rejected": -1.242219090461731,
+      "step": 365
+    },
+    {
+      "epoch": 0.7930341594105828,
+      "grad_norm": 24.714287844140323,
+      "learning_rate": 6.201430216410985e-08,
+      "logits/chosen": -3.0034446716308594,
+      "logits/rejected": -3.001354217529297,
+      "logps/chosen": -1.1147871017456055,
+      "logps/rejected": -1.200984239578247,
+      "loss": -0.081,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.1147871017456055,
+      "rewards/margins": 0.08619704097509384,
+      "rewards/rejected": -1.200984239578247,
+      "step": 370
+    },
+    {
+      "epoch": 0.8037508372404555,
+      "grad_norm": 20.89166421346418,
+      "learning_rate": 5.596935348479029e-08,
+      "logits/chosen": -2.9388105869293213,
+      "logits/rejected": -2.948521137237549,
+      "logps/chosen": -1.2442572116851807,
+      "logps/rejected": -1.393157958984375,
+      "loss": -0.0861,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2442572116851807,
+      "rewards/margins": 0.1489005833864212,
+      "rewards/rejected": -1.393157958984375,
+      "step": 375
+    },
+    {
+      "epoch": 0.8144675150703282,
+      "grad_norm": 22.406198534031674,
+      "learning_rate": 5.01970709226808e-08,
+      "logits/chosen": -3.028179883956909,
+      "logits/rejected": -3.009589672088623,
+      "logps/chosen": -1.0915172100067139,
+      "logps/rejected": -1.243607997894287,
+      "loss": -0.0837,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.0915172100067139,
+      "rewards/margins": 0.1520906686782837,
+      "rewards/rejected": -1.243607997894287,
+      "step": 380
+    },
+    {
+      "epoch": 0.825184192900201,
+      "grad_norm": 24.898962036749563,
+      "learning_rate": 4.4705566113246635e-08,
+      "logits/chosen": -2.9715943336486816,
+      "logits/rejected": -2.949638605117798,
+      "logps/chosen": -1.228461503982544,
+      "logps/rejected": -1.3865272998809814,
+      "loss": -0.0897,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.228461503982544,
+      "rewards/margins": 0.15806564688682556,
+      "rewards/rejected": -1.3865272998809814,
+      "step": 385
+    },
+    {
+      "epoch": 0.8359008707300737,
+      "grad_norm": 19.449984589836117,
+      "learning_rate": 3.950255612243042e-08,
+      "logits/chosen": -2.951904058456421,
+      "logits/rejected": -2.928208827972412,
+      "logps/chosen": -1.2001017332077026,
+      "logps/rejected": -1.3440535068511963,
+      "loss": -0.0724,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2001017332077026,
+      "rewards/margins": 0.1439516693353653,
+      "rewards/rejected": -1.3440535068511963,
+      "step": 390
+    },
+    {
+      "epoch": 0.8466175485599464,
+      "grad_norm": 19.857222600639766,
+      "learning_rate": 3.459535260206384e-08,
+      "logits/chosen": -2.9977169036865234,
+      "logits/rejected": -3.020946979522705,
+      "logps/chosen": -1.096103549003601,
+      "logps/rejected": -1.21526300907135,
+      "loss": -0.0898,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.096103549003601,
+      "rewards/margins": 0.11915931850671768,
+      "rewards/rejected": -1.21526300907135,
+      "step": 395
+    },
+    {
+      "epoch": 0.8573342263898192,
+      "grad_norm": 21.73532568075866,
+      "learning_rate": 2.999085151499764e-08,
+      "logits/chosen": -2.9534425735473633,
+      "logits/rejected": -2.9359939098358154,
+      "logps/chosen": -1.0908204317092896,
+      "logps/rejected": -1.356081485748291,
+      "loss": -0.0905,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.0908204317092896,
+      "rewards/margins": 0.2652612626552582,
+      "rewards/rejected": -1.356081485748291,
+      "step": 400
+    },
+    {
+      "epoch": 0.8573342263898192,
+      "eval_logits/chosen": -2.8999884128570557,
+      "eval_logits/rejected": -2.892754316329956,
+      "eval_logps/chosen": -1.1346372365951538,
+      "eval_logps/rejected": -1.2466005086898804,
+      "eval_loss": -0.07080772519111633,
+      "eval_rewards/accuracies": 0.5757978558540344,
+      "eval_rewards/chosen": -1.1346372365951538,
+      "eval_rewards/margins": 0.11196334660053253,
+      "eval_rewards/rejected": -1.2466005086898804,
+      "eval_runtime": 432.4273,
+      "eval_samples_per_second": 6.924,
+      "eval_steps_per_second": 0.435,
+      "step": 400
+    },
+    {
+      "epoch": 0.8680509042196919,
+      "grad_norm": 19.440021908588918,
+      "learning_rate": 2.5695523444387747e-08,
+      "logits/chosen": -2.9388387203216553,
+      "logits/rejected": -2.9628615379333496,
+      "logps/chosen": -1.111882209777832,
+      "logps/rejected": -1.2351758480072021,
+      "loss": -0.09,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.111882209777832,
+      "rewards/margins": 0.12329357862472534,
+      "rewards/rejected": -1.2351758480072021,
+      "step": 405
+    },
+    {
+      "epoch": 0.8787675820495646,
+      "grad_norm": 19.454852119565878,
+      "learning_rate": 2.1715404500756875e-08,
+      "logits/chosen": -2.98136305809021,
+      "logits/rejected": -3.0059518814086914,
+      "logps/chosen": -1.090047836303711,
+      "logps/rejected": -1.203460693359375,
+      "loss": -0.0823,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.090047836303711,
+      "rewards/margins": 0.11341279745101929,
+      "rewards/rejected": -1.203460693359375,
+      "step": 410
+    },
+    {
+      "epoch": 0.8894842598794374,
+      "grad_norm": 27.119138813457198,
+      "learning_rate": 1.8056087839608546e-08,
+      "logits/chosen": -2.9424166679382324,
+      "logits/rejected": -2.9254748821258545,
+      "logps/chosen": -1.1260377168655396,
+      "logps/rejected": -1.3184921741485596,
+      "loss": -0.1086,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.1260377168655396,
+      "rewards/margins": 0.19245454668998718,
+      "rewards/rejected": -1.3184921741485596,
+      "step": 415
+    },
+    {
+      "epoch": 0.9002009377093101,
+      "grad_norm": 19.3060602023721,
+      "learning_rate": 1.4722715801513935e-08,
+      "logits/chosen": -2.9605276584625244,
+      "logits/rejected": -2.967475414276123,
+      "logps/chosen": -1.0344493389129639,
+      "logps/rejected": -1.0916310548782349,
+      "loss": -0.0802,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.0344493389129639,
+      "rewards/margins": 0.057181768119335175,
+      "rewards/rejected": -1.0916310548782349,
+      "step": 420
+    },
+    {
+      "epoch": 0.9109176155391828,
+      "grad_norm": 20.048601077954377,
+      "learning_rate": 1.1719972685717395e-08,
+      "logits/chosen": -2.970942258834839,
+      "logits/rejected": -2.9757351875305176,
+      "logps/chosen": -1.0653189420700073,
+      "logps/rejected": -1.3184258937835693,
+      "loss": -0.1043,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.0653189420700073,
+      "rewards/margins": 0.2531070411205292,
+      "rewards/rejected": -1.3184258937835693,
+      "step": 425
+    },
+    {
+      "epoch": 0.9216342933690556,
+      "grad_norm": 21.66418470338373,
+      "learning_rate": 9.05207816741485e-09,
+      "logits/chosen": -2.939943313598633,
+      "logits/rejected": -2.9273343086242676,
+      "logps/chosen": -1.2503459453582764,
+      "logps/rejected": -1.4009029865264893,
+      "loss": -0.1013,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2503459453582764,
+      "rewards/margins": 0.15055687725543976,
+      "rewards/rejected": -1.4009029865264893,
+      "step": 430
+    },
+    {
+      "epoch": 0.9323509711989283,
+      "grad_norm": 21.774320932145653,
+      "learning_rate": 6.722781367956481e-09,
+      "logits/chosen": -3.0610954761505127,
+      "logits/rejected": -3.0484843254089355,
+      "logps/chosen": -1.0789310932159424,
+      "logps/rejected": -1.2742266654968262,
+      "loss": -0.1009,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.0789310932159424,
+      "rewards/margins": 0.19529545307159424,
+      "rewards/rejected": -1.2742266654968262,
+      "step": 435
+    },
+    {
+      "epoch": 0.943067649028801,
+      "grad_norm": 19.319233646112725,
+      "learning_rate": 4.735355586305861e-09,
+      "logits/chosen": -2.94116473197937,
+      "logits/rejected": -2.949489116668701,
+      "logps/chosen": -1.1311179399490356,
+      "logps/rejected": -1.3173315525054932,
+      "loss": -0.0993,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.1311179399490356,
+      "rewards/margins": 0.1862136423587799,
+      "rewards/rejected": -1.3173315525054932,
+      "step": 440
+    },
+    {
+      "epoch": 0.9537843268586738,
+      "grad_norm": 19.556600822562356,
+      "learning_rate": 3.0925936991598454e-09,
+      "logits/chosen": -2.9415977001190186,
+      "logits/rejected": -2.944352388381958,
+      "logps/chosen": -1.1981102228164673,
+      "logps/rejected": -1.329925775527954,
+      "loss": -0.0746,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.1981102228164673,
+      "rewards/margins": 0.1318153738975525,
+      "rewards/rejected": -1.329925775527954,
+      "step": 445
+    },
+    {
+      "epoch": 0.9645010046885466,
+      "grad_norm": 24.43813858171124,
+      "learning_rate": 1.7968042361930225e-09,
+      "logits/chosen": -2.958519697189331,
+      "logits/rejected": -2.9468445777893066,
+      "logps/chosen": -1.1451342105865479,
+      "logps/rejected": -1.265005111694336,
+      "loss": -0.0894,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.1451342105865479,
+      "rewards/margins": 0.11987074464559555,
+      "rewards/rejected": -1.265005111694336,
+      "step": 450
+    },
+    {
+      "epoch": 0.9752176825184193,
+      "grad_norm": 23.495311684580507,
+      "learning_rate": 8.498081359423759e-10,
+      "logits/chosen": -2.9030869007110596,
+      "logits/rejected": -2.932225465774536,
+      "logps/chosen": -1.1186686754226685,
+      "logps/rejected": -1.335756540298462,
+      "loss": -0.1008,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.1186686754226685,
+      "rewards/margins": 0.21708789467811584,
+      "rewards/rejected": -1.335756540298462,
+      "step": 455
+    },
+    {
+      "epoch": 0.9859343603482921,
+      "grad_norm": 20.358552883416746,
+      "learning_rate": 2.5293618689056774e-10,
+      "logits/chosen": -2.9370243549346924,
+      "logits/rejected": -2.953622341156006,
+      "logps/chosen": -1.1718332767486572,
+      "logps/rejected": -1.4025189876556396,
+      "loss": -0.0805,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.1718332767486572,
+      "rewards/margins": 0.230685755610466,
+      "rewards/rejected": -1.4025189876556396,
+      "step": 460
+    },
+    {
+      "epoch": 0.9966510381781648,
+      "grad_norm": 27.657325299971138,
+      "learning_rate": 7.02715734424153e-12,
+      "logits/chosen": -2.9429261684417725,
+      "logits/rejected": -2.951002359390259,
+      "logps/chosen": -1.2534146308898926,
+      "logps/rejected": -1.3702634572982788,
+      "loss": -0.0688,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2534146308898926,
+      "rewards/margins": 0.11684894561767578,
+      "rewards/rejected": -1.3702634572982788,
+      "step": 465
+    },
+    {
+      "epoch": 0.9987943737441393,
+      "step": 466,
+      "total_flos": 0.0,
+      "train_loss": 0.0,
+      "train_runtime": 4.3106,
+      "train_samples_per_second": 13854.197,
+      "train_steps_per_second": 108.105
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 466,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be465117a6d2106022526ee1bd1d741b052c43e8c7af05d0aa7ae9ba7ef3f4b7
+size 7416