Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

all_results.json

@@ -0,0 +1,20 @@
+{
+    "epoch": 0.9998706171561651,
+    "eval_kl": 0.01532898098230362,
+    "eval_logps/chosen": -340.4056263858093,
+    "eval_logps/rejected": -394.53469279661016,
+    "eval_loss": 0.40464144945144653,
+    "eval_rewards/chosen": -0.636441926469824,
+    "eval_rewards/margins": 1.5609318031713308,
+    "eval_rewards/rejected": -2.197373729641155,
+    "eval_runtime": 260.8191,
+    "eval_samples": 1846,
+    "eval_samples_per_second": 7.078,
+    "eval_steps_per_second": 3.539,
+    "total_flos": 0.0,
+    "train_loss": 0.420091498218955,
+    "train_runtime": 6442.4359,
+    "train_samples": 15458,
+    "train_samples_per_second": 2.399,
+    "train_steps_per_second": 0.15
+}

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "experiments/gemma-2b-zephyr-sft/kto-mix-14k-lf-response-f1_0.75-fg",
+  "architectures": [
+    "GemmaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 106,
+  "eos_token_id": 107,
+  "head_dim": 256,
+  "hidden_act": "gelu",
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 16384,
+  "max_position_embeddings": 8192,
+  "model_type": "gemma",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 18,
+  "num_key_value_heads": 1,
+  "pad_token_id": 107,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "torch_dtype": "float16",
+  "transformers_version": "4.41.1",
+  "use_cache": true,
+  "vocab_size": 256000
+}

eval_results.json

@@ -0,0 +1,14 @@
+{
+    "epoch": 0.9998706171561651,
+    "eval_kl": 0.01532898098230362,
+    "eval_logps/chosen": -340.4056263858093,
+    "eval_logps/rejected": -394.53469279661016,
+    "eval_loss": 0.40464144945144653,
+    "eval_rewards/chosen": -0.636441926469824,
+    "eval_rewards/margins": 1.5609318031713308,
+    "eval_rewards/rejected": -2.197373729641155,
+    "eval_runtime": 260.8191,
+    "eval_samples": 1846,
+    "eval_samples_per_second": 7.078,
+    "eval_steps_per_second": 3.539
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 106,
+  "eos_token_id": 107,
+  "pad_token_id": 107,
+  "transformers_version": "4.41.1"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c4aed628a8fe01eb701d36f1f0c0380fb01a849a2ded5faaf846ce7d483d52e
+size 4945242104

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df65b7cd6e4dfa43d9dfae7f6eaf58a8846de61aa3aefa34bb910f7aaf34f5f6
+size 1115697720

model.safetensors.index.json

@@ -0,0 +1,172 @@
+{
+  "metadata": {
+    "total_size": 6060920832
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

runs/Jun15_18-00-51_yme9r2ctr1718003783591-jbt67/events.out.tfevents.1718445923.yme9r2ctr1718003783591-jbt67.24351.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f7303497ebd3ebf7cf2023f7848cd627a067ff707662a1c4edcc56b1311f0e6
+size 61635

runs/Jun15_18-00-51_yme9r2ctr1718003783591-jbt67/events.out.tfevents.1718452635.yme9r2ctr1718003783591-jbt67.24351.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e9ce5a98a4e3e4ffe804211ca56753e60fdac3a8b24a404846d13184b2dffec
+size 696

special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<|im_start|>",
+  "eos_token": "<|im_end|>",
+  "pad_token": "<|im_end|>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:322a5f52ab5cab196761ab397a022d6fa3a2e1418585e532bb6efb2fedd2ae94
+size 17477501

tokenizer_config.json

@@ -0,0 +1,70 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "106": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "legacy": null,
+  "model_max_length": 2048,
+  "pad_token": "<|im_end|>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9998706171561651,
+    "total_flos": 0.0,
+    "train_loss": 0.420091498218955,
+    "train_runtime": 6442.4359,
+    "train_samples": 15458,
+    "train_samples_per_second": 2.399,
+    "train_steps_per_second": 0.15
+}

trainer_state.json

@@ -0,0 +1,1346 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9998706171561651,
+  "eval_steps": 200,
+  "global_step": 966,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010350627506792599,
+      "grad_norm": 36.896687952176364,
+      "kl": 0.006904316134750843,
+      "learning_rate": 2.126812117966759e-07,
+      "logps/chosen": -417.2011954066265,
+      "logps/rejected": -375.1744622564935,
+      "loss": 0.4997,
+      "rewards/chosen": -0.003417713455407016,
+      "rewards/margins": 0.002690252778750596,
+      "rewards/rejected": -0.006107966234157612,
+      "step": 10
+    },
+    {
+      "epoch": 0.020701255013585197,
+      "grad_norm": 36.344980205107255,
+      "kl": 0.004612588789314032,
+      "learning_rate": 3.096603651432316e-07,
+      "logps/chosen": -277.7969021267361,
+      "logps/rejected": -407.0654296875,
+      "loss": 0.4892,
+      "rewards/chosen": -0.04762052165137397,
+      "rewards/margins": 0.07216862355819856,
+      "rewards/rejected": -0.11978914520957253,
+      "step": 20
+    },
+    {
+      "epoch": 0.0310518825203778,
+      "grad_norm": 36.71259965236908,
+      "kl": 0.0,
+      "learning_rate": 3.602235071779947e-07,
+      "logps/chosen": -355.01975574712645,
+      "logps/rejected": -431.1890785530822,
+      "loss": 0.4788,
+      "rewards/chosen": -0.2371558485359981,
+      "rewards/margins": 0.25850485494016306,
+      "rewards/rejected": -0.49566070347616115,
+      "step": 30
+    },
+    {
+      "epoch": 0.041402510027170394,
+      "grad_norm": 38.99807359143627,
+      "kl": 0.016344498842954636,
+      "learning_rate": 3.9466076978545386e-07,
+      "logps/chosen": -347.33896998355266,
+      "logps/rejected": -404.498046875,
+      "loss": 0.4398,
+      "rewards/chosen": -0.5145087995027241,
+      "rewards/margins": 0.5702773885320601,
+      "rewards/rejected": -1.0847861880347842,
+      "step": 40
+    },
+    {
+      "epoch": 0.051753137533962996,
+      "grad_norm": 34.4366373643818,
+      "kl": 0.0,
+      "learning_rate": 4.208077428062608e-07,
+      "logps/chosen": -401.31200610632186,
+      "logps/rejected": -408.78579837328766,
+      "loss": 0.4906,
+      "rewards/chosen": -1.0610687431247754,
+      "rewards/margins": 0.6205490982038848,
+      "rewards/rejected": -1.6816178413286602,
+      "step": 50
+    },
+    {
+      "epoch": 0.0621037650407556,
+      "grad_norm": 40.6191164803455,
+      "kl": 0.0,
+      "learning_rate": 4.4189144263242994e-07,
+      "logps/chosen": -292.2215844131098,
+      "logps/rejected": -431.56860977564105,
+      "loss": 0.4649,
+      "rewards/chosen": -0.6987755007860137,
+      "rewards/margins": 0.5288097293321754,
+      "rewards/rejected": -1.227585230118189,
+      "step": 60
+    },
+    {
+      "epoch": 0.0724543925475482,
+      "grad_norm": 30.15586760876392,
+      "kl": 0.0,
+      "learning_rate": 4.5955828020052655e-07,
+      "logps/chosen": -355.0156035370879,
+      "logps/rejected": -401.8425045289855,
+      "loss": 0.4658,
+      "rewards/chosen": -0.5742165701729911,
+      "rewards/margins": 0.8813798008004576,
+      "rewards/rejected": -1.4555963709734487,
+      "step": 70
+    },
+    {
+      "epoch": 0.08280502005434079,
+      "grad_norm": 28.330817825249255,
+      "kl": 0.0,
+      "learning_rate": 4.7476282570257156e-07,
+      "logps/chosen": -394.8970209478022,
+      "logps/rejected": -387.24026268115944,
+      "loss": 0.4731,
+      "rewards/chosen": -0.7301217383080787,
+      "rewards/margins": 0.8021065933268852,
+      "rewards/rejected": -1.5322283316349639,
+      "step": 80
+    },
+    {
+      "epoch": 0.0931556475611334,
+      "grad_norm": 30.67181137678842,
+      "kl": 0.0,
+      "learning_rate": 4.881082258136016e-07,
+      "logps/chosen": -294.80318509615387,
+      "logps/rejected": -385.4679163490854,
+      "loss": 0.4613,
+      "rewards/chosen": -0.6603363232734876,
+      "rewards/margins": 0.5270779856001309,
+      "rewards/rejected": -1.1874143088736184,
+      "step": 90
+    },
+    {
+      "epoch": 0.10350627506792599,
+      "grad_norm": 31.541642718713373,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -356.5553466796875,
+      "logps/rejected": -427.236279296875,
+      "loss": 0.4466,
+      "rewards/chosen": -0.5922697067260743,
+      "rewards/margins": 0.8280625343322754,
+      "rewards/rejected": -1.4203322410583497,
+      "step": 100
+    },
+    {
+      "epoch": 0.11385690257471859,
+      "grad_norm": 28.188607938438196,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -349.71470997431504,
+      "logps/rejected": -431.29777298850576,
+      "loss": 0.445,
+      "rewards/chosen": -0.7172038457165025,
+      "rewards/margins": 0.5990428885518614,
+      "rewards/rejected": -1.316246734268364,
+      "step": 110
+    },
+    {
+      "epoch": 0.1242075300815112,
+      "grad_norm": 28.777259577988843,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -317.6811767578125,
+      "logps/rejected": -400.8,
+      "loss": 0.4577,
+      "rewards/chosen": -0.748396921157837,
+      "rewards/margins": 0.6573972225189209,
+      "rewards/rejected": -1.4057941436767578,
+      "step": 120
+    },
+    {
+      "epoch": 0.13455815758830378,
+      "grad_norm": 27.073111094683828,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -334.3945529513889,
+      "logps/rejected": -424.9839564732143,
+      "loss": 0.4654,
+      "rewards/chosen": -0.797715589735243,
+      "rewards/margins": 0.9449826437329489,
+      "rewards/rejected": -1.742698233468192,
+      "step": 130
+    },
+    {
+      "epoch": 0.1449087850950964,
+      "grad_norm": 29.19719970356803,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -368.06354166666665,
+      "logps/rejected": -338.69952566964287,
+      "loss": 0.487,
+      "rewards/chosen": -0.6288536071777344,
+      "rewards/margins": 0.4991338457380021,
+      "rewards/rejected": -1.1279874529157365,
+      "step": 140
+    },
+    {
+      "epoch": 0.155259412601889,
+      "grad_norm": 31.37561442050933,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -308.6735341061828,
+      "logps/rejected": -390.50953241604475,
+      "loss": 0.4775,
+      "rewards/chosen": -0.5661141180223034,
+      "rewards/margins": 0.7382167362512507,
+      "rewards/rejected": -1.304330854273554,
+      "step": 150
+    },
+    {
+      "epoch": 0.16561004010868158,
+      "grad_norm": 36.002916810630985,
+      "kl": 0.07424011081457138,
+      "learning_rate": 5e-07,
+      "logps/chosen": -395.24665850903614,
+      "logps/rejected": -402.71707589285717,
+      "loss": 0.4599,
+      "rewards/chosen": -0.5169859277196678,
+      "rewards/margins": 0.6465992892484227,
+      "rewards/rejected": -1.1635852169680905,
+      "step": 160
+    },
+    {
+      "epoch": 0.1759606676154742,
+      "grad_norm": 27.151915007789793,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -380.0856370192308,
+      "logps/rejected": -437.3331269054878,
+      "loss": 0.434,
+      "rewards/chosen": -0.5822516710330279,
+      "rewards/margins": 0.9586351846739081,
+      "rewards/rejected": -1.540886855706936,
+      "step": 170
+    },
+    {
+      "epoch": 0.1863112951222668,
+      "grad_norm": 22.70085892654007,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -352.72755281690144,
+      "logps/rejected": -438.0743504213483,
+      "loss": 0.4081,
+      "rewards/chosen": -0.6838695364938655,
+      "rewards/margins": 0.9942577951862609,
+      "rewards/rejected": -1.6781273316801264,
+      "step": 180
+    },
+    {
+      "epoch": 0.19666192262905938,
+      "grad_norm": 27.14667046044915,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -320.2009880514706,
+      "logps/rejected": -407.538046875,
+      "loss": 0.449,
+      "rewards/chosen": -1.0842503267176011,
+      "rewards/margins": 1.4791047758214615,
+      "rewards/rejected": -2.5633551025390626,
+      "step": 190
+    },
+    {
+      "epoch": 0.20701255013585199,
+      "grad_norm": 27.694967881656005,
+      "kl": 0.005686330609023571,
+      "learning_rate": 5e-07,
+      "logps/chosen": -337.79836856617646,
+      "logps/rejected": -427.5978645833333,
+      "loss": 0.4522,
+      "rewards/chosen": -0.9811132094439339,
+      "rewards/margins": 1.1809233302696076,
+      "rewards/rejected": -2.1620365397135415,
+      "step": 200
+    },
+    {
+      "epoch": 0.20701255013585199,
+      "eval_kl": 0.0010393437696620822,
+      "eval_logps/chosen": -345.2487181263858,
+      "eval_logps/rejected": -393.2139168432203,
+      "eval_loss": 0.44461360573768616,
+      "eval_rewards/chosen": -1.1207509516612388,
+      "eval_rewards/margins": 0.9445489068199584,
+      "eval_rewards/rejected": -2.0652998584811972,
+      "eval_runtime": 261.4133,
+      "eval_samples_per_second": 7.062,
+      "eval_steps_per_second": 3.531,
+      "step": 200
+    },
+    {
+      "epoch": 0.2173631776426446,
+      "grad_norm": 30.464898770807604,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -365.0289713541667,
+      "logps/rejected": -427.5989879261364,
+      "loss": 0.4016,
+      "rewards/chosen": -0.8537895944383409,
+      "rewards/margins": 1.3811903818689213,
+      "rewards/rejected": -2.234979976307262,
+      "step": 210
+    },
+    {
+      "epoch": 0.22771380514943718,
+      "grad_norm": 23.41466055625897,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -345.475933908046,
+      "logps/rejected": -393.412189640411,
+      "loss": 0.4443,
+      "rewards/chosen": -0.9509018645889458,
+      "rewards/margins": 1.2836219608605406,
+      "rewards/rejected": -2.2345238254494864,
+      "step": 220
+    },
+    {
+      "epoch": 0.23806443265622979,
+      "grad_norm": 26.98695760593119,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -323.52855282738096,
+      "logps/rejected": -349.73843544407896,
+      "loss": 0.4764,
+      "rewards/chosen": -0.9678686232793898,
+      "rewards/margins": 0.599584660733254,
+      "rewards/rejected": -1.5674532840126438,
+      "step": 230
+    },
+    {
+      "epoch": 0.2484150601630224,
+      "grad_norm": 26.51688363505412,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -339.72511772260276,
+      "logps/rejected": -392.6373024425287,
+      "loss": 0.4254,
+      "rewards/chosen": -0.7914297548058915,
+      "rewards/margins": 0.7201490352668456,
+      "rewards/rejected": -1.511578790072737,
+      "step": 240
+    },
+    {
+      "epoch": 0.258765687669815,
+      "grad_norm": 38.3631109147077,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -362.05623478084414,
+      "logps/rejected": -361.7480233433735,
+      "loss": 0.4123,
+      "rewards/chosen": -0.5259268129026735,
+      "rewards/margins": 1.3802443193803648,
+      "rewards/rejected": -1.9061711322830384,
+      "step": 250
+    },
+    {
+      "epoch": 0.26911631517660756,
+      "grad_norm": 28.783330131851603,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -318.40223672945206,
+      "logps/rejected": -472.43588362068965,
+      "loss": 0.4135,
+      "rewards/chosen": -0.8065869579576466,
+      "rewards/margins": 1.0660637146918686,
+      "rewards/rejected": -1.8726506726495151,
+      "step": 260
+    },
+    {
+      "epoch": 0.27946694268340017,
+      "grad_norm": 25.596794311830312,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -367.7001139322917,
+      "logps/rejected": -436.5064808238636,
+      "loss": 0.4093,
+      "rewards/chosen": -0.8103501001993815,
+      "rewards/margins": 1.0771059267448657,
+      "rewards/rejected": -1.887456026944247,
+      "step": 270
+    },
+    {
+      "epoch": 0.2898175701901928,
+      "grad_norm": 31.49395928287787,
+      "kl": 0.0206025131046772,
+      "learning_rate": 5e-07,
+      "logps/chosen": -456.68581081081084,
+      "logps/rejected": -399.91547056686045,
+      "loss": 0.4345,
+      "rewards/chosen": -1.0405741511164486,
+      "rewards/margins": 0.9193381711718871,
+      "rewards/rejected": -1.9599123222883357,
+      "step": 280
+    },
+    {
+      "epoch": 0.3001681976969854,
+      "grad_norm": 27.507156588516853,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -354.3474633487654,
+      "logps/rejected": -430.79647943037975,
+      "loss": 0.4246,
+      "rewards/chosen": -0.8167637954523534,
+      "rewards/margins": 1.6583199540531843,
+      "rewards/rejected": -2.4750837495055378,
+      "step": 290
+    },
+    {
+      "epoch": 0.310518825203778,
+      "grad_norm": 27.243120877089865,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -369.0854611280488,
+      "logps/rejected": -411.7598407451923,
+      "loss": 0.4515,
+      "rewards/chosen": -0.9885020372344226,
+      "rewards/margins": 1.07946980364849,
+      "rewards/rejected": -2.0679718408829126,
+      "step": 300
+    },
+    {
+      "epoch": 0.3208694527105706,
+      "grad_norm": 26.41738454716243,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -328.5754642210145,
+      "logps/rejected": -473.32679429945057,
+      "loss": 0.4,
+      "rewards/chosen": -0.8000211853911912,
+      "rewards/margins": 1.4340147676694202,
+      "rewards/rejected": -2.2340359530606113,
+      "step": 310
+    },
+    {
+      "epoch": 0.33122008021736316,
+      "grad_norm": 27.90947815196134,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -379.9532833614865,
+      "logps/rejected": -414.0909792877907,
+      "loss": 0.4206,
+      "rewards/chosen": -0.707832078675966,
+      "rewards/margins": 1.2218697929741975,
+      "rewards/rejected": -1.9297018716501635,
+      "step": 320
+    },
+    {
+      "epoch": 0.34157070772415576,
+      "grad_norm": 34.323093394556274,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -302.920654296875,
+      "logps/rejected": -365.2463107638889,
+      "loss": 0.4599,
+      "rewards/chosen": -0.8200391422618519,
+      "rewards/margins": 1.4710271334407303,
+      "rewards/rejected": -2.2910662757025824,
+      "step": 330
+    },
+    {
+      "epoch": 0.3519213352309484,
+      "grad_norm": 35.00194682599148,
+      "kl": 0.020750045776367188,
+      "learning_rate": 5e-07,
+      "logps/chosen": -370.7525414156627,
+      "logps/rejected": -407.6445819805195,
+      "loss": 0.4401,
+      "rewards/chosen": -0.4423764699912933,
+      "rewards/margins": 0.795768243571511,
+      "rewards/rejected": -1.2381447135628043,
+      "step": 340
+    },
+    {
+      "epoch": 0.362271962737741,
+      "grad_norm": 28.74449923281838,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -369.2329220655488,
+      "logps/rejected": -412.28390424679486,
+      "loss": 0.443,
+      "rewards/chosen": -0.5598751161156631,
+      "rewards/margins": 0.7612275152820732,
+      "rewards/rejected": -1.3211026313977363,
+      "step": 350
+    },
+    {
+      "epoch": 0.3726225902445336,
+      "grad_norm": 33.80893504974849,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -333.06757269965277,
+      "logps/rejected": -476.0582386363636,
+      "loss": 0.3942,
+      "rewards/chosen": -0.8373040093315972,
+      "rewards/margins": 1.5102612081200184,
+      "rewards/rejected": -2.3475652174516157,
+      "step": 360
+    },
+    {
+      "epoch": 0.3829732177513262,
+      "grad_norm": 23.093501234844034,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -344.985234375,
+      "logps/rejected": -478.50422794117645,
+      "loss": 0.3956,
+      "rewards/chosen": -1.264248046875,
+      "rewards/margins": 1.5773571059283087,
+      "rewards/rejected": -2.8416051528033086,
+      "step": 370
+    },
+    {
+      "epoch": 0.39332384525811875,
+      "grad_norm": 23.6165146626171,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -331.19694890202703,
+      "logps/rejected": -458.86123728197674,
+      "loss": 0.4153,
+      "rewards/chosen": -1.1533899049501162,
+      "rewards/margins": 1.4770645798563884,
+      "rewards/rejected": -2.6304544848065046,
+      "step": 380
+    },
+    {
+      "epoch": 0.40367447276491136,
+      "grad_norm": 24.677426766885045,
+      "kl": 0.045375823974609375,
+      "learning_rate": 5e-07,
+      "logps/chosen": -335.65542204483694,
+      "logps/rejected": -439.2108800551471,
+      "loss": 0.4565,
+      "rewards/chosen": -0.7980768784232761,
+      "rewards/margins": 1.7186397981765629,
+      "rewards/rejected": -2.516716676599839,
+      "step": 390
+    },
+    {
+      "epoch": 0.41402510027170397,
+      "grad_norm": 32.96461257238746,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -313.4358512581169,
+      "logps/rejected": -439.3407379518072,
+      "loss": 0.4056,
+      "rewards/chosen": -0.6841482187246347,
+      "rewards/margins": 1.2257567208579143,
+      "rewards/rejected": -1.909904939582549,
+      "step": 400
+    },
+    {
+      "epoch": 0.41402510027170397,
+      "eval_kl": 0.02226920612156391,
+      "eval_logps/chosen": -340.0967987804878,
+      "eval_logps/rejected": -387.47169623940675,
+      "eval_loss": 0.44011881947517395,
+      "eval_rewards/chosen": -0.6055575284090909,
+      "eval_rewards/margins": 0.885514011000999,
+      "eval_rewards/rejected": -1.49107153941009,
+      "eval_runtime": 260.8826,
+      "eval_samples_per_second": 7.076,
+      "eval_steps_per_second": 3.538,
+      "step": 400
+    },
+    {
+      "epoch": 0.4243757277784966,
+      "grad_norm": 26.751958968613145,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -372.82060185185185,
+      "logps/rejected": -399.93740110759495,
+      "loss": 0.4265,
+      "rewards/chosen": -0.5553302058467159,
+      "rewards/margins": 1.1236735458839013,
+      "rewards/rejected": -1.6790037517306171,
+      "step": 410
+    },
+    {
+      "epoch": 0.4347263552852892,
+      "grad_norm": 32.246235152731096,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -348.120418595679,
+      "logps/rejected": -402.0041287579114,
+      "loss": 0.4239,
+      "rewards/chosen": -0.7425044495382427,
+      "rewards/margins": 1.1616015940238618,
+      "rewards/rejected": -1.9041060435621044,
+      "step": 420
+    },
+    {
+      "epoch": 0.44507698279208174,
+      "grad_norm": 27.212254824473547,
+      "kl": 0.04713239520788193,
+      "learning_rate": 5e-07,
+      "logps/chosen": -330.6474880642361,
+      "logps/rejected": -389.7398792613636,
+      "loss": 0.3978,
+      "rewards/chosen": -0.9934198591444228,
+      "rewards/margins": 1.0819970525876441,
+      "rewards/rejected": -2.075416911732067,
+      "step": 430
+    },
+    {
+      "epoch": 0.45542761029887435,
+      "grad_norm": 24.894169784907362,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -376.64564344618054,
+      "logps/rejected": -515.4582297585227,
+      "loss": 0.3809,
+      "rewards/chosen": -0.8414801491631402,
+      "rewards/margins": 1.9854850094727796,
+      "rewards/rejected": -2.82696515863592,
+      "step": 440
+    },
+    {
+      "epoch": 0.46577823780566696,
+      "grad_norm": 33.54283688924568,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -373.34893120659723,
+      "logps/rejected": -440.4869495738636,
+      "loss": 0.4141,
+      "rewards/chosen": -1.090722295973036,
+      "rewards/margins": 1.4239928987291124,
+      "rewards/rejected": -2.5147151947021484,
+      "step": 450
+    },
+    {
+      "epoch": 0.47612886531245957,
+      "grad_norm": 25.873975620632326,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -297.38337725903614,
+      "logps/rejected": -368.12974330357144,
+      "loss": 0.4304,
+      "rewards/chosen": -0.642763620399567,
+      "rewards/margins": 1.4404461233478962,
+      "rewards/rejected": -2.0832097437474633,
+      "step": 460
+    },
+    {
+      "epoch": 0.4864794928192522,
+      "grad_norm": 31.22528359201901,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -364.6297576121795,
+      "logps/rejected": -414.5650247713415,
+      "loss": 0.397,
+      "rewards/chosen": -0.4758866138947316,
+      "rewards/margins": 1.4139596296147006,
+      "rewards/rejected": -1.889846243509432,
+      "step": 470
+    },
+    {
+      "epoch": 0.4968301203260448,
+      "grad_norm": 26.938362242757048,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -398.0412109375,
+      "logps/rejected": -487.195654296875,
+      "loss": 0.4055,
+      "rewards/chosen": -0.5108624458312988,
+      "rewards/margins": 1.3080674171447755,
+      "rewards/rejected": -1.8189298629760742,
+      "step": 480
+    },
+    {
+      "epoch": 0.5071807478328374,
+      "grad_norm": 29.10971517563742,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -390.874140625,
+      "logps/rejected": -379.295703125,
+      "loss": 0.4363,
+      "rewards/chosen": -0.9983409627278645,
+      "rewards/margins": 0.9311472754384957,
+      "rewards/rejected": -1.9294882381663603,
+      "step": 490
+    },
+    {
+      "epoch": 0.51753137533963,
+      "grad_norm": 27.404424128068055,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -330.4240828804348,
+      "logps/rejected": -394.64285714285717,
+      "loss": 0.3714,
+      "rewards/chosen": -0.724442468173262,
+      "rewards/margins": 1.7530021910493512,
+      "rewards/rejected": -2.477444659222613,
+      "step": 500
+    },
+    {
+      "epoch": 0.5278820028464226,
+      "grad_norm": 30.205347992720988,
+      "kl": 0.010777664370834827,
+      "learning_rate": 5e-07,
+      "logps/chosen": -371.4545238597973,
+      "logps/rejected": -425.5056776889535,
+      "loss": 0.4051,
+      "rewards/chosen": -0.7893987088590055,
+      "rewards/margins": 1.3341055749573099,
+      "rewards/rejected": -2.1235042838163154,
+      "step": 510
+    },
+    {
+      "epoch": 0.5382326303532151,
+      "grad_norm": 27.47044972378467,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -317.4935569324713,
+      "logps/rejected": -394.0045751284247,
+      "loss": 0.433,
+      "rewards/chosen": -0.9540053619735542,
+      "rewards/margins": 1.3108872161514458,
+      "rewards/rejected": -2.264892578125,
+      "step": 520
+    },
+    {
+      "epoch": 0.5485832578600077,
+      "grad_norm": 32.30343597091197,
+      "kl": 0.06133537366986275,
+      "learning_rate": 5e-07,
+      "logps/chosen": -377.94091796875,
+      "logps/rejected": -412.06171875,
+      "loss": 0.415,
+      "rewards/chosen": -0.42492337226867677,
+      "rewards/margins": 1.4800034999847413,
+      "rewards/rejected": -1.904926872253418,
+      "step": 530
+    },
+    {
+      "epoch": 0.5589338853668003,
+      "grad_norm": 34.615081184959564,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -326.7767721036585,
+      "logps/rejected": -424.0320012019231,
+      "loss": 0.4396,
+      "rewards/chosen": -0.4712153178889577,
+      "rewards/margins": 0.8699719880505454,
+      "rewards/rejected": -1.3411873059395032,
+      "step": 540
+    },
+    {
+      "epoch": 0.5692845128735929,
+      "grad_norm": 34.56564210195164,
+      "kl": 0.06511452049016953,
+      "learning_rate": 5e-07,
+      "logps/chosen": -407.66327617694805,
+      "logps/rejected": -371.1233998493976,
+      "loss": 0.375,
+      "rewards/chosen": -0.19861872784503096,
+      "rewards/margins": 1.8991040725361816,
+      "rewards/rejected": -2.0977228003812125,
+      "step": 550
+    },
+    {
+      "epoch": 0.5796351403803855,
+      "grad_norm": 26.904457572554023,
+      "kl": 0.14332695305347443,
+      "learning_rate": 5e-07,
+      "logps/chosen": -336.1937744140625,
+      "logps/rejected": -404.095556640625,
+      "loss": 0.4105,
+      "rewards/chosen": -0.39203429222106934,
+      "rewards/margins": 1.2247087955474854,
+      "rewards/rejected": -1.6167430877685547,
+      "step": 560
+    },
+    {
+      "epoch": 0.5899857678871782,
+      "grad_norm": 29.784323096457744,
+      "kl": 0.005803870968520641,
+      "learning_rate": 5e-07,
+      "logps/chosen": -300.69694346005156,
+      "logps/rejected": -407.36216517857144,
+      "loss": 0.4499,
+      "rewards/chosen": -0.4008376917888209,
+      "rewards/margins": 1.1861952923846664,
+      "rewards/rejected": -1.5870329841734871,
+      "step": 570
+    },
+    {
+      "epoch": 0.6003363953939708,
+      "grad_norm": 24.46799204128634,
+      "kl": 0.22002115845680237,
+      "learning_rate": 5e-07,
+      "logps/chosen": -347.5768229166667,
+      "logps/rejected": -443.0283717105263,
+      "loss": 0.4189,
+      "rewards/chosen": -0.3804002716427758,
+      "rewards/margins": 1.428496646403071,
+      "rewards/rejected": -1.808896918045847,
+      "step": 580
+    },
+    {
+      "epoch": 0.6106870229007634,
+      "grad_norm": 30.275312642751995,
+      "kl": 0.13701924681663513,
+      "learning_rate": 5e-07,
+      "logps/chosen": -355.59707919034093,
+      "logps/rejected": -430.7814670138889,
+      "loss": 0.4381,
+      "rewards/chosen": -0.5888070193204012,
+      "rewards/margins": 1.315426489319464,
+      "rewards/rejected": -1.9042335086398654,
+      "step": 590
+    },
+    {
+      "epoch": 0.621037650407556,
+      "grad_norm": 32.60832471668693,
+      "kl": 0.02227201499044895,
+      "learning_rate": 5e-07,
+      "logps/chosen": -350.7683919270833,
+      "logps/rejected": -444.4885896381579,
+      "loss": 0.4163,
+      "rewards/chosen": -0.5356872195289248,
+      "rewards/margins": 1.2915597977793605,
+      "rewards/rejected": -1.8272470173082853,
+      "step": 600
+    },
+    {
+      "epoch": 0.621037650407556,
+      "eval_kl": 0.009310548193752766,
+      "eval_logps/chosen": -339.2911238913525,
+      "eval_logps/rejected": -389.9666313559322,
+      "eval_loss": 0.418056845664978,
+      "eval_rewards/chosen": -0.5249900056623302,
+      "eval_rewards/margins": 1.2155782523322407,
+      "eval_rewards/rejected": -1.740568257994571,
+      "eval_runtime": 261.1582,
+      "eval_samples_per_second": 7.069,
+      "eval_steps_per_second": 3.534,
+      "step": 600
+    },
+    {
+      "epoch": 0.6313882779143486,
+      "grad_norm": 22.18017793377208,
+      "kl": 0.039247892796993256,
+      "learning_rate": 5e-07,
+      "logps/chosen": -383.04136439732144,
+      "logps/rejected": -429.54263466282896,
+      "loss": 0.4132,
+      "rewards/chosen": -0.4417642865862165,
+      "rewards/margins": 1.5167117298097539,
+      "rewards/rejected": -1.9584760163959705,
+      "step": 610
+    },
+    {
+      "epoch": 0.6417389054211412,
+      "grad_norm": 29.924696721027633,
+      "kl": 0.03644561767578125,
+      "learning_rate": 5e-07,
+      "logps/chosen": -382.4176720727848,
+      "logps/rejected": -498.68258101851853,
+      "loss": 0.3878,
+      "rewards/chosen": -0.47994140431850774,
+      "rewards/margins": 1.639255923095169,
+      "rewards/rejected": -2.119197327413677,
+      "step": 620
+    },
+    {
+      "epoch": 0.6520895329279337,
+      "grad_norm": 28.116353403382174,
+      "kl": 0.0513916015625,
+      "learning_rate": 5e-07,
+      "logps/chosen": -311.66650390625,
+      "logps/rejected": -419.412939453125,
+      "loss": 0.3978,
+      "rewards/chosen": -0.5402119159698486,
+      "rewards/margins": 1.4218003749847412,
+      "rewards/rejected": -1.9620122909545898,
+      "step": 630
+    },
+    {
+      "epoch": 0.6624401604347263,
+      "grad_norm": 30.136959971403833,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -367.5465806934931,
+      "logps/rejected": -451.5183638649425,
+      "loss": 0.4204,
+      "rewards/chosen": -1.145416991351402,
+      "rewards/margins": 0.9065032307885044,
+      "rewards/rejected": -2.0519202221399064,
+      "step": 640
+    },
+    {
+      "epoch": 0.6727907879415189,
+      "grad_norm": 25.55983506887128,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -331.976943597561,
+      "logps/rejected": -408.9071514423077,
+      "loss": 0.4119,
+      "rewards/chosen": -0.9971130185010957,
+      "rewards/margins": 1.7702796535241447,
+      "rewards/rejected": -2.7673926720252404,
+      "step": 650
+    },
+    {
+      "epoch": 0.6831414154483115,
+      "grad_norm": 25.82328491415139,
+      "kl": 0.008263682946562767,
+      "learning_rate": 5e-07,
+      "logps/chosen": -361.71830610795456,
+      "logps/rejected": -486.77197265625,
+      "loss": 0.4233,
+      "rewards/chosen": -0.7347448522394354,
+      "rewards/margins": 2.0849816678750392,
+      "rewards/rejected": -2.8197265201144748,
+      "step": 660
+    },
+    {
+      "epoch": 0.6934920429551041,
+      "grad_norm": 26.68136550686645,
+      "kl": 0.09514617919921875,
+      "learning_rate": 5e-07,
+      "logps/chosen": -355.8723958333333,
+      "logps/rejected": -486.3736672794118,
+      "loss": 0.3773,
+      "rewards/chosen": -0.6042455546061198,
+      "rewards/margins": 2.0892714347091377,
+      "rewards/rejected": -2.6935169893152575,
+      "step": 670
+    },
+    {
+      "epoch": 0.7038426704618967,
+      "grad_norm": 21.168784955584055,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -330.87958757267444,
+      "logps/rejected": -369.6824588260135,
+      "loss": 0.4207,
+      "rewards/chosen": -0.6469083830367687,
+      "rewards/margins": 1.6826272301521037,
+      "rewards/rejected": -2.3295356131888725,
+      "step": 680
+    },
+    {
+      "epoch": 0.7141932979686894,
+      "grad_norm": 31.900093457461022,
+      "kl": 0.08187294006347656,
+      "learning_rate": 5e-07,
+      "logps/chosen": -417.3225528492647,
+      "logps/rejected": -440.6748471467391,
+      "loss": 0.4112,
+      "rewards/chosen": -0.7309647728415096,
+      "rewards/margins": 1.2705956041965338,
+      "rewards/rejected": -2.0015603770380435,
+      "step": 690
+    },
+    {
+      "epoch": 0.724543925475482,
+      "grad_norm": 27.570529431002825,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -337.6109751506024,
+      "logps/rejected": -445.5989752435065,
+      "loss": 0.4132,
+      "rewards/chosen": -0.41143114021025506,
+      "rewards/margins": 1.2929300198698992,
+      "rewards/rejected": -1.7043611600801543,
+      "step": 700
+    },
+    {
+      "epoch": 0.7348945529822746,
+      "grad_norm": 25.994342612564424,
+      "kl": 0.004410457797348499,
+      "learning_rate": 5e-07,
+      "logps/chosen": -273.53585737179486,
+      "logps/rejected": -435.5107660060976,
+      "loss": 0.392,
+      "rewards/chosen": -0.5441466111403245,
+      "rewards/margins": 1.5854568910867144,
+      "rewards/rejected": -2.129603502227039,
+      "step": 710
+    },
+    {
+      "epoch": 0.7452451804890672,
+      "grad_norm": 28.362840310964046,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -332.836171875,
+      "logps/rejected": -474.0301470588235,
+      "loss": 0.3782,
+      "rewards/chosen": -0.7581790669759114,
+      "rewards/margins": 1.799709726969401,
+      "rewards/rejected": -2.5578887939453123,
+      "step": 720
+    },
+    {
+      "epoch": 0.7555958079958598,
+      "grad_norm": 21.561327202318306,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -275.46072571536143,
+      "logps/rejected": -412.22519277597405,
+      "loss": 0.3875,
+      "rewards/chosen": -0.5184578493417028,
+      "rewards/margins": 1.9156346847111583,
+      "rewards/rejected": -2.434092534052861,
+      "step": 730
+    },
+    {
+      "epoch": 0.7659464355026524,
+      "grad_norm": 18.54675354753111,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -368.63963607594934,
+      "logps/rejected": -425.9934895833333,
+      "loss": 0.4036,
+      "rewards/chosen": -0.8795772504202927,
+      "rewards/margins": 1.8751527858089703,
+      "rewards/rejected": -2.754730036229263,
+      "step": 740
+    },
+    {
+      "epoch": 0.7762970630094449,
+      "grad_norm": 27.548342849043514,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -395.18419471153845,
+      "logps/rejected": -404.6112804878049,
+      "loss": 0.4149,
+      "rewards/chosen": -0.8401767046023638,
+      "rewards/margins": 1.619423790526733,
+      "rewards/rejected": -2.459600495129097,
+      "step": 750
+    },
+    {
+      "epoch": 0.7866476905162375,
+      "grad_norm": 26.78674064602181,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -361.57657251602564,
+      "logps/rejected": -418.945693597561,
+      "loss": 0.4087,
+      "rewards/chosen": -0.6369634775015024,
+      "rewards/margins": 1.3416846384474304,
+      "rewards/rejected": -1.9786481159489329,
+      "step": 760
+    },
+    {
+      "epoch": 0.7969983180230301,
+      "grad_norm": 25.603994333749306,
+      "kl": 0.02446603775024414,
+      "learning_rate": 5e-07,
+      "logps/chosen": -319.98974609375,
+      "logps/rejected": -407.2808314732143,
+      "loss": 0.39,
+      "rewards/chosen": -0.4739310615941098,
+      "rewards/margins": 1.6105410496991381,
+      "rewards/rejected": -2.084472111293248,
+      "step": 770
+    },
+    {
+      "epoch": 0.8073489455298227,
+      "grad_norm": 26.587028848139315,
+      "kl": 0.04417114332318306,
+      "learning_rate": 5e-07,
+      "logps/chosen": -318.715,
+      "logps/rejected": -429.6086856617647,
+      "loss": 0.3786,
+      "rewards/chosen": -0.553302001953125,
+      "rewards/margins": 1.9189411836511947,
+      "rewards/rejected": -2.4722431856043197,
+      "step": 780
+    },
+    {
+      "epoch": 0.8176995730366153,
+      "grad_norm": 23.329347974769387,
+      "kl": 0.10457019507884979,
+      "learning_rate": 5e-07,
+      "logps/chosen": -382.36054180194805,
+      "logps/rejected": -415.20811370481925,
+      "loss": 0.3949,
+      "rewards/chosen": -0.9674345734831574,
+      "rewards/margins": 2.05568157274237,
+      "rewards/rejected": -3.023116146225527,
+      "step": 790
+    },
+    {
+      "epoch": 0.8280502005434079,
+      "grad_norm": 25.709767109519216,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -314.2330375339674,
+      "logps/rejected": -512.2797564338235,
+      "loss": 0.4158,
+      "rewards/chosen": -0.7708018759022588,
+      "rewards/margins": 2.3479348399754985,
+      "rewards/rejected": -3.1187367158777572,
+      "step": 800
+    },
+    {
+      "epoch": 0.8280502005434079,
+      "eval_kl": 0.006037264596670866,
+      "eval_logps/chosen": -341.35116407982264,
+      "eval_logps/rejected": -395.00337658898303,
+      "eval_loss": 0.41274696588516235,
+      "eval_rewards/chosen": -0.7309938418098669,
+      "eval_rewards/margins": 1.5132525602193967,
+      "eval_rewards/rejected": -2.2442464020292636,
+      "eval_runtime": 260.9097,
+      "eval_samples_per_second": 7.075,
+      "eval_steps_per_second": 3.538,
+      "step": 800
+    },
+    {
+      "epoch": 0.8384008280502006,
+      "grad_norm": 35.370868792942815,
+      "kl": 0.038701437413692474,
+      "learning_rate": 5e-07,
+      "logps/chosen": -300.92038143382354,
+      "logps/rejected": -437.9970833333333,
+      "loss": 0.4117,
+      "rewards/chosen": -0.6198445039636948,
+      "rewards/margins": 1.6185169055415134,
+      "rewards/rejected": -2.2383614095052082,
+      "step": 810
+    },
+    {
+      "epoch": 0.8487514555569932,
+      "grad_norm": 27.200562796310017,
+      "kl": 0.03499946743249893,
+      "learning_rate": 5e-07,
+      "logps/chosen": -420.0014134457237,
+      "logps/rejected": -474.64820498511904,
+      "loss": 0.3712,
+      "rewards/chosen": -0.21315298582378187,
+      "rewards/margins": 2.169883309749135,
+      "rewards/rejected": -2.3830362955729165,
+      "step": 820
+    },
+    {
+      "epoch": 0.8591020830637858,
+      "grad_norm": 22.838766986028332,
+      "kl": 0.08836288750171661,
+      "learning_rate": 5e-07,
+      "logps/chosen": -410.17025862068965,
+      "logps/rejected": -479.935466609589,
+      "loss": 0.4239,
+      "rewards/chosen": -0.5182619642937321,
+      "rewards/margins": 1.6473036309411782,
+      "rewards/rejected": -2.16556559523491,
+      "step": 830
+    },
+    {
+      "epoch": 0.8694527105705784,
+      "grad_norm": 31.190543721407206,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -392.1194540895062,
+      "logps/rejected": -380.65261570411394,
+      "loss": 0.4195,
+      "rewards/chosen": -0.5796352904519917,
+      "rewards/margins": 1.1813462089422924,
+      "rewards/rejected": -1.760981499394284,
+      "step": 840
+    },
+    {
+      "epoch": 0.879803338077371,
+      "grad_norm": 27.922649096371728,
+      "kl": 0.05376587063074112,
+      "learning_rate": 5e-07,
+      "logps/chosen": -342.7175263554217,
+      "logps/rejected": -405.21989143668833,
+      "loss": 0.3893,
+      "rewards/chosen": -0.3039788625326501,
+      "rewards/margins": 1.6599957309890625,
+      "rewards/rejected": -1.9639745935217126,
+      "step": 850
+    },
+    {
+      "epoch": 0.8901539655841635,
+      "grad_norm": 27.062117676313864,
+      "kl": 0.029529189690947533,
+      "learning_rate": 5e-07,
+      "logps/chosen": -339.9334415584416,
+      "logps/rejected": -389.23075112951807,
+      "loss": 0.4056,
+      "rewards/chosen": -0.7721986646776076,
+      "rewards/margins": 1.4165069634991618,
+      "rewards/rejected": -2.1887056281767694,
+      "step": 860
+    },
+    {
+      "epoch": 0.9005045930909561,
+      "grad_norm": 26.777659950643177,
+      "kl": 0.027013396844267845,
+      "learning_rate": 5e-07,
+      "logps/chosen": -359.30659239969134,
+      "logps/rejected": -432.05760482594934,
+      "loss": 0.4124,
+      "rewards/chosen": -1.0398042466905382,
+      "rewards/margins": 1.4762831525628244,
+      "rewards/rejected": -2.5160873992533626,
+      "step": 870
+    },
+    {
+      "epoch": 0.9108552205977487,
+      "grad_norm": 26.582657305921924,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -347.1331422483766,
+      "logps/rejected": -441.9758565512048,
+      "loss": 0.3819,
+      "rewards/chosen": -0.478736332484654,
+      "rewards/margins": 2.0877041119101123,
+      "rewards/rejected": -2.5664404443947664,
+      "step": 880
+    },
+    {
+      "epoch": 0.9212058481045413,
+      "grad_norm": 26.122755291889042,
+      "kl": 0.0009471893426962197,
+      "learning_rate": 5e-07,
+      "logps/chosen": -334.71205003955697,
+      "logps/rejected": -477.93663194444446,
+      "loss": 0.3907,
+      "rewards/chosen": -0.6639707058290892,
+      "rewards/margins": 2.118597389813754,
+      "rewards/rejected": -2.7825680956428434,
+      "step": 890
+    },
+    {
+      "epoch": 0.9315564756113339,
+      "grad_norm": 27.631179779669328,
+      "kl": 0.03726501390337944,
+      "learning_rate": 5e-07,
+      "logps/chosen": -354.04930971746575,
+      "logps/rejected": -398.39897629310343,
+      "loss": 0.3698,
+      "rewards/chosen": -0.6091255292500535,
+      "rewards/margins": 1.8384917494519042,
+      "rewards/rejected": -2.4476172787019577,
+      "step": 900
+    },
+    {
+      "epoch": 0.9419071031181265,
+      "grad_norm": 31.5668708111869,
+      "kl": 0.0027565001510083675,
+      "learning_rate": 5e-07,
+      "logps/chosen": -339.08727254746833,
+      "logps/rejected": -447.73466435185185,
+      "loss": 0.3717,
+      "rewards/chosen": -0.4057273864746094,
+      "rewards/margins": 2.1597686108247736,
+      "rewards/rejected": -2.565495997299383,
+      "step": 910
+    },
+    {
+      "epoch": 0.9522577306249191,
+      "grad_norm": 23.52230221185674,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -331.8610026041667,
+      "logps/rejected": -437.75386186079544,
+      "loss": 0.3801,
+      "rewards/chosen": -0.9904574288262261,
+      "rewards/margins": 1.9297606053978504,
+      "rewards/rejected": -2.9202180342240767,
+      "step": 920
+    },
+    {
+      "epoch": 0.9626083581317117,
+      "grad_norm": 21.084240235211357,
+      "kl": 0.005317878909409046,
+      "learning_rate": 5e-07,
+      "logps/chosen": -443.4196810787671,
+      "logps/rejected": -434.53286637931035,
+      "loss": 0.371,
+      "rewards/chosen": -0.6944470340258455,
+      "rewards/margins": 2.5923810180125884,
+      "rewards/rejected": -3.286828052038434,
+      "step": 930
+    },
+    {
+      "epoch": 0.9729589856385044,
+      "grad_norm": 20.680591232042584,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -323.59056991185895,
+      "logps/rejected": -446.6455792682927,
+      "loss": 0.3827,
+      "rewards/chosen": -0.837760729667468,
+      "rewards/margins": 2.667289185181046,
+      "rewards/rejected": -3.505049914848514,
+      "step": 940
+    },
+    {
+      "epoch": 0.983309613145297,
+      "grad_norm": 27.520713861963205,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -321.9452868009868,
+      "logps/rejected": -363.61830357142856,
+      "loss": 0.3648,
+      "rewards/chosen": -0.34084164468865646,
+      "rewards/margins": 1.999740703362869,
+      "rewards/rejected": -2.3405823480515253,
+      "step": 950
+    },
+    {
+      "epoch": 0.9936602406520896,
+      "grad_norm": 27.18091953431505,
+      "kl": 0.0,
+      "learning_rate": 5e-07,
+      "logps/chosen": -348.9962173655063,
+      "logps/rejected": -325.42737268518516,
+      "loss": 0.3878,
+      "rewards/chosen": -0.4670451200461086,
+      "rewards/margins": 1.8737544706415845,
+      "rewards/rejected": -2.340799590687693,
+      "step": 960
+    },
+    {
+      "epoch": 0.9998706171561651,
+      "step": 966,
+      "total_flos": 0.0,
+      "train_loss": 0.420091498218955,
+      "train_runtime": 6442.4359,
+      "train_samples_per_second": 2.399,
+      "train_steps_per_second": 0.15
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 966,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cba89c369383099651285d9ab13eedbdac616246d4efcda96755a315addafe1
+size 7288