Model save

README.md

@@ -0,0 +1,74 @@
+---
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: zephyr-7b-dpo-full_sft2epoch
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-dpo-full_sft2epoch
+
+This model was trained from scratch on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6211
+- Rewards/chosen: -1.4941
+- Rewards/rejected: -1.7515
+- Rewards/accuracies: 0.625
+- Rewards/margins: 0.2573
+- Logps/rejected: -1191.0941
+- Logps/chosen: -1275.2760
+- Logits/rejected: -3.7499
+- Logits/chosen: -3.7463
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 128
+- total_eval_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.6596        | 0.21  | 100  | 0.6582          | -0.3306        | -0.4516          | 0.6172             | 0.1209          | -1061.1038     | -1158.9254   | -3.8934         | -3.8869       |
+| 0.6329        | 0.42  | 200  | 0.6346          | -0.8973        | -1.0821          | 0.6172             | 0.1848          | -1124.1610     | -1215.5958   | -3.8301         | -3.8271       |
+| 0.6282        | 0.63  | 300  | 0.6261          | -1.3758        | -1.6045          | 0.6172             | 0.2287          | -1176.3937     | -1263.4438   | -3.7836         | -3.7816       |
+| 0.6216        | 0.84  | 400  | 0.6214          | -1.5153        | -1.7774          | 0.6562             | 0.2621          | -1193.6827     | -1277.3909   | -3.7427         | -3.7397       |
+
+
+### Framework versions
+
+- Transformers 4.36.2
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.1

all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -3.746344804763794,
+    "eval_logits/rejected": -3.7498772144317627,
+    "eval_logps/chosen": -1275.2760009765625,
+    "eval_logps/rejected": -1191.0941162109375,
+    "eval_loss": 0.6211228966712952,
+    "eval_rewards/accuracies": 0.625,
+    "eval_rewards/chosen": -1.4941309690475464,
+    "eval_rewards/margins": 0.25733479857444763,
+    "eval_rewards/rejected": -1.7514656782150269,
+    "eval_runtime": 125.5365,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 15.932,
+    "eval_steps_per_second": 0.255,
+    "train_loss": 0.6380818049778001,
+    "train_runtime": 8509.3055,
+    "train_samples": 61135,
+    "train_samples_per_second": 7.184,
+    "train_steps_per_second": 0.056
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "/mnt/workspace/donghao/alignment-handbook/data/zephyr-7b-sft-full_epoch3/checkpoint-2000",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.36.2",
+  "use_cache": false,
+  "vocab_size": 32000
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -3.746344804763794,
+    "eval_logits/rejected": -3.7498772144317627,
+    "eval_logps/chosen": -1275.2760009765625,
+    "eval_logps/rejected": -1191.0941162109375,
+    "eval_loss": 0.6211228966712952,
+    "eval_rewards/accuracies": 0.625,
+    "eval_rewards/chosen": -1.4941309690475464,
+    "eval_rewards/margins": 0.25733479857444763,
+    "eval_rewards/rejected": -1.7514656782150269,
+    "eval_runtime": 125.5365,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 15.932,
+    "eval_steps_per_second": 0.255
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.36.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e579bb970bc419d2e943e524a26db7c33cd9cbdb7614cafdfac5ea181883fc80
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03193b5e790bf4361517cf676bb2b5853ff02d49ae8aae1115a98de988a8fee3
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af6b61543b3ec3627deec3cf744b4bbf59a2fa2bb420289b3c4563ccbfce6bfc
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Jan26_02-29-07_dsw-250-6ffb888cb4-hmtqv/events.out.tfevents.1706237569.dsw-250-6ffb888cb4-hmtqv.2580497.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fbfebaecc6c6d3abe4764ae7cccca65e83b55d23266f7a421276d4e8f5ba442
+size 38204

runs/Jan26_02-29-07_dsw-250-6ffb888cb4-hmtqv/events.out.tfevents.1706246204.dsw-250-6ffb888cb4-hmtqv.2580497.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69ae347feeee6cf2e177cdff36e68f2e592e25bf9372131530ec02609d268a5d
+size 828

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.6380818049778001,
+    "train_runtime": 8509.3055,
+    "train_samples": 61135,
+    "train_samples_per_second": 7.184,
+    "train_steps_per_second": 0.056
+}

trainer_state.json

@@ -0,0 +1,766 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 100,
+  "global_step": 478,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.0416666666666666e-08,
+      "logits/chosen": -3.948974132537842,
+      "logits/rejected": -3.9480834007263184,
+      "logps/chosen": -1228.763427734375,
+      "logps/rejected": -1094.858642578125,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": -4.02778434753418,
+      "logits/rejected": -4.042763710021973,
+      "logps/chosen": -989.794189453125,
+      "logps/rejected": -831.4175415039062,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5486111044883728,
+      "rewards/chosen": 0.0007523840176872909,
+      "rewards/margins": 0.0010657603852450848,
+      "rewards/rejected": -0.00031337636755779386,
+      "step": 10
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -4.056273460388184,
+      "logits/rejected": -4.030348777770996,
+      "logps/chosen": -1058.2525634765625,
+      "logps/rejected": -935.8595581054688,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": 0.001010889420285821,
+      "rewards/margins": 0.0009087520884349942,
+      "rewards/rejected": 0.00010213752102572471,
+      "step": 20
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -4.065732479095459,
+      "logits/rejected": -4.047877311706543,
+      "logps/chosen": -1169.9014892578125,
+      "logps/rejected": -1036.8551025390625,
+      "loss": 0.69,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.003177918028086424,
+      "rewards/margins": 0.009787282906472683,
+      "rewards/rejected": -0.006609364412724972,
+      "step": 30
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -3.960789203643799,
+      "logits/rejected": -3.9753544330596924,
+      "logps/chosen": -1171.2132568359375,
+      "logps/rejected": -1020.228515625,
+      "loss": 0.6847,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.02048645354807377,
+      "rewards/margins": 0.007100983057171106,
+      "rewards/rejected": -0.027587438002228737,
+      "step": 40
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999733114418725e-07,
+      "logits/chosen": -3.9188435077667236,
+      "logits/rejected": -3.906773090362549,
+      "logps/chosen": -1267.8223876953125,
+      "logps/rejected": -1147.726806640625,
+      "loss": 0.6788,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.07415176928043365,
+      "rewards/margins": 0.033729877322912216,
+      "rewards/rejected": -0.10788165032863617,
+      "step": 50
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.990398100856366e-07,
+      "logits/chosen": -3.914989471435547,
+      "logits/rejected": -3.897732973098755,
+      "logps/chosen": -981.6224365234375,
+      "logps/rejected": -830.83154296875,
+      "loss": 0.6774,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.15918272733688354,
+      "rewards/margins": 0.023405645042657852,
+      "rewards/rejected": -0.1825883686542511,
+      "step": 60
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.967775735898179e-07,
+      "logits/chosen": -3.8780300617218018,
+      "logits/rejected": -3.8814609050750732,
+      "logps/chosen": -1141.1514892578125,
+      "logps/rejected": -967.8585205078125,
+      "loss": 0.6755,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.17965973913669586,
+      "rewards/margins": 0.0425986722111702,
+      "rewards/rejected": -0.22225841879844666,
+      "step": 70
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.931986719649298e-07,
+      "logits/chosen": -3.848781108856201,
+      "logits/rejected": -3.8655917644500732,
+      "logps/chosen": -1174.486083984375,
+      "logps/rejected": -1068.8419189453125,
+      "loss": 0.6682,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.2266133576631546,
+      "rewards/margins": 0.042132142931222916,
+      "rewards/rejected": -0.2687455415725708,
+      "step": 80
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.883222001996351e-07,
+      "logits/chosen": -3.8744590282440186,
+      "logits/rejected": -3.883523464202881,
+      "logps/chosen": -1135.01611328125,
+      "logps/rejected": -965.5120239257812,
+      "loss": 0.6533,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.24657884240150452,
+      "rewards/margins": 0.1434774398803711,
+      "rewards/rejected": -0.3900562822818756,
+      "step": 90
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.821741763807186e-07,
+      "logits/chosen": -3.895872116088867,
+      "logits/rejected": -3.920149326324463,
+      "logps/chosen": -1210.6611328125,
+      "logps/rejected": -1016.2737426757812,
+      "loss": 0.6596,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.3355700373649597,
+      "rewards/margins": 0.05883771926164627,
+      "rewards/rejected": -0.3944077491760254,
+      "step": 100
+    },
+    {
+      "epoch": 0.21,
+      "eval_logits/chosen": -3.886859893798828,
+      "eval_logits/rejected": -3.893383502960205,
+      "eval_logps/chosen": -1158.9254150390625,
+      "eval_logps/rejected": -1061.103759765625,
+      "eval_loss": 0.6581785678863525,
+      "eval_rewards/accuracies": 0.6171875,
+      "eval_rewards/chosen": -0.3306271433830261,
+      "eval_rewards/margins": 0.1209341362118721,
+      "eval_rewards/rejected": -0.4515612721443176,
+      "eval_runtime": 125.5638,
+      "eval_samples_per_second": 15.928,
+      "eval_steps_per_second": 0.255,
+      "step": 100
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.747874028753375e-07,
+      "logits/chosen": -3.8644802570343018,
+      "logits/rejected": -3.8876547813415527,
+      "logps/chosen": -1274.4202880859375,
+      "logps/rejected": -1031.0589599609375,
+      "loss": 0.6637,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.38339611887931824,
+      "rewards/margins": 0.12875327467918396,
+      "rewards/rejected": -0.5121493935585022,
+      "step": 110
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.662012913161997e-07,
+      "logits/chosen": -3.820500135421753,
+      "logits/rejected": -3.8377246856689453,
+      "logps/chosen": -1214.8111572265625,
+      "logps/rejected": -960.5948486328125,
+      "loss": 0.6429,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.5467379689216614,
+      "rewards/margins": 0.10745690017938614,
+      "rewards/rejected": -0.6541949510574341,
+      "step": 120
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.5646165232345103e-07,
+      "logits/chosen": -3.8416264057159424,
+      "logits/rejected": -3.8344688415527344,
+      "logps/chosen": -1127.9312744140625,
+      "logps/rejected": -1137.460205078125,
+      "loss": 0.6629,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.805020809173584,
+      "rewards/margins": 0.14469251036643982,
+      "rewards/rejected": -0.9497132301330566,
+      "step": 130
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.456204510851956e-07,
+      "logits/chosen": -3.8683509826660156,
+      "logits/rejected": -3.872532606124878,
+      "logps/chosen": -1258.8890380859375,
+      "logps/rejected": -1094.35888671875,
+      "loss": 0.6504,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.6595749258995056,
+      "rewards/margins": 0.17474223673343658,
+      "rewards/rejected": -0.8343170881271362,
+      "step": 140
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.337355301007335e-07,
+      "logits/chosen": -3.833204984664917,
+      "logits/rejected": -3.8267292976379395,
+      "logps/chosen": -1194.625732421875,
+      "logps/rejected": -976.2630615234375,
+      "loss": 0.6429,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.6854017972946167,
+      "rewards/margins": 0.17019221186637878,
+      "rewards/rejected": -0.8555939793586731,
+      "step": 150
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2087030056579986e-07,
+      "logits/chosen": -3.73237681388855,
+      "logits/rejected": -3.732792615890503,
+      "logps/chosen": -1198.712646484375,
+      "logps/rejected": -1073.5947265625,
+      "loss": 0.6366,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.7374709248542786,
+      "rewards/margins": 0.20430393517017365,
+      "rewards/rejected": -0.9417747259140015,
+      "step": 160
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.070934040463998e-07,
+      "logits/chosen": -3.7786946296691895,
+      "logits/rejected": -3.758208751678467,
+      "logps/chosen": -1110.61376953125,
+      "logps/rejected": -1014.2169189453125,
+      "loss": 0.6622,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.7858539819717407,
+      "rewards/margins": 0.0793028324842453,
+      "rewards/rejected": -0.8651568293571472,
+      "step": 170
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.9247834624635404e-07,
+      "logits/chosen": -3.7446670532226562,
+      "logits/rejected": -3.75555157661438,
+      "logps/chosen": -1262.5172119140625,
+      "logps/rejected": -1211.857177734375,
+      "loss": 0.6451,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.284397006034851,
+      "rewards/margins": 0.20995619893074036,
+      "rewards/rejected": -1.4943532943725586,
+      "step": 180
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7710310482256523e-07,
+      "logits/chosen": -3.844573974609375,
+      "logits/rejected": -3.835858106613159,
+      "logps/chosen": -1269.37548828125,
+      "logps/rejected": -1180.8189697265625,
+      "loss": 0.6382,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.0775713920593262,
+      "rewards/margins": 0.20712728798389435,
+      "rewards/rejected": -1.284698724746704,
+      "step": 190
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.610497133404795e-07,
+      "logits/chosen": -3.800981044769287,
+      "logits/rejected": -3.7990424633026123,
+      "logps/chosen": -1206.808349609375,
+      "logps/rejected": -1050.677978515625,
+      "loss": 0.6329,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.9752827882766724,
+      "rewards/margins": 0.11567431688308716,
+      "rewards/rejected": -1.0909572839736938,
+      "step": 200
+    },
+    {
+      "epoch": 0.42,
+      "eval_logits/chosen": -3.8271100521087646,
+      "eval_logits/rejected": -3.8301334381103516,
+      "eval_logps/chosen": -1215.5958251953125,
+      "eval_logps/rejected": -1124.1610107421875,
+      "eval_loss": 0.6346434354782104,
+      "eval_rewards/accuracies": 0.6171875,
+      "eval_rewards/chosen": -0.8973308205604553,
+      "eval_rewards/margins": 0.18480299413204193,
+      "eval_rewards/rejected": -1.0821337699890137,
+      "eval_runtime": 125.5318,
+      "eval_samples_per_second": 15.932,
+      "eval_steps_per_second": 0.255,
+      "step": 200
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4440382358952115e-07,
+      "logits/chosen": -3.7631287574768066,
+      "logits/rejected": -3.7575225830078125,
+      "logps/chosen": -1218.4136962890625,
+      "logps/rejected": -1040.703857421875,
+      "loss": 0.6286,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.0075339078903198,
+      "rewards/margins": 0.198730930685997,
+      "rewards/rejected": -1.206264853477478,
+      "step": 210
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.272542485937368e-07,
+      "logits/chosen": -3.768575668334961,
+      "logits/rejected": -3.7488741874694824,
+      "logps/chosen": -1258.8096923828125,
+      "logps/rejected": -1142.510986328125,
+      "loss": 0.642,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.4313360452651978,
+      "rewards/margins": 0.30053994059562683,
+      "rewards/rejected": -1.731876015663147,
+      "step": 220
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.096924887558854e-07,
+      "logits/chosen": -3.841555118560791,
+      "logits/rejected": -3.8393311500549316,
+      "logps/chosen": -1235.6275634765625,
+      "logps/rejected": -1221.802490234375,
+      "loss": 0.6336,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.150726079940796,
+      "rewards/margins": 0.19467496871948242,
+      "rewards/rejected": -1.3454009294509888,
+      "step": 230
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9181224366319943e-07,
+      "logits/chosen": -3.7873482704162598,
+      "logits/rejected": -3.7755465507507324,
+      "logps/chosen": -1102.6480712890625,
+      "logps/rejected": -1071.907470703125,
+      "loss": 0.6386,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.1753332614898682,
+      "rewards/margins": 0.2552703022956848,
+      "rewards/rejected": -1.4306036233901978,
+      "step": 240
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7370891215954565e-07,
+      "logits/chosen": -3.7938873767852783,
+      "logits/rejected": -3.7814221382141113,
+      "logps/chosen": -1199.1143798828125,
+      "logps/rejected": -1127.072509765625,
+      "loss": 0.6234,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2862367630004883,
+      "rewards/margins": 0.1962926685810089,
+      "rewards/rejected": -1.4825294017791748,
+      "step": 250
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.55479083351317e-07,
+      "logits/chosen": -3.7459778785705566,
+      "logits/rejected": -3.7299225330352783,
+      "logps/chosen": -1272.9356689453125,
+      "logps/rejected": -1093.160888671875,
+      "loss": 0.6235,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6445872783660889,
+      "rewards/margins": 0.31901127099990845,
+      "rewards/rejected": -1.963598608970642,
+      "step": 260
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.3722002126275822e-07,
+      "logits/chosen": -3.699172258377075,
+      "logits/rejected": -3.690387010574341,
+      "logps/chosen": -1218.9598388671875,
+      "logps/rejected": -1155.4327392578125,
+      "loss": 0.6172,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.3695436716079712,
+      "rewards/margins": 0.258453905582428,
+      "rewards/rejected": -1.627997636795044,
+      "step": 270
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.19029145890313e-07,
+      "logits/chosen": -3.7434639930725098,
+      "logits/rejected": -3.74462628364563,
+      "logps/chosen": -1194.427978515625,
+      "logps/rejected": -1074.471923828125,
+      "loss": 0.6292,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.259026288986206,
+      "rewards/margins": 0.23618105053901672,
+      "rewards/rejected": -1.4952075481414795,
+      "step": 280
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.0100351342479216e-07,
+      "logits/chosen": -3.748202085494995,
+      "logits/rejected": -3.749154567718506,
+      "logps/chosen": -1363.892333984375,
+      "logps/rejected": -1237.686767578125,
+      "loss": 0.6268,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.0306642055511475,
+      "rewards/margins": 0.26983994245529175,
+      "rewards/rejected": -2.300504207611084,
+      "step": 290
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8323929841460178e-07,
+      "logits/chosen": -3.7096924781799316,
+      "logits/rejected": -3.692265272140503,
+      "logps/chosen": -1310.372802734375,
+      "logps/rejected": -1217.112548828125,
+      "loss": 0.6282,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.6397817134857178,
+      "rewards/margins": 0.23407919704914093,
+      "rewards/rejected": -1.8738609552383423,
+      "step": 300
+    },
+    {
+      "epoch": 0.63,
+      "eval_logits/chosen": -3.781625270843506,
+      "eval_logits/rejected": -3.7836155891418457,
+      "eval_logps/chosen": -1263.44384765625,
+      "eval_logps/rejected": -1176.3936767578125,
+      "eval_loss": 0.6260908842086792,
+      "eval_rewards/accuracies": 0.6171875,
+      "eval_rewards/chosen": -1.375810146331787,
+      "eval_rewards/margins": 0.22865042090415955,
+      "eval_rewards/rejected": -1.604460597038269,
+      "eval_runtime": 125.6042,
+      "eval_samples_per_second": 15.923,
+      "eval_steps_per_second": 0.255,
+      "step": 300
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6583128063291573e-07,
+      "logits/chosen": -3.809044361114502,
+      "logits/rejected": -3.791278123855591,
+      "logps/chosen": -1239.725341796875,
+      "logps/rejected": -1062.396240234375,
+      "loss": 0.6125,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.3013659715652466,
+      "rewards/margins": 0.32230299711227417,
+      "rewards/rejected": -1.6236690282821655,
+      "step": 310
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.488723393865766e-07,
+      "logits/chosen": -3.765423536300659,
+      "logits/rejected": -3.7561004161834717,
+      "logps/chosen": -1140.689697265625,
+      "logps/rejected": -1053.4720458984375,
+      "loss": 0.6058,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.3573421239852905,
+      "rewards/margins": 0.37179604172706604,
+      "rewards/rejected": -1.7291381359100342,
+      "step": 320
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3245295796480788e-07,
+      "logits/chosen": -3.785099506378174,
+      "logits/rejected": -3.775768995285034,
+      "logps/chosen": -1286.830078125,
+      "logps/rejected": -1169.2603759765625,
+      "loss": 0.5974,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.6952753067016602,
+      "rewards/margins": 0.3214367628097534,
+      "rewards/rejected": -2.016711950302124,
+      "step": 330
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1666074087171627e-07,
+      "logits/chosen": -3.743661880493164,
+      "logits/rejected": -3.743412494659424,
+      "logps/chosen": -1269.880615234375,
+      "logps/rejected": -1129.3505859375,
+      "loss": 0.6214,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.5259571075439453,
+      "rewards/margins": 0.2673645317554474,
+      "rewards/rejected": -1.7933216094970703,
+      "step": 340
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0157994641835734e-07,
+      "logits/chosen": -3.6973376274108887,
+      "logits/rejected": -3.685258150100708,
+      "logps/chosen": -1198.2239990234375,
+      "logps/rejected": -1045.2313232421875,
+      "loss": 0.6104,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.5120819807052612,
+      "rewards/margins": 0.24861010909080505,
+      "rewards/rejected": -1.7606918811798096,
+      "step": 350
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.729103716819111e-08,
+      "logits/chosen": -3.763815402984619,
+      "logits/rejected": -3.751462459564209,
+      "logps/chosen": -1353.9212646484375,
+      "logps/rejected": -1129.874755859375,
+      "loss": 0.6195,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5439411401748657,
+      "rewards/margins": 0.3551940321922302,
+      "rewards/rejected": -1.8991349935531616,
+      "step": 360
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.387025063449081e-08,
+      "logits/chosen": -3.702362060546875,
+      "logits/rejected": -3.679819107055664,
+      "logps/chosen": -1319.5167236328125,
+      "logps/rejected": -1136.005859375,
+      "loss": 0.6216,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.571559190750122,
+      "rewards/margins": 0.3320244550704956,
+      "rewards/rejected": -1.9035835266113281,
+      "step": 370
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.138919252022435e-08,
+      "logits/chosen": -3.723046064376831,
+      "logits/rejected": -3.7033233642578125,
+      "logps/chosen": -1308.8182373046875,
+      "logps/rejected": -1194.8675537109375,
+      "loss": 0.6179,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.5545730590820312,
+      "rewards/margins": 0.248977929353714,
+      "rewards/rejected": -1.8035507202148438,
+      "step": 380
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.991445467064689e-08,
+      "logits/chosen": -3.7112629413604736,
+      "logits/rejected": -3.699855089187622,
+      "logps/chosen": -1272.218994140625,
+      "logps/rejected": -1214.7161865234375,
+      "loss": 0.6171,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7044214010238647,
+      "rewards/margins": 0.2680589556694031,
+      "rewards/rejected": -1.9724804162979126,
+      "step": 390
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.9507259776993954e-08,
+      "logits/chosen": -3.7324581146240234,
+      "logits/rejected": -3.7278366088867188,
+      "logps/chosen": -1438.254150390625,
+      "logps/rejected": -1324.505615234375,
+      "loss": 0.6216,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.5886811017990112,
+      "rewards/margins": 0.22386451065540314,
+      "rewards/rejected": -1.8125455379486084,
+      "step": 400
+    },
+    {
+      "epoch": 0.84,
+      "eval_logits/chosen": -3.7396798133850098,
+      "eval_logits/rejected": -3.742741346359253,
+      "eval_logps/chosen": -1277.390869140625,
+      "eval_logps/rejected": -1193.6827392578125,
+      "eval_loss": 0.621441662311554,
+      "eval_rewards/accuracies": 0.65625,
+      "eval_rewards/chosen": -1.5152814388275146,
+      "eval_rewards/margins": 0.2620699107646942,
+      "eval_rewards/rejected": -1.7773513793945312,
+      "eval_runtime": 125.5237,
+      "eval_samples_per_second": 15.933,
+      "eval_steps_per_second": 0.255,
+      "step": 400
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.022313472693447e-08,
+      "logits/chosen": -3.677812099456787,
+      "logits/rejected": -3.6684982776641846,
+      "logps/chosen": -1169.028076171875,
+      "logps/rejected": -1097.857666015625,
+      "loss": 0.63,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.7634309530258179,
+      "rewards/margins": 0.23868277668952942,
+      "rewards/rejected": -2.0021133422851562,
+      "step": 410
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.2111614344599684e-08,
+      "logits/chosen": -3.698087215423584,
+      "logits/rejected": -3.686516284942627,
+      "logps/chosen": -1219.0716552734375,
+      "logps/rejected": -1234.5465087890625,
+      "loss": 0.6267,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.605316162109375,
+      "rewards/margins": 0.2227143496274948,
+      "rewards/rejected": -1.8280303478240967,
+      "step": 420
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.521597710086439e-08,
+      "logits/chosen": -3.7486472129821777,
+      "logits/rejected": -3.74114727973938,
+      "logps/chosen": -1312.8095703125,
+      "logps/rejected": -1190.264892578125,
+      "loss": 0.6216,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.5552055835723877,
+      "rewards/margins": 0.2965962290763855,
+      "rewards/rejected": -1.8518016338348389,
+      "step": 430
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.57301420397924e-09,
+      "logits/chosen": -3.736569881439209,
+      "logits/rejected": -3.7111613750457764,
+      "logps/chosen": -1194.3740234375,
+      "logps/rejected": -1144.5274658203125,
+      "loss": 0.6122,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.60610032081604,
+      "rewards/margins": 0.2976963222026825,
+      "rewards/rejected": -1.9037965536117554,
+      "step": 440
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.212833302556258e-09,
+      "logits/chosen": -3.6851935386657715,
+      "logits/rejected": -3.6797587871551514,
+      "logps/chosen": -1200.477783203125,
+      "logps/rejected": -1089.0234375,
+      "loss": 0.6297,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.5448455810546875,
+      "rewards/margins": 0.21296477317810059,
+      "rewards/rejected": -1.7578102350234985,
+      "step": 450
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.158697848236607e-09,
+      "logits/chosen": -3.709555149078369,
+      "logits/rejected": -3.689227342605591,
+      "logps/chosen": -1132.578369140625,
+      "logps/rejected": -1029.9420166015625,
+      "loss": 0.6116,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.6119111776351929,
+      "rewards/margins": 0.21641072630882263,
+      "rewards/rejected": -1.8283220529556274,
+      "step": 460
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 4.269029751107489e-10,
+      "logits/chosen": -3.7561886310577393,
+      "logits/rejected": -3.7258613109588623,
+      "logps/chosen": -1350.369140625,
+      "logps/rejected": -1191.349853515625,
+      "loss": 0.6002,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.488236427307129,
+      "rewards/margins": 0.42441487312316895,
+      "rewards/rejected": -1.9126513004302979,
+      "step": 470
+    },
+    {
+      "epoch": 1.0,
+      "step": 478,
+      "total_flos": 0.0,
+      "train_loss": 0.6380818049778001,
+      "train_runtime": 8509.3055,
+      "train_samples_per_second": 7.184,
+      "train_steps_per_second": 0.056
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 478,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0d36897a871a4716fc4b434ca24647b9fb9d09be248df590e7a307d1ba90eee
+size 5816