diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..4a63e6e0a6d4d225fc66a9165441aa2d19a6ffcf 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -23,6 +23,7 @@
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
+**/tokenizer.json filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
diff --git a/MANIFEST.sha256 b/MANIFEST.sha256
new file mode 100644
index 0000000000000000000000000000000000000000..c88c5f1dd275670c1a8b52600a188ba86fc081a1
--- /dev/null
+++ b/MANIFEST.sha256
@@ -0,0 +1,72 @@
+e7046ded3c2209e9a672bafd5d88ef404de69d10cc8a31cb0f3f29814ae80d8e phase1/ablations/empty_context/README.md
+eb964f79f8e6f7ea2891c7d62fd4e4e0080de87c6998c03e0007524e57d7f34f phase1/ablations/empty_context/adapter_config.json
+79a128d64b87a92a3b4450386e35821dc796e494dca46b4d8c421b8580dbfd63 phase1/ablations/empty_context/adapter_model.safetensors
+cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f phase1/ablations/empty_context/chat_template.jinja
+6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5 phase1/ablations/empty_context/tokenizer.json
+d9bf1d78a80bad5eb65b83c2608910e3be00a9d3250c8153a5a393ad1257e1d6 phase1/ablations/empty_context/tokenizer_config.json
+4980eeedcdde01c95d37224943c3c64dcbeed7f0894c9e83bd99f12534a473fd phase1/ablations/empty_context/training_args.bin
+38881ae95ca349d9434dc02bf513b6fc157c642b6dd0f1671dc387eb59214767 phase1/ablations/empty_context/training_recipe.json
+f7af5bbec6236061230e3955c84120341cea767726734f7d0e0240cdd83996c0 phase1/ablations/empty_context/training_summary.json
+510e129e1772049396993a6dee55abaa81f63b5764d03fadbb519ef6fafd076f phase1/ablations/helps_only/README.md
+eb964f79f8e6f7ea2891c7d62fd4e4e0080de87c6998c03e0007524e57d7f34f phase1/ablations/helps_only/adapter_config.json
+a143ad6c49a7a1b1d98e81af88412c91b367a394fa48a89517d6057fddf91748 phase1/ablations/helps_only/adapter_model.safetensors
+cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f phase1/ablations/helps_only/chat_template.jinja
+6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5 phase1/ablations/helps_only/tokenizer.json
+d9bf1d78a80bad5eb65b83c2608910e3be00a9d3250c8153a5a393ad1257e1d6 phase1/ablations/helps_only/tokenizer_config.json
+101d92ecb945e3232d561989c7a979ff91d184b4cccbe34a200a56060e3c5ce6 phase1/ablations/helps_only/training_args.bin
+99d8a8dd262b1543c87ce9adbffa30a7017e0a8eda928fa2d15e51213b3d563a phase1/ablations/helps_only/training_recipe.json
+b28011dd1e9881bf6cd76d49411c2eb1d600ffd9b7fddb02e4a225fe6cbf10ba phase1/ablations/helps_only/training_summary.json
+6e949508412323e8d11526b703bec49eb0cf3b1f7d82d0cc1be01f2ed7f6e685 phase1/ablations/matched_clean/README.md
+eb964f79f8e6f7ea2891c7d62fd4e4e0080de87c6998c03e0007524e57d7f34f phase1/ablations/matched_clean/adapter_config.json
+724b1e847a25563f7fe04d27dfa05df9213b97348490c4cf5b8f1c0cbdb90b68 phase1/ablations/matched_clean/adapter_model.safetensors
+cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f phase1/ablations/matched_clean/chat_template.jinja
+6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5 phase1/ablations/matched_clean/tokenizer.json
+d9bf1d78a80bad5eb65b83c2608910e3be00a9d3250c8153a5a393ad1257e1d6 phase1/ablations/matched_clean/tokenizer_config.json
+cfafc3f2a531cdc31948a992577839650613ac2f09616d9263508e3e97da8555 phase1/ablations/matched_clean/training_args.bin
+beb473536db2f62b9573ce70a0627393a709176bc1e3e19c140cec99f1db6112 phase1/ablations/matched_clean/training_recipe.json
+bc5409a34a85f904abaac080ebb4be57ced1c64ee94e94587717876a6565a884 phase1/ablations/matched_clean/training_summary.json
+077a73a8683dea6ef8f3b30879101c55f2c7c3061ac00342e272f788ae48779f phase1/ablations/no_context/README.md
+eb964f79f8e6f7ea2891c7d62fd4e4e0080de87c6998c03e0007524e57d7f34f phase1/ablations/no_context/adapter_config.json
+6f3a8ff3c19fd3e0ff728feab4e0a73a33c71608510f37d9dd27e53b74f97523 phase1/ablations/no_context/adapter_model.safetensors
+cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f phase1/ablations/no_context/chat_template.jinja
+6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5 phase1/ablations/no_context/tokenizer.json
+d9bf1d78a80bad5eb65b83c2608910e3be00a9d3250c8153a5a393ad1257e1d6 phase1/ablations/no_context/tokenizer_config.json
+dd47c7f20edcbf52ada253a96ebd7ddcb8045bdd2a75f52fdd0f51ad7bc0183e phase1/ablations/no_context/training_args.bin
+1c7e4ab6940afc79f279336e739811171ce4a85c1198fb2a8baf4fd4cc6160b7 phase1/ablations/no_context/training_recipe.json
+91d8ec7a27b0ffe87b52ac7370ceffd78c910bacf142b7a9e942274ce26c8d0b phase1/ablations/no_context/training_summary.json
+81302140e113697d25464f222778a32a3a55dc576752d834b3b120e690833e5e phase1/ablations/random_labels/README.md
+eb964f79f8e6f7ea2891c7d62fd4e4e0080de87c6998c03e0007524e57d7f34f phase1/ablations/random_labels/adapter_config.json
+7bebded9e10f63603bfd9d6a135dda87c8bcac25b4a43b365e2b1ce882bf4c93 phase1/ablations/random_labels/adapter_model.safetensors
+cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f phase1/ablations/random_labels/chat_template.jinja
+6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5 phase1/ablations/random_labels/tokenizer.json
+d9bf1d78a80bad5eb65b83c2608910e3be00a9d3250c8153a5a393ad1257e1d6 phase1/ablations/random_labels/tokenizer_config.json
+ad84a00370d0c977fcd38a1fce033b130494f3fffd4a3c0cbc44a9dd43c0bdbf phase1/ablations/random_labels/training_args.bin
+986dd7375539ac325bec654e90e7d077b4bf7738a471d83d7aa47c4d9f1898fb phase1/ablations/random_labels/training_recipe.json
+4fcf1c2cc9c584de1ac096a5e7cb48babbd74b25d74c9973e654976f42d0be74 phase1/ablations/random_labels/training_summary.json
+6399ea44e270928871d5a50c1bc2c06eeb83d7ca10d5deaaeb78e4d41e859933 phase1/ablations/shuffled_documents/README.md
+eb964f79f8e6f7ea2891c7d62fd4e4e0080de87c6998c03e0007524e57d7f34f phase1/ablations/shuffled_documents/adapter_config.json
+c9d8b830ce2a3b583eb54eab7470a7b3acb9c163dee812de0313c454462de064 phase1/ablations/shuffled_documents/adapter_model.safetensors
+cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f phase1/ablations/shuffled_documents/chat_template.jinja
+6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5 phase1/ablations/shuffled_documents/tokenizer.json
+d9bf1d78a80bad5eb65b83c2608910e3be00a9d3250c8153a5a393ad1257e1d6 phase1/ablations/shuffled_documents/tokenizer_config.json
+eaedd299a54d4c58a5c10cd9e379b80d44bbfd7922ccd578d5731f3e7adeb7cb phase1/ablations/shuffled_documents/training_args.bin
+d37a5d995c8264d0abffeff9171478803b7b22412780c4e8226bf0b857f96235 phase1/ablations/shuffled_documents/training_recipe.json
+2e8d728d08c0bcc0212162c3cdd9295f5c982d92f65316f19f78c055a8c5589c phase1/ablations/shuffled_documents/training_summary.json
+4d1f59c2d09544c2e0397996d8032202fff701109f5e7320c48a72c7fa005b70 phase1/llama-dpo-v3/README.md
+18239fcf982739e03a128d680ac81ec7f05e0f058c032bed56a51376d7da731e phase1/llama-dpo-v3/adapter_config.json
+2633d2be48f838d5c4404786ca9baa38ff2b0ed66d2fa5d7d11eb74fe63c178d phase1/llama-dpo-v3/adapter_model.safetensors
+5816fce10444e03c2e9ee1ef8a4a1ea61ae7e69e438613f3b17b69d0426223a4 phase1/llama-dpo-v3/chat_template.jinja
+b20d148821d1e209454ef0d54a1d2bb1ff12cd365a0ff5391d96010e897b9644 phase1/llama-dpo-v3/tokenizer.json
+85795daec85614d3918187e7bd6d3548a0de064dc66e365db30bdea9d7b71dc3 phase1/llama-dpo-v3/tokenizer_config.json
+fced63a90802dbed1aa84b5b0863e967f730e08882f30d6a2cec2f76954c2675 phase1/llama-dpo-v3/training_args.bin
+a8d5a947873a04a1bbbc7972465db555eee1cf238d06c7f30f3d698056ee3d4e phase1/llama-dpo-v3/training_recipe.json
+55c87632a89ed152627178af35294d6c5c11096f8af740c682141925ad0aa122 phase1/llama-dpo-v3/training_summary.json
+512f4df0376b902685fe95ad685ea8b06c1112bec0cdf3ae7c7b7db7eb462664 phase1/qwen-main-dpo-v3/README.md
+f59cbec9982518824aa800f9fbc56b764fc73098422cb4e81f2e70d6099d26a2 phase1/qwen-main-dpo-v3/adapter_config.json
+cc60e5fe47be9bf4f64d734d40d89310a4826d8156d0ab8221b466f21c5efd58 phase1/qwen-main-dpo-v3/adapter_model.safetensors
+cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f phase1/qwen-main-dpo-v3/chat_template.jinja
+6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5 phase1/qwen-main-dpo-v3/tokenizer.json
+d9bf1d78a80bad5eb65b83c2608910e3be00a9d3250c8153a5a393ad1257e1d6 phase1/qwen-main-dpo-v3/tokenizer_config.json
+1578389f7f0fe332b393d7d42285d2a1a804b2efc9a174e73d1805a5ca67e41b phase1/qwen-main-dpo-v3/training_args.bin
+3e8d64f2b8fcf2d9259644c9b919fc9597a450d3ec89eae738a467fded895a88 phase1/qwen-main-dpo-v3/training_recipe.json
+5263a105fa846c7cf3ddbb03ede58f4956c25f0e7a8ae38f9f8974930d16518b phase1/qwen-main-dpo-v3/training_summary.json
diff --git a/README.md b/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..960c1e02e180f734ac955bbb995cd4c97640f4f5
--- /dev/null
+++ b/README.md
@@ -0,0 +1,88 @@
+---
+library_name: peft
+tags:
+- dpo
+- lora
+- peft
+- self-training
+- transformers
+- trl
+pipeline_tag: text-generation
+---
+
+# York Milestone Project — Phase 1 adapters
+
+Hugging Face 仓库：
+[`geyuxu/york-milestone-project-self-traing-loop-model`](https://huggingface.co/geyuxu/york-milestone-project-self-traing-loop-model)
+
+本仓库保存一期 v3 的最终 PEFT LoRA-DPO adapter，用于复现实验和后续阶段复用。
+它不包含基础模型权重、完整训练 checkpoint、合成数据、评测输出或原始 API 响应。
+
+## 内容
+
+| 路径 | 基础模型 | 用途 | 训练 / 验证偏好对 |
+| --- | --- | --- | ---: |
+| `phase1/qwen-main-dpo-v3/` | `Qwen/Qwen2.5-1.5B-Instruct` | 一期主实验 | 99,986 / 11,110 |
+| `phase1/llama-dpo-v3/` | `meta-llama/Llama-3.2-1B-Instruct` | 跨模型迁移实验 | 99,986 / 11,110 |
+| `phase1/ablations/empty_context/` | Qwen2.5-1.5B-Instruct | 诊断消融 | 77,124 / 8,563 |
+| `phase1/ablations/helps_only/` | Qwen2.5-1.5B-Instruct | 诊断消融 | 77,124 / 8,563 |
+| `phase1/ablations/matched_clean/` | Qwen2.5-1.5B-Instruct | 诊断消融 | 77,124 / 8,563 |
+| `phase1/ablations/no_context/` | Qwen2.5-1.5B-Instruct | 诊断消融 | 77,124 / 8,563 |
+| `phase1/ablations/random_labels/` | Qwen2.5-1.5B-Instruct | 诊断消融 | 77,124 / 8,563 |
+| `phase1/ablations/shuffled_documents/` | Qwen2.5-1.5B-Instruct | 诊断消融 | 77,124 / 8,563 |
+
+每个目录包含：
+
+- `adapter_model.safetensors` 和 `adapter_config.json`；
+- 与导出时一致的 tokenizer 和 chat template；
+- `training_recipe.json`：冻结的数据成员哈希及核心训练参数；
+- `training_summary.json`：训练步数、日志和最终摘要；
+- `training_args.bin`：Transformers 训练参数快照。
+
+## 加载 adapter
+
+以下示例加载 Qwen 主实验。其他 adapter 只需替换 `subfolder` 和对应的
+`base_id`：
+
+```python
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo_id = "geyuxu/york-milestone-project-self-traing-loop-model"
+subfolder = "phase1/qwen-main-dpo-v3"
+base_id = "Qwen/Qwen2.5-1.5B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(repo_id, subfolder=subfolder)
+base_model = AutoModelForCausalLM.from_pretrained(
+ base_id,
+ torch_dtype="auto",
+ device_map="auto",
+)
+model = PeftModel.from_pretrained(
+ base_model,
+ repo_id,
+ subfolder=subfolder,
+)
+```
+
+Llama adapter 需要先在 Hugging Face 接受
+`meta-llama/Llama-3.2-1B-Instruct` 的访问条款，并使用有权限的账户登录。
+
+## 完整性验证
+
+从仓库根目录执行：
+
+```bash
+sha256sum -c MANIFEST.sha256
+```
+
+`MANIFEST.sha256` 覆盖 `phase1/` 下发布的全部文件。Safetensors 通过 Git LFS
+管理；基础模型权重不会被提交到本仓库。
+
+## 许可证与限制
+
+- Qwen adapter 依赖 Apache-2.0 许可的 Qwen2.5 基础模型；
+- Llama adapter 的使用受 Llama 3.2 Community License 及其访问条款约束；
+- 训练数据的来源、许可和限制以配套数据仓库的 dataset card 为准。
+
+这些 adapter 是研究实验产物，未针对生产安全、事实准确性或具体行业用途做保证。
diff --git a/phase1/ablations/empty_context/README.md b/phase1/ablations/empty_context/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..418cfb6032416f4697e763e02d1d7dcd5849ef38
--- /dev/null
+++ b/phase1/ablations/empty_context/README.md
@@ -0,0 +1,80 @@
+---
+library_name: peft
+model_name: phase1-qwen-empty-context-ablation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-1.5B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+license: apache-2.0
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+pipeline_tag: text-generation
+---
+
+# Phase 1 Qwen `empty_context` diagnostic ablation
+
+This is the Phase 1 `empty_context` diagnostic LoRA-DPO adapter fine-tuned from
+[Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct).
+The frozen training recipe and membership hashes are included in this directory.
+
+## Quick start
+
+```python
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo_id = "geyuxu/york-milestone-project-self-traing-loop-model"
+subfolder = "phase1/ablations/empty_context"
+base_id = "Qwen/Qwen2.5-1.5B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(repo_id, subfolder=subfolder)
+base_model = AutoModelForCausalLM.from_pretrained(
+ base_id, torch_dtype="auto", device_map="auto"
+)
+model = PeftModel.from_pretrained(base_model, repo_id, subfolder=subfolder)
+```
+
+## Training procedure
+
+
+
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 0.29.0
+- Transformers: 5.3.0
+- Pytorch: 2.10.0
+- Datasets: 4.6.1
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+ title = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+ author = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+ year = 2023,
+ booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+ url = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+ editor = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+ title = {{TRL: Transformers Reinforcement Learning}},
+ author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+ license = {Apache-2.0},
+ url = {https://github.com/huggingface/trl},
+ year = {2020}
+}
+```
diff --git a/phase1/ablations/empty_context/adapter_config.json b/phase1/ablations/empty_context/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..fa47a2292cd8a23ccf1b4d101d7f7a03e3f08871
--- /dev/null
+++ b/phase1/ablations/empty_context/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "k_proj",
+ "o_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
diff --git a/phase1/ablations/empty_context/adapter_model.safetensors b/phase1/ablations/empty_context/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c4ea5bcda49d32002dcadbbbbcb099c30e2ed44e
--- /dev/null
+++ b/phase1/ablations/empty_context/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79a128d64b87a92a3b4450386e35821dc796e494dca46b4d8c421b8580dbfd63
+size 17462432
diff --git a/phase1/ablations/empty_context/chat_template.jinja b/phase1/ablations/empty_context/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e
--- /dev/null
+++ b/phase1/ablations/empty_context/chat_template.jinja
@@ -0,0 +1,54 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- messages[0]['content'] }}
+ {%- else %}
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+ {%- endif %}
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+ {%- else %}
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {{- '<|im_start|>' + message.role }}
+ {%- if message.content %}
+ {{- '\n' + message.content }}
+ {%- endif %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- '}\n' }}
+ {%- endfor %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- message.content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}
diff --git a/phase1/ablations/empty_context/tokenizer.json b/phase1/ablations/empty_context/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..d73428d91463b495bc017fb6a2fb3a656646482b
--- /dev/null
+++ b/phase1/ablations/empty_context/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5
+size 11422166
diff --git a/phase1/ablations/empty_context/tokenizer_config.json b/phase1/ablations/empty_context/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9fd0fb48e73786f54fb04e98892f5f5a0ebeebdb
--- /dev/null
+++ b/phase1/ablations/empty_context/tokenizer_config.json
@@ -0,0 +1,29 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": true,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/phase1/ablations/empty_context/training_args.bin b/phase1/ablations/empty_context/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..cd93f3052761a17eab1c70bf77cf946db6f593a1
--- /dev/null
+++ b/phase1/ablations/empty_context/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4980eeedcdde01c95d37224943c3c64dcbeed7f0894c9e83bd99f12534a473fd
+size 5841
diff --git a/phase1/ablations/empty_context/training_recipe.json b/phase1/ablations/empty_context/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..7cf28f78be36e7fdd024c29e76741d2c0e0b180c
--- /dev/null
+++ b/phase1/ablations/empty_context/training_recipe.json
@@ -0,0 +1,22 @@
+{
+ "format_version": 1,
+ "experiment": "confusion_ablation",
+ "ablation": "empty_context",
+ "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+ "train_membership_sha256": "b497fa765223a9810ad784f31e1598edfa992a162146d6f94d9cdf4adcedd87d",
+ "validation_membership_sha256": "2f609467438cd6a01fd84b9d80c0fcd84c44767e3b2ed98f4f6a83d68a62c181",
+ "num_train_pairs": 77124,
+ "num_validation_pairs": 8563,
+ "transformation_seed": 22,
+ "validation_prompt_mode": "clean_rag",
+ "learning_rate": 1e-05,
+ "beta": 0.1,
+ "num_epochs": 1.0,
+ "seed": 22,
+ "max_length": 512,
+ "max_prompt_length": 384,
+ "gradient_accumulation_steps": 80,
+ "save_steps": 125,
+ "save_total_limit": 12,
+ "preference_pairs_format_version": 2
+}
diff --git a/phase1/ablations/empty_context/training_summary.json b/phase1/ablations/empty_context/training_summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..327a8d452fd85eb40274cb41da8fa7207f708fa9
--- /dev/null
+++ b/phase1/ablations/empty_context/training_summary.json
@@ -0,0 +1,2925 @@
+{
+ "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+ "resolved_model": "Qwen/Qwen2.5-1.5B-Instruct",
+ "num_train_records": 77124,
+ "num_validation_records_available": 8563,
+ "training_config": {
+ "lora_rank": 16,
+ "lora_alpha": 32,
+ "learning_rate": 1e-05,
+ "num_epochs": 1.0,
+ "per_device_batch_size": 1,
+ "per_device_eval_batch_size": 1,
+ "gradient_accumulation_steps": 80,
+ "beta": 0.1,
+ "max_length": 512,
+ "max_prompt_length": 384,
+ "seed": 22,
+ "save_steps": 125,
+ "save_total_limit": 12,
+ "resume_from_checkpoint": null,
+ "precision_dtype": "torch.bfloat16",
+ "bf16": true,
+ "fp16": false,
+ "tokenizer_add_bos_token": false
+ },
+ "global_step": 965,
+ "best_metric": 0.5264487266540527,
+ "best_model_checkpoint": "outputs/confusion_ablation_phase1_repair_v3/empty_context/lora/checkpoint-875",
+ "train_metrics": {
+ "train_runtime": 22398.3946,
+ "train_samples_per_second": 3.443,
+ "train_steps_per_second": 0.043,
+ "total_flos": 1.3308435135621734e+17,
+ "train_loss": 0.48160764293967134
+ },
+ "log_history": [
+ {
+ "loss": 0.6885997772216796,
+ "grad_norm": 0.012969385832548141,
+ "learning_rate": 9.906735751295338e-06,
+ "entropy": 1.0798783786967396,
+ "num_tokens": 156534.0,
+ "logits/chosen": -0.7346464429003529,
+ "logits/rejected": -0.7633830132317224,
+ "mean_token_accuracy": 0.689994627982378,
+ "rewards/chosen": 0.004463997096299863,
+ "rewards/rejected": -0.005229698287648716,
+ "rewards/accuracies": 0.5475,
+ "rewards/margins": 0.009693695359455887,
+ "logps/chosen": -53.64696934700012,
+ "logps/rejected": -45.47284424066544,
+ "epoch": 0.010372905969607386,
+ "step": 10
+ },
+ {
+ "loss": 0.6514244079589844,
+ "grad_norm": 0.01852896623313427,
+ "learning_rate": 9.803108808290156e-06,
+ "entropy": 1.1224612561706453,
+ "num_tokens": 313607.0,
+ "logits/chosen": -0.6341513052007935,
+ "logits/rejected": -0.6562745898278561,
+ "mean_token_accuracy": 0.6891892428323626,
+ "rewards/chosen": 0.03406709198377939,
+ "rewards/rejected": -0.05670502469384701,
+ "rewards/accuracies": 0.7825,
+ "rewards/margins": 0.09077211681949848,
+ "logps/chosen": -55.87430178999901,
+ "logps/rejected": -44.03523090600967,
+ "epoch": 0.02074581193921477,
+ "step": 20
+ },
+ {
+ "loss": 0.6161895275115967,
+ "grad_norm": 0.009809216484427452,
+ "learning_rate": 9.699481865284975e-06,
+ "entropy": 1.2153802848607302,
+ "num_tokens": 471990.0,
+ "logits/chosen": -0.5183433123187651,
+ "logits/rejected": -0.48201045809287074,
+ "mean_token_accuracy": 0.6826715680211782,
+ "rewards/chosen": 0.03547020315836562,
+ "rewards/rejected": -0.15197590536190547,
+ "rewards/accuracies": 0.76625,
+ "rewards/margins": 0.18744610846129944,
+ "logps/chosen": -55.871621309518815,
+ "logps/rejected": -47.164069904088976,
+ "epoch": 0.031118717908822157,
+ "step": 30
+ },
+ {
+ "loss": 0.5837482929229736,
+ "grad_norm": 0.009694655425846577,
+ "learning_rate": 9.595854922279793e-06,
+ "entropy": 1.2649927862174808,
+ "num_tokens": 630457.0,
+ "logits/chosen": -0.41558199378068605,
+ "logits/rejected": -0.36016731520911593,
+ "mean_token_accuracy": 0.6745159946288913,
+ "rewards/chosen": 0.0030272094297106376,
+ "rewards/rejected": -0.28438122774517977,
+ "rewards/accuracies": 0.78,
+ "rewards/margins": 0.2874084369814955,
+ "logps/chosen": -56.05272542119026,
+ "logps/rejected": -47.54765802979469,
+ "epoch": 0.04149162387842954,
+ "step": 40
+ },
+ {
+ "loss": 0.5699157238006591,
+ "grad_norm": 0.00790126621723175,
+ "learning_rate": 9.492227979274612e-06,
+ "entropy": 1.3643021282646806,
+ "num_tokens": 788279.0,
+ "logits/chosen": -0.2938073242511318,
+ "logits/rejected": -0.2156574113712712,
+ "mean_token_accuracy": 0.6769538759440183,
+ "rewards/chosen": -0.057574519852860245,
+ "rewards/rejected": -0.4283463936719636,
+ "rewards/accuracies": 0.77,
+ "rewards/margins": 0.37077187476679685,
+ "logps/chosen": -57.178628134727475,
+ "logps/rejected": -48.340812134742734,
+ "epoch": 0.051864529848036925,
+ "step": 50
+ },
+ {
+ "loss": 0.5565647125244141,
+ "grad_norm": 0.007377359550446272,
+ "learning_rate": 9.388601036269432e-06,
+ "entropy": 1.4374694915488362,
+ "num_tokens": 944837.0,
+ "logits/chosen": -0.24100234533637963,
+ "logits/rejected": -0.15933266229535895,
+ "mean_token_accuracy": 0.6763953970745206,
+ "rewards/chosen": -0.11264537643779476,
+ "rewards/rejected": -0.5672522658561502,
+ "rewards/accuracies": 0.77125,
+ "rewards/margins": 0.4546068909333553,
+ "logps/chosen": -56.07041595697403,
+ "logps/rejected": -49.03205164551735,
+ "epoch": 0.062237435817644314,
+ "step": 60
+ },
+ {
+ "loss": 0.5234037876129151,
+ "grad_norm": 0.008119487203657627,
+ "learning_rate": 9.284974093264249e-06,
+ "entropy": 1.4613975173421203,
+ "num_tokens": 1102438.0,
+ "logits/chosen": -0.2570220845928096,
+ "logits/rejected": -0.08345125228558167,
+ "mean_token_accuracy": 0.6671836027130484,
+ "rewards/chosen": -0.20022683466155286,
+ "rewards/rejected": -0.7679086389223813,
+ "rewards/accuracies": 0.7875,
+ "rewards/margins": 0.5676818046835251,
+ "logps/chosen": -58.67170790791511,
+ "logps/rejected": -51.08670456886291,
+ "epoch": 0.0726103417872517,
+ "step": 70
+ },
+ {
+ "loss": 0.5317386150360107,
+ "grad_norm": 0.008479549549520016,
+ "learning_rate": 9.181347150259067e-06,
+ "entropy": 1.5662438894622028,
+ "num_tokens": 1259718.0,
+ "logits/chosen": -0.19741540588436432,
+ "logits/rejected": -0.05688106650755805,
+ "mean_token_accuracy": 0.6640376079455018,
+ "rewards/chosen": -0.30314797887514944,
+ "rewards/rejected": -0.9170537282113219,
+ "rewards/accuracies": 0.775,
+ "rewards/margins": 0.6139057491905987,
+ "logps/chosen": -58.94739377617836,
+ "logps/rejected": -54.774017692804335,
+ "epoch": 0.08298324775685909,
+ "step": 80
+ },
+ {
+ "loss": 0.524826192855835,
+ "grad_norm": 0.008542119525372982,
+ "learning_rate": 9.077720207253888e-06,
+ "entropy": 1.5676167696528136,
+ "num_tokens": 1416640.0,
+ "logits/chosen": -0.19087733562866938,
+ "logits/rejected": -0.06961324446158064,
+ "mean_token_accuracy": 0.6602621851768344,
+ "rewards/chosen": -0.32906408929731695,
+ "rewards/rejected": -1.0247976127301808,
+ "rewards/accuracies": 0.78,
+ "rewards/margins": 0.6957335224747658,
+ "logps/chosen": -59.972526935338976,
+ "logps/rejected": -54.91353523254394,
+ "epoch": 0.09335615372646647,
+ "step": 90
+ },
+ {
+ "loss": 0.5366811752319336,
+ "grad_norm": 0.01028946042060852,
+ "learning_rate": 8.974093264248706e-06,
+ "entropy": 1.6082658682204782,
+ "num_tokens": 1578582.0,
+ "logits/chosen": -0.13524073540751783,
+ "logits/rejected": -0.03532241128049209,
+ "mean_token_accuracy": 0.6632538431324064,
+ "rewards/chosen": -0.45744188758188103,
+ "rewards/rejected": -1.1608894589025294,
+ "rewards/accuracies": 0.75625,
+ "rewards/margins": 0.7034475702582859,
+ "logps/chosen": -65.19879947781563,
+ "logps/rejected": -61.238030140399935,
+ "epoch": 0.10372905969607385,
+ "step": 100
+ },
+ {
+ "loss": 0.5021177291870117,
+ "grad_norm": 0.009549791924655437,
+ "learning_rate": 8.870466321243523e-06,
+ "entropy": 1.580866142194718,
+ "num_tokens": 1736902.0,
+ "logits/chosen": -0.16430359268125203,
+ "logits/rejected": -0.02982257553996645,
+ "mean_token_accuracy": 0.6615490332618356,
+ "rewards/chosen": -0.4121622362524067,
+ "rewards/rejected": -1.1860957287461498,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 0.7739334937836975,
+ "logps/chosen": -60.39180130243302,
+ "logps/rejected": -55.96378324866295,
+ "epoch": 0.11410196566568125,
+ "step": 110
+ },
+ {
+ "loss": 0.5247333526611329,
+ "grad_norm": 0.007295758463442326,
+ "learning_rate": 8.766839378238343e-06,
+ "entropy": 1.5716826527751981,
+ "num_tokens": 1894607.0,
+ "logits/chosen": -0.21342730308044772,
+ "logits/rejected": -0.08283230979493357,
+ "mean_token_accuracy": 0.668023902028799,
+ "rewards/chosen": -0.4445982250826205,
+ "rewards/rejected": -1.1845778720156523,
+ "rewards/accuracies": 0.76875,
+ "rewards/margins": 0.7399796470795991,
+ "logps/chosen": -61.616042573452,
+ "logps/rejected": -53.51006484746933,
+ "epoch": 0.12447487163528863,
+ "step": 120
+ },
+ {
+ "eval_loss": 0.5866048336029053,
+ "eval_runtime": 186.3701,
+ "eval_samples_per_second": 5.366,
+ "eval_steps_per_second": 5.366,
+ "eval_entropy": 1.026585803925991,
+ "eval_num_tokens": 1973585.0,
+ "eval_logits/chosen": -0.27179382896787163,
+ "eval_logits/rejected": -0.1471519449829466,
+ "eval_mean_token_accuracy": 0.7038057380318642,
+ "eval_rewards/chosen": -0.4332459507464664,
+ "eval_rewards/rejected": -0.8382879723370134,
+ "eval_rewards/accuracies": 0.673,
+ "eval_rewards/margins": 0.40504202180169524,
+ "eval_logps/chosen": -54.505446267127994,
+ "eval_logps/rejected": -61.022751308441165,
+ "epoch": 0.1296613246200923,
+ "step": 125
+ },
+ {
+ "loss": 0.5197542190551758,
+ "grad_norm": 0.007421356160193682,
+ "learning_rate": 8.663212435233162e-06,
+ "entropy": 1.6153548054210842,
+ "num_tokens": 2053549.0,
+ "logits/chosen": -0.16180727962310634,
+ "logits/rejected": -0.02674481546655947,
+ "mean_token_accuracy": 0.6631045359373092,
+ "rewards/chosen": -0.44111566214839515,
+ "rewards/rejected": -1.2121200907422462,
+ "rewards/accuracies": 0.7875,
+ "rewards/margins": 0.7710044272383675,
+ "logps/chosen": -60.93198619604111,
+ "logps/rejected": -57.747130882740024,
+ "epoch": 0.13484777760489602,
+ "step": 130
+ },
+ {
+ "loss": 0.5147777080535889,
+ "grad_norm": 0.009197528474032879,
+ "learning_rate": 8.55958549222798e-06,
+ "entropy": 1.5458942975103855,
+ "num_tokens": 2209979.0,
+ "logits/chosen": -0.2903924655131609,
+ "logits/rejected": -0.10041574268765137,
+ "mean_token_accuracy": 0.6736215281113982,
+ "rewards/chosen": -0.420290390338123,
+ "rewards/rejected": -1.1833371208855532,
+ "rewards/accuracies": 0.78875,
+ "rewards/margins": 0.7630467313993722,
+ "logps/chosen": -55.64869555354118,
+ "logps/rejected": -56.70225902199745,
+ "epoch": 0.1452206835745034,
+ "step": 140
+ },
+ {
+ "loss": 0.48405823707580564,
+ "grad_norm": 0.009587124921381474,
+ "learning_rate": 8.455958549222799e-06,
+ "entropy": 1.5990170714166014,
+ "num_tokens": 2369571.0,
+ "logits/chosen": -0.24899731487552476,
+ "logits/rejected": -0.0959463068576649,
+ "mean_token_accuracy": 0.6553366997931153,
+ "rewards/chosen": -0.3976931418222512,
+ "rewards/rejected": -1.2890572238096503,
+ "rewards/accuracies": 0.8025,
+ "rewards/margins": 0.8913640801142901,
+ "logps/chosen": -60.26506376743317,
+ "logps/rejected": -59.52328753948212,
+ "epoch": 0.1555935895441108,
+ "step": 150
+ },
+ {
+ "loss": 0.5164113998413086,
+ "grad_norm": 0.009895331226289272,
+ "learning_rate": 8.352331606217617e-06,
+ "entropy": 1.6323789210617543,
+ "num_tokens": 2528769.0,
+ "logits/chosen": -0.2223647205235664,
+ "logits/rejected": -0.09436181620109207,
+ "mean_token_accuracy": 0.6549394633993506,
+ "rewards/chosen": -0.5111256973049604,
+ "rewards/rejected": -1.2863853847890276,
+ "rewards/accuracies": 0.77875,
+ "rewards/margins": 0.7752596874302253,
+ "logps/chosen": -64.33949285626412,
+ "logps/rejected": -59.65175414085388,
+ "epoch": 0.16596649551371817,
+ "step": 160
+ },
+ {
+ "loss": 0.5000368118286133,
+ "grad_norm": 0.008498546667397022,
+ "learning_rate": 8.248704663212436e-06,
+ "entropy": 1.6361813547555357,
+ "num_tokens": 2688485.0,
+ "logits/chosen": -0.16995661589474917,
+ "logits/rejected": -0.058628515103994625,
+ "mean_token_accuracy": 0.6581607956252992,
+ "rewards/chosen": -0.4723435838901787,
+ "rewards/rejected": -1.3314759727602359,
+ "rewards/accuracies": 0.79,
+ "rewards/margins": 0.8591323888232,
+ "logps/chosen": -63.09673968315124,
+ "logps/rejected": -58.408678278923034,
+ "epoch": 0.17633940148332555,
+ "step": 170
+ },
+ {
+ "loss": 0.48834381103515623,
+ "grad_norm": 0.0076238978654146194,
+ "learning_rate": 8.145077720207254e-06,
+ "entropy": 1.6382397083751856,
+ "num_tokens": 2845521.0,
+ "logits/chosen": -0.15718930734615982,
+ "logits/rejected": -0.05206926899191467,
+ "mean_token_accuracy": 0.6570399883762001,
+ "rewards/chosen": -0.44634571393646183,
+ "rewards/rejected": -1.3426366519206203,
+ "rewards/accuracies": 0.80125,
+ "rewards/margins": 0.8962909340648912,
+ "logps/chosen": -60.25443920731544,
+ "logps/rejected": -57.17802233457565,
+ "epoch": 0.18671230745293294,
+ "step": 180
+ },
+ {
+ "loss": 0.47557435035705564,
+ "grad_norm": 0.008195850066840649,
+ "learning_rate": 8.041450777202073e-06,
+ "entropy": 1.664605896770954,
+ "num_tokens": 3002074.0,
+ "logits/chosen": -0.18363723049103992,
+ "logits/rejected": -0.02745318001992498,
+ "mean_token_accuracy": 0.6470572968758642,
+ "rewards/chosen": -0.4189180153416373,
+ "rewards/rejected": -1.3144572579705618,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 0.895539241922088,
+ "logps/chosen": -62.842660636901854,
+ "logps/rejected": -55.66383082151413,
+ "epoch": 0.19708521342254032,
+ "step": 190
+ },
+ {
+ "loss": 0.534032154083252,
+ "grad_norm": 0.009718852117657661,
+ "learning_rate": 7.937823834196891e-06,
+ "entropy": 1.624278563093394,
+ "num_tokens": 3158718.0,
+ "logits/chosen": -0.1727993139137808,
+ "logits/rejected": -0.03551834474392872,
+ "mean_token_accuracy": 0.659155095871538,
+ "rewards/chosen": -0.4744971431684098,
+ "rewards/rejected": -1.2906594037846661,
+ "rewards/accuracies": 0.7525,
+ "rewards/margins": 0.8161622601863928,
+ "logps/chosen": -59.0803443980217,
+ "logps/rejected": -56.69621436238289,
+ "epoch": 0.2074581193921477,
+ "step": 200
+ },
+ {
+ "loss": 0.49621829986572263,
+ "grad_norm": 0.006965890992432833,
+ "learning_rate": 7.834196891191712e-06,
+ "entropy": 1.6065182481892406,
+ "num_tokens": 3312685.0,
+ "logits/chosen": -0.27075287980432916,
+ "logits/rejected": -0.0948328996504896,
+ "mean_token_accuracy": 0.655901060551405,
+ "rewards/chosen": -0.37993197983159915,
+ "rewards/rejected": -1.2266587249445728,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 0.8467267453856766,
+ "logps/chosen": -56.4505591404438,
+ "logps/rejected": -53.79004474401474,
+ "epoch": 0.2178310253617551,
+ "step": 210
+ },
+ {
+ "loss": 0.48090167045593263,
+ "grad_norm": 0.009547212161123753,
+ "learning_rate": 7.730569948186528e-06,
+ "entropy": 1.6623543649539352,
+ "num_tokens": 3472099.0,
+ "logits/chosen": -0.13968818712728476,
+ "logits/rejected": 0.0031978122884744786,
+ "mean_token_accuracy": 0.6451264442130923,
+ "rewards/chosen": -0.482780237980769,
+ "rewards/rejected": -1.413478313506348,
+ "rewards/accuracies": 0.795,
+ "rewards/margins": 0.930698074856773,
+ "logps/chosen": -60.84456259608269,
+ "logps/rejected": -58.714372800588606,
+ "epoch": 0.2282039313313625,
+ "step": 220
+ },
+ {
+ "loss": 0.4559622764587402,
+ "grad_norm": 0.007603341713547707,
+ "learning_rate": 7.626943005181348e-06,
+ "entropy": 1.6722249686904251,
+ "num_tokens": 3632168.0,
+ "logits/chosen": -0.10667086636674211,
+ "logits/rejected": -0.022223443209434823,
+ "mean_token_accuracy": 0.6569089805148542,
+ "rewards/chosen": -0.4016924969325191,
+ "rewards/rejected": -1.4414992470349535,
+ "rewards/accuracies": 0.8125,
+ "rewards/margins": 1.0398067526565864,
+ "logps/chosen": -65.72550685882568,
+ "logps/rejected": -59.37691457271576,
+ "epoch": 0.23857683730096987,
+ "step": 230
+ },
+ {
+ "loss": 0.4861691951751709,
+ "grad_norm": 0.0118019487708807,
+ "learning_rate": 7.523316062176167e-06,
+ "entropy": 1.663213079739362,
+ "num_tokens": 3790720.0,
+ "logits/chosen": -0.084852375856096,
+ "logits/rejected": 0.020669056782071628,
+ "mean_token_accuracy": 0.6473133432120085,
+ "rewards/chosen": -0.48122090814737023,
+ "rewards/rejected": -1.4429112731330678,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 0.9616903657466174,
+ "logps/chosen": -63.50809263586998,
+ "logps/rejected": -59.43233494758606,
+ "epoch": 0.24894974327057726,
+ "step": 240
+ },
+ {
+ "loss": 0.4908440589904785,
+ "grad_norm": 0.009277455508708954,
+ "learning_rate": 7.419689119170985e-06,
+ "entropy": 1.6366083236783744,
+ "num_tokens": 3949401.0,
+ "logits/chosen": -0.1602433276424981,
+ "logits/rejected": 0.01983137979948086,
+ "mean_token_accuracy": 0.6576688695885241,
+ "rewards/chosen": -0.4815865662365832,
+ "rewards/rejected": -1.3985355464673193,
+ "rewards/accuracies": 0.78375,
+ "rewards/margins": 0.9169489816110581,
+ "logps/chosen": -62.95811867237091,
+ "logps/rejected": -59.38419282078743,
+ "epoch": 0.2593226492401846,
+ "step": 250
+ },
+ {
+ "eval_loss": 0.5700510144233704,
+ "eval_runtime": 215.5827,
+ "eval_samples_per_second": 4.639,
+ "eval_steps_per_second": 4.639,
+ "eval_entropy": 1.0522404039800166,
+ "eval_num_tokens": 3949401.0,
+ "eval_logits/chosen": -0.2592554757373893,
+ "eval_logits/rejected": -0.12886324215802,
+ "eval_mean_token_accuracy": 0.6999065904319286,
+ "eval_rewards/chosen": -0.5199006232493557,
+ "eval_rewards/rejected": -1.0152434906987473,
+ "eval_rewards/accuracies": 0.689,
+ "eval_rewards/margins": 0.4953428685963154,
+ "eval_logps/chosen": -55.371993000030514,
+ "eval_logps/rejected": -62.79230645370483,
+ "epoch": 0.2593226492401846,
+ "step": 250
+ },
+ {
+ "loss": 0.5332072734832763,
+ "grad_norm": 0.01310813333839178,
+ "learning_rate": 7.3160621761658035e-06,
+ "entropy": 1.6763470254838466,
+ "num_tokens": 4109572.0,
+ "logits/chosen": -0.11223109996092892,
+ "logits/rejected": 0.00783857225474676,
+ "mean_token_accuracy": 0.6527093886770308,
+ "rewards/chosen": -0.5290841975083458,
+ "rewards/rejected": -1.4143161330505973,
+ "rewards/accuracies": 0.76875,
+ "rewards/margins": 0.8852319360524415,
+ "logps/chosen": -65.61116208314895,
+ "logps/rejected": -60.50024994492531,
+ "epoch": 0.26969555520979205,
+ "step": 260
+ },
+ {
+ "loss": 0.46380929946899413,
+ "grad_norm": 0.008016095496714115,
+ "learning_rate": 7.212435233160623e-06,
+ "entropy": 1.6493263538647442,
+ "num_tokens": 4264194.0,
+ "logits/chosen": -0.15574262186655452,
+ "logits/rejected": -0.00402961174777503,
+ "mean_token_accuracy": 0.6535617489926517,
+ "rewards/chosen": -0.39940022186754504,
+ "rewards/rejected": -1.3337873129447688,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 0.9343870897591114,
+ "logps/chosen": -58.40460694432259,
+ "logps/rejected": -56.3271455013752,
+ "epoch": 0.28006846117939943,
+ "step": 270
+ },
+ {
+ "loss": 0.46946086883544924,
+ "grad_norm": 0.010439381934702396,
+ "learning_rate": 7.108808290155441e-06,
+ "entropy": 1.6875111198704689,
+ "num_tokens": 4422991.0,
+ "logits/chosen": -0.15225440851382163,
+ "logits/rejected": -0.012922279572286165,
+ "mean_token_accuracy": 0.6428867661487311,
+ "rewards/chosen": -0.42678490347389014,
+ "rewards/rejected": -1.3960492441387031,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 0.9692643392784521,
+ "logps/chosen": -63.95781509041786,
+ "logps/rejected": -57.44645619392395,
+ "epoch": 0.2904413671490068,
+ "step": 280
+ },
+ {
+ "loss": 0.4867039680480957,
+ "grad_norm": 0.008255927823483944,
+ "learning_rate": 7.005181347150259e-06,
+ "entropy": 1.6560454944707452,
+ "num_tokens": 4582080.0,
+ "logits/chosen": -0.12475022342462339,
+ "logits/rejected": -0.004146134714328987,
+ "mean_token_accuracy": 0.6578386729583144,
+ "rewards/chosen": -0.4762678133821464,
+ "rewards/rejected": -1.3799970698577817,
+ "rewards/accuracies": 0.78875,
+ "rewards/margins": 0.9037292560562491,
+ "logps/chosen": -62.563043652772905,
+ "logps/rejected": -58.98709165453911,
+ "epoch": 0.3008142731186142,
+ "step": 290
+ },
+ {
+ "loss": 0.46946206092834475,
+ "grad_norm": 0.008176690898835659,
+ "learning_rate": 6.9015544041450784e-06,
+ "entropy": 1.636359941866249,
+ "num_tokens": 4740383.0,
+ "logits/chosen": -0.14494295129202733,
+ "logits/rejected": -0.012990473919638954,
+ "mean_token_accuracy": 0.6567367980629206,
+ "rewards/chosen": -0.5025469059059469,
+ "rewards/rejected": -1.4605811326974072,
+ "rewards/accuracies": 0.80125,
+ "rewards/margins": 0.9580342254508287,
+ "logps/chosen": -61.66068306684494,
+ "logps/rejected": -59.60216732621193,
+ "epoch": 0.3111871790882216,
+ "step": 300
+ },
+ {
+ "loss": 0.4926762580871582,
+ "grad_norm": 0.008404894731938839,
+ "learning_rate": 6.797927461139897e-06,
+ "entropy": 1.6317085930332542,
+ "num_tokens": 4901454.0,
+ "logits/chosen": -0.10420462296034713,
+ "logits/rejected": -0.0032643015884895763,
+ "mean_token_accuracy": 0.6517730862647295,
+ "rewards/chosen": -0.4740483855601633,
+ "rewards/rejected": -1.467887537285569,
+ "rewards/accuracies": 0.78625,
+ "rewards/margins": 0.9938391536381096,
+ "logps/chosen": -62.19328094601631,
+ "logps/rejected": -62.36802620172501,
+ "epoch": 0.32156008505782896,
+ "step": 310
+ },
+ {
+ "loss": 0.4758056640625,
+ "grad_norm": 0.01086408831179142,
+ "learning_rate": 6.6943005181347155e-06,
+ "entropy": 1.6321182049531489,
+ "num_tokens": 5058889.0,
+ "logits/chosen": -0.1815479056351674,
+ "logits/rejected": -0.07081081249938541,
+ "mean_token_accuracy": 0.6414232835173607,
+ "rewards/chosen": -0.5062945262032008,
+ "rewards/rejected": -1.4949601662519854,
+ "rewards/accuracies": 0.81,
+ "rewards/margins": 0.9886656388547271,
+ "logps/chosen": -63.52185977697373,
+ "logps/rejected": -59.69021632432938,
+ "epoch": 0.33193299102743634,
+ "step": 320
+ },
+ {
+ "loss": 0.4784576892852783,
+ "grad_norm": 0.010599741712212563,
+ "learning_rate": 6.590673575129535e-06,
+ "entropy": 1.6721400913968683,
+ "num_tokens": 5216549.0,
+ "logits/chosen": -0.1229586102306177,
+ "logits/rejected": -0.033501552710855156,
+ "mean_token_accuracy": 0.639937344957143,
+ "rewards/chosen": -0.5294519958324235,
+ "rewards/rejected": -1.616222132177645,
+ "rewards/accuracies": 0.78,
+ "rewards/margins": 1.086770132854581,
+ "logps/chosen": -63.12015713572502,
+ "logps/rejected": -62.12025535821915,
+ "epoch": 0.3423058969970437,
+ "step": 330
+ },
+ {
+ "loss": 0.4864365100860596,
+ "grad_norm": 0.010677792131900787,
+ "learning_rate": 6.487046632124353e-06,
+ "entropy": 1.5796756833419203,
+ "num_tokens": 5374161.0,
+ "logits/chosen": -0.17744578636097688,
+ "logits/rejected": -0.08292268708566802,
+ "mean_token_accuracy": 0.6436819550767541,
+ "rewards/chosen": -0.5906799642526311,
+ "rewards/rejected": -1.6022152152087075,
+ "rewards/accuracies": 0.78375,
+ "rewards/margins": 1.0115352519135923,
+ "logps/chosen": -62.16209160447121,
+ "logps/rejected": -60.97210006713867,
+ "epoch": 0.3526788029666511,
+ "step": 340
+ },
+ {
+ "loss": 0.5056374549865723,
+ "grad_norm": 0.013692691922187805,
+ "learning_rate": 6.383419689119171e-06,
+ "entropy": 1.5922023140452801,
+ "num_tokens": 5533185.0,
+ "logits/chosen": -0.11285970908028921,
+ "logits/rejected": -0.018185535407032168,
+ "mean_token_accuracy": 0.6443661257252097,
+ "rewards/chosen": -0.580222937125145,
+ "rewards/rejected": -1.5960858852404636,
+ "rewards/accuracies": 0.7725,
+ "rewards/margins": 1.0158629460725934,
+ "logps/chosen": -64.89270892024041,
+ "logps/rejected": -62.29878480195999,
+ "epoch": 0.3630517089362585,
+ "step": 350
+ },
+ {
+ "loss": 0.48989334106445315,
+ "grad_norm": 0.012253508903086185,
+ "learning_rate": 6.2797927461139905e-06,
+ "entropy": 1.5395906928181649,
+ "num_tokens": 5691159.0,
+ "logits/chosen": -0.14245793212958643,
+ "logits/rejected": -0.015230929526604408,
+ "mean_token_accuracy": 0.6611733642220498,
+ "rewards/chosen": -0.5437271582013921,
+ "rewards/rejected": -1.5548808443109738,
+ "rewards/accuracies": 0.7975,
+ "rewards/margins": 1.011153682768345,
+ "logps/chosen": -60.83840570926666,
+ "logps/rejected": -59.6216096830368,
+ "epoch": 0.37342461490586587,
+ "step": 360
+ },
+ {
+ "loss": 0.48537001609802244,
+ "grad_norm": 0.00956793874502182,
+ "learning_rate": 6.176165803108809e-06,
+ "entropy": 1.6442958949133755,
+ "num_tokens": 5850088.0,
+ "logits/chosen": -0.058320485277038855,
+ "logits/rejected": 0.06270940886949956,
+ "mean_token_accuracy": 0.6433443369530142,
+ "rewards/chosen": -0.5535192088356417,
+ "rewards/rejected": -1.5760743238392751,
+ "rewards/accuracies": 0.7975,
+ "rewards/margins": 1.0225551136396824,
+ "logps/chosen": -66.1269180560112,
+ "logps/rejected": -60.73978069186211,
+ "epoch": 0.38379752087547325,
+ "step": 370
+ },
+ {
+ "eval_loss": 0.5541799664497375,
+ "eval_runtime": 245.9984,
+ "eval_samples_per_second": 4.065,
+ "eval_steps_per_second": 4.065,
+ "eval_entropy": 1.0474999970644712,
+ "eval_num_tokens": 5928735.0,
+ "eval_logits/chosen": -0.23271217653526577,
+ "eval_logits/rejected": -0.10208672570441625,
+ "eval_mean_token_accuracy": 0.6993389547616243,
+ "eval_rewards/chosen": -0.571883928276773,
+ "eval_rewards/rejected": -1.1227601580685005,
+ "eval_rewards/accuracies": 0.707,
+ "eval_rewards/margins": 0.5508762280121445,
+ "eval_logps/chosen": -55.89182602119446,
+ "eval_logps/rejected": -63.867473125457764,
+ "epoch": 0.38898397386027694,
+ "step": 375
+ },
+ {
+ "loss": 0.447767972946167,
+ "grad_norm": 0.010045935399830341,
+ "learning_rate": 6.0725388601036275e-06,
+ "entropy": 1.5958718929998577,
+ "num_tokens": 6007948.0,
+ "logits/chosen": -0.010203895356886221,
+ "logits/rejected": 0.0953480252296025,
+ "mean_token_accuracy": 0.6510641277581454,
+ "rewards/chosen": -0.46026305966945075,
+ "rewards/rejected": -1.566630380146671,
+ "rewards/accuracies": 0.825,
+ "rewards/margins": 1.1063673190772534,
+ "logps/chosen": -61.75375435590744,
+ "logps/rejected": -58.965001332759854,
+ "epoch": 0.39417042684508063,
+ "step": 380
+ },
+ {
+ "loss": 0.4722391128540039,
+ "grad_norm": 0.010498611256480217,
+ "learning_rate": 5.968911917098445e-06,
+ "entropy": 1.6217002650536596,
+ "num_tokens": 6167000.0,
+ "logits/chosen": -0.010942938609718112,
+ "logits/rejected": 0.13150225704237622,
+ "mean_token_accuracy": 0.6487077697180211,
+ "rewards/chosen": -0.4893380562632228,
+ "rewards/rejected": -1.546268537521828,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.056930480999872,
+ "logps/chosen": -63.818650953769684,
+ "logps/rejected": -61.81397884130478,
+ "epoch": 0.404543332814688,
+ "step": 390
+ },
+ {
+ "loss": 0.4782541275024414,
+ "grad_norm": 0.009464209899306297,
+ "learning_rate": 5.865284974093265e-06,
+ "entropy": 1.6495763343665748,
+ "num_tokens": 6325205.0,
+ "logits/chosen": -0.011458384051150232,
+ "logits/rejected": 0.09236410504123445,
+ "mean_token_accuracy": 0.6448470102529973,
+ "rewards/chosen": -0.51347439962934,
+ "rewards/rejected": -1.5123917219531722,
+ "rewards/accuracies": 0.79125,
+ "rewards/margins": 0.9989173209294677,
+ "logps/chosen": -63.590197974443434,
+ "logps/rejected": -61.28543629407883,
+ "epoch": 0.4149162387842954,
+ "step": 400
+ },
+ {
+ "loss": 0.4769914627075195,
+ "grad_norm": 0.011201824992895126,
+ "learning_rate": 5.761658031088083e-06,
+ "entropy": 1.6099769476987422,
+ "num_tokens": 6483981.0,
+ "logits/chosen": -0.008573553675108867,
+ "logits/rejected": 0.10661449974217808,
+ "mean_token_accuracy": 0.6476613377220929,
+ "rewards/chosen": -0.5158215398409811,
+ "rewards/rejected": -1.5142960718111136,
+ "rewards/accuracies": 0.795,
+ "rewards/margins": 0.9984745322261006,
+ "logps/chosen": -62.939886302948,
+ "logps/rejected": -60.21168664813042,
+ "epoch": 0.4252891447539028,
+ "step": 410
+ },
+ {
+ "loss": 0.4447061061859131,
+ "grad_norm": 0.01015845313668251,
+ "learning_rate": 5.658031088082902e-06,
+ "entropy": 1.6073184362612665,
+ "num_tokens": 6642561.0,
+ "logits/chosen": 0.007230915169619996,
+ "logits/rejected": 0.10328242233161539,
+ "mean_token_accuracy": 0.6508273026905954,
+ "rewards/chosen": -0.4930744762084214,
+ "rewards/rejected": -1.6134738429238495,
+ "rewards/accuracies": 0.82,
+ "rewards/margins": 1.1203993632458151,
+ "logps/chosen": -65.08479910731316,
+ "logps/rejected": -60.65289543390274,
+ "epoch": 0.4356620507235102,
+ "step": 420
+ },
+ {
+ "loss": 0.44136753082275393,
+ "grad_norm": 0.008634388446807861,
+ "learning_rate": 5.554404145077721e-06,
+ "entropy": 1.62525453383103,
+ "num_tokens": 6802172.0,
+ "logits/chosen": 0.09677644223113024,
+ "logits/rejected": 0.19686068222909975,
+ "mean_token_accuracy": 0.6414820049889386,
+ "rewards/chosen": -0.530476398501778,
+ "rewards/rejected": -1.7060055632909643,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.1755291634099558,
+ "logps/chosen": -63.63275898098946,
+ "logps/rejected": -61.548430292606355,
+ "epoch": 0.4460349566931176,
+ "step": 430
+ },
+ {
+ "loss": 0.44923830032348633,
+ "grad_norm": 0.011443796567618847,
+ "learning_rate": 5.4507772020725395e-06,
+ "entropy": 1.6008457892294974,
+ "num_tokens": 6960698.0,
+ "logits/chosen": 0.03864486058133284,
+ "logits/rejected": 0.16183756452396064,
+ "mean_token_accuracy": 0.6556359087582677,
+ "rewards/chosen": -0.5446211060311179,
+ "rewards/rejected": -1.6678383790072986,
+ "rewards/accuracies": 0.8275,
+ "rewards/margins": 1.1232172738574446,
+ "logps/chosen": -61.80046648740768,
+ "logps/rejected": -60.58844954848289,
+ "epoch": 0.456407862662725,
+ "step": 440
+ },
+ {
+ "loss": 0.4775514602661133,
+ "grad_norm": 0.015888305380940437,
+ "learning_rate": 5.347150259067357e-06,
+ "entropy": 1.6431135883461685,
+ "num_tokens": 7120292.0,
+ "logits/chosen": 0.058064731861634356,
+ "logits/rejected": 0.2062258352022907,
+ "mean_token_accuracy": 0.6481080191396177,
+ "rewards/chosen": -0.5665322134363828,
+ "rewards/rejected": -1.696948098014691,
+ "rewards/accuracies": 0.79375,
+ "rewards/margins": 1.1304158817767165,
+ "logps/chosen": -62.31974795341492,
+ "logps/rejected": -62.94689394712448,
+ "epoch": 0.46678076863233237,
+ "step": 450
+ },
+ {
+ "loss": 0.45407419204711913,
+ "grad_norm": 0.011245348490774632,
+ "learning_rate": 5.243523316062177e-06,
+ "entropy": 1.6142372595332564,
+ "num_tokens": 7277503.0,
+ "logits/chosen": 0.027211253980977537,
+ "logits/rejected": 0.14270901371261632,
+ "mean_token_accuracy": 0.6483659755066037,
+ "rewards/chosen": -0.5206575704316492,
+ "rewards/rejected": -1.627284916813951,
+ "rewards/accuracies": 0.82875,
+ "rewards/margins": 1.1066273492295295,
+ "logps/chosen": -62.51640429496765,
+ "logps/rejected": -59.92736347913742,
+ "epoch": 0.47715367460193975,
+ "step": 460
+ },
+ {
+ "loss": 0.4705537796020508,
+ "grad_norm": 0.017906704917550087,
+ "learning_rate": 5.139896373056995e-06,
+ "entropy": 1.6305865264125168,
+ "num_tokens": 7433576.0,
+ "logits/chosen": -0.01868226836908457,
+ "logits/rejected": 0.10559852450806628,
+ "mean_token_accuracy": 0.6382040186412632,
+ "rewards/chosen": -0.5739967311238433,
+ "rewards/rejected": -1.6218135237134994,
+ "rewards/accuracies": 0.78125,
+ "rewards/margins": 1.0478167911106722,
+ "logps/chosen": -61.798225450515744,
+ "logps/rejected": -59.17569625377655,
+ "epoch": 0.48752658057154713,
+ "step": 470
+ },
+ {
+ "loss": 0.44884982109069826,
+ "grad_norm": 0.009567538276314735,
+ "learning_rate": 5.036269430051814e-06,
+ "entropy": 1.6556390350870789,
+ "num_tokens": 7593962.0,
+ "logits/chosen": 0.02396342091178786,
+ "logits/rejected": 0.15151965562177133,
+ "mean_token_accuracy": 0.6423015125840902,
+ "rewards/chosen": -0.6105934929932119,
+ "rewards/rejected": -1.7286542325050687,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.1180607356689871,
+ "logps/chosen": -65.96984986424447,
+ "logps/rejected": -63.567516083717344,
+ "epoch": 0.4978994865411545,
+ "step": 480
+ },
+ {
+ "loss": 0.46419124603271483,
+ "grad_norm": 0.010610009543597698,
+ "learning_rate": 4.932642487046633e-06,
+ "entropy": 1.63694656169042,
+ "num_tokens": 7753841.0,
+ "logits/chosen": 0.05542139131963341,
+ "logits/rejected": 0.1531588473091942,
+ "mean_token_accuracy": 0.6532461035437882,
+ "rewards/chosen": -0.6174891357401794,
+ "rewards/rejected": -1.7478586566343437,
+ "rewards/accuracies": 0.8125,
+ "rewards/margins": 1.130369521221146,
+ "logps/chosen": -63.278802993297575,
+ "logps/rejected": -63.9350430393219,
+ "epoch": 0.5082723925107618,
+ "step": 490
+ },
+ {
+ "loss": 0.4718832015991211,
+ "grad_norm": 0.01074894331395626,
+ "learning_rate": 4.829015544041451e-06,
+ "entropy": 1.6344914321228863,
+ "num_tokens": 7911511.0,
+ "logits/chosen": 0.039370719731205275,
+ "logits/rejected": 0.16598465687796668,
+ "mean_token_accuracy": 0.6492240923829377,
+ "rewards/chosen": -0.5642123090758104,
+ "rewards/rejected": -1.6763065649938653,
+ "rewards/accuracies": 0.78625,
+ "rewards/margins": 1.1120942557882518,
+ "logps/chosen": -61.73503879547119,
+ "logps/rejected": -62.51426592111588,
+ "epoch": 0.5186452984803692,
+ "step": 500
+ },
+ {
+ "eval_loss": 0.5400257706642151,
+ "eval_runtime": 278.2919,
+ "eval_samples_per_second": 3.593,
+ "eval_steps_per_second": 3.593,
+ "eval_entropy": 1.0458819408267737,
+ "eval_num_tokens": 7911511.0,
+ "eval_logits/chosen": -0.1978596412920271,
+ "eval_logits/rejected": -0.06171821412221096,
+ "eval_mean_token_accuracy": 0.6972633041292429,
+ "eval_rewards/chosen": -0.6398490693312487,
+ "eval_rewards/rejected": -1.2539186353033873,
+ "eval_rewards/accuracies": 0.727,
+ "eval_rewards/margins": 0.6140695666372776,
+ "eval_logps/chosen": -56.57147743225098,
+ "eval_logps/rejected": -65.17905788230895,
+ "epoch": 0.5186452984803692,
+ "step": 500
+ },
+ {
+ "loss": 0.4755256175994873,
+ "grad_norm": 0.011379857547581196,
+ "learning_rate": 4.72538860103627e-06,
+ "entropy": 1.6379237968847156,
+ "num_tokens": 8070081.0,
+ "logits/chosen": 0.08480375218267698,
+ "logits/rejected": 0.18248930696878024,
+ "mean_token_accuracy": 0.6486436153575778,
+ "rewards/chosen": -0.6162076763511868,
+ "rewards/rejected": -1.6715471999935108,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.0553395241731778,
+ "logps/chosen": -64.42356903076171,
+ "logps/rejected": -59.52313063383102,
+ "epoch": 0.5290182044499767,
+ "step": 510
+ },
+ {
+ "loss": 0.4912434101104736,
+ "grad_norm": 0.009543896652758121,
+ "learning_rate": 4.621761658031089e-06,
+ "entropy": 1.62886246021837,
+ "num_tokens": 8226032.0,
+ "logits/chosen": 0.04373571989101422,
+ "logits/rejected": 0.1632708972191895,
+ "mean_token_accuracy": 0.6450024632364512,
+ "rewards/chosen": -0.5994134752946411,
+ "rewards/rejected": -1.5973126511012379,
+ "rewards/accuracies": 0.79,
+ "rewards/margins": 0.9978991763386875,
+ "logps/chosen": -61.06594479799271,
+ "logps/rejected": -59.51982655286789,
+ "epoch": 0.5393911104195841,
+ "step": 520
+ },
+ {
+ "loss": 0.4588636875152588,
+ "grad_norm": 0.010315664112567902,
+ "learning_rate": 4.518134715025907e-06,
+ "entropy": 1.6835303687490524,
+ "num_tokens": 8384377.0,
+ "logits/chosen": 0.06022681958823317,
+ "logits/rejected": 0.182036032657229,
+ "mean_token_accuracy": 0.63875184844248,
+ "rewards/chosen": -0.5799533515534131,
+ "rewards/rejected": -1.7233658448676579,
+ "rewards/accuracies": 0.80875,
+ "rewards/margins": 1.143412495199591,
+ "logps/chosen": -64.0477586555481,
+ "logps/rejected": -62.54838394999504,
+ "epoch": 0.5497640163891915,
+ "step": 530
+ },
+ {
+ "loss": 0.4572054386138916,
+ "grad_norm": 0.01254233904182911,
+ "learning_rate": 4.414507772020726e-06,
+ "entropy": 1.6456316580064594,
+ "num_tokens": 8543279.0,
+ "logits/chosen": 0.06499475886537846,
+ "logits/rejected": 0.19926284731697655,
+ "mean_token_accuracy": 0.6476211673207581,
+ "rewards/chosen": -0.6441993828896193,
+ "rewards/rejected": -1.7510575192118996,
+ "rewards/accuracies": 0.83,
+ "rewards/margins": 1.106858134865761,
+ "logps/chosen": -62.7225538957119,
+ "logps/rejected": -62.64689356803894,
+ "epoch": 0.5601369223587989,
+ "step": 540
+ },
+ {
+ "loss": 0.46482481956481936,
+ "grad_norm": 0.033692456781864166,
+ "learning_rate": 4.310880829015544e-06,
+ "entropy": 1.694882277622819,
+ "num_tokens": 8703115.0,
+ "logits/chosen": 0.15109038640147246,
+ "logits/rejected": 0.2297669311708141,
+ "mean_token_accuracy": 0.6399877128005028,
+ "rewards/chosen": -0.661507485556067,
+ "rewards/rejected": -1.8317748800176197,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.170267395619303,
+ "logps/chosen": -66.47220434308052,
+ "logps/rejected": -66.16992118120193,
+ "epoch": 0.5705098283284062,
+ "step": 550
+ },
+ {
+ "loss": 0.46529335975646974,
+ "grad_norm": 0.012274126522243023,
+ "learning_rate": 4.207253886010363e-06,
+ "entropy": 1.6411442287266254,
+ "num_tokens": 8862439.0,
+ "logits/chosen": 0.07749967645058822,
+ "logits/rejected": 0.17964104042378107,
+ "mean_token_accuracy": 0.6407143748924136,
+ "rewards/chosen": -0.6461584532729012,
+ "rewards/rejected": -1.7939777009031967,
+ "rewards/accuracies": 0.8,
+ "rewards/margins": 1.1478192471479998,
+ "logps/chosen": -64.53955813646317,
+ "logps/rejected": -62.56636171579361,
+ "epoch": 0.5808827342980136,
+ "step": 560
+ },
+ {
+ "loss": 0.4430724620819092,
+ "grad_norm": 0.014654034748673439,
+ "learning_rate": 4.103626943005182e-06,
+ "entropy": 1.641110150022432,
+ "num_tokens": 9019532.0,
+ "logits/chosen": 0.1006488560296452,
+ "logits/rejected": 0.21211194322628596,
+ "mean_token_accuracy": 0.6409156301990151,
+ "rewards/chosen": -0.6455532481150612,
+ "rewards/rejected": -1.7706630855146797,
+ "rewards/accuracies": 0.835,
+ "rewards/margins": 1.1251098392903804,
+ "logps/chosen": -63.84049278497696,
+ "logps/rejected": -60.347349853515624,
+ "epoch": 0.591255640267621,
+ "step": 570
+ },
+ {
+ "loss": 0.46980915069580076,
+ "grad_norm": 0.015018550679087639,
+ "learning_rate": 4.000000000000001e-06,
+ "entropy": 1.6142543618567289,
+ "num_tokens": 9176151.0,
+ "logits/chosen": 0.008260599334212472,
+ "logits/rejected": 0.13603789421792953,
+ "mean_token_accuracy": 0.6421346751507372,
+ "rewards/chosen": -0.6516643855780512,
+ "rewards/rejected": -1.7954043288715185,
+ "rewards/accuracies": 0.805,
+ "rewards/margins": 1.14373994121328,
+ "logps/chosen": -61.326625975370405,
+ "logps/rejected": -61.17676810503006,
+ "epoch": 0.6016285462372284,
+ "step": 580
+ },
+ {
+ "loss": 0.5050342559814454,
+ "grad_norm": 0.018550831824541092,
+ "learning_rate": 3.896373056994819e-06,
+ "entropy": 1.6450188556872307,
+ "num_tokens": 9335435.0,
+ "logits/chosen": 0.03664537931858989,
+ "logits/rejected": 0.1512234776259551,
+ "mean_token_accuracy": 0.6435695023182779,
+ "rewards/chosen": -0.7144207920093322,
+ "rewards/rejected": -1.727617413054686,
+ "rewards/accuracies": 0.77375,
+ "rewards/margins": 1.013196619511582,
+ "logps/chosen": -65.46060695648194,
+ "logps/rejected": -63.5906271135807,
+ "epoch": 0.6120014522068358,
+ "step": 590
+ },
+ {
+ "loss": 0.4836301326751709,
+ "grad_norm": 0.016474206000566483,
+ "learning_rate": 3.7927461139896377e-06,
+ "entropy": 1.6528891836851836,
+ "num_tokens": 9491363.0,
+ "logits/chosen": 0.08386782463631755,
+ "logits/rejected": 0.20943303131643703,
+ "mean_token_accuracy": 0.6487930232100189,
+ "rewards/chosen": -0.6163708032899012,
+ "rewards/rejected": -1.6733668212150223,
+ "rewards/accuracies": 0.78375,
+ "rewards/margins": 1.0569960164930672,
+ "logps/chosen": -60.280444753170016,
+ "logps/rejected": -61.18290127515793,
+ "epoch": 0.6223743581764432,
+ "step": 600
+ },
+ {
+ "loss": 0.4617151260375977,
+ "grad_norm": 0.010820780880749226,
+ "learning_rate": 3.6891191709844567e-06,
+ "entropy": 1.6939242084044963,
+ "num_tokens": 9648643.0,
+ "logits/chosen": 0.09977307296605359,
+ "logits/rejected": 0.20379420233565199,
+ "mean_token_accuracy": 0.6439293037727475,
+ "rewards/chosen": -0.6212675093274447,
+ "rewards/rejected": -1.7044480502128136,
+ "rewards/accuracies": 0.82,
+ "rewards/margins": 1.0831805411481765,
+ "logps/chosen": -60.97681099057198,
+ "logps/rejected": -62.301487598419186,
+ "epoch": 0.6327472641460505,
+ "step": 610
+ },
+ {
+ "loss": 0.45804176330566404,
+ "grad_norm": 0.012919292785227299,
+ "learning_rate": 3.5854922279792748e-06,
+ "entropy": 1.619003531029448,
+ "num_tokens": 9806029.0,
+ "logits/chosen": 0.0483610800015232,
+ "logits/rejected": 0.1797610236274009,
+ "mean_token_accuracy": 0.6475096028484404,
+ "rewards/chosen": -0.5888219272701827,
+ "rewards/rejected": -1.706802941111964,
+ "rewards/accuracies": 0.8075,
+ "rewards/margins": 1.1179810122167693,
+ "logps/chosen": -59.50200087547302,
+ "logps/rejected": -61.049637752771375,
+ "epoch": 0.6431201701156579,
+ "step": 620
+ },
+ {
+ "eval_loss": 0.5324422717094421,
+ "eval_runtime": 288.7421,
+ "eval_samples_per_second": 3.463,
+ "eval_steps_per_second": 3.463,
+ "eval_entropy": 1.0448875862136484,
+ "eval_num_tokens": 9884893.0,
+ "eval_logits/chosen": -0.19975289350595532,
+ "eval_logits/rejected": -0.060755203565891945,
+ "eval_mean_token_accuracy": 0.6964090894907713,
+ "eval_rewards/chosen": -0.6912122842722456,
+ "eval_rewards/rejected": -1.345464039585553,
+ "eval_rewards/accuracies": 0.736,
+ "eval_rewards/margins": 0.6542517534568906,
+ "eval_logps/chosen": -57.08510956954956,
+ "eval_logps/rejected": -66.09451188468933,
+ "epoch": 0.6483066231004616,
+ "step": 625
+ },
+ {
+ "loss": 0.45544919967651365,
+ "grad_norm": 0.010571232065558434,
+ "learning_rate": 3.4818652849740937e-06,
+ "entropy": 1.6706199841760099,
+ "num_tokens": 9965103.0,
+ "logits/chosen": 0.1180553175480971,
+ "logits/rejected": 0.21143161932841895,
+ "mean_token_accuracy": 0.6404067935794592,
+ "rewards/chosen": -0.6081898983999418,
+ "rewards/rejected": -1.7734553063547356,
+ "rewards/accuracies": 0.81,
+ "rewards/margins": 1.1652654066774994,
+ "logps/chosen": -62.56314527273178,
+ "logps/rejected": -63.61036643981934,
+ "epoch": 0.6534930760852653,
+ "step": 630
+ },
+ {
+ "loss": 0.44363651275634763,
+ "grad_norm": 0.011373443529009819,
+ "learning_rate": 3.3782383419689123e-06,
+ "entropy": 1.661266395803541,
+ "num_tokens": 10122828.0,
+ "logits/chosen": 0.13310990911343398,
+ "logits/rejected": 0.2430608346820568,
+ "mean_token_accuracy": 0.6414054480008781,
+ "rewards/chosen": -0.645503295796052,
+ "rewards/rejected": -1.7806003371396217,
+ "rewards/accuracies": 0.81,
+ "rewards/margins": 1.1350970419961959,
+ "logps/chosen": -61.5342090690136,
+ "logps/rejected": -63.98482904434204,
+ "epoch": 0.6638659820548727,
+ "step": 640
+ },
+ {
+ "loss": 0.4796154499053955,
+ "grad_norm": 0.012532204389572144,
+ "learning_rate": 3.274611398963731e-06,
+ "entropy": 1.6701466926187276,
+ "num_tokens": 10279859.0,
+ "logits/chosen": 0.07329429847103937,
+ "logits/rejected": 0.1878599203293891,
+ "mean_token_accuracy": 0.6414687449485064,
+ "rewards/chosen": -0.661306283445374,
+ "rewards/rejected": -1.7610867334110663,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.099780449680984,
+ "logps/chosen": -62.595525816679,
+ "logps/rejected": -61.356811988353726,
+ "epoch": 0.6742388880244801,
+ "step": 650
+ },
+ {
+ "loss": 0.4748543739318848,
+ "grad_norm": 0.01547182071954012,
+ "learning_rate": 3.1709844559585493e-06,
+ "entropy": 1.6658864275645464,
+ "num_tokens": 10436075.0,
+ "logits/chosen": 0.10011936781303017,
+ "logits/rejected": 0.2041365355242361,
+ "mean_token_accuracy": 0.6389276959933341,
+ "rewards/chosen": -0.6364252381109691,
+ "rewards/rejected": -1.735760998390615,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.099335762821138,
+ "logps/chosen": -61.83362093210221,
+ "logps/rejected": -61.003912779092786,
+ "epoch": 0.6846117939940874,
+ "step": 660
+ },
+ {
+ "loss": 0.4296245098114014,
+ "grad_norm": 0.011170756071805954,
+ "learning_rate": 3.0673575129533683e-06,
+ "entropy": 1.6508124286774546,
+ "num_tokens": 10594936.0,
+ "logits/chosen": 0.10534840408886,
+ "logits/rejected": 0.20408601168915652,
+ "mean_token_accuracy": 0.6367748867347837,
+ "rewards/chosen": -0.6433876353609412,
+ "rewards/rejected": -1.8510402401175816,
+ "rewards/accuracies": 0.8175,
+ "rewards/margins": 1.2076526060514152,
+ "logps/chosen": -64.62543644189834,
+ "logps/rejected": -62.8101417517662,
+ "epoch": 0.6949846999636948,
+ "step": 670
+ },
+ {
+ "loss": 0.4698948383331299,
+ "grad_norm": 0.019366616383194923,
+ "learning_rate": 2.963730569948187e-06,
+ "entropy": 1.6687841359246522,
+ "num_tokens": 10754603.0,
+ "logits/chosen": 0.13530835426877952,
+ "logits/rejected": 0.25161180215337653,
+ "mean_token_accuracy": 0.6364573692902923,
+ "rewards/chosen": -0.6632168094060035,
+ "rewards/rejected": -1.8288732153433376,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.165656405221671,
+ "logps/chosen": -65.38578713893891,
+ "logps/rejected": -64.20987098813058,
+ "epoch": 0.7053576059333022,
+ "step": 680
+ },
+ {
+ "loss": 0.5016210079193115,
+ "grad_norm": 0.022646328434348106,
+ "learning_rate": 2.8601036269430053e-06,
+ "entropy": 1.711660223044455,
+ "num_tokens": 10913690.0,
+ "logits/chosen": 0.16971737857777713,
+ "logits/rejected": 0.2760859463634132,
+ "mean_token_accuracy": 0.6390431644208729,
+ "rewards/chosen": -0.6508824050683597,
+ "rewards/rejected": -1.6932820503300172,
+ "rewards/accuracies": 0.79375,
+ "rewards/margins": 1.0423996420949697,
+ "logps/chosen": -66.28588567495346,
+ "logps/rejected": -62.060355219841,
+ "epoch": 0.7157305119029096,
+ "step": 690
+ },
+ {
+ "loss": 0.43436241149902344,
+ "grad_norm": 0.009509176947176456,
+ "learning_rate": 2.7564766839378243e-06,
+ "entropy": 1.6765110883302987,
+ "num_tokens": 11071134.0,
+ "logits/chosen": 0.12124103042581971,
+ "logits/rejected": 0.2600804952933655,
+ "mean_token_accuracy": 0.6427443787083029,
+ "rewards/chosen": -0.5906095821626696,
+ "rewards/rejected": -1.720989261372597,
+ "rewards/accuracies": 0.835,
+ "rewards/margins": 1.130379677421879,
+ "logps/chosen": -63.95436416387558,
+ "logps/rejected": -59.14868986845016,
+ "epoch": 0.726103417872517,
+ "step": 700
+ },
+ {
+ "loss": 0.4556713581085205,
+ "grad_norm": 0.01146204024553299,
+ "learning_rate": 2.6528497409326424e-06,
+ "entropy": 1.6694943796936423,
+ "num_tokens": 11230405.0,
+ "logits/chosen": 0.07533675440714994,
+ "logits/rejected": 0.19347860103992787,
+ "mean_token_accuracy": 0.6436189501732588,
+ "rewards/chosen": -0.5904611392628294,
+ "rewards/rejected": -1.7462449892124277,
+ "rewards/accuracies": 0.8025,
+ "rewards/margins": 1.155783847218845,
+ "logps/chosen": -66.84839742302894,
+ "logps/rejected": -61.99807296037674,
+ "epoch": 0.7364763238421244,
+ "step": 710
+ },
+ {
+ "loss": 0.4577175140380859,
+ "grad_norm": 0.012849503196775913,
+ "learning_rate": 2.5492227979274614e-06,
+ "entropy": 1.6657723085489125,
+ "num_tokens": 11386296.0,
+ "logits/chosen": 0.05922319803504693,
+ "logits/rejected": 0.17808647771296027,
+ "mean_token_accuracy": 0.6431331102643162,
+ "rewards/chosen": -0.5677624635772373,
+ "rewards/rejected": -1.6767527183049242,
+ "rewards/accuracies": 0.8225,
+ "rewards/margins": 1.1089902543462813,
+ "logps/chosen": -61.68554986834526,
+ "logps/rejected": -58.83755177259445,
+ "epoch": 0.7468492298117317,
+ "step": 720
+ },
+ {
+ "loss": 0.4547234535217285,
+ "grad_norm": 0.014395375736057758,
+ "learning_rate": 2.44559585492228e-06,
+ "entropy": 1.6862239858694374,
+ "num_tokens": 11546081.0,
+ "logits/chosen": 0.1350684718563505,
+ "logits/rejected": 0.2427451158459563,
+ "mean_token_accuracy": 0.64147017583251,
+ "rewards/chosen": -0.5967396693397313,
+ "rewards/rejected": -1.765362592941492,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.1686229225434364,
+ "logps/chosen": -63.325855791568756,
+ "logps/rejected": -65.51477008223533,
+ "epoch": 0.7572221357813391,
+ "step": 730
+ },
+ {
+ "loss": 0.4426294803619385,
+ "grad_norm": 0.012110771611332893,
+ "learning_rate": 2.3419689119170984e-06,
+ "entropy": 1.707785174669698,
+ "num_tokens": 11703833.0,
+ "logits/chosen": 0.0927098814711561,
+ "logits/rejected": 0.23603759404490104,
+ "mean_token_accuracy": 0.6414807749167085,
+ "rewards/chosen": -0.5864860777647118,
+ "rewards/rejected": -1.7438211480446626,
+ "rewards/accuracies": 0.83125,
+ "rewards/margins": 1.1573350698873401,
+ "logps/chosen": -63.334676113128666,
+ "logps/rejected": -63.40963178634644,
+ "epoch": 0.7675950417509465,
+ "step": 740
+ },
+ {
+ "loss": 0.46600141525268557,
+ "grad_norm": 0.013532786630094051,
+ "learning_rate": 2.2383419689119174e-06,
+ "entropy": 1.6768190996162593,
+ "num_tokens": 11865175.0,
+ "logits/chosen": 0.1273617797525129,
+ "logits/rejected": 0.22872866333573835,
+ "mean_token_accuracy": 0.642769878860563,
+ "rewards/chosen": -0.6710381170455002,
+ "rewards/rejected": -1.8427071001776494,
+ "rewards/accuracies": 0.805,
+ "rewards/margins": 1.1716689813882113,
+ "logps/chosen": -66.42985866189002,
+ "logps/rejected": -66.3072749710083,
+ "epoch": 0.7779679477205539,
+ "step": 750
+ },
+ {
+ "eval_loss": 0.5291131734848022,
+ "eval_runtime": 316.7312,
+ "eval_samples_per_second": 3.157,
+ "eval_steps_per_second": 3.157,
+ "eval_entropy": 1.0425387567952276,
+ "eval_num_tokens": 11865175.0,
+ "eval_logits/chosen": -0.19368413704948595,
+ "eval_logits/rejected": -0.05676454023564469,
+ "eval_mean_token_accuracy": 0.6966218997985124,
+ "eval_rewards/chosen": -0.6922309857774526,
+ "eval_rewards/rejected": -1.3544460057471879,
+ "eval_rewards/accuracies": 0.747,
+ "eval_rewards/margins": 0.6622150189429522,
+ "eval_logps/chosen": -57.09529658699036,
+ "eval_logps/rejected": -66.18433157348633,
+ "epoch": 0.7779679477205539,
+ "step": 750
+ },
+ {
+ "loss": 0.43166389465332033,
+ "grad_norm": 0.012500453740358353,
+ "learning_rate": 2.134715025906736e-06,
+ "entropy": 1.6828834046982228,
+ "num_tokens": 12024093.0,
+ "logits/chosen": 0.09826941010263189,
+ "logits/rejected": 0.215735024556398,
+ "mean_token_accuracy": 0.6342808033898473,
+ "rewards/chosen": -0.6309049091633642,
+ "rewards/rejected": -1.7712371364238242,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.1403322232328355,
+ "logps/chosen": -64.024556722641,
+ "logps/rejected": -62.57202324867249,
+ "epoch": 0.7883408536901613,
+ "step": 760
+ },
+ {
+ "loss": 0.4471421718597412,
+ "grad_norm": 0.014410309493541718,
+ "learning_rate": 2.0310880829015544e-06,
+ "entropy": 1.6857702764961868,
+ "num_tokens": 12185744.0,
+ "logits/chosen": 0.14331582088471329,
+ "logits/rejected": 0.2682991323037694,
+ "mean_token_accuracy": 0.6441437931358814,
+ "rewards/chosen": -0.6643577039731394,
+ "rewards/rejected": -1.8807749817200239,
+ "rewards/accuracies": 0.81125,
+ "rewards/margins": 1.216417273581028,
+ "logps/chosen": -67.24417539000511,
+ "logps/rejected": -66.79223707199097,
+ "epoch": 0.7987137596597687,
+ "step": 770
+ },
+ {
+ "loss": 0.4785769939422607,
+ "grad_norm": 0.010570192709565163,
+ "learning_rate": 1.9274611398963734e-06,
+ "entropy": 1.6633455219957978,
+ "num_tokens": 12343401.0,
+ "logits/chosen": 0.07438493724116783,
+ "logits/rejected": 0.23364041511381434,
+ "mean_token_accuracy": 0.639512750543654,
+ "rewards/chosen": -0.6736341681498743,
+ "rewards/rejected": -1.796194753185846,
+ "rewards/accuracies": 0.79,
+ "rewards/margins": 1.1225605825753882,
+ "logps/chosen": -62.37282539129257,
+ "logps/rejected": -63.56037598967552,
+ "epoch": 0.809086665629376,
+ "step": 780
+ },
+ {
+ "loss": 0.45286383628845217,
+ "grad_norm": 0.011962756514549255,
+ "learning_rate": 1.823834196891192e-06,
+ "entropy": 1.6700926853902638,
+ "num_tokens": 12502157.0,
+ "logits/chosen": 0.1286325604504252,
+ "logits/rejected": 0.2370263101476654,
+ "mean_token_accuracy": 0.6403315839730204,
+ "rewards/chosen": -0.6551112483125325,
+ "rewards/rejected": -1.8369905388413463,
+ "rewards/accuracies": 0.81625,
+ "rewards/margins": 1.1818792873900383,
+ "logps/chosen": -63.591456497907636,
+ "logps/rejected": -63.02011847734451,
+ "epoch": 0.8194595715989834,
+ "step": 790
+ },
+ {
+ "loss": 0.4665197849273682,
+ "grad_norm": 0.015845410525798798,
+ "learning_rate": 1.7202072538860104e-06,
+ "entropy": 1.7264064208138734,
+ "num_tokens": 12659952.0,
+ "logits/chosen": 0.12747972368676874,
+ "logits/rejected": 0.2234084493421016,
+ "mean_token_accuracy": 0.6323709175549448,
+ "rewards/chosen": -0.670276442007671,
+ "rewards/rejected": -1.800412250665977,
+ "rewards/accuracies": 0.81375,
+ "rewards/margins": 1.1301358060259372,
+ "logps/chosen": -65.14966633319855,
+ "logps/rejected": -62.890252923965456,
+ "epoch": 0.8298324775685908,
+ "step": 800
+ },
+ {
+ "loss": 0.46672878265380857,
+ "grad_norm": 0.011275322176516056,
+ "learning_rate": 1.6165803108808292e-06,
+ "entropy": 1.688749819751829,
+ "num_tokens": 12818219.0,
+ "logits/chosen": 0.08963352847012644,
+ "logits/rejected": 0.1937745526364166,
+ "mean_token_accuracy": 0.6342837125249207,
+ "rewards/chosen": -0.6473926620587008,
+ "rewards/rejected": -1.7900561995629687,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 1.142663535233587,
+ "logps/chosen": -63.99472352027893,
+ "logps/rejected": -63.061451687812806,
+ "epoch": 0.8402053835381982,
+ "step": 810
+ },
+ {
+ "loss": 0.4595985412597656,
+ "grad_norm": 0.009922225028276443,
+ "learning_rate": 1.5129533678756477e-06,
+ "entropy": 1.6839555408246816,
+ "num_tokens": 12975834.0,
+ "logits/chosen": 0.14187639603561716,
+ "logits/rejected": 0.2228106795045077,
+ "mean_token_accuracy": 0.6386101646535098,
+ "rewards/chosen": -0.6652093456028524,
+ "rewards/rejected": -1.8065621317276963,
+ "rewards/accuracies": 0.8,
+ "rewards/margins": 1.1413527865428477,
+ "logps/chosen": -65.14974474191666,
+ "logps/rejected": -61.74604295730591,
+ "epoch": 0.8505782895078056,
+ "step": 820
+ },
+ {
+ "loss": 0.4635880470275879,
+ "grad_norm": 0.015552631579339504,
+ "learning_rate": 1.4093264248704663e-06,
+ "entropy": 1.684992496855557,
+ "num_tokens": 13133432.0,
+ "logits/chosen": 0.09258615948240319,
+ "logits/rejected": 0.23631826346334864,
+ "mean_token_accuracy": 0.6466719186119735,
+ "rewards/chosen": -0.6487690168057452,
+ "rewards/rejected": -1.7847515585264773,
+ "rewards/accuracies": 0.81125,
+ "rewards/margins": 1.1359825418051333,
+ "logps/chosen": -61.15254833102226,
+ "logps/rejected": -62.26296893358231,
+ "epoch": 0.860951195477413,
+ "step": 830
+ },
+ {
+ "loss": 0.44598889350891113,
+ "grad_norm": 0.015216046944260597,
+ "learning_rate": 1.3056994818652852e-06,
+ "entropy": 1.709428556431085,
+ "num_tokens": 13291850.0,
+ "logits/chosen": 0.15097671106129862,
+ "logits/rejected": 0.25655990051725586,
+ "mean_token_accuracy": 0.6395855396427215,
+ "rewards/chosen": -0.5992260871930921,
+ "rewards/rejected": -1.8031220447564555,
+ "rewards/accuracies": 0.81125,
+ "rewards/margins": 1.2038959555141628,
+ "logps/chosen": -65.10634205937386,
+ "logps/rejected": -63.04532386302948,
+ "epoch": 0.8713241014470204,
+ "step": 840
+ },
+ {
+ "loss": 0.46810121536254884,
+ "grad_norm": 0.009856506250798702,
+ "learning_rate": 1.2020725388601037e-06,
+ "entropy": 1.70108500065282,
+ "num_tokens": 13450070.0,
+ "logits/chosen": 0.15703350726422605,
+ "logits/rejected": 0.2533505205489093,
+ "mean_token_accuracy": 0.643168338034302,
+ "rewards/chosen": -0.62836980179698,
+ "rewards/rejected": -1.7627987958928224,
+ "rewards/accuracies": 0.80125,
+ "rewards/margins": 1.1344289934798144,
+ "logps/chosen": -64.12525810718536,
+ "logps/rejected": -62.792922837734224,
+ "epoch": 0.8816970074166278,
+ "step": 850
+ },
+ {
+ "loss": 0.44599761962890627,
+ "grad_norm": 0.012146789580583572,
+ "learning_rate": 1.0984455958549225e-06,
+ "entropy": 1.70398797435686,
+ "num_tokens": 13610138.0,
+ "logits/chosen": 0.17104202048628422,
+ "logits/rejected": 0.262595645620993,
+ "mean_token_accuracy": 0.642651722766459,
+ "rewards/chosen": -0.6636459323119197,
+ "rewards/rejected": -1.85319078174929,
+ "rewards/accuracies": 0.8125,
+ "rewards/margins": 1.1895448501838837,
+ "logps/chosen": -67.19200572252274,
+ "logps/rejected": -64.02687373161316,
+ "epoch": 0.8920699133862352,
+ "step": 860
+ },
+ {
+ "loss": 0.4463825702667236,
+ "grad_norm": 0.013926991261541843,
+ "learning_rate": 9.94818652849741e-07,
+ "entropy": 1.6875281669571995,
+ "num_tokens": 13769613.0,
+ "logits/chosen": 0.1617350362536588,
+ "logits/rejected": 0.25974264117481755,
+ "mean_token_accuracy": 0.6424587191455067,
+ "rewards/chosen": -0.6411357878561103,
+ "rewards/rejected": -1.7520794819842558,
+ "rewards/accuracies": 0.8275,
+ "rewards/margins": 1.1109436923218892,
+ "logps/chosen": -66.13908497691155,
+ "logps/rejected": -62.40538552284241,
+ "epoch": 0.9024428193558426,
+ "step": 870
+ },
+ {
+ "eval_loss": 0.5264487266540527,
+ "eval_runtime": 346.8502,
+ "eval_samples_per_second": 2.883,
+ "eval_steps_per_second": 2.883,
+ "eval_entropy": 1.0440044164210558,
+ "eval_num_tokens": 13848578.0,
+ "eval_logits/chosen": -0.18929922542314412,
+ "eval_logits/rejected": -0.05010199907599402,
+ "eval_mean_token_accuracy": 0.6949034848362208,
+ "eval_rewards/chosen": -0.7382756498559029,
+ "eval_rewards/rejected": -1.426999929712445,
+ "eval_rewards/accuracies": 0.743,
+ "eval_rewards/margins": 0.6887242792025209,
+ "eval_logps/chosen": -57.555743215560916,
+ "eval_logps/rejected": -66.9098708076477,
+ "epoch": 0.9076292723406463,
+ "step": 875
+ },
+ {
+ "loss": 0.43689889907836915,
+ "grad_norm": 0.010173802264034748,
+ "learning_rate": 8.911917098445596e-07,
+ "entropy": 1.6774525323137641,
+ "num_tokens": 13928284.0,
+ "logits/chosen": 0.15889503262817226,
+ "logits/rejected": 0.27439704581314006,
+ "mean_token_accuracy": 0.6466628183331341,
+ "rewards/chosen": -0.6104124534751736,
+ "rewards/rejected": -1.7897700341788003,
+ "rewards/accuracies": 0.81625,
+ "rewards/margins": 1.1793575775902718,
+ "logps/chosen": -62.02614854454994,
+ "logps/rejected": -64.84975938558578,
+ "epoch": 0.91281572532545,
+ "step": 880
+ },
+ {
+ "loss": 0.48273696899414065,
+ "grad_norm": 0.017326118424534798,
+ "learning_rate": 7.875647668393784e-07,
+ "entropy": 1.6614145183190703,
+ "num_tokens": 14086112.0,
+ "logits/chosen": 0.07985394351840137,
+ "logits/rejected": 0.20892557638470582,
+ "mean_token_accuracy": 0.6420493371598422,
+ "rewards/chosen": -0.7163908848223582,
+ "rewards/rejected": -1.8062069240648997,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.0898160382080824,
+ "logps/chosen": -63.230327110290524,
+ "logps/rejected": -64.38345015048981,
+ "epoch": 0.9231886312950573,
+ "step": 890
+ },
+ {
+ "loss": 0.43041396141052246,
+ "grad_norm": 0.017108725383877754,
+ "learning_rate": 6.839378238341969e-07,
+ "entropy": 1.6680658238008619,
+ "num_tokens": 14245558.0,
+ "logits/chosen": 0.14648505909070886,
+ "logits/rejected": 0.26673792896430265,
+ "mean_token_accuracy": 0.6451807205565274,
+ "rewards/chosen": -0.6325454073216497,
+ "rewards/rejected": -1.9007032794272527,
+ "rewards/accuracies": 0.82125,
+ "rewards/margins": 1.2681578750582412,
+ "logps/chosen": -62.82921484231949,
+ "logps/rejected": -66.73553317070008,
+ "epoch": 0.9335615372646647,
+ "step": 900
+ },
+ {
+ "loss": 0.4457756519317627,
+ "grad_norm": 0.010929541662335396,
+ "learning_rate": 5.803108808290156e-07,
+ "entropy": 1.7130034917313606,
+ "num_tokens": 14404540.0,
+ "logits/chosen": 0.12710906584716752,
+ "logits/rejected": 0.2675685432938819,
+ "mean_token_accuracy": 0.6400158200226724,
+ "rewards/chosen": -0.6709339278266998,
+ "rewards/rejected": -1.8822100719693118,
+ "rewards/accuracies": 0.825,
+ "rewards/margins": 1.2112761446926743,
+ "logps/chosen": -64.97942503213882,
+ "logps/rejected": -64.39793292522431,
+ "epoch": 0.9439344432342721,
+ "step": 910
+ },
+ {
+ "loss": 0.45868711471557616,
+ "grad_norm": 0.012205246835947037,
+ "learning_rate": 4.7668393782383424e-07,
+ "entropy": 1.6709410886280238,
+ "num_tokens": 14560645.0,
+ "logits/chosen": 0.10928400241500846,
+ "logits/rejected": 0.2426133711223395,
+ "mean_token_accuracy": 0.6370881532691419,
+ "rewards/chosen": -0.6589926348004519,
+ "rewards/rejected": -1.8069635758572258,
+ "rewards/accuracies": 0.81375,
+ "rewards/margins": 1.14797093979083,
+ "logps/chosen": -61.491244453191754,
+ "logps/rejected": -61.11656880378723,
+ "epoch": 0.9543073492038795,
+ "step": 920
+ },
+ {
+ "loss": 0.4510962963104248,
+ "grad_norm": 0.016680743545293808,
+ "learning_rate": 3.730569948186528e-07,
+ "entropy": 1.7099607919715345,
+ "num_tokens": 14720454.0,
+ "logits/chosen": 0.11883571728281689,
+ "logits/rejected": 0.20641075253811259,
+ "mean_token_accuracy": 0.6349475438706577,
+ "rewards/chosen": -0.6969622136335238,
+ "rewards/rejected": -1.859341476371046,
+ "rewards/accuracies": 0.81,
+ "rewards/margins": 1.162379261488095,
+ "logps/chosen": -65.8184747338295,
+ "logps/rejected": -65.03840645551682,
+ "epoch": 0.9646802551734869,
+ "step": 930
+ },
+ {
+ "loss": 0.44940953254699706,
+ "grad_norm": 0.013420112431049347,
+ "learning_rate": 2.694300518134715e-07,
+ "entropy": 1.6909510315582157,
+ "num_tokens": 14879971.0,
+ "logits/chosen": 0.18477739616368072,
+ "logits/rejected": 0.2645026210979778,
+ "mean_token_accuracy": 0.6382351936399937,
+ "rewards/chosen": -0.6513557667902206,
+ "rewards/rejected": -1.7696480820770375,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.1182923125289381,
+ "logps/chosen": -65.54289688110352,
+ "logps/rejected": -62.99382516860962,
+ "epoch": 0.9750531611430943,
+ "step": 940
+ },
+ {
+ "loss": 0.47352304458618166,
+ "grad_norm": 0.013681693933904171,
+ "learning_rate": 1.6580310880829015e-07,
+ "entropy": 1.661503377771005,
+ "num_tokens": 15038091.0,
+ "logits/chosen": 0.11480745743066594,
+ "logits/rejected": 0.22102661224680678,
+ "mean_token_accuracy": 0.6446126988902688,
+ "rewards/chosen": -0.7158770331276173,
+ "rewards/rejected": -1.8211175910846213,
+ "rewards/accuracies": 0.78875,
+ "rewards/margins": 1.1052405576594173,
+ "logps/chosen": -63.61789976716042,
+ "logps/rejected": -62.64833914756775,
+ "epoch": 0.9854260671127016,
+ "step": 950
+ },
+ {
+ "loss": 0.46666803359985354,
+ "grad_norm": 0.014694225043058395,
+ "learning_rate": 6.217616580310881e-08,
+ "entropy": 1.6944513383321465,
+ "num_tokens": 15198513.0,
+ "logits/chosen": 0.1733613867125754,
+ "logits/rejected": 0.27222349129564544,
+ "mean_token_accuracy": 0.6361824345402419,
+ "rewards/chosen": -0.7022936144633786,
+ "rewards/rejected": -1.8512264592042629,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 1.148932844074443,
+ "logps/chosen": -66.14034503817558,
+ "logps/rejected": -63.98270012617111,
+ "epoch": 0.995798973082309,
+ "step": 960
+ },
+ {
+ "train_runtime": 22398.3946,
+ "train_samples_per_second": 3.443,
+ "train_steps_per_second": 0.043,
+ "total_flos": 1.3308435135621734e+17,
+ "train_loss": 0.48160764293967134,
+ "entropy": 1.7446463192686623,
+ "num_tokens": 15263900.0,
+ "logits/chosen": 0.2563561171205477,
+ "logits/rejected": 0.3359260186024246,
+ "mean_token_accuracy": 0.6413786758059337,
+ "rewards/chosen": -0.7268365635874359,
+ "rewards/rejected": -1.856159317700974,
+ "rewards/accuracies": 0.8209876543209876,
+ "rewards/margins": 1.1293227543801436,
+ "logps/chosen": -67.83627563052707,
+ "logps/rejected": -68.1628461767126,
+ "epoch": 1.0,
+ "step": 965
+ }
+ ],
+ "validation_monitor_size": 1000,
+ "validation_monitor_selection": "fixed_seed_random_without_replacement",
+ "validation_monitor_seed": 22,
+ "validation_monitor_indices": [
+ 2,
+ 10,
+ 14,
+ 21,
+ 55,
+ 63,
+ 66,
+ 69,
+ 107,
+ 110,
+ 142,
+ 144,
+ 149,
+ 150,
+ 151,
+ 152,
+ 160,
+ 163,
+ 166,
+ 167,
+ 176,
+ 181,
+ 206,
+ 207,
+ 259,
+ 267,
+ 281,
+ 295,
+ 298,
+ 306,
+ 307,
+ 317,
+ 321,
+ 331,
+ 336,
+ 341,
+ 346,
+ 349,
+ 351,
+ 352,
+ 357,
+ 358,
+ 369,
+ 370,
+ 382,
+ 386,
+ 391,
+ 401,
+ 411,
+ 412,
+ 432,
+ 437,
+ 452,
+ 462,
+ 465,
+ 488,
+ 490,
+ 491,
+ 492,
+ 494,
+ 503,
+ 504,
+ 508,
+ 528,
+ 538,
+ 540,
+ 551,
+ 553,
+ 567,
+ 586,
+ 605,
+ 606,
+ 625,
+ 627,
+ 661,
+ 663,
+ 666,
+ 677,
+ 685,
+ 690,
+ 692,
+ 704,
+ 708,
+ 714,
+ 715,
+ 727,
+ 728,
+ 733,
+ 745,
+ 752,
+ 753,
+ 755,
+ 764,
+ 773,
+ 779,
+ 783,
+ 793,
+ 796,
+ 799,
+ 803,
+ 804,
+ 805,
+ 813,
+ 816,
+ 818,
+ 823,
+ 827,
+ 829,
+ 830,
+ 837,
+ 842,
+ 845,
+ 850,
+ 853,
+ 856,
+ 889,
+ 890,
+ 905,
+ 915,
+ 924,
+ 930,
+ 939,
+ 951,
+ 988,
+ 1002,
+ 1005,
+ 1023,
+ 1029,
+ 1038,
+ 1049,
+ 1050,
+ 1054,
+ 1056,
+ 1067,
+ 1068,
+ 1079,
+ 1088,
+ 1091,
+ 1103,
+ 1114,
+ 1118,
+ 1133,
+ 1140,
+ 1144,
+ 1145,
+ 1155,
+ 1186,
+ 1195,
+ 1206,
+ 1223,
+ 1251,
+ 1252,
+ 1257,
+ 1259,
+ 1270,
+ 1271,
+ 1295,
+ 1303,
+ 1304,
+ 1305,
+ 1329,
+ 1335,
+ 1336,
+ 1343,
+ 1367,
+ 1373,
+ 1377,
+ 1403,
+ 1412,
+ 1429,
+ 1443,
+ 1457,
+ 1459,
+ 1460,
+ 1476,
+ 1483,
+ 1486,
+ 1494,
+ 1507,
+ 1510,
+ 1519,
+ 1521,
+ 1522,
+ 1545,
+ 1551,
+ 1570,
+ 1582,
+ 1593,
+ 1595,
+ 1603,
+ 1607,
+ 1614,
+ 1615,
+ 1625,
+ 1634,
+ 1639,
+ 1648,
+ 1654,
+ 1657,
+ 1662,
+ 1667,
+ 1673,
+ 1690,
+ 1704,
+ 1746,
+ 1756,
+ 1781,
+ 1783,
+ 1796,
+ 1799,
+ 1809,
+ 1814,
+ 1827,
+ 1829,
+ 1832,
+ 1844,
+ 1847,
+ 1854,
+ 1856,
+ 1857,
+ 1858,
+ 1874,
+ 1883,
+ 1889,
+ 1924,
+ 1925,
+ 1931,
+ 1932,
+ 1934,
+ 1935,
+ 1938,
+ 1950,
+ 1957,
+ 1962,
+ 1967,
+ 1975,
+ 1982,
+ 1983,
+ 1991,
+ 1998,
+ 2003,
+ 2008,
+ 2017,
+ 2021,
+ 2026,
+ 2035,
+ 2040,
+ 2050,
+ 2056,
+ 2077,
+ 2079,
+ 2082,
+ 2098,
+ 2100,
+ 2108,
+ 2121,
+ 2130,
+ 2133,
+ 2134,
+ 2138,
+ 2143,
+ 2166,
+ 2167,
+ 2180,
+ 2181,
+ 2185,
+ 2204,
+ 2205,
+ 2212,
+ 2221,
+ 2224,
+ 2226,
+ 2230,
+ 2233,
+ 2256,
+ 2261,
+ 2263,
+ 2269,
+ 2273,
+ 2290,
+ 2291,
+ 2299,
+ 2301,
+ 2321,
+ 2323,
+ 2330,
+ 2384,
+ 2401,
+ 2417,
+ 2420,
+ 2421,
+ 2424,
+ 2430,
+ 2435,
+ 2456,
+ 2488,
+ 2502,
+ 2504,
+ 2519,
+ 2527,
+ 2532,
+ 2538,
+ 2542,
+ 2553,
+ 2555,
+ 2558,
+ 2559,
+ 2562,
+ 2578,
+ 2583,
+ 2585,
+ 2590,
+ 2592,
+ 2594,
+ 2596,
+ 2600,
+ 2606,
+ 2607,
+ 2618,
+ 2630,
+ 2637,
+ 2647,
+ 2650,
+ 2657,
+ 2679,
+ 2685,
+ 2705,
+ 2706,
+ 2712,
+ 2713,
+ 2714,
+ 2727,
+ 2740,
+ 2742,
+ 2755,
+ 2780,
+ 2784,
+ 2792,
+ 2807,
+ 2808,
+ 2810,
+ 2820,
+ 2831,
+ 2839,
+ 2860,
+ 2862,
+ 2863,
+ 2866,
+ 2875,
+ 2878,
+ 2898,
+ 2905,
+ 2921,
+ 2922,
+ 2926,
+ 2930,
+ 2934,
+ 2944,
+ 2955,
+ 2957,
+ 2959,
+ 2973,
+ 2978,
+ 2998,
+ 3018,
+ 3022,
+ 3028,
+ 3031,
+ 3061,
+ 3085,
+ 3090,
+ 3104,
+ 3105,
+ 3111,
+ 3115,
+ 3121,
+ 3122,
+ 3129,
+ 3133,
+ 3135,
+ 3161,
+ 3162,
+ 3169,
+ 3173,
+ 3194,
+ 3195,
+ 3215,
+ 3225,
+ 3226,
+ 3241,
+ 3247,
+ 3250,
+ 3253,
+ 3265,
+ 3268,
+ 3293,
+ 3301,
+ 3312,
+ 3329,
+ 3352,
+ 3361,
+ 3362,
+ 3376,
+ 3396,
+ 3401,
+ 3403,
+ 3410,
+ 3419,
+ 3432,
+ 3443,
+ 3452,
+ 3467,
+ 3469,
+ 3475,
+ 3485,
+ 3503,
+ 3514,
+ 3515,
+ 3524,
+ 3528,
+ 3538,
+ 3544,
+ 3557,
+ 3560,
+ 3565,
+ 3600,
+ 3637,
+ 3642,
+ 3658,
+ 3659,
+ 3692,
+ 3693,
+ 3699,
+ 3722,
+ 3725,
+ 3728,
+ 3731,
+ 3740,
+ 3742,
+ 3762,
+ 3766,
+ 3780,
+ 3788,
+ 3794,
+ 3796,
+ 3798,
+ 3805,
+ 3817,
+ 3819,
+ 3822,
+ 3837,
+ 3839,
+ 3843,
+ 3846,
+ 3851,
+ 3854,
+ 3865,
+ 3870,
+ 3871,
+ 3884,
+ 3894,
+ 3895,
+ 3896,
+ 3907,
+ 3911,
+ 3915,
+ 3919,
+ 3920,
+ 3923,
+ 3933,
+ 3955,
+ 3967,
+ 3972,
+ 3974,
+ 3975,
+ 3984,
+ 3985,
+ 3997,
+ 4002,
+ 4010,
+ 4034,
+ 4044,
+ 4063,
+ 4073,
+ 4079,
+ 4082,
+ 4088,
+ 4121,
+ 4145,
+ 4148,
+ 4159,
+ 4161,
+ 4164,
+ 4177,
+ 4188,
+ 4199,
+ 4203,
+ 4207,
+ 4208,
+ 4213,
+ 4221,
+ 4225,
+ 4233,
+ 4241,
+ 4243,
+ 4247,
+ 4264,
+ 4274,
+ 4282,
+ 4286,
+ 4290,
+ 4308,
+ 4310,
+ 4313,
+ 4321,
+ 4324,
+ 4327,
+ 4349,
+ 4362,
+ 4370,
+ 4374,
+ 4380,
+ 4407,
+ 4409,
+ 4411,
+ 4415,
+ 4417,
+ 4418,
+ 4427,
+ 4431,
+ 4433,
+ 4451,
+ 4466,
+ 4477,
+ 4478,
+ 4479,
+ 4493,
+ 4494,
+ 4514,
+ 4527,
+ 4539,
+ 4550,
+ 4558,
+ 4568,
+ 4579,
+ 4583,
+ 4584,
+ 4586,
+ 4587,
+ 4590,
+ 4599,
+ 4602,
+ 4610,
+ 4626,
+ 4627,
+ 4630,
+ 4641,
+ 4647,
+ 4655,
+ 4656,
+ 4657,
+ 4661,
+ 4685,
+ 4714,
+ 4715,
+ 4731,
+ 4749,
+ 4752,
+ 4769,
+ 4777,
+ 4782,
+ 4791,
+ 4805,
+ 4818,
+ 4829,
+ 4833,
+ 4837,
+ 4839,
+ 4843,
+ 4871,
+ 4875,
+ 4879,
+ 4880,
+ 4885,
+ 4888,
+ 4895,
+ 4900,
+ 4923,
+ 4966,
+ 4968,
+ 4972,
+ 4989,
+ 4994,
+ 4998,
+ 5001,
+ 5013,
+ 5019,
+ 5026,
+ 5032,
+ 5034,
+ 5046,
+ 5055,
+ 5085,
+ 5086,
+ 5088,
+ 5089,
+ 5090,
+ 5095,
+ 5108,
+ 5110,
+ 5119,
+ 5120,
+ 5121,
+ 5133,
+ 5148,
+ 5154,
+ 5160,
+ 5169,
+ 5178,
+ 5222,
+ 5223,
+ 5232,
+ 5233,
+ 5240,
+ 5256,
+ 5259,
+ 5264,
+ 5271,
+ 5276,
+ 5279,
+ 5294,
+ 5300,
+ 5303,
+ 5321,
+ 5335,
+ 5337,
+ 5345,
+ 5348,
+ 5365,
+ 5373,
+ 5378,
+ 5384,
+ 5422,
+ 5442,
+ 5443,
+ 5445,
+ 5477,
+ 5485,
+ 5495,
+ 5497,
+ 5504,
+ 5526,
+ 5533,
+ 5542,
+ 5564,
+ 5570,
+ 5579,
+ 5587,
+ 5592,
+ 5608,
+ 5610,
+ 5624,
+ 5630,
+ 5638,
+ 5651,
+ 5663,
+ 5670,
+ 5675,
+ 5691,
+ 5700,
+ 5706,
+ 5707,
+ 5715,
+ 5720,
+ 5721,
+ 5722,
+ 5732,
+ 5738,
+ 5741,
+ 5769,
+ 5771,
+ 5775,
+ 5795,
+ 5796,
+ 5800,
+ 5809,
+ 5810,
+ 5815,
+ 5819,
+ 5827,
+ 5848,
+ 5849,
+ 5852,
+ 5859,
+ 5880,
+ 5884,
+ 5907,
+ 5909,
+ 5912,
+ 5919,
+ 5931,
+ 5932,
+ 5934,
+ 5941,
+ 5948,
+ 5950,
+ 5952,
+ 5953,
+ 5969,
+ 5981,
+ 6000,
+ 6003,
+ 6010,
+ 6018,
+ 6041,
+ 6065,
+ 6075,
+ 6090,
+ 6103,
+ 6106,
+ 6109,
+ 6114,
+ 6123,
+ 6131,
+ 6136,
+ 6138,
+ 6139,
+ 6164,
+ 6165,
+ 6171,
+ 6173,
+ 6180,
+ 6185,
+ 6189,
+ 6190,
+ 6221,
+ 6223,
+ 6237,
+ 6255,
+ 6262,
+ 6265,
+ 6293,
+ 6296,
+ 6305,
+ 6318,
+ 6331,
+ 6336,
+ 6340,
+ 6355,
+ 6366,
+ 6371,
+ 6396,
+ 6399,
+ 6402,
+ 6408,
+ 6432,
+ 6433,
+ 6441,
+ 6456,
+ 6465,
+ 6478,
+ 6486,
+ 6489,
+ 6507,
+ 6508,
+ 6520,
+ 6522,
+ 6528,
+ 6537,
+ 6551,
+ 6564,
+ 6589,
+ 6590,
+ 6598,
+ 6599,
+ 6611,
+ 6613,
+ 6615,
+ 6621,
+ 6634,
+ 6647,
+ 6649,
+ 6654,
+ 6676,
+ 6680,
+ 6690,
+ 6708,
+ 6729,
+ 6736,
+ 6744,
+ 6749,
+ 6756,
+ 6761,
+ 6763,
+ 6773,
+ 6788,
+ 6790,
+ 6796,
+ 6797,
+ 6811,
+ 6824,
+ 6850,
+ 6869,
+ 6871,
+ 6882,
+ 6892,
+ 6895,
+ 6906,
+ 6911,
+ 6912,
+ 6914,
+ 6927,
+ 6952,
+ 6966,
+ 6985,
+ 7001,
+ 7021,
+ 7031,
+ 7035,
+ 7038,
+ 7041,
+ 7045,
+ 7052,
+ 7057,
+ 7058,
+ 7072,
+ 7073,
+ 7076,
+ 7086,
+ 7102,
+ 7107,
+ 7109,
+ 7117,
+ 7122,
+ 7125,
+ 7166,
+ 7182,
+ 7199,
+ 7207,
+ 7212,
+ 7215,
+ 7232,
+ 7243,
+ 7246,
+ 7250,
+ 7253,
+ 7258,
+ 7263,
+ 7267,
+ 7270,
+ 7274,
+ 7280,
+ 7286,
+ 7293,
+ 7298,
+ 7302,
+ 7306,
+ 7316,
+ 7325,
+ 7326,
+ 7334,
+ 7356,
+ 7357,
+ 7363,
+ 7364,
+ 7367,
+ 7385,
+ 7392,
+ 7399,
+ 7405,
+ 7416,
+ 7420,
+ 7421,
+ 7426,
+ 7452,
+ 7476,
+ 7481,
+ 7519,
+ 7534,
+ 7542,
+ 7546,
+ 7573,
+ 7585,
+ 7601,
+ 7604,
+ 7606,
+ 7609,
+ 7629,
+ 7649,
+ 7653,
+ 7667,
+ 7682,
+ 7699,
+ 7738,
+ 7750,
+ 7786,
+ 7798,
+ 7800,
+ 7801,
+ 7805,
+ 7806,
+ 7811,
+ 7813,
+ 7832,
+ 7837,
+ 7849,
+ 7856,
+ 7876,
+ 7877,
+ 7887,
+ 7905,
+ 7916,
+ 7919,
+ 7920,
+ 7922,
+ 7923,
+ 7926,
+ 7944,
+ 7961,
+ 7966,
+ 7970,
+ 7973,
+ 7974,
+ 7976,
+ 7986,
+ 7999,
+ 8027,
+ 8028,
+ 8034,
+ 8047,
+ 8068,
+ 8074,
+ 8077,
+ 8091,
+ 8120,
+ 8122,
+ 8125,
+ 8152,
+ 8153,
+ 8164,
+ 8167,
+ 8169,
+ 8171,
+ 8177,
+ 8184,
+ 8189,
+ 8192,
+ 8196,
+ 8202,
+ 8212,
+ 8217,
+ 8231,
+ 8242,
+ 8244,
+ 8250,
+ 8256,
+ 8257,
+ 8265,
+ 8270,
+ 8274,
+ 8283,
+ 8290,
+ 8294,
+ 8301,
+ 8302,
+ 8307,
+ 8318,
+ 8326,
+ 8338,
+ 8349,
+ 8350,
+ 8353,
+ 8357,
+ 8371,
+ 8374,
+ 8377,
+ 8380,
+ 8387,
+ 8388,
+ 8396,
+ 8402,
+ 8419,
+ 8423,
+ 8428,
+ 8435,
+ 8439,
+ 8442,
+ 8444,
+ 8453,
+ 8461,
+ 8475,
+ 8481,
+ 8485,
+ 8493,
+ 8495,
+ 8498,
+ 8523,
+ 8530,
+ 8531,
+ 8562
+ ],
+ "early_stopping_patience": 3
+}
diff --git a/phase1/ablations/helps_only/README.md b/phase1/ablations/helps_only/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..e15407bf0578dc3b02fd9bb8c09f2df2fdaa6621
--- /dev/null
+++ b/phase1/ablations/helps_only/README.md
@@ -0,0 +1,80 @@
+---
+library_name: peft
+model_name: phase1-qwen-helps-only-ablation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-1.5B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+license: apache-2.0
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+pipeline_tag: text-generation
+---
+
+# Phase 1 Qwen `helps_only` diagnostic ablation
+
+This is the Phase 1 `helps_only` diagnostic LoRA-DPO adapter fine-tuned from
+[Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct).
+The frozen training recipe and membership hashes are included in this directory.
+
+## Quick start
+
+```python
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo_id = "geyuxu/york-milestone-project-self-traing-loop-model"
+subfolder = "phase1/ablations/helps_only"
+base_id = "Qwen/Qwen2.5-1.5B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(repo_id, subfolder=subfolder)
+base_model = AutoModelForCausalLM.from_pretrained(
+ base_id, torch_dtype="auto", device_map="auto"
+)
+model = PeftModel.from_pretrained(base_model, repo_id, subfolder=subfolder)
+```
+
+## Training procedure
+
+
+
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 0.29.0
+- Transformers: 5.3.0
+- Pytorch: 2.10.0
+- Datasets: 4.6.1
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+ title = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+ author = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+ year = 2023,
+ booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+ url = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+ editor = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+ title = {{TRL: Transformers Reinforcement Learning}},
+ author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+ license = {Apache-2.0},
+ url = {https://github.com/huggingface/trl},
+ year = {2020}
+}
+```
diff --git a/phase1/ablations/helps_only/adapter_config.json b/phase1/ablations/helps_only/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..fa47a2292cd8a23ccf1b4d101d7f7a03e3f08871
--- /dev/null
+++ b/phase1/ablations/helps_only/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "k_proj",
+ "o_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
diff --git a/phase1/ablations/helps_only/adapter_model.safetensors b/phase1/ablations/helps_only/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2d205b95d4ab86d4d996c737054b25bcb7d7d4ab
--- /dev/null
+++ b/phase1/ablations/helps_only/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a143ad6c49a7a1b1d98e81af88412c91b367a394fa48a89517d6057fddf91748
+size 17462432
diff --git a/phase1/ablations/helps_only/chat_template.jinja b/phase1/ablations/helps_only/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e
--- /dev/null
+++ b/phase1/ablations/helps_only/chat_template.jinja
@@ -0,0 +1,54 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- messages[0]['content'] }}
+ {%- else %}
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+ {%- endif %}
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+ {%- else %}
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {{- '<|im_start|>' + message.role }}
+ {%- if message.content %}
+ {{- '\n' + message.content }}
+ {%- endif %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- '}\n' }}
+ {%- endfor %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- message.content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}
diff --git a/phase1/ablations/helps_only/tokenizer.json b/phase1/ablations/helps_only/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..d73428d91463b495bc017fb6a2fb3a656646482b
--- /dev/null
+++ b/phase1/ablations/helps_only/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5
+size 11422166
diff --git a/phase1/ablations/helps_only/tokenizer_config.json b/phase1/ablations/helps_only/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9fd0fb48e73786f54fb04e98892f5f5a0ebeebdb
--- /dev/null
+++ b/phase1/ablations/helps_only/tokenizer_config.json
@@ -0,0 +1,29 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": true,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/phase1/ablations/helps_only/training_args.bin b/phase1/ablations/helps_only/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..a0e8c600d39d5a9f9542fd7088abfb7ec6bf7d1b
--- /dev/null
+++ b/phase1/ablations/helps_only/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:101d92ecb945e3232d561989c7a979ff91d184b4cccbe34a200a56060e3c5ce6
+size 5841
diff --git a/phase1/ablations/helps_only/training_recipe.json b/phase1/ablations/helps_only/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..ca058e2120ded3b68670d6aace62f00d623b8deb
--- /dev/null
+++ b/phase1/ablations/helps_only/training_recipe.json
@@ -0,0 +1,22 @@
+{
+ "format_version": 1,
+ "experiment": "confusion_ablation",
+ "ablation": "helps_only",
+ "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+ "train_membership_sha256": "5febb0b71375db4da6a27fdaa6d3226681631022eab8e57a816d93519c4854ab",
+ "validation_membership_sha256": "2f609467438cd6a01fd84b9d80c0fcd84c44767e3b2ed98f4f6a83d68a62c181",
+ "num_train_pairs": 77124,
+ "num_validation_pairs": 8563,
+ "transformation_seed": 22,
+ "validation_prompt_mode": "clean_rag",
+ "learning_rate": 1e-05,
+ "beta": 0.1,
+ "num_epochs": 1.0,
+ "seed": 22,
+ "max_length": 512,
+ "max_prompt_length": 384,
+ "gradient_accumulation_steps": 80,
+ "save_steps": 125,
+ "save_total_limit": 12,
+ "preference_pairs_format_version": 2
+}
diff --git a/phase1/ablations/helps_only/training_summary.json b/phase1/ablations/helps_only/training_summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..e960cc5b20fc2baf0d98c901ad9b7ab0b73deb47
--- /dev/null
+++ b/phase1/ablations/helps_only/training_summary.json
@@ -0,0 +1,2029 @@
+{
+ "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+ "resolved_model": "Qwen/Qwen2.5-1.5B-Instruct",
+ "num_train_records": 77124,
+ "num_validation_records_available": 8563,
+ "training_config": {
+ "lora_rank": 16,
+ "lora_alpha": 32,
+ "learning_rate": 1e-05,
+ "num_epochs": 1.0,
+ "per_device_batch_size": 1,
+ "per_device_eval_batch_size": 1,
+ "gradient_accumulation_steps": 80,
+ "beta": 0.1,
+ "max_length": 512,
+ "max_prompt_length": 384,
+ "seed": 22,
+ "save_steps": 125,
+ "save_total_limit": 12,
+ "resume_from_checkpoint": null,
+ "precision_dtype": "torch.bfloat16",
+ "bf16": true,
+ "fp16": false,
+ "tokenizer_add_bos_token": false
+ },
+ "global_step": 500,
+ "best_metric": 0.5006195306777954,
+ "best_model_checkpoint": "outputs/confusion_ablation_phase1_repair_v3/helps_only/lora/checkpoint-125",
+ "train_metrics": {
+ "train_runtime": 16864.5913,
+ "train_samples_per_second": 4.573,
+ "train_steps_per_second": 0.057,
+ "total_flos": 2.7163725934432666e+17,
+ "train_loss": 0.23282903826236725
+ },
+ "log_history": [
+ {
+ "loss": 0.6858583450317383,
+ "grad_norm": 0.016611717641353607,
+ "learning_rate": 9.906735751295338e-06,
+ "entropy": 0.794021682399325,
+ "num_tokens": 670561.0,
+ "logits/chosen": -0.9187091639020697,
+ "logits/rejected": -0.9434830672895622,
+ "mean_token_accuracy": 0.736259245146066,
+ "rewards/chosen": 0.0011638513133243578,
+ "rewards/rejected": -0.013860612378184669,
+ "rewards/accuracies": 0.5925,
+ "rewards/margins": 0.015024463631707476,
+ "logps/chosen": -54.79746845722198,
+ "logps/rejected": -49.96550277948379,
+ "epoch": 0.010372905969607386,
+ "step": 10
+ },
+ {
+ "loss": 0.6576845169067382,
+ "grad_norm": 0.016666823998093605,
+ "learning_rate": 9.803108808290156e-06,
+ "entropy": 0.7863238480500877,
+ "num_tokens": 1340958.0,
+ "logits/chosen": -0.9552881883230906,
+ "logits/rejected": -0.9189709572546179,
+ "mean_token_accuracy": 0.7423679579421878,
+ "rewards/chosen": -0.013600413688568552,
+ "rewards/rejected": -0.08799497530219376,
+ "rewards/accuracies": 0.85875,
+ "rewards/margins": 0.07439456147782039,
+ "logps/chosen": -55.26659148097038,
+ "logps/rejected": -49.557674720287324,
+ "epoch": 0.02074581193921477,
+ "step": 20
+ },
+ {
+ "loss": 0.6105106353759766,
+ "grad_norm": 0.015847327187657356,
+ "learning_rate": 9.699481865284975e-06,
+ "entropy": 0.846787225897424,
+ "num_tokens": 2013796.0,
+ "logits/chosen": -0.8573241884593181,
+ "logits/rejected": -0.8453699552830984,
+ "mean_token_accuracy": 0.727089679390192,
+ "rewards/chosen": -0.06516332936625986,
+ "rewards/rejected": -0.24710432796498935,
+ "rewards/accuracies": 0.8575,
+ "rewards/margins": 0.18194099859509152,
+ "logps/chosen": -59.86930260896683,
+ "logps/rejected": -53.61334676504135,
+ "epoch": 0.031118717908822157,
+ "step": 30
+ },
+ {
+ "loss": 0.560779619216919,
+ "grad_norm": 0.01469502318650484,
+ "learning_rate": 9.595854922279793e-06,
+ "entropy": 0.7988823162578046,
+ "num_tokens": 2684543.0,
+ "logits/chosen": -0.9062229187635736,
+ "logits/rejected": -0.8287031519573034,
+ "mean_token_accuracy": 0.726212237663567,
+ "rewards/chosen": -0.15750733469684747,
+ "rewards/rejected": -0.47408700099374984,
+ "rewards/accuracies": 0.8625,
+ "rewards/margins": 0.3165796657791361,
+ "logps/chosen": -57.612799718379975,
+ "logps/rejected": -53.58559124469757,
+ "epoch": 0.04149162387842954,
+ "step": 40
+ },
+ {
+ "loss": 0.5105207920074463,
+ "grad_norm": 0.014577334746718407,
+ "learning_rate": 9.492227979274612e-06,
+ "entropy": 0.7784375699353404,
+ "num_tokens": 3356031.0,
+ "logits/chosen": -0.913959642386141,
+ "logits/rejected": -0.8075734227458465,
+ "mean_token_accuracy": 0.7303825634159148,
+ "rewards/chosen": -0.32432046058405833,
+ "rewards/rejected": -0.8036991727564601,
+ "rewards/accuracies": 0.8625,
+ "rewards/margins": 0.4793787114601582,
+ "logps/chosen": -60.06104364156723,
+ "logps/rejected": -56.74762117385864,
+ "epoch": 0.051864529848036925,
+ "step": 50
+ },
+ {
+ "loss": 0.4511364459991455,
+ "grad_norm": 0.01305594015866518,
+ "learning_rate": 9.388601036269432e-06,
+ "entropy": 0.7942980694322614,
+ "num_tokens": 4026577.0,
+ "logits/chosen": -0.9856768630883611,
+ "logits/rejected": -0.8317867858234134,
+ "mean_token_accuracy": 0.7197631917707622,
+ "rewards/chosen": -0.5297682328737574,
+ "rewards/rejected": -1.217431449344149,
+ "rewards/accuracies": 0.88625,
+ "rewards/margins": 0.6876632150541991,
+ "logps/chosen": -60.84162630081177,
+ "logps/rejected": -61.641977578401566,
+ "epoch": 0.062237435817644314,
+ "step": 60
+ },
+ {
+ "loss": 0.422989559173584,
+ "grad_norm": 0.01258748210966587,
+ "learning_rate": 9.284974093264249e-06,
+ "entropy": 0.7553169909078861,
+ "num_tokens": 4696820.0,
+ "logits/chosen": -1.1534164984013175,
+ "logits/rejected": -0.9453616152186384,
+ "mean_token_accuracy": 0.7059021528251469,
+ "rewards/chosen": -0.8479185369245533,
+ "rewards/rejected": -1.733439953867346,
+ "rewards/accuracies": 0.85125,
+ "rewards/margins": 0.8855214151740074,
+ "logps/chosen": -64.60678691148757,
+ "logps/rejected": -65.6156524682045,
+ "epoch": 0.0726103417872517,
+ "step": 70
+ },
+ {
+ "loss": 0.37308950424194337,
+ "grad_norm": 0.013154606334865093,
+ "learning_rate": 9.181347150259067e-06,
+ "entropy": 0.7961554935819004,
+ "num_tokens": 5368225.0,
+ "logits/chosen": -1.1909632763159101,
+ "logits/rejected": -0.9626243884256602,
+ "mean_token_accuracy": 0.7057545288372785,
+ "rewards/chosen": -1.0665127522181137,
+ "rewards/rejected": -2.248069805242121,
+ "rewards/accuracies": 0.8875,
+ "rewards/margins": 1.1815570489689708,
+ "logps/chosen": -68.12463062763214,
+ "logps/rejected": -72.84404395580292,
+ "epoch": 0.08298324775685909,
+ "step": 80
+ },
+ {
+ "loss": 0.36935536861419677,
+ "grad_norm": 0.015311975963413715,
+ "learning_rate": 9.077720207253888e-06,
+ "entropy": 0.7894015382009093,
+ "num_tokens": 6040068.0,
+ "logits/chosen": -1.3107291135601724,
+ "logits/rejected": -1.0484886964785651,
+ "mean_token_accuracy": 0.7019536628946662,
+ "rewards/chosen": -1.3442938781030533,
+ "rewards/rejected": -2.594835725099547,
+ "rewards/accuracies": 0.875,
+ "rewards/margins": 1.2505418445914984,
+ "logps/chosen": -72.38709788918496,
+ "logps/rejected": -75.64441309213639,
+ "epoch": 0.09335615372646647,
+ "step": 90
+ },
+ {
+ "loss": 0.32566983699798585,
+ "grad_norm": 0.013686278834939003,
+ "learning_rate": 8.974093264248706e-06,
+ "entropy": 0.7407993375349906,
+ "num_tokens": 6710899.0,
+ "logits/chosen": -1.491155676700035,
+ "logits/rejected": -1.1507826984501721,
+ "mean_token_accuracy": 0.7010396635532379,
+ "rewards/chosen": -1.472484585774364,
+ "rewards/rejected": -3.0050534189864995,
+ "rewards/accuracies": 0.87375,
+ "rewards/margins": 1.5325688306987286,
+ "logps/chosen": -71.50509068489075,
+ "logps/rejected": -79.74594541549682,
+ "epoch": 0.10372905969607385,
+ "step": 100
+ },
+ {
+ "loss": 0.3260276556015015,
+ "grad_norm": 0.015970611944794655,
+ "learning_rate": 8.870466321243523e-06,
+ "entropy": 0.755189473812934,
+ "num_tokens": 7384308.0,
+ "logits/chosen": -1.4866158399686538,
+ "logits/rejected": -1.2556732947348208,
+ "mean_token_accuracy": 0.6936355204880238,
+ "rewards/chosen": -1.6455462214258296,
+ "rewards/rejected": -3.314337354162708,
+ "rewards/accuracies": 0.875,
+ "rewards/margins": 1.6687911279685794,
+ "logps/chosen": -75.27282932758331,
+ "logps/rejected": -83.60757318258285,
+ "epoch": 0.11410196566568125,
+ "step": 110
+ },
+ {
+ "loss": 0.29123826026916505,
+ "grad_norm": 0.011552904732525349,
+ "learning_rate": 8.766839378238343e-06,
+ "entropy": 0.7474201882537455,
+ "num_tokens": 8056453.0,
+ "logits/chosen": -1.6142062577777776,
+ "logits/rejected": -1.264399125865953,
+ "mean_token_accuracy": 0.7005275747552514,
+ "rewards/chosen": -1.5997523294587155,
+ "rewards/rejected": -3.4206406278908252,
+ "rewards/accuracies": 0.89875,
+ "rewards/margins": 1.8208882979303598,
+ "logps/chosen": -72.69924384593963,
+ "logps/rejected": -84.9409653043747,
+ "epoch": 0.12447487163528863,
+ "step": 120
+ },
+ {
+ "eval_loss": 0.5006195306777954,
+ "eval_runtime": 185.028,
+ "eval_samples_per_second": 5.405,
+ "eval_steps_per_second": 5.405,
+ "eval_entropy": 0.7257204860020429,
+ "eval_num_tokens": 8393345.0,
+ "eval_logits/chosen": -1.6364654488183503,
+ "eval_logits/rejected": -1.3224665456826323,
+ "eval_mean_token_accuracy": 0.6896886819005013,
+ "eval_rewards/chosen": -1.8563280573266092,
+ "eval_rewards/rejected": -3.186325779806066,
+ "eval_rewards/accuracies": 0.791,
+ "eval_rewards/margins": 1.329997718065977,
+ "eval_logps/chosen": -68.73626706314087,
+ "eval_logps/rejected": -84.50312891960144,
+ "epoch": 0.1296613246200923,
+ "step": 125
+ },
+ {
+ "loss": 0.2636831045150757,
+ "grad_norm": 0.013076480478048325,
+ "learning_rate": 8.663212435233162e-06,
+ "entropy": 0.7181003772222903,
+ "num_tokens": 8728634.0,
+ "logits/chosen": -1.671642872950834,
+ "logits/rejected": -1.343902642880392,
+ "mean_token_accuracy": 0.705937882065773,
+ "rewards/chosen": -1.5789886895833478,
+ "rewards/rejected": -3.5362877251021563,
+ "rewards/accuracies": 0.90625,
+ "rewards/margins": 1.9572990308701992,
+ "logps/chosen": -72.90494430541992,
+ "logps/rejected": -84.75396874666214,
+ "epoch": 0.13484777760489602,
+ "step": 130
+ },
+ {
+ "loss": 0.2740552186965942,
+ "grad_norm": 0.021369587630033493,
+ "learning_rate": 8.55958549222798e-06,
+ "entropy": 0.7297478528990178,
+ "num_tokens": 9401159.0,
+ "logits/chosen": -1.7250535840926815,
+ "logits/rejected": -1.360942676206459,
+ "mean_token_accuracy": 0.7058176286891102,
+ "rewards/chosen": -1.711061445976993,
+ "rewards/rejected": -3.760642220210284,
+ "rewards/accuracies": 0.89875,
+ "rewards/margins": 2.0495807684585454,
+ "logps/chosen": -75.34392234086991,
+ "logps/rejected": -86.37021635055542,
+ "epoch": 0.1452206835745034,
+ "step": 140
+ },
+ {
+ "loss": 0.22246203422546387,
+ "grad_norm": 0.011216296814382076,
+ "learning_rate": 8.455958549222799e-06,
+ "entropy": 0.7156421507499181,
+ "num_tokens": 10070639.0,
+ "logits/chosen": -1.8058072450366334,
+ "logits/rejected": -1.453452492851942,
+ "mean_token_accuracy": 0.6968013681657612,
+ "rewards/chosen": -1.7734466221980256,
+ "rewards/rejected": -3.9852739233896135,
+ "rewards/accuracies": 0.935,
+ "rewards/margins": 2.2118273005262017,
+ "logps/chosen": -71.75837573051453,
+ "logps/rejected": -88.40371996164322,
+ "epoch": 0.1555935895441108,
+ "step": 150
+ },
+ {
+ "loss": 0.22982723712921144,
+ "grad_norm": 0.012776419520378113,
+ "learning_rate": 8.352331606217617e-06,
+ "entropy": 0.6933300006289209,
+ "num_tokens": 10739089.0,
+ "logits/chosen": -1.9515460256771149,
+ "logits/rejected": -1.5484639940247698,
+ "mean_token_accuracy": 0.7057104521989822,
+ "rewards/chosen": -1.9016535378903792,
+ "rewards/rejected": -4.284137947391718,
+ "rewards/accuracies": 0.92625,
+ "rewards/margins": 2.3824844037741424,
+ "logps/chosen": -71.04089045763016,
+ "logps/rejected": -91.5341923880577,
+ "epoch": 0.16596649551371817,
+ "step": 160
+ },
+ {
+ "loss": 0.23764348030090332,
+ "grad_norm": 0.014385406859219074,
+ "learning_rate": 8.248704663212436e-06,
+ "entropy": 0.7189149663675926,
+ "num_tokens": 11409834.0,
+ "logits/chosen": -1.891959825395243,
+ "logits/rejected": -1.548625273180476,
+ "mean_token_accuracy": 0.6981369163282216,
+ "rewards/chosen": -1.8744462743058101,
+ "rewards/rejected": -4.231724939988926,
+ "rewards/accuracies": 0.90875,
+ "rewards/margins": 2.3572786602377893,
+ "logps/chosen": -74.21431750059128,
+ "logps/rejected": -91.71514132022858,
+ "epoch": 0.17633940148332555,
+ "step": 170
+ },
+ {
+ "loss": 0.20213358402252196,
+ "grad_norm": 0.01709628663957119,
+ "learning_rate": 8.145077720207254e-06,
+ "entropy": 0.7331272099440684,
+ "num_tokens": 12080449.0,
+ "logits/chosen": -1.8974689940297265,
+ "logits/rejected": -1.5145953081096013,
+ "mean_token_accuracy": 0.6983240643888712,
+ "rewards/chosen": -1.801844048615967,
+ "rewards/rejected": -4.443616501316428,
+ "rewards/accuracies": 0.92875,
+ "rewards/margins": 2.6417724495008588,
+ "logps/chosen": -74.37266374826432,
+ "logps/rejected": -93.39013815402984,
+ "epoch": 0.18671230745293294,
+ "step": 180
+ },
+ {
+ "loss": 0.22085981369018554,
+ "grad_norm": 0.017351916059851646,
+ "learning_rate": 8.041450777202073e-06,
+ "entropy": 0.7038006630554446,
+ "num_tokens": 12750629.0,
+ "logits/chosen": -2.036204467140802,
+ "logits/rejected": -1.5767073775724174,
+ "mean_token_accuracy": 0.7034011324681342,
+ "rewards/chosen": -1.9227157408563653,
+ "rewards/rejected": -4.568975618286058,
+ "rewards/accuracies": 0.9175,
+ "rewards/margins": 2.6462598730251194,
+ "logps/chosen": -71.0266849398613,
+ "logps/rejected": -95.72787975311279,
+ "epoch": 0.19708521342254032,
+ "step": 190
+ },
+ {
+ "loss": 0.21095876693725585,
+ "grad_norm": 0.011923568323254585,
+ "learning_rate": 7.937823834196891e-06,
+ "entropy": 0.7554844116524327,
+ "num_tokens": 13423352.0,
+ "logits/chosen": -1.86974235439135,
+ "logits/rejected": -1.501746707999648,
+ "mean_token_accuracy": 0.6907317889854312,
+ "rewards/chosen": -1.9836193190991616,
+ "rewards/rejected": -4.729873430356383,
+ "rewards/accuracies": 0.92,
+ "rewards/margins": 2.7462541125901043,
+ "logps/chosen": -76.15088132619857,
+ "logps/rejected": -98.10042838811874,
+ "epoch": 0.2074581193921477,
+ "step": 200
+ },
+ {
+ "loss": 0.20679113864898682,
+ "grad_norm": 0.016088780015707016,
+ "learning_rate": 7.834196891191712e-06,
+ "entropy": 0.7792452401274932,
+ "num_tokens": 14095948.0,
+ "logits/chosen": -1.8705466560278834,
+ "logits/rejected": -1.461863531116033,
+ "mean_token_accuracy": 0.6862335817702114,
+ "rewards/chosen": -1.8493201506307742,
+ "rewards/rejected": -4.571774728436139,
+ "rewards/accuracies": 0.9275,
+ "rewards/margins": 2.7224545743037014,
+ "logps/chosen": -74.57652376651764,
+ "logps/rejected": -96.64638544797897,
+ "epoch": 0.2178310253617551,
+ "step": 210
+ },
+ {
+ "loss": 0.1841462254524231,
+ "grad_norm": 0.018083656206727028,
+ "learning_rate": 7.730569948186528e-06,
+ "entropy": 0.7562577223840344,
+ "num_tokens": 14765355.0,
+ "logits/chosen": -1.9466678828790673,
+ "logits/rejected": -1.556764617717985,
+ "mean_token_accuracy": 0.6736517621856183,
+ "rewards/chosen": -1.9498732457583536,
+ "rewards/rejected": -4.814249985329806,
+ "rewards/accuracies": 0.94375,
+ "rewards/margins": 2.8643767323717473,
+ "logps/chosen": -74.6550524687767,
+ "logps/rejected": -96.928872423172,
+ "epoch": 0.2282039313313625,
+ "step": 220
+ },
+ {
+ "loss": 0.18351367712020875,
+ "grad_norm": 0.02809528261423111,
+ "learning_rate": 7.626943005181348e-06,
+ "entropy": 0.7285784264818358,
+ "num_tokens": 15435816.0,
+ "logits/chosen": -2.0795930087592787,
+ "logits/rejected": -1.636840952128652,
+ "mean_token_accuracy": 0.6832175821252168,
+ "rewards/chosen": -2.231121272182354,
+ "rewards/rejected": -5.6053673730045555,
+ "rewards/accuracies": 0.935,
+ "rewards/margins": 3.374246101528406,
+ "logps/chosen": -77.43112219810486,
+ "logps/rejected": -106.47521061182022,
+ "epoch": 0.23857683730096987,
+ "step": 230
+ },
+ {
+ "loss": 0.19073803424835206,
+ "grad_norm": 0.01874386891722679,
+ "learning_rate": 7.523316062176167e-06,
+ "entropy": 0.7100621155637782,
+ "num_tokens": 16103129.0,
+ "logits/chosen": -2.111952753872817,
+ "logits/rejected": -1.7125841187317576,
+ "mean_token_accuracy": 0.6768746449425816,
+ "rewards/chosen": -2.3892388375883455,
+ "rewards/rejected": -5.755352979376912,
+ "rewards/accuracies": 0.935,
+ "rewards/margins": 3.366114140301943,
+ "logps/chosen": -76.3891685450077,
+ "logps/rejected": -104.74123107671738,
+ "epoch": 0.24894974327057726,
+ "step": 240
+ },
+ {
+ "loss": 0.15556421279907226,
+ "grad_norm": 0.01747226156294346,
+ "learning_rate": 7.419689119170985e-06,
+ "entropy": 0.733566192787257,
+ "num_tokens": 16774426.0,
+ "logits/chosen": -1.9979488002914059,
+ "logits/rejected": -1.5990273070071686,
+ "mean_token_accuracy": 0.6710057172738015,
+ "rewards/chosen": -2.2422293889999856,
+ "rewards/rejected": -5.71058324098587,
+ "rewards/accuracies": 0.95125,
+ "rewards/margins": 3.4683538476377724,
+ "logps/chosen": -77.04140710353852,
+ "logps/rejected": -106.12069250583649,
+ "epoch": 0.2593226492401846,
+ "step": 250
+ },
+ {
+ "eval_loss": 0.6056406497955322,
+ "eval_runtime": 214.9765,
+ "eval_samples_per_second": 4.652,
+ "eval_steps_per_second": 4.652,
+ "eval_entropy": 0.7675776538113133,
+ "eval_num_tokens": 16774426.0,
+ "eval_logits/chosen": -1.9016790324834714,
+ "eval_logits/rejected": -1.5678728323483735,
+ "eval_mean_token_accuracy": 0.638426756888628,
+ "eval_rewards/chosen": -2.858136704711709,
+ "eval_rewards/rejected": -5.103179955605418,
+ "eval_rewards/accuracies": 0.779,
+ "eval_rewards/margins": 2.245043248832226,
+ "eval_logps/chosen": -78.75435346412658,
+ "eval_logps/rejected": -103.67167038726807,
+ "epoch": 0.2593226492401846,
+ "step": 250
+ },
+ {
+ "loss": 0.15468125343322753,
+ "grad_norm": 0.023065408691763878,
+ "learning_rate": 7.3160621761658035e-06,
+ "entropy": 0.7432888356089825,
+ "num_tokens": 17443716.0,
+ "logits/chosen": -2.0815265223306243,
+ "logits/rejected": -1.6500056931946074,
+ "mean_token_accuracy": 0.665775734577328,
+ "rewards/chosen": -2.3973246609413765,
+ "rewards/rejected": -5.997232149764895,
+ "rewards/accuracies": 0.95,
+ "rewards/margins": 3.599907489940524,
+ "logps/chosen": -77.37100106954574,
+ "logps/rejected": -108.72272551059723,
+ "epoch": 0.26969555520979205,
+ "step": 260
+ },
+ {
+ "loss": 0.17242192029953002,
+ "grad_norm": 0.018100790679454803,
+ "learning_rate": 7.212435233160623e-06,
+ "entropy": 0.7815824018983403,
+ "num_tokens": 18115456.0,
+ "logits/chosen": -1.8929626691100117,
+ "logits/rejected": -1.568634429940542,
+ "mean_token_accuracy": 0.6482958744466305,
+ "rewards/chosen": -2.8240217579288767,
+ "rewards/rejected": -6.417929985076189,
+ "rewards/accuracies": 0.93875,
+ "rewards/margins": 3.593908224105835,
+ "logps/chosen": -86.06786829471588,
+ "logps/rejected": -114.08773995637894,
+ "epoch": 0.28006846117939943,
+ "step": 270
+ },
+ {
+ "loss": 0.13711202144622803,
+ "grad_norm": 0.013445369899272919,
+ "learning_rate": 7.108808290155441e-06,
+ "entropy": 0.7558632073167246,
+ "num_tokens": 18787182.0,
+ "logits/chosen": -1.978579450117466,
+ "logits/rejected": -1.6631533533075546,
+ "mean_token_accuracy": 0.6569508682191372,
+ "rewards/chosen": -2.7565071246563457,
+ "rewards/rejected": -6.632811545878649,
+ "rewards/accuracies": 0.9575,
+ "rewards/margins": 3.8763044214993716,
+ "logps/chosen": -84.94609524965286,
+ "logps/rejected": -115.75064901828766,
+ "epoch": 0.2904413671490068,
+ "step": 280
+ },
+ {
+ "loss": 0.14471136331558226,
+ "grad_norm": 0.03034352697432041,
+ "learning_rate": 7.005181347150259e-06,
+ "entropy": 0.7719684389990289,
+ "num_tokens": 19458826.0,
+ "logits/chosen": -1.9281082628928359,
+ "logits/rejected": -1.5931914351338883,
+ "mean_token_accuracy": 0.6512335392460227,
+ "rewards/chosen": -3.03531946905714,
+ "rewards/rejected": -7.0661579599231485,
+ "rewards/accuracies": 0.94875,
+ "rewards/margins": 4.030838481113315,
+ "logps/chosen": -85.57453865528106,
+ "logps/rejected": -120.38242918252945,
+ "epoch": 0.3008142731186142,
+ "step": 290
+ },
+ {
+ "loss": 0.14207189083099364,
+ "grad_norm": 0.01976640149950981,
+ "learning_rate": 6.9015544041450784e-06,
+ "entropy": 0.775840242926497,
+ "num_tokens": 20129807.0,
+ "logits/chosen": -1.9291908069467354,
+ "logits/rejected": -1.529736851381803,
+ "mean_token_accuracy": 0.647318363133818,
+ "rewards/chosen": -3.2047973467002158,
+ "rewards/rejected": -7.190337324440479,
+ "rewards/accuracies": 0.94375,
+ "rewards/margins": 3.985539976321161,
+ "logps/chosen": -88.11508201122284,
+ "logps/rejected": -121.52326109170913,
+ "epoch": 0.3111871790882216,
+ "step": 300
+ },
+ {
+ "loss": 0.13882720470428467,
+ "grad_norm": 0.019125595688819885,
+ "learning_rate": 6.797927461139897e-06,
+ "entropy": 0.7558516392938327,
+ "num_tokens": 20801777.0,
+ "logits/chosen": -1.9366754550881657,
+ "logits/rejected": -1.5732930849415423,
+ "mean_token_accuracy": 0.6629200987983495,
+ "rewards/chosen": -3.214626377870736,
+ "rewards/rejected": -7.299282197505236,
+ "rewards/accuracies": 0.9475,
+ "rewards/margins": 4.084655814617872,
+ "logps/chosen": -88.43090007543564,
+ "logps/rejected": -121.64797079086304,
+ "epoch": 0.32156008505782896,
+ "step": 310
+ },
+ {
+ "loss": 0.15239068269729614,
+ "grad_norm": 0.026703625917434692,
+ "learning_rate": 6.6943005181347155e-06,
+ "entropy": 0.759519019573927,
+ "num_tokens": 21473384.0,
+ "logits/chosen": -1.9384710945696861,
+ "logits/rejected": -1.6229014066134837,
+ "mean_token_accuracy": 0.6460824762284756,
+ "rewards/chosen": -3.5289745971094817,
+ "rewards/rejected": -7.814981915429234,
+ "rewards/accuracies": 0.9425,
+ "rewards/margins": 4.286007315814495,
+ "logps/chosen": -95.04349534749984,
+ "logps/rejected": -127.0398188662529,
+ "epoch": 0.33193299102743634,
+ "step": 320
+ },
+ {
+ "loss": 0.13009344339370726,
+ "grad_norm": 0.018585270270705223,
+ "learning_rate": 6.590673575129535e-06,
+ "entropy": 0.7342186298122396,
+ "num_tokens": 22142477.0,
+ "logits/chosen": -2.0363266451123923,
+ "logits/rejected": -1.67308820389407,
+ "mean_token_accuracy": 0.6532345769926906,
+ "rewards/chosen": -3.3065084332806873,
+ "rewards/rejected": -7.662135322336107,
+ "rewards/accuracies": 0.95625,
+ "rewards/margins": 4.355626890957356,
+ "logps/chosen": -86.15242535829545,
+ "logps/rejected": -125.1262379026413,
+ "epoch": 0.3423058969970437,
+ "step": 330
+ },
+ {
+ "loss": 0.1294613003730774,
+ "grad_norm": 0.024174772202968597,
+ "learning_rate": 6.487046632124353e-06,
+ "entropy": 0.778732093811268,
+ "num_tokens": 22814406.0,
+ "logits/chosen": -1.8744846753614508,
+ "logits/rejected": -1.557779354695435,
+ "mean_token_accuracy": 0.6483545247651636,
+ "rewards/chosen": -3.3343913850979243,
+ "rewards/rejected": -7.493394564837217,
+ "rewards/accuracies": 0.95625,
+ "rewards/margins": 4.1590031705796715,
+ "logps/chosen": -91.52018271446228,
+ "logps/rejected": -125.49517280578613,
+ "epoch": 0.3526788029666511,
+ "step": 340
+ },
+ {
+ "loss": 0.12839078903198242,
+ "grad_norm": 0.021780716255307198,
+ "learning_rate": 6.383419689119171e-06,
+ "entropy": 0.7516101838089526,
+ "num_tokens": 23486707.0,
+ "logits/chosen": -1.9700945184752572,
+ "logits/rejected": -1.6397432545695403,
+ "mean_token_accuracy": 0.6522356096096337,
+ "rewards/chosen": -3.294995879915514,
+ "rewards/rejected": -7.7126641423255204,
+ "rewards/accuracies": 0.9575,
+ "rewards/margins": 4.4176682551950215,
+ "logps/chosen": -90.83600348949432,
+ "logps/rejected": -126.77610003948212,
+ "epoch": 0.3630517089362585,
+ "step": 350
+ },
+ {
+ "loss": 0.1488456130027771,
+ "grad_norm": 0.020714374259114265,
+ "learning_rate": 6.2797927461139905e-06,
+ "entropy": 0.7544214495958295,
+ "num_tokens": 24156469.0,
+ "logits/chosen": -1.92354159972201,
+ "logits/rejected": -1.5932112088234438,
+ "mean_token_accuracy": 0.6473389553651213,
+ "rewards/chosen": -3.2932736888423095,
+ "rewards/rejected": -7.400682945102453,
+ "rewards/accuracies": 0.945,
+ "rewards/margins": 4.107409240156412,
+ "logps/chosen": -88.0268507552147,
+ "logps/rejected": -122.07945943832398,
+ "epoch": 0.37342461490586587,
+ "step": 360
+ },
+ {
+ "loss": 0.12627313137054444,
+ "grad_norm": 0.01646515727043152,
+ "learning_rate": 6.176165803108809e-06,
+ "entropy": 0.8060684028314427,
+ "num_tokens": 24831471.0,
+ "logits/chosen": -1.7544659981832322,
+ "logits/rejected": -1.463259412146846,
+ "mean_token_accuracy": 0.6474123366177083,
+ "rewards/chosen": -3.231792522999458,
+ "rewards/rejected": -7.640311172902584,
+ "rewards/accuracies": 0.95875,
+ "rewards/margins": 4.408518632799387,
+ "logps/chosen": -94.10357024669648,
+ "logps/rejected": -128.06225858926774,
+ "epoch": 0.38379752087547325,
+ "step": 370
+ },
+ {
+ "eval_loss": 0.7162045836448669,
+ "eval_runtime": 245.5758,
+ "eval_samples_per_second": 4.072,
+ "eval_steps_per_second": 4.072,
+ "eval_entropy": 0.7526722595356404,
+ "eval_num_tokens": 25165755.0,
+ "eval_logits/chosen": -1.90159736974585,
+ "eval_logits/rejected": -1.5791501459591346,
+ "eval_mean_token_accuracy": 0.6217855967730284,
+ "eval_rewards/chosen": -4.221462745887227,
+ "eval_rewards/rejected": -7.149939286857843,
+ "eval_rewards/accuracies": 0.783,
+ "eval_rewards/margins": 2.9284765325784683,
+ "eval_logps/chosen": -92.38761359214783,
+ "eval_logps/rejected": -124.13926333999633,
+ "epoch": 0.38898397386027694,
+ "step": 375
+ },
+ {
+ "loss": 0.12565295696258544,
+ "grad_norm": 0.023654216900467873,
+ "learning_rate": 6.0725388601036275e-06,
+ "entropy": 0.7168624554201961,
+ "num_tokens": 25500679.0,
+ "logits/chosen": -2.056052749523911,
+ "logits/rejected": -1.643372895075378,
+ "mean_token_accuracy": 0.6568564863596111,
+ "rewards/chosen": -3.4278560562586065,
+ "rewards/rejected": -7.977554326504469,
+ "rewards/accuracies": 0.95,
+ "rewards/margins": 4.549698264449835,
+ "logps/chosen": -86.61842272043228,
+ "logps/rejected": -128.30167908906935,
+ "epoch": 0.39417042684508063,
+ "step": 380
+ },
+ {
+ "loss": 0.11409251689910889,
+ "grad_norm": 0.024783629924058914,
+ "learning_rate": 5.968911917098445e-06,
+ "entropy": 0.7501401096500921,
+ "num_tokens": 26171757.0,
+ "logits/chosen": -1.9333745257284571,
+ "logits/rejected": -1.567348165840012,
+ "mean_token_accuracy": 0.6470369586162269,
+ "rewards/chosen": -3.536213577263552,
+ "rewards/rejected": -8.32627578780055,
+ "rewards/accuracies": 0.96,
+ "rewards/margins": 4.790062201619148,
+ "logps/chosen": -91.70740163564682,
+ "logps/rejected": -133.4269912481308,
+ "epoch": 0.404543332814688,
+ "step": 390
+ },
+ {
+ "loss": 0.10429913997650146,
+ "grad_norm": 0.01578158512711525,
+ "learning_rate": 5.865284974093265e-06,
+ "entropy": 0.7956635004619602,
+ "num_tokens": 26844715.0,
+ "logits/chosen": -1.8049366638144833,
+ "logits/rejected": -1.5104901431761746,
+ "mean_token_accuracy": 0.6389342303480953,
+ "rewards/chosen": -3.4948488865431866,
+ "rewards/rejected": -8.024183837622404,
+ "rewards/accuracies": 0.96375,
+ "rewards/margins": 4.529334946647286,
+ "logps/chosen": -94.648166680336,
+ "logps/rejected": -130.52285103082656,
+ "epoch": 0.4149162387842954,
+ "step": 400
+ },
+ {
+ "loss": 0.13800268173217772,
+ "grad_norm": 0.019957033917307854,
+ "learning_rate": 5.761658031088083e-06,
+ "entropy": 0.7793120911484585,
+ "num_tokens": 27516725.0,
+ "logits/chosen": -1.8549602335857083,
+ "logits/rejected": -1.5267175160987139,
+ "mean_token_accuracy": 0.6423193990625441,
+ "rewards/chosen": -3.492969937432499,
+ "rewards/rejected": -8.126144999023527,
+ "rewards/accuracies": 0.96125,
+ "rewards/margins": 4.633175050020218,
+ "logps/chosen": -93.58816482782363,
+ "logps/rejected": -131.41816306352615,
+ "epoch": 0.4252891447539028,
+ "step": 410
+ },
+ {
+ "loss": 0.11805713176727295,
+ "grad_norm": 0.0187994334846735,
+ "learning_rate": 5.658031088082902e-06,
+ "entropy": 0.7545757652306929,
+ "num_tokens": 28188135.0,
+ "logits/chosen": -1.873550110920591,
+ "logits/rejected": -1.5043320525754367,
+ "mean_token_accuracy": 0.6394615587592125,
+ "rewards/chosen": -3.3145995734280405,
+ "rewards/rejected": -7.947769758999348,
+ "rewards/accuracies": 0.9575,
+ "rewards/margins": 4.633170171380043,
+ "logps/chosen": -87.63653787136077,
+ "logps/rejected": -129.56220105171204,
+ "epoch": 0.4356620507235102,
+ "step": 420
+ },
+ {
+ "loss": 0.12740592956542968,
+ "grad_norm": 0.01738053746521473,
+ "learning_rate": 5.554404145077721e-06,
+ "entropy": 0.75883277257788,
+ "num_tokens": 28858804.0,
+ "logits/chosen": -1.9298376085920284,
+ "logits/rejected": -1.560056586818597,
+ "mean_token_accuracy": 0.6475845011882484,
+ "rewards/chosen": -3.515412328981329,
+ "rewards/rejected": -8.071005233377218,
+ "rewards/accuracies": 0.96,
+ "rewards/margins": 4.555592897236347,
+ "logps/chosen": -92.735348944664,
+ "logps/rejected": -129.75473901987075,
+ "epoch": 0.4460349566931176,
+ "step": 430
+ },
+ {
+ "loss": 0.14025732278823852,
+ "grad_norm": 0.03447360917925835,
+ "learning_rate": 5.4507772020725395e-06,
+ "entropy": 0.7607411430426874,
+ "num_tokens": 29529999.0,
+ "logits/chosen": -1.8672980078366863,
+ "logits/rejected": -1.5365591751796719,
+ "mean_token_accuracy": 0.6453877515532076,
+ "rewards/chosen": -3.5431255162111484,
+ "rewards/rejected": -8.013637196272612,
+ "rewards/accuracies": 0.94625,
+ "rewards/margins": 4.4705116748809814,
+ "logps/chosen": -93.00428878068924,
+ "logps/rejected": -129.0281007218361,
+ "epoch": 0.456407862662725,
+ "step": 440
+ },
+ {
+ "loss": 0.09569562077522278,
+ "grad_norm": 0.019115449860692024,
+ "learning_rate": 5.347150259067357e-06,
+ "entropy": 0.787961185129825,
+ "num_tokens": 30203505.0,
+ "logits/chosen": -1.9089318864847349,
+ "logits/rejected": -1.4739854222647117,
+ "mean_token_accuracy": 0.654497133269906,
+ "rewards/chosen": -3.3072420401062117,
+ "rewards/rejected": -8.128699697032571,
+ "rewards/accuracies": 0.97375,
+ "rewards/margins": 4.821457646489144,
+ "logps/chosen": -90.26756436109542,
+ "logps/rejected": -133.00522322177886,
+ "epoch": 0.46678076863233237,
+ "step": 450
+ },
+ {
+ "loss": 0.12792205810546875,
+ "grad_norm": 0.0281841903924942,
+ "learning_rate": 5.243523316062177e-06,
+ "entropy": 0.766239798675524,
+ "num_tokens": 30875568.0,
+ "logits/chosen": -1.9169137057827605,
+ "logits/rejected": -1.5443000783572756,
+ "mean_token_accuracy": 0.6505569527018815,
+ "rewards/chosen": -3.42417212592205,
+ "rewards/rejected": -8.022385044395923,
+ "rewards/accuracies": 0.95375,
+ "rewards/margins": 4.598212912380696,
+ "logps/chosen": -88.34398028373718,
+ "logps/rejected": -130.9107288122177,
+ "epoch": 0.47715367460193975,
+ "step": 460
+ },
+ {
+ "loss": 0.10438011884689331,
+ "grad_norm": 0.012769816443324089,
+ "learning_rate": 5.139896373056995e-06,
+ "entropy": 0.754261478689732,
+ "num_tokens": 31547801.0,
+ "logits/chosen": -1.9733235777624498,
+ "logits/rejected": -1.5496449682143498,
+ "mean_token_accuracy": 0.6587452093698084,
+ "rewards/chosen": -3.4046586474310607,
+ "rewards/rejected": -8.238689427375794,
+ "rewards/accuracies": 0.96,
+ "rewards/margins": 4.8340307791531085,
+ "logps/chosen": -90.22236561775208,
+ "logps/rejected": -133.67569626569747,
+ "epoch": 0.48752658057154713,
+ "step": 470
+ },
+ {
+ "loss": 0.11527338027954101,
+ "grad_norm": 0.04143095761537552,
+ "learning_rate": 5.036269430051814e-06,
+ "entropy": 0.8031201446475461,
+ "num_tokens": 32220905.0,
+ "logits/chosen": -1.8379997780593569,
+ "logits/rejected": -1.5025386930087672,
+ "mean_token_accuracy": 0.6306095580849796,
+ "rewards/chosen": -3.4276347397238713,
+ "rewards/rejected": -8.276202333420516,
+ "rewards/accuracies": 0.96,
+ "rewards/margins": 4.848567594140768,
+ "logps/chosen": -95.14371522426605,
+ "logps/rejected": -134.20426798820495,
+ "epoch": 0.4978994865411545,
+ "step": 480
+ },
+ {
+ "loss": 0.13075348138809204,
+ "grad_norm": 0.011864494532346725,
+ "learning_rate": 4.932642487046633e-06,
+ "entropy": 0.7831255796179175,
+ "num_tokens": 32894872.0,
+ "logits/chosen": -1.8320905121355995,
+ "logits/rejected": -1.4693764522144144,
+ "mean_token_accuracy": 0.6478395171277225,
+ "rewards/chosen": -3.2756898268085206,
+ "rewards/rejected": -7.780137736238539,
+ "rewards/accuracies": 0.955,
+ "rewards/margins": 4.504447904080153,
+ "logps/chosen": -91.73629876852036,
+ "logps/rejected": -129.0320829319954,
+ "epoch": 0.5082723925107618,
+ "step": 490
+ },
+ {
+ "loss": 0.12714189291000366,
+ "grad_norm": 0.02473694644868374,
+ "learning_rate": 4.829015544041451e-06,
+ "entropy": 0.7955848192726261,
+ "num_tokens": 33565691.0,
+ "logits/chosen": -1.8834938774520926,
+ "logits/rejected": -1.4981447820382991,
+ "mean_token_accuracy": 0.6382771720923484,
+ "rewards/chosen": -3.1717117047600913,
+ "rewards/rejected": -7.624671882167458,
+ "rewards/accuracies": 0.9575,
+ "rewards/margins": 4.452960164994002,
+ "logps/chosen": -87.47686364412307,
+ "logps/rejected": -125.46468779563904,
+ "epoch": 0.5186452984803692,
+ "step": 500
+ },
+ {
+ "eval_loss": 0.6964932084083557,
+ "eval_runtime": 279.8952,
+ "eval_samples_per_second": 3.573,
+ "eval_steps_per_second": 3.573,
+ "eval_entropy": 0.7739749677795916,
+ "eval_num_tokens": 33565691.0,
+ "eval_logits/chosen": -1.8410994007704304,
+ "eval_logits/rejected": -1.4860494630066636,
+ "eval_mean_token_accuracy": 0.6207611305266618,
+ "eval_rewards/chosen": -3.7967804961130023,
+ "eval_rewards/rejected": -6.769779369160533,
+ "eval_rewards/accuracies": 0.793,
+ "eval_rewards/margins": 2.9729988664388656,
+ "eval_logps/chosen": -88.14079112052917,
+ "eval_logps/rejected": -120.33766422653198,
+ "epoch": 0.5186452984803692,
+ "step": 500
+ },
+ {
+ "train_runtime": 16864.5913,
+ "train_samples_per_second": 4.573,
+ "train_steps_per_second": 0.057,
+ "total_flos": 2.7163725934432666e+17,
+ "train_loss": 0.23282903826236725,
+ "epoch": 0.5186452984803692,
+ "step": 500
+ }
+ ],
+ "validation_monitor_size": 1000,
+ "validation_monitor_selection": "fixed_seed_random_without_replacement",
+ "validation_monitor_seed": 22,
+ "validation_monitor_indices": [
+ 2,
+ 10,
+ 14,
+ 21,
+ 55,
+ 63,
+ 66,
+ 69,
+ 107,
+ 110,
+ 142,
+ 144,
+ 149,
+ 150,
+ 151,
+ 152,
+ 160,
+ 163,
+ 166,
+ 167,
+ 176,
+ 181,
+ 206,
+ 207,
+ 259,
+ 267,
+ 281,
+ 295,
+ 298,
+ 306,
+ 307,
+ 317,
+ 321,
+ 331,
+ 336,
+ 341,
+ 346,
+ 349,
+ 351,
+ 352,
+ 357,
+ 358,
+ 369,
+ 370,
+ 382,
+ 386,
+ 391,
+ 401,
+ 411,
+ 412,
+ 432,
+ 437,
+ 452,
+ 462,
+ 465,
+ 488,
+ 490,
+ 491,
+ 492,
+ 494,
+ 503,
+ 504,
+ 508,
+ 528,
+ 538,
+ 540,
+ 551,
+ 553,
+ 567,
+ 586,
+ 605,
+ 606,
+ 625,
+ 627,
+ 661,
+ 663,
+ 666,
+ 677,
+ 685,
+ 690,
+ 692,
+ 704,
+ 708,
+ 714,
+ 715,
+ 727,
+ 728,
+ 733,
+ 745,
+ 752,
+ 753,
+ 755,
+ 764,
+ 773,
+ 779,
+ 783,
+ 793,
+ 796,
+ 799,
+ 803,
+ 804,
+ 805,
+ 813,
+ 816,
+ 818,
+ 823,
+ 827,
+ 829,
+ 830,
+ 837,
+ 842,
+ 845,
+ 850,
+ 853,
+ 856,
+ 889,
+ 890,
+ 905,
+ 915,
+ 924,
+ 930,
+ 939,
+ 951,
+ 988,
+ 1002,
+ 1005,
+ 1023,
+ 1029,
+ 1038,
+ 1049,
+ 1050,
+ 1054,
+ 1056,
+ 1067,
+ 1068,
+ 1079,
+ 1088,
+ 1091,
+ 1103,
+ 1114,
+ 1118,
+ 1133,
+ 1140,
+ 1144,
+ 1145,
+ 1155,
+ 1186,
+ 1195,
+ 1206,
+ 1223,
+ 1251,
+ 1252,
+ 1257,
+ 1259,
+ 1270,
+ 1271,
+ 1295,
+ 1303,
+ 1304,
+ 1305,
+ 1329,
+ 1335,
+ 1336,
+ 1343,
+ 1367,
+ 1373,
+ 1377,
+ 1403,
+ 1412,
+ 1429,
+ 1443,
+ 1457,
+ 1459,
+ 1460,
+ 1476,
+ 1483,
+ 1486,
+ 1494,
+ 1507,
+ 1510,
+ 1519,
+ 1521,
+ 1522,
+ 1545,
+ 1551,
+ 1570,
+ 1582,
+ 1593,
+ 1595,
+ 1603,
+ 1607,
+ 1614,
+ 1615,
+ 1625,
+ 1634,
+ 1639,
+ 1648,
+ 1654,
+ 1657,
+ 1662,
+ 1667,
+ 1673,
+ 1690,
+ 1704,
+ 1746,
+ 1756,
+ 1781,
+ 1783,
+ 1796,
+ 1799,
+ 1809,
+ 1814,
+ 1827,
+ 1829,
+ 1832,
+ 1844,
+ 1847,
+ 1854,
+ 1856,
+ 1857,
+ 1858,
+ 1874,
+ 1883,
+ 1889,
+ 1924,
+ 1925,
+ 1931,
+ 1932,
+ 1934,
+ 1935,
+ 1938,
+ 1950,
+ 1957,
+ 1962,
+ 1967,
+ 1975,
+ 1982,
+ 1983,
+ 1991,
+ 1998,
+ 2003,
+ 2008,
+ 2017,
+ 2021,
+ 2026,
+ 2035,
+ 2040,
+ 2050,
+ 2056,
+ 2077,
+ 2079,
+ 2082,
+ 2098,
+ 2100,
+ 2108,
+ 2121,
+ 2130,
+ 2133,
+ 2134,
+ 2138,
+ 2143,
+ 2166,
+ 2167,
+ 2180,
+ 2181,
+ 2185,
+ 2204,
+ 2205,
+ 2212,
+ 2221,
+ 2224,
+ 2226,
+ 2230,
+ 2233,
+ 2256,
+ 2261,
+ 2263,
+ 2269,
+ 2273,
+ 2290,
+ 2291,
+ 2299,
+ 2301,
+ 2321,
+ 2323,
+ 2330,
+ 2384,
+ 2401,
+ 2417,
+ 2420,
+ 2421,
+ 2424,
+ 2430,
+ 2435,
+ 2456,
+ 2488,
+ 2502,
+ 2504,
+ 2519,
+ 2527,
+ 2532,
+ 2538,
+ 2542,
+ 2553,
+ 2555,
+ 2558,
+ 2559,
+ 2562,
+ 2578,
+ 2583,
+ 2585,
+ 2590,
+ 2592,
+ 2594,
+ 2596,
+ 2600,
+ 2606,
+ 2607,
+ 2618,
+ 2630,
+ 2637,
+ 2647,
+ 2650,
+ 2657,
+ 2679,
+ 2685,
+ 2705,
+ 2706,
+ 2712,
+ 2713,
+ 2714,
+ 2727,
+ 2740,
+ 2742,
+ 2755,
+ 2780,
+ 2784,
+ 2792,
+ 2807,
+ 2808,
+ 2810,
+ 2820,
+ 2831,
+ 2839,
+ 2860,
+ 2862,
+ 2863,
+ 2866,
+ 2875,
+ 2878,
+ 2898,
+ 2905,
+ 2921,
+ 2922,
+ 2926,
+ 2930,
+ 2934,
+ 2944,
+ 2955,
+ 2957,
+ 2959,
+ 2973,
+ 2978,
+ 2998,
+ 3018,
+ 3022,
+ 3028,
+ 3031,
+ 3061,
+ 3085,
+ 3090,
+ 3104,
+ 3105,
+ 3111,
+ 3115,
+ 3121,
+ 3122,
+ 3129,
+ 3133,
+ 3135,
+ 3161,
+ 3162,
+ 3169,
+ 3173,
+ 3194,
+ 3195,
+ 3215,
+ 3225,
+ 3226,
+ 3241,
+ 3247,
+ 3250,
+ 3253,
+ 3265,
+ 3268,
+ 3293,
+ 3301,
+ 3312,
+ 3329,
+ 3352,
+ 3361,
+ 3362,
+ 3376,
+ 3396,
+ 3401,
+ 3403,
+ 3410,
+ 3419,
+ 3432,
+ 3443,
+ 3452,
+ 3467,
+ 3469,
+ 3475,
+ 3485,
+ 3503,
+ 3514,
+ 3515,
+ 3524,
+ 3528,
+ 3538,
+ 3544,
+ 3557,
+ 3560,
+ 3565,
+ 3600,
+ 3637,
+ 3642,
+ 3658,
+ 3659,
+ 3692,
+ 3693,
+ 3699,
+ 3722,
+ 3725,
+ 3728,
+ 3731,
+ 3740,
+ 3742,
+ 3762,
+ 3766,
+ 3780,
+ 3788,
+ 3794,
+ 3796,
+ 3798,
+ 3805,
+ 3817,
+ 3819,
+ 3822,
+ 3837,
+ 3839,
+ 3843,
+ 3846,
+ 3851,
+ 3854,
+ 3865,
+ 3870,
+ 3871,
+ 3884,
+ 3894,
+ 3895,
+ 3896,
+ 3907,
+ 3911,
+ 3915,
+ 3919,
+ 3920,
+ 3923,
+ 3933,
+ 3955,
+ 3967,
+ 3972,
+ 3974,
+ 3975,
+ 3984,
+ 3985,
+ 3997,
+ 4002,
+ 4010,
+ 4034,
+ 4044,
+ 4063,
+ 4073,
+ 4079,
+ 4082,
+ 4088,
+ 4121,
+ 4145,
+ 4148,
+ 4159,
+ 4161,
+ 4164,
+ 4177,
+ 4188,
+ 4199,
+ 4203,
+ 4207,
+ 4208,
+ 4213,
+ 4221,
+ 4225,
+ 4233,
+ 4241,
+ 4243,
+ 4247,
+ 4264,
+ 4274,
+ 4282,
+ 4286,
+ 4290,
+ 4308,
+ 4310,
+ 4313,
+ 4321,
+ 4324,
+ 4327,
+ 4349,
+ 4362,
+ 4370,
+ 4374,
+ 4380,
+ 4407,
+ 4409,
+ 4411,
+ 4415,
+ 4417,
+ 4418,
+ 4427,
+ 4431,
+ 4433,
+ 4451,
+ 4466,
+ 4477,
+ 4478,
+ 4479,
+ 4493,
+ 4494,
+ 4514,
+ 4527,
+ 4539,
+ 4550,
+ 4558,
+ 4568,
+ 4579,
+ 4583,
+ 4584,
+ 4586,
+ 4587,
+ 4590,
+ 4599,
+ 4602,
+ 4610,
+ 4626,
+ 4627,
+ 4630,
+ 4641,
+ 4647,
+ 4655,
+ 4656,
+ 4657,
+ 4661,
+ 4685,
+ 4714,
+ 4715,
+ 4731,
+ 4749,
+ 4752,
+ 4769,
+ 4777,
+ 4782,
+ 4791,
+ 4805,
+ 4818,
+ 4829,
+ 4833,
+ 4837,
+ 4839,
+ 4843,
+ 4871,
+ 4875,
+ 4879,
+ 4880,
+ 4885,
+ 4888,
+ 4895,
+ 4900,
+ 4923,
+ 4966,
+ 4968,
+ 4972,
+ 4989,
+ 4994,
+ 4998,
+ 5001,
+ 5013,
+ 5019,
+ 5026,
+ 5032,
+ 5034,
+ 5046,
+ 5055,
+ 5085,
+ 5086,
+ 5088,
+ 5089,
+ 5090,
+ 5095,
+ 5108,
+ 5110,
+ 5119,
+ 5120,
+ 5121,
+ 5133,
+ 5148,
+ 5154,
+ 5160,
+ 5169,
+ 5178,
+ 5222,
+ 5223,
+ 5232,
+ 5233,
+ 5240,
+ 5256,
+ 5259,
+ 5264,
+ 5271,
+ 5276,
+ 5279,
+ 5294,
+ 5300,
+ 5303,
+ 5321,
+ 5335,
+ 5337,
+ 5345,
+ 5348,
+ 5365,
+ 5373,
+ 5378,
+ 5384,
+ 5422,
+ 5442,
+ 5443,
+ 5445,
+ 5477,
+ 5485,
+ 5495,
+ 5497,
+ 5504,
+ 5526,
+ 5533,
+ 5542,
+ 5564,
+ 5570,
+ 5579,
+ 5587,
+ 5592,
+ 5608,
+ 5610,
+ 5624,
+ 5630,
+ 5638,
+ 5651,
+ 5663,
+ 5670,
+ 5675,
+ 5691,
+ 5700,
+ 5706,
+ 5707,
+ 5715,
+ 5720,
+ 5721,
+ 5722,
+ 5732,
+ 5738,
+ 5741,
+ 5769,
+ 5771,
+ 5775,
+ 5795,
+ 5796,
+ 5800,
+ 5809,
+ 5810,
+ 5815,
+ 5819,
+ 5827,
+ 5848,
+ 5849,
+ 5852,
+ 5859,
+ 5880,
+ 5884,
+ 5907,
+ 5909,
+ 5912,
+ 5919,
+ 5931,
+ 5932,
+ 5934,
+ 5941,
+ 5948,
+ 5950,
+ 5952,
+ 5953,
+ 5969,
+ 5981,
+ 6000,
+ 6003,
+ 6010,
+ 6018,
+ 6041,
+ 6065,
+ 6075,
+ 6090,
+ 6103,
+ 6106,
+ 6109,
+ 6114,
+ 6123,
+ 6131,
+ 6136,
+ 6138,
+ 6139,
+ 6164,
+ 6165,
+ 6171,
+ 6173,
+ 6180,
+ 6185,
+ 6189,
+ 6190,
+ 6221,
+ 6223,
+ 6237,
+ 6255,
+ 6262,
+ 6265,
+ 6293,
+ 6296,
+ 6305,
+ 6318,
+ 6331,
+ 6336,
+ 6340,
+ 6355,
+ 6366,
+ 6371,
+ 6396,
+ 6399,
+ 6402,
+ 6408,
+ 6432,
+ 6433,
+ 6441,
+ 6456,
+ 6465,
+ 6478,
+ 6486,
+ 6489,
+ 6507,
+ 6508,
+ 6520,
+ 6522,
+ 6528,
+ 6537,
+ 6551,
+ 6564,
+ 6589,
+ 6590,
+ 6598,
+ 6599,
+ 6611,
+ 6613,
+ 6615,
+ 6621,
+ 6634,
+ 6647,
+ 6649,
+ 6654,
+ 6676,
+ 6680,
+ 6690,
+ 6708,
+ 6729,
+ 6736,
+ 6744,
+ 6749,
+ 6756,
+ 6761,
+ 6763,
+ 6773,
+ 6788,
+ 6790,
+ 6796,
+ 6797,
+ 6811,
+ 6824,
+ 6850,
+ 6869,
+ 6871,
+ 6882,
+ 6892,
+ 6895,
+ 6906,
+ 6911,
+ 6912,
+ 6914,
+ 6927,
+ 6952,
+ 6966,
+ 6985,
+ 7001,
+ 7021,
+ 7031,
+ 7035,
+ 7038,
+ 7041,
+ 7045,
+ 7052,
+ 7057,
+ 7058,
+ 7072,
+ 7073,
+ 7076,
+ 7086,
+ 7102,
+ 7107,
+ 7109,
+ 7117,
+ 7122,
+ 7125,
+ 7166,
+ 7182,
+ 7199,
+ 7207,
+ 7212,
+ 7215,
+ 7232,
+ 7243,
+ 7246,
+ 7250,
+ 7253,
+ 7258,
+ 7263,
+ 7267,
+ 7270,
+ 7274,
+ 7280,
+ 7286,
+ 7293,
+ 7298,
+ 7302,
+ 7306,
+ 7316,
+ 7325,
+ 7326,
+ 7334,
+ 7356,
+ 7357,
+ 7363,
+ 7364,
+ 7367,
+ 7385,
+ 7392,
+ 7399,
+ 7405,
+ 7416,
+ 7420,
+ 7421,
+ 7426,
+ 7452,
+ 7476,
+ 7481,
+ 7519,
+ 7534,
+ 7542,
+ 7546,
+ 7573,
+ 7585,
+ 7601,
+ 7604,
+ 7606,
+ 7609,
+ 7629,
+ 7649,
+ 7653,
+ 7667,
+ 7682,
+ 7699,
+ 7738,
+ 7750,
+ 7786,
+ 7798,
+ 7800,
+ 7801,
+ 7805,
+ 7806,
+ 7811,
+ 7813,
+ 7832,
+ 7837,
+ 7849,
+ 7856,
+ 7876,
+ 7877,
+ 7887,
+ 7905,
+ 7916,
+ 7919,
+ 7920,
+ 7922,
+ 7923,
+ 7926,
+ 7944,
+ 7961,
+ 7966,
+ 7970,
+ 7973,
+ 7974,
+ 7976,
+ 7986,
+ 7999,
+ 8027,
+ 8028,
+ 8034,
+ 8047,
+ 8068,
+ 8074,
+ 8077,
+ 8091,
+ 8120,
+ 8122,
+ 8125,
+ 8152,
+ 8153,
+ 8164,
+ 8167,
+ 8169,
+ 8171,
+ 8177,
+ 8184,
+ 8189,
+ 8192,
+ 8196,
+ 8202,
+ 8212,
+ 8217,
+ 8231,
+ 8242,
+ 8244,
+ 8250,
+ 8256,
+ 8257,
+ 8265,
+ 8270,
+ 8274,
+ 8283,
+ 8290,
+ 8294,
+ 8301,
+ 8302,
+ 8307,
+ 8318,
+ 8326,
+ 8338,
+ 8349,
+ 8350,
+ 8353,
+ 8357,
+ 8371,
+ 8374,
+ 8377,
+ 8380,
+ 8387,
+ 8388,
+ 8396,
+ 8402,
+ 8419,
+ 8423,
+ 8428,
+ 8435,
+ 8439,
+ 8442,
+ 8444,
+ 8453,
+ 8461,
+ 8475,
+ 8481,
+ 8485,
+ 8493,
+ 8495,
+ 8498,
+ 8523,
+ 8530,
+ 8531,
+ 8562
+ ],
+ "early_stopping_patience": 3
+}
diff --git a/phase1/ablations/matched_clean/README.md b/phase1/ablations/matched_clean/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..febbf874e976c96e0a8a8821f50aea3c8d064002
--- /dev/null
+++ b/phase1/ablations/matched_clean/README.md
@@ -0,0 +1,80 @@
+---
+library_name: peft
+model_name: phase1-qwen-matched-clean-ablation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-1.5B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+license: apache-2.0
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+pipeline_tag: text-generation
+---
+
+# Phase 1 Qwen `matched_clean` diagnostic ablation
+
+This is the Phase 1 `matched_clean` diagnostic LoRA-DPO adapter fine-tuned from
+[Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct).
+The frozen training recipe and membership hashes are included in this directory.
+
+## Quick start
+
+```python
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo_id = "geyuxu/york-milestone-project-self-traing-loop-model"
+subfolder = "phase1/ablations/matched_clean"
+base_id = "Qwen/Qwen2.5-1.5B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(repo_id, subfolder=subfolder)
+base_model = AutoModelForCausalLM.from_pretrained(
+ base_id, torch_dtype="auto", device_map="auto"
+)
+model = PeftModel.from_pretrained(base_model, repo_id, subfolder=subfolder)
+```
+
+## Training procedure
+
+
+
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 0.29.0
+- Transformers: 5.3.0
+- Pytorch: 2.10.0
+- Datasets: 4.6.1
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+ title = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+ author = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+ year = 2023,
+ booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+ url = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+ editor = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+ title = {{TRL: Transformers Reinforcement Learning}},
+ author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+ license = {Apache-2.0},
+ url = {https://github.com/huggingface/trl},
+ year = {2020}
+}
+```
diff --git a/phase1/ablations/matched_clean/adapter_config.json b/phase1/ablations/matched_clean/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..fa47a2292cd8a23ccf1b4d101d7f7a03e3f08871
--- /dev/null
+++ b/phase1/ablations/matched_clean/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "k_proj",
+ "o_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
diff --git a/phase1/ablations/matched_clean/adapter_model.safetensors b/phase1/ablations/matched_clean/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..abfc79b7fe73687b578faaf3335859e6b570f361
--- /dev/null
+++ b/phase1/ablations/matched_clean/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:724b1e847a25563f7fe04d27dfa05df9213b97348490c4cf5b8f1c0cbdb90b68
+size 17462432
diff --git a/phase1/ablations/matched_clean/chat_template.jinja b/phase1/ablations/matched_clean/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e
--- /dev/null
+++ b/phase1/ablations/matched_clean/chat_template.jinja
@@ -0,0 +1,54 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- messages[0]['content'] }}
+ {%- else %}
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+ {%- endif %}
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+ {%- else %}
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {{- '<|im_start|>' + message.role }}
+ {%- if message.content %}
+ {{- '\n' + message.content }}
+ {%- endif %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- '}\n' }}
+ {%- endfor %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- message.content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}
diff --git a/phase1/ablations/matched_clean/tokenizer.json b/phase1/ablations/matched_clean/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..d73428d91463b495bc017fb6a2fb3a656646482b
--- /dev/null
+++ b/phase1/ablations/matched_clean/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5
+size 11422166
diff --git a/phase1/ablations/matched_clean/tokenizer_config.json b/phase1/ablations/matched_clean/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9fd0fb48e73786f54fb04e98892f5f5a0ebeebdb
--- /dev/null
+++ b/phase1/ablations/matched_clean/tokenizer_config.json
@@ -0,0 +1,29 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": true,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/phase1/ablations/matched_clean/training_args.bin b/phase1/ablations/matched_clean/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..2f437b1196bf4c269d26fd793e13c8b59ca7bcf4
--- /dev/null
+++ b/phase1/ablations/matched_clean/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfafc3f2a531cdc31948a992577839650613ac2f09616d9263508e3e97da8555
+size 5841
diff --git a/phase1/ablations/matched_clean/training_recipe.json b/phase1/ablations/matched_clean/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..fe45675645e1a6da622734791f465e7e0b917d20
--- /dev/null
+++ b/phase1/ablations/matched_clean/training_recipe.json
@@ -0,0 +1,22 @@
+{
+ "format_version": 1,
+ "experiment": "confusion_ablation",
+ "ablation": "matched_clean",
+ "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+ "train_membership_sha256": "b497fa765223a9810ad784f31e1598edfa992a162146d6f94d9cdf4adcedd87d",
+ "validation_membership_sha256": "2f609467438cd6a01fd84b9d80c0fcd84c44767e3b2ed98f4f6a83d68a62c181",
+ "num_train_pairs": 77124,
+ "num_validation_pairs": 8563,
+ "transformation_seed": 22,
+ "validation_prompt_mode": "clean_rag",
+ "learning_rate": 1e-05,
+ "beta": 0.1,
+ "num_epochs": 1.0,
+ "seed": 22,
+ "max_length": 512,
+ "max_prompt_length": 384,
+ "gradient_accumulation_steps": 80,
+ "save_steps": 125,
+ "save_total_limit": 12,
+ "preference_pairs_format_version": 2
+}
diff --git a/phase1/ablations/matched_clean/training_summary.json b/phase1/ablations/matched_clean/training_summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..c0b25a0c830c40ac50d0837a8da6d9ded9b5b668
--- /dev/null
+++ b/phase1/ablations/matched_clean/training_summary.json
@@ -0,0 +1,2925 @@
+{
+ "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+ "resolved_model": "Qwen/Qwen2.5-1.5B-Instruct",
+ "num_train_records": 77124,
+ "num_validation_records_available": 8563,
+ "training_config": {
+ "lora_rank": 16,
+ "lora_alpha": 32,
+ "learning_rate": 1e-05,
+ "num_epochs": 1.0,
+ "per_device_batch_size": 1,
+ "per_device_eval_batch_size": 1,
+ "gradient_accumulation_steps": 80,
+ "beta": 0.1,
+ "max_length": 512,
+ "max_prompt_length": 384,
+ "seed": 22,
+ "save_steps": 125,
+ "save_total_limit": 12,
+ "resume_from_checkpoint": null,
+ "precision_dtype": "torch.bfloat16",
+ "bf16": true,
+ "fp16": false,
+ "tokenizer_add_bos_token": false
+ },
+ "global_step": 965,
+ "best_metric": 0.4090946316719055,
+ "best_model_checkpoint": "outputs/confusion_ablation_phase1_repair_v3/matched_clean/lora/checkpoint-875",
+ "train_metrics": {
+ "train_runtime": 36967.0361,
+ "train_samples_per_second": 2.086,
+ "train_steps_per_second": 0.026,
+ "total_flos": 5.2405488888805786e+17,
+ "train_loss": 0.453640728787437
+ },
+ "log_history": [
+ {
+ "loss": 0.6900316715240479,
+ "grad_norm": 0.011662614531815052,
+ "learning_rate": 9.906735751295338e-06,
+ "entropy": 0.8081745196832344,
+ "num_tokens": 669528.0,
+ "logits/chosen": -0.9878667544984245,
+ "logits/rejected": -0.8929429136538265,
+ "mean_token_accuracy": 0.7366863564029336,
+ "rewards/chosen": 0.0014809935365531145,
+ "rewards/rejected": -0.0051326466447972055,
+ "rewards/accuracies": 0.495,
+ "rewards/margins": 0.00661364022129419,
+ "logps/chosen": -48.20964523553848,
+ "logps/rejected": -54.25052870512009,
+ "epoch": 0.010372905969607386,
+ "step": 10
+ },
+ {
+ "loss": 0.6709239482879639,
+ "grad_norm": 0.01787947677075863,
+ "learning_rate": 9.803108808290156e-06,
+ "entropy": 0.7905870282277465,
+ "num_tokens": 1338759.0,
+ "logits/chosen": -0.9877784900726513,
+ "logits/rejected": -0.9284377500817239,
+ "mean_token_accuracy": 0.7397855435311794,
+ "rewards/chosen": -0.016295835754335714,
+ "rewards/rejected": -0.06304080443188653,
+ "rewards/accuracies": 0.74,
+ "rewards/margins": 0.046744968572747896,
+ "logps/chosen": -49.98522471666336,
+ "logps/rejected": -53.08380969285965,
+ "epoch": 0.02074581193921477,
+ "step": 20
+ },
+ {
+ "loss": 0.6449037075042725,
+ "grad_norm": 0.012443667277693748,
+ "learning_rate": 9.699481865284975e-06,
+ "entropy": 0.8007427603192627,
+ "num_tokens": 2009918.0,
+ "logits/chosen": -1.0123033629117577,
+ "logits/rejected": -0.9116259647175791,
+ "mean_token_accuracy": 0.7288216127734631,
+ "rewards/chosen": -0.08125005806276021,
+ "rewards/rejected": -0.18847612114805087,
+ "rewards/accuracies": 0.7525,
+ "rewards/margins": 0.10722606290772091,
+ "logps/chosen": -51.017456583976745,
+ "logps/rejected": -55.82100034952164,
+ "epoch": 0.031118717908822157,
+ "step": 30
+ },
+ {
+ "loss": 0.6104300498962403,
+ "grad_norm": 0.011565622873604298,
+ "learning_rate": 9.595854922279793e-06,
+ "entropy": 0.7688610723486636,
+ "num_tokens": 2680367.0,
+ "logits/chosen": -1.0477489025836138,
+ "logits/rejected": -0.9185756571968596,
+ "mean_token_accuracy": 0.7304606804065407,
+ "rewards/chosen": -0.18015499103314597,
+ "rewards/rejected": -0.377999417107676,
+ "rewards/accuracies": 0.76125,
+ "rewards/margins": 0.19784442565287463,
+ "logps/chosen": -52.41810529708862,
+ "logps/rejected": -56.31561070203781,
+ "epoch": 0.04149162387842954,
+ "step": 40
+ },
+ {
+ "loss": 0.5890812397003173,
+ "grad_norm": 0.011881239712238312,
+ "learning_rate": 9.492227979274612e-06,
+ "entropy": 0.767548923434224,
+ "num_tokens": 3350695.0,
+ "logits/chosen": -1.045900867553522,
+ "logits/rejected": -0.8987545206323213,
+ "mean_token_accuracy": 0.7290248272754252,
+ "rewards/chosen": -0.3571337752114414,
+ "rewards/rejected": -0.635303623637883,
+ "rewards/accuracies": 0.76125,
+ "rewards/margins": 0.27816984807141126,
+ "logps/chosen": -53.70132049441337,
+ "logps/rejected": -58.7293958568573,
+ "epoch": 0.051864529848036925,
+ "step": 50
+ },
+ {
+ "loss": 0.5672520160675049,
+ "grad_norm": 0.012122954241931438,
+ "learning_rate": 9.388601036269432e-06,
+ "entropy": 0.7509493381145876,
+ "num_tokens": 4019867.0,
+ "logits/chosen": -1.0931640275508827,
+ "logits/rejected": -0.9542402869856921,
+ "mean_token_accuracy": 0.7216887871176004,
+ "rewards/chosen": -0.5263037721261935,
+ "rewards/rejected": -0.9088172687360202,
+ "rewards/accuracies": 0.74875,
+ "rewards/margins": 0.38251349702011794,
+ "logps/chosen": -54.063412301540374,
+ "logps/rejected": -60.20355456352234,
+ "epoch": 0.062237435817644314,
+ "step": 60
+ },
+ {
+ "loss": 0.5313051700592041,
+ "grad_norm": 0.012870470061898232,
+ "learning_rate": 9.284974093264249e-06,
+ "entropy": 0.7521841926735943,
+ "num_tokens": 4689830.0,
+ "logits/chosen": -1.1636649836163147,
+ "logits/rejected": -0.9546643791863164,
+ "mean_token_accuracy": 0.7130510857887566,
+ "rewards/chosen": -0.7180539178028266,
+ "rewards/rejected": -1.2522271308361086,
+ "rewards/accuracies": 0.765,
+ "rewards/margins": 0.5341732117999345,
+ "logps/chosen": -57.92338786125183,
+ "logps/rejected": -64.00875183820725,
+ "epoch": 0.0726103417872517,
+ "step": 70
+ },
+ {
+ "loss": 0.524173355102539,
+ "grad_norm": 0.012850801460444927,
+ "learning_rate": 9.181347150259067e-06,
+ "entropy": 0.7584943031892181,
+ "num_tokens": 5359790.0,
+ "logits/chosen": -1.257342141058688,
+ "logits/rejected": -1.0016241015611054,
+ "mean_token_accuracy": 0.7126817052811384,
+ "rewards/chosen": -0.9431407463727374,
+ "rewards/rejected": -1.5808365274948302,
+ "rewards/accuracies": 0.76375,
+ "rewards/margins": 0.6376957796374336,
+ "logps/chosen": -58.12816961526871,
+ "logps/rejected": -69.45178006649017,
+ "epoch": 0.08298324775685909,
+ "step": 80
+ },
+ {
+ "loss": 0.5096397399902344,
+ "grad_norm": 0.015030446462333202,
+ "learning_rate": 9.077720207253888e-06,
+ "entropy": 0.7446463107218733,
+ "num_tokens": 6029642.0,
+ "logits/chosen": -1.3131360929541611,
+ "logits/rejected": -1.12108701233509,
+ "mean_token_accuracy": 0.6976540317758918,
+ "rewards/chosen": -1.0887427906056837,
+ "rewards/rejected": -1.8297204279349535,
+ "rewards/accuracies": 0.765,
+ "rewards/margins": 0.7409776375815272,
+ "logps/chosen": -62.075138006210324,
+ "logps/rejected": -71.068751745224,
+ "epoch": 0.09335615372646647,
+ "step": 90
+ },
+ {
+ "loss": 0.5227419376373291,
+ "grad_norm": 0.017649168148636818,
+ "learning_rate": 8.974093264248706e-06,
+ "entropy": 0.774682844899653,
+ "num_tokens": 6703750.0,
+ "logits/chosen": -1.3070892244508543,
+ "logits/rejected": -1.0911262790664193,
+ "mean_token_accuracy": 0.698255299879238,
+ "rewards/chosen": -1.2929798694234342,
+ "rewards/rejected": -2.059200689474819,
+ "rewards/accuracies": 0.755,
+ "rewards/margins": 0.7662208179756999,
+ "logps/chosen": -67.27675194501877,
+ "logps/rejected": -78.23825345277787,
+ "epoch": 0.10372905969607385,
+ "step": 100
+ },
+ {
+ "loss": 0.5076948642730713,
+ "grad_norm": 0.018913043662905693,
+ "learning_rate": 8.870466321243523e-06,
+ "entropy": 0.723871832547884,
+ "num_tokens": 7374932.0,
+ "logits/chosen": -1.3768008463722332,
+ "logits/rejected": -1.1314466745219074,
+ "mean_token_accuracy": 0.7054882103763521,
+ "rewards/chosen": -1.247365918380383,
+ "rewards/rejected": -2.083569277639035,
+ "rewards/accuracies": 0.775,
+ "rewards/margins": 0.8362033596914261,
+ "logps/chosen": -62.56643625497818,
+ "logps/rejected": -73.74503843307495,
+ "epoch": 0.11410196566568125,
+ "step": 110
+ },
+ {
+ "loss": 0.518116807937622,
+ "grad_norm": 0.014157967641949654,
+ "learning_rate": 8.766839378238343e-06,
+ "entropy": 0.7146987286728108,
+ "num_tokens": 8045025.0,
+ "logits/chosen": -1.392513233808292,
+ "logits/rejected": -1.167585582525032,
+ "mean_token_accuracy": 0.7107785574346781,
+ "rewards/chosen": -1.1991379019781017,
+ "rewards/rejected": -1.9976002290286123,
+ "rewards/accuracies": 0.76625,
+ "rewards/margins": 0.7984623239561915,
+ "logps/chosen": -63.023617169857026,
+ "logps/rejected": -70.3282685637474,
+ "epoch": 0.12447487163528863,
+ "step": 120
+ },
+ {
+ "eval_loss": 0.49094030261039734,
+ "eval_runtime": 195.9122,
+ "eval_samples_per_second": 5.104,
+ "eval_steps_per_second": 5.104,
+ "eval_entropy": 0.7422072718068957,
+ "eval_num_tokens": 8380297.0,
+ "eval_logits/chosen": -1.3606685028205767,
+ "eval_logits/rejected": -1.1272819430340941,
+ "eval_mean_token_accuracy": 0.7042824859470129,
+ "eval_rewards/chosen": -1.0899753504878027,
+ "eval_rewards/rejected": -1.9470994755723514,
+ "eval_rewards/accuracies": 0.777,
+ "eval_rewards/margins": 0.8571241236105561,
+ "eval_logps/chosen": -61.07274015617371,
+ "eval_logps/rejected": -72.11086616706848,
+ "epoch": 0.1296613246200923,
+ "step": 125
+ },
+ {
+ "loss": 0.4915438652038574,
+ "grad_norm": 0.014024928212165833,
+ "learning_rate": 8.663212435233162e-06,
+ "entropy": 0.7629431400250177,
+ "num_tokens": 8716283.0,
+ "logits/chosen": -1.3493070406504677,
+ "logits/rejected": -1.1303618494014056,
+ "mean_token_accuracy": 0.7068962034024299,
+ "rewards/chosen": -1.064063529693958,
+ "rewards/rejected": -1.9368204824090935,
+ "rewards/accuracies": 0.7825,
+ "rewards/margins": 0.8727569509390741,
+ "logps/chosen": -61.36201237678528,
+ "logps/rejected": -73.24028332233429,
+ "epoch": 0.13484777760489602,
+ "step": 130
+ },
+ {
+ "loss": 0.5040828704833984,
+ "grad_norm": 0.01371421106159687,
+ "learning_rate": 8.55958549222798e-06,
+ "entropy": 0.7165489907818846,
+ "num_tokens": 9385323.0,
+ "logits/chosen": -1.4613930482520017,
+ "logits/rejected": -1.211856862455939,
+ "mean_token_accuracy": 0.7094380159396678,
+ "rewards/chosen": -1.0889925239968579,
+ "rewards/rejected": -1.8759657187038101,
+ "rewards/accuracies": 0.77875,
+ "rewards/margins": 0.7869731936138123,
+ "logps/chosen": -57.32686332225799,
+ "logps/rejected": -71.04226065397262,
+ "epoch": 0.1452206835745034,
+ "step": 140
+ },
+ {
+ "loss": 0.45884413719177247,
+ "grad_norm": 0.01413357350975275,
+ "learning_rate": 8.455958549222799e-06,
+ "entropy": 0.701725748301833,
+ "num_tokens": 10057097.0,
+ "logits/chosen": -1.5608966569387421,
+ "logits/rejected": -1.247274744986142,
+ "mean_token_accuracy": 0.7153122393228114,
+ "rewards/chosen": -1.071364169051376,
+ "rewards/rejected": -2.1302177897136425,
+ "rewards/accuracies": 0.8075,
+ "rewards/margins": 1.0588536195317284,
+ "logps/chosen": -58.66886381149292,
+ "logps/rejected": -75.77571431159973,
+ "epoch": 0.1555935895441108,
+ "step": 150
+ },
+ {
+ "loss": 0.5141475677490235,
+ "grad_norm": 0.01554640382528305,
+ "learning_rate": 8.352331606217617e-06,
+ "entropy": 0.7456542236352106,
+ "num_tokens": 10728653.0,
+ "logits/chosen": -1.4681753702348763,
+ "logits/rejected": -1.2457289726855978,
+ "mean_token_accuracy": 0.70392991816625,
+ "rewards/chosen": -1.161004657074118,
+ "rewards/rejected": -2.000340641771909,
+ "rewards/accuracies": 0.77375,
+ "rewards/margins": 0.8393359830789268,
+ "logps/chosen": -64.61760116338729,
+ "logps/rejected": -74.73108973264694,
+ "epoch": 0.16596649551371817,
+ "step": 160
+ },
+ {
+ "loss": 0.48798179626464844,
+ "grad_norm": 0.013569124042987823,
+ "learning_rate": 8.248704663212436e-06,
+ "entropy": 0.7485733293113299,
+ "num_tokens": 11400551.0,
+ "logits/chosen": -1.469850987453924,
+ "logits/rejected": -1.2418553626350932,
+ "mean_token_accuracy": 0.7040991749428213,
+ "rewards/chosen": -1.1290287345201069,
+ "rewards/rejected": -2.089886896072421,
+ "rewards/accuracies": 0.775,
+ "rewards/margins": 0.9608581608976238,
+ "logps/chosen": -63.84155872344971,
+ "logps/rejected": -74.28893199920654,
+ "epoch": 0.17633940148332555,
+ "step": 170
+ },
+ {
+ "loss": 0.4588949203491211,
+ "grad_norm": 0.013312213122844696,
+ "learning_rate": 8.145077720207254e-06,
+ "entropy": 0.7363077022106154,
+ "num_tokens": 12070305.0,
+ "logits/chosen": -1.5014831557472343,
+ "logits/rejected": -1.253561978212398,
+ "mean_token_accuracy": 0.709209191892296,
+ "rewards/chosen": -1.082961096275685,
+ "rewards/rejected": -2.0920695228746626,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.009108423003927,
+ "logps/chosen": -60.205562086105346,
+ "logps/rejected": -73.08745456457137,
+ "epoch": 0.18671230745293294,
+ "step": 180
+ },
+ {
+ "loss": 0.44720139503479006,
+ "grad_norm": 0.019025955349206924,
+ "learning_rate": 8.041450777202073e-06,
+ "entropy": 0.7329158629218,
+ "num_tokens": 12738966.0,
+ "logits/chosen": -1.6102334910947507,
+ "logits/rejected": -1.2938961518114829,
+ "mean_token_accuracy": 0.7022849645651877,
+ "rewards/chosen": -1.096955630929151,
+ "rewards/rejected": -2.191100225120317,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.0941445926297455,
+ "logps/chosen": -62.84091185808182,
+ "logps/rejected": -73.56753115415573,
+ "epoch": 0.19708521342254032,
+ "step": 190
+ },
+ {
+ "loss": 0.5053049564361572,
+ "grad_norm": 0.022256096825003624,
+ "learning_rate": 7.937823834196891e-06,
+ "entropy": 0.7230936869091238,
+ "num_tokens": 13408002.0,
+ "logits/chosen": -1.6278242340657059,
+ "logits/rejected": -1.3487957538640436,
+ "mean_token_accuracy": 0.7035108367726207,
+ "rewards/chosen": -1.2608012669734308,
+ "rewards/rejected": -2.296834750812268,
+ "rewards/accuracies": 0.7525,
+ "rewards/margins": 1.036033486686647,
+ "logps/chosen": -61.396967930793764,
+ "logps/rejected": -75.2872954583168,
+ "epoch": 0.2074581193921477,
+ "step": 200
+ },
+ {
+ "loss": 0.4660433292388916,
+ "grad_norm": 0.014599275775253773,
+ "learning_rate": 7.834196891191712e-06,
+ "entropy": 0.6921609364774486,
+ "num_tokens": 14074115.0,
+ "logits/chosen": -1.7416979060664273,
+ "logits/rejected": -1.418815718305467,
+ "mean_token_accuracy": 0.7174021677859127,
+ "rewards/chosen": -1.077466004992384,
+ "rewards/rejected": -2.168546776890289,
+ "rewards/accuracies": 0.78125,
+ "rewards/margins": 1.0910807690175717,
+ "logps/chosen": -57.26282028913498,
+ "logps/rejected": -71.84579543352127,
+ "epoch": 0.2178310253617551,
+ "step": 210
+ },
+ {
+ "loss": 0.44950242042541505,
+ "grad_norm": 0.015949787572026253,
+ "learning_rate": 7.730569948186528e-06,
+ "entropy": 0.718045612655842,
+ "num_tokens": 14745903.0,
+ "logits/chosen": -1.6029292718575974,
+ "logits/rejected": -1.2788201063427131,
+ "mean_token_accuracy": 0.7201293612085283,
+ "rewards/chosen": -1.034126255120791,
+ "rewards/rejected": -2.162680851666955,
+ "rewards/accuracies": 0.8025,
+ "rewards/margins": 1.1285545978834852,
+ "logps/chosen": -58.2793731212616,
+ "logps/rejected": -75.22644976615906,
+ "epoch": 0.2282039313313625,
+ "step": 220
+ },
+ {
+ "loss": 0.43929591178894045,
+ "grad_norm": 0.013934735208749771,
+ "learning_rate": 7.626943005181348e-06,
+ "entropy": 0.7872297932501533,
+ "num_tokens": 15418570.0,
+ "logits/chosen": -1.4825574599936209,
+ "logits/rejected": -1.2401306349486072,
+ "mean_token_accuracy": 0.7127851650305093,
+ "rewards/chosen": -0.9212520667978242,
+ "rewards/rejected": -2.088999391239631,
+ "rewards/accuracies": 0.7975,
+ "rewards/margins": 1.1677473206445574,
+ "logps/chosen": -64.23240270137786,
+ "logps/rejected": -75.61436933040619,
+ "epoch": 0.23857683730096987,
+ "step": 230
+ },
+ {
+ "loss": 0.46149606704711915,
+ "grad_norm": 0.016272729262709618,
+ "learning_rate": 7.523316062176167e-06,
+ "entropy": 0.7730766562063945,
+ "num_tokens": 16089276.0,
+ "logits/chosen": -1.4998074947450468,
+ "logits/rejected": -1.2344424531786968,
+ "mean_token_accuracy": 0.7029178227111698,
+ "rewards/chosen": -0.9342721946419624,
+ "rewards/rejected": -2.0036902719875798,
+ "rewards/accuracies": 0.805,
+ "rewards/margins": 1.069418080030009,
+ "logps/chosen": -61.67156839847565,
+ "logps/rejected": -73.40640575170517,
+ "epoch": 0.24894974327057726,
+ "step": 240
+ },
+ {
+ "loss": 0.46842288970947266,
+ "grad_norm": 0.01317604910582304,
+ "learning_rate": 7.419689119170985e-06,
+ "entropy": 0.7556795614853036,
+ "num_tokens": 16760081.0,
+ "logits/chosen": -1.5251408018321724,
+ "logits/rejected": -1.2550001884745545,
+ "mean_token_accuracy": 0.7026587814092636,
+ "rewards/chosen": -0.9681878225761466,
+ "rewards/rejected": -1.9993022468869457,
+ "rewards/accuracies": 0.79625,
+ "rewards/margins": 1.031114422827959,
+ "logps/chosen": -62.68858078241348,
+ "logps/rejected": -72.8400109577179,
+ "epoch": 0.2593226492401846,
+ "step": 250
+ },
+ {
+ "eval_loss": 0.45063868165016174,
+ "eval_runtime": 228.335,
+ "eval_samples_per_second": 4.38,
+ "eval_steps_per_second": 4.38,
+ "eval_entropy": 0.7599611716154031,
+ "eval_num_tokens": 16760081.0,
+ "eval_logits/chosen": -1.5228878121420686,
+ "eval_logits/rejected": -1.2457822987193723,
+ "eval_mean_token_accuracy": 0.7089936109930277,
+ "eval_rewards/chosen": -0.9581855501902755,
+ "eval_rewards/rejected": -2.058153331893496,
+ "eval_rewards/accuracies": 0.804,
+ "eval_rewards/margins": 1.0999677814096211,
+ "eval_logps/chosen": -59.75484214401245,
+ "eval_logps/rejected": -73.22140468788147,
+ "epoch": 0.2593226492401846,
+ "step": 250
+ },
+ {
+ "loss": 0.4814570426940918,
+ "grad_norm": 0.015276545658707619,
+ "learning_rate": 7.3160621761658035e-06,
+ "entropy": 0.7907758677133825,
+ "num_tokens": 17432770.0,
+ "logits/chosen": -1.5061012207082294,
+ "logits/rejected": -1.2302418451410948,
+ "mean_token_accuracy": 0.7116011143475771,
+ "rewards/chosen": -0.9151591785412165,
+ "rewards/rejected": -2.000397002430982,
+ "rewards/accuracies": 0.78375,
+ "rewards/margins": 1.085237822062336,
+ "logps/chosen": -62.86761567592621,
+ "logps/rejected": -74.82630295515061,
+ "epoch": 0.26969555520979205,
+ "step": 260
+ },
+ {
+ "loss": 0.4540716648101807,
+ "grad_norm": 0.013052938506007195,
+ "learning_rate": 7.212435233160623e-06,
+ "entropy": 0.7655438208338455,
+ "num_tokens": 18100438.0,
+ "logits/chosen": -1.525467093384173,
+ "logits/rejected": -1.2448767667260814,
+ "mean_token_accuracy": 0.7059390800073743,
+ "rewards/chosen": -0.8308528269545059,
+ "rewards/rejected": -1.8842440211544453,
+ "rewards/accuracies": 0.79375,
+ "rewards/margins": 1.0533911933761555,
+ "logps/chosen": -56.91530933380127,
+ "logps/rejected": -70.32416380643845,
+ "epoch": 0.28006846117939943,
+ "step": 270
+ },
+ {
+ "loss": 0.44190540313720705,
+ "grad_norm": 0.014065246097743511,
+ "learning_rate": 7.108808290155441e-06,
+ "entropy": 0.7839024822821375,
+ "num_tokens": 18772061.0,
+ "logits/chosen": -1.5250317827718283,
+ "logits/rejected": -1.227351313367144,
+ "mean_token_accuracy": 0.7068624695949256,
+ "rewards/chosen": -0.8610654402987712,
+ "rewards/rejected": -2.0157007031119427,
+ "rewards/accuracies": 0.81875,
+ "rewards/margins": 1.1546352597884835,
+ "logps/chosen": -61.52414802789688,
+ "logps/rejected": -72.9297788310051,
+ "epoch": 0.2904413671490068,
+ "step": 280
+ },
+ {
+ "loss": 0.4456183910369873,
+ "grad_norm": 0.015716280788183212,
+ "learning_rate": 7.005181347150259e-06,
+ "entropy": 0.773276298265555,
+ "num_tokens": 19443506.0,
+ "logits/chosen": -1.5330978743938402,
+ "logits/rejected": -1.2171561791639127,
+ "mean_token_accuracy": 0.7161024253070355,
+ "rewards/chosen": -0.9044717278318422,
+ "rewards/rejected": -2.01870841772703,
+ "rewards/accuracies": 0.8025,
+ "rewards/margins": 1.1142366893729194,
+ "logps/chosen": -60.47999572515488,
+ "logps/rejected": -74.31323668241501,
+ "epoch": 0.3008142731186142,
+ "step": 290
+ },
+ {
+ "loss": 0.44274072647094725,
+ "grad_norm": 0.014205160550773144,
+ "learning_rate": 6.9015544041450784e-06,
+ "entropy": 0.7546117506347946,
+ "num_tokens": 20114567.0,
+ "logits/chosen": -1.6212462502874232,
+ "logits/rejected": -1.3379802284295708,
+ "mean_token_accuracy": 0.7209940696135163,
+ "rewards/chosen": -0.9589981533990612,
+ "rewards/rejected": -2.169916975693777,
+ "rewards/accuracies": 0.795,
+ "rewards/margins": 1.2109188246540725,
+ "logps/chosen": -60.32763281106949,
+ "logps/rejected": -75.60327394008637,
+ "epoch": 0.3111871790882216,
+ "step": 300
+ },
+ {
+ "loss": 0.4552000522613525,
+ "grad_norm": 0.014621925540268421,
+ "learning_rate": 6.797927461139897e-06,
+ "entropy": 0.7755605199559068,
+ "num_tokens": 20787766.0,
+ "logits/chosen": -1.612899054190608,
+ "logits/rejected": -1.3232876722515345,
+ "mean_token_accuracy": 0.7147772766277194,
+ "rewards/chosen": -0.9240679290609841,
+ "rewards/rejected": -2.1595666578254895,
+ "rewards/accuracies": 0.79,
+ "rewards/margins": 1.2354987265914679,
+ "logps/chosen": -61.32829741239548,
+ "logps/rejected": -77.28044722557068,
+ "epoch": 0.32156008505782896,
+ "step": 310
+ },
+ {
+ "loss": 0.4410384178161621,
+ "grad_norm": 0.01790008693933487,
+ "learning_rate": 6.6943005181347155e-06,
+ "entropy": 0.7489672241648077,
+ "num_tokens": 21457559.0,
+ "logits/chosen": -1.6415965131088623,
+ "logits/rejected": -1.347490645506002,
+ "mean_token_accuracy": 0.7051716044172645,
+ "rewards/chosen": -0.9709404561911651,
+ "rewards/rejected": -2.2215542565838406,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 1.250613798997365,
+ "logps/chosen": -61.40900204181671,
+ "logps/rejected": -75.62910031557084,
+ "epoch": 0.33193299102743634,
+ "step": 320
+ },
+ {
+ "loss": 0.4336719512939453,
+ "grad_norm": 0.014728070236742496,
+ "learning_rate": 6.590673575129535e-06,
+ "entropy": 0.7923685038526309,
+ "num_tokens": 22128277.0,
+ "logits/chosen": -1.5480857679147186,
+ "logits/rejected": -1.2608543306081368,
+ "mean_token_accuracy": 0.703275697156787,
+ "rewards/chosen": -0.9459266294135409,
+ "rewards/rejected": -2.2482709953014273,
+ "rewards/accuracies": 0.80875,
+ "rewards/margins": 1.3023443661583587,
+ "logps/chosen": -60.81426377058029,
+ "logps/rejected": -77.32076614141464,
+ "epoch": 0.3423058969970437,
+ "step": 330
+ },
+ {
+ "loss": 0.4488088130950928,
+ "grad_norm": 0.01915821246802807,
+ "learning_rate": 6.487046632124353e-06,
+ "entropy": 0.7655984267906751,
+ "num_tokens": 22798429.0,
+ "logits/chosen": -1.5794928447393075,
+ "logits/rejected": -1.3028102903528602,
+ "mean_token_accuracy": 0.7063317270204424,
+ "rewards/chosen": -0.9481951130661764,
+ "rewards/rejected": -2.169792889961391,
+ "rewards/accuracies": 0.7975,
+ "rewards/margins": 1.2215977760369423,
+ "logps/chosen": -60.23724600553513,
+ "logps/rejected": -74.96227306127548,
+ "epoch": 0.3526788029666511,
+ "step": 340
+ },
+ {
+ "loss": 0.4490936756134033,
+ "grad_norm": 0.017944717779755592,
+ "learning_rate": 6.383419689119171e-06,
+ "entropy": 0.7542878538439982,
+ "num_tokens": 23470239.0,
+ "logits/chosen": -1.5727406857545416,
+ "logits/rejected": -1.276317261399869,
+ "mean_token_accuracy": 0.7159937589988112,
+ "rewards/chosen": -0.9953277947314564,
+ "rewards/rejected": -2.3134128075395712,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.3180850110016764,
+ "logps/chosen": -61.82909578323364,
+ "logps/rejected": -78.39685034513474,
+ "epoch": 0.3630517089362585,
+ "step": 350
+ },
+ {
+ "loss": 0.4634250640869141,
+ "grad_norm": 0.017308367416262627,
+ "learning_rate": 6.2797927461139905e-06,
+ "entropy": 0.7325191626208835,
+ "num_tokens": 24140239.0,
+ "logits/chosen": -1.6208129462287795,
+ "logits/rejected": -1.3357805451866935,
+ "mean_token_accuracy": 0.7167608435824513,
+ "rewards/chosen": -0.9554049956912786,
+ "rewards/rejected": -2.160907984265359,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 1.205502986907959,
+ "logps/chosen": -60.00006797075272,
+ "logps/rejected": -73.66487602233887,
+ "epoch": 0.37342461490586587,
+ "step": 360
+ },
+ {
+ "loss": 0.4462597846984863,
+ "grad_norm": 0.012704321183264256,
+ "learning_rate": 6.176165803108809e-06,
+ "entropy": 0.7730704579531448,
+ "num_tokens": 24812178.0,
+ "logits/chosen": -1.5574609696072523,
+ "logits/rejected": -1.260951689955208,
+ "mean_token_accuracy": 0.7079000303149223,
+ "rewards/chosen": -0.9217178278604843,
+ "rewards/rejected": -2.219055590032367,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.2973377590533346,
+ "logps/chosen": -63.260009841918944,
+ "logps/rejected": -75.37872500896454,
+ "epoch": 0.38379752087547325,
+ "step": 370
+ },
+ {
+ "eval_loss": 0.4316043555736542,
+ "eval_runtime": 247.4497,
+ "eval_samples_per_second": 4.041,
+ "eval_steps_per_second": 4.041,
+ "eval_entropy": 0.7676939936636481,
+ "eval_num_tokens": 25147105.0,
+ "eval_logits/chosen": -1.5026115886795766,
+ "eval_logits/rejected": -1.2167814175886924,
+ "eval_mean_token_accuracy": 0.7118223344534635,
+ "eval_rewards/chosen": -0.7611177709406475,
+ "eval_rewards/rejected": -1.9837484061150754,
+ "eval_rewards/accuracies": 0.812,
+ "eval_rewards/margins": 1.2226306333914398,
+ "eval_logps/chosen": -57.784164419174196,
+ "eval_logps/rejected": -72.47735544586182,
+ "epoch": 0.38898397386027694,
+ "step": 375
+ },
+ {
+ "loss": 0.42072787284851076,
+ "grad_norm": 0.01512940414249897,
+ "learning_rate": 6.0725388601036275e-06,
+ "entropy": 0.7662378180027009,
+ "num_tokens": 25482532.0,
+ "logits/chosen": -1.4455535287130075,
+ "logits/rejected": -1.1827597378991437,
+ "mean_token_accuracy": 0.7134129768237472,
+ "rewards/chosen": -0.7430938966096438,
+ "rewards/rejected": -1.9754590333898159,
+ "rewards/accuracies": 0.81625,
+ "rewards/margins": 1.2323651346145197,
+ "logps/chosen": -59.126625483036044,
+ "logps/rejected": -71.10404143095016,
+ "epoch": 0.39417042684508063,
+ "step": 380
+ },
+ {
+ "loss": 0.4370901107788086,
+ "grad_norm": 0.013698432594537735,
+ "learning_rate": 5.968911917098445e-06,
+ "entropy": 0.7876232456968865,
+ "num_tokens": 26154266.0,
+ "logits/chosen": -1.4235855556360628,
+ "logits/rejected": -1.1775204862100577,
+ "mean_token_accuracy": 0.7055446618422866,
+ "rewards/chosen": -0.7474835715969129,
+ "rewards/rejected": -1.9929245413161698,
+ "rewards/accuracies": 0.81125,
+ "rewards/margins": 1.2454409682005645,
+ "logps/chosen": -60.49559247493744,
+ "logps/rejected": -74.38287235736847,
+ "epoch": 0.404543332814688,
+ "step": 390
+ },
+ {
+ "loss": 0.43750705718994143,
+ "grad_norm": 0.014688065275549889,
+ "learning_rate": 5.865284974093265e-06,
+ "entropy": 0.8005011603605817,
+ "num_tokens": 26825159.0,
+ "logits/chosen": -1.4366348824569963,
+ "logits/rejected": -1.1432329035500064,
+ "mean_token_accuracy": 0.7120773574337363,
+ "rewards/chosen": -0.7654138012278418,
+ "rewards/rejected": -1.9986112341369153,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.2331974321743473,
+ "logps/chosen": -59.77998896837234,
+ "logps/rejected": -74.2601730799675,
+ "epoch": 0.4149162387842954,
+ "step": 400
+ },
+ {
+ "loss": 0.4483372211456299,
+ "grad_norm": 0.01656300760805607,
+ "learning_rate": 5.761658031088083e-06,
+ "entropy": 0.7781640422128839,
+ "num_tokens": 27495663.0,
+ "logits/chosen": -1.469463162115489,
+ "logits/rejected": -1.1635286970905576,
+ "mean_token_accuracy": 0.7104245729371905,
+ "rewards/chosen": -0.7733871162302784,
+ "rewards/rejected": -2.009647502012085,
+ "rewards/accuracies": 0.7775,
+ "rewards/margins": 1.236260382984765,
+ "logps/chosen": -59.314085538387296,
+ "logps/rejected": -73.74020329475402,
+ "epoch": 0.4252891447539028,
+ "step": 410
+ },
+ {
+ "loss": 0.40839343070983886,
+ "grad_norm": 0.014693827368319035,
+ "learning_rate": 5.658031088082902e-06,
+ "entropy": 0.7646654536006827,
+ "num_tokens": 28167389.0,
+ "logits/chosen": -1.5559907884752369,
+ "logits/rejected": -1.2529527889366994,
+ "mean_token_accuracy": 0.7122585397399962,
+ "rewards/chosen": -0.812723506633156,
+ "rewards/rejected": -2.2211731961916668,
+ "rewards/accuracies": 0.8325,
+ "rewards/margins": 1.4084496867936105,
+ "logps/chosen": -61.519875913858414,
+ "logps/rejected": -76.1291908788681,
+ "epoch": 0.4356620507235102,
+ "step": 420
+ },
+ {
+ "loss": 0.4050635814666748,
+ "grad_norm": 0.014569880440831184,
+ "learning_rate": 5.554404145077721e-06,
+ "entropy": 0.7813236166906427,
+ "num_tokens": 28839388.0,
+ "logits/chosen": -1.4355108492016981,
+ "logits/rejected": -1.1415682707849564,
+ "mean_token_accuracy": 0.7073199293017387,
+ "rewards/chosen": -0.8523474504226578,
+ "rewards/rejected": -2.3254094777817955,
+ "rewards/accuracies": 0.82,
+ "rewards/margins": 1.4730620284751057,
+ "logps/chosen": -60.116721353530885,
+ "logps/rejected": -76.6921053647995,
+ "epoch": 0.4460349566931176,
+ "step": 430
+ },
+ {
+ "loss": 0.40834870338439944,
+ "grad_norm": 0.01712818816304207,
+ "learning_rate": 5.4507772020725395e-06,
+ "entropy": 0.7444785924613825,
+ "num_tokens": 29510096.0,
+ "logits/chosen": -1.6494087471154035,
+ "logits/rejected": -1.2686231425397783,
+ "mean_token_accuracy": 0.7219790226966143,
+ "rewards/chosen": -0.9436784289986826,
+ "rewards/rejected": -2.380959646940464,
+ "rewards/accuracies": 0.82875,
+ "rewards/margins": 1.4372812157822772,
+ "logps/chosen": -59.11840226888657,
+ "logps/rejected": -76.61158362627029,
+ "epoch": 0.456407862662725,
+ "step": 440
+ },
+ {
+ "loss": 0.44087815284729004,
+ "grad_norm": 0.022262893617153168,
+ "learning_rate": 5.347150259067357e-06,
+ "entropy": 0.7486801335980999,
+ "num_tokens": 30181792.0,
+ "logits/chosen": -1.6525915875287975,
+ "logits/rejected": -1.292702118780194,
+ "mean_token_accuracy": 0.7041960602998734,
+ "rewards/chosen": -1.0031295236017104,
+ "rewards/rejected": -2.4976175978421815,
+ "rewards/accuracies": 0.805,
+ "rewards/margins": 1.4944880706071855,
+ "logps/chosen": -61.45527629375458,
+ "logps/rejected": -78.80332666873932,
+ "epoch": 0.46678076863233237,
+ "step": 450
+ },
+ {
+ "loss": 0.41168813705444335,
+ "grad_norm": 0.01639724150300026,
+ "learning_rate": 5.243523316062177e-06,
+ "entropy": 0.7348720514599699,
+ "num_tokens": 30851409.0,
+ "logits/chosen": -1.6804476245129396,
+ "logits/rejected": -1.3568483163933152,
+ "mean_token_accuracy": 0.7089731366932392,
+ "rewards/chosen": -0.9868619644123828,
+ "rewards/rejected": -2.5387522551673465,
+ "rewards/accuracies": 0.8125,
+ "rewards/margins": 1.5518902849871665,
+ "logps/chosen": -61.486987377405164,
+ "logps/rejected": -78.15051867246628,
+ "epoch": 0.47715367460193975,
+ "step": 460
+ },
+ {
+ "loss": 0.44434194564819335,
+ "grad_norm": 0.021561825647950172,
+ "learning_rate": 5.139896373056995e-06,
+ "entropy": 0.7410113723303948,
+ "num_tokens": 31519564.0,
+ "logits/chosen": -1.640166155420613,
+ "logits/rejected": -1.3236098037357882,
+ "mean_token_accuracy": 0.7085670954920351,
+ "rewards/chosen": -0.9577470733088558,
+ "rewards/rejected": -2.3723132372720284,
+ "rewards/accuracies": 0.80125,
+ "rewards/margins": 1.4145661635301077,
+ "logps/chosen": -59.36082991600037,
+ "logps/rejected": -74.87610804796219,
+ "epoch": 0.48752658057154713,
+ "step": 470
+ },
+ {
+ "loss": 0.4187155723571777,
+ "grad_norm": 0.015293825417757034,
+ "learning_rate": 5.036269430051814e-06,
+ "entropy": 0.819734820156591,
+ "num_tokens": 32192056.0,
+ "logits/chosen": -1.4117972345061844,
+ "logits/rejected": -1.1251919844105895,
+ "mean_token_accuracy": 0.699143321532756,
+ "rewards/chosen": -0.827813671599506,
+ "rewards/rejected": -2.1753858611593024,
+ "rewards/accuracies": 0.82125,
+ "rewards/margins": 1.347572191953659,
+ "logps/chosen": -62.310746512413026,
+ "logps/rejected": -76.46800751209258,
+ "epoch": 0.4978994865411545,
+ "step": 480
+ },
+ {
+ "loss": 0.41543288230895997,
+ "grad_norm": 0.016549421474337578,
+ "learning_rate": 4.932642487046633e-06,
+ "entropy": 0.793799487358192,
+ "num_tokens": 32864077.0,
+ "logits/chosen": -1.45461478674727,
+ "logits/rejected": -1.1643940857008355,
+ "mean_token_accuracy": 0.7187336018308997,
+ "rewards/chosen": -0.8021746008354603,
+ "rewards/rejected": -2.22868984402754,
+ "rewards/accuracies": 0.82,
+ "rewards/margins": 1.4265152450464667,
+ "logps/chosen": -58.781578176021576,
+ "logps/rejected": -78.34760097026825,
+ "epoch": 0.5082723925107618,
+ "step": 490
+ },
+ {
+ "loss": 0.4357460498809814,
+ "grad_norm": 0.015738829970359802,
+ "learning_rate": 4.829015544041451e-06,
+ "entropy": 0.7886733054189244,
+ "num_tokens": 33534279.0,
+ "logits/chosen": -1.482443230472497,
+ "logits/rejected": -1.1571150498996416,
+ "mean_token_accuracy": 0.7196742885001004,
+ "rewards/chosen": -0.7262393125944072,
+ "rewards/rejected": -2.081412044418103,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.3551727333851158,
+ "logps/chosen": -56.740478208065035,
+ "logps/rejected": -74.49606896877289,
+ "epoch": 0.5186452984803692,
+ "step": 500
+ },
+ {
+ "eval_loss": 0.42026275396347046,
+ "eval_runtime": 281.7739,
+ "eval_samples_per_second": 3.549,
+ "eval_steps_per_second": 3.549,
+ "eval_entropy": 0.7882621039540972,
+ "eval_num_tokens": 33534279.0,
+ "eval_logits/chosen": -1.4541776191798497,
+ "eval_logits/rejected": -1.1470192806607813,
+ "eval_mean_token_accuracy": 0.7118717866092921,
+ "eval_rewards/chosen": -0.7887642608310853,
+ "eval_rewards/rejected": -2.164067000001669,
+ "eval_rewards/accuracies": 0.818,
+ "eval_rewards/margins": 1.3753027384020389,
+ "eval_logps/chosen": -58.06062930107117,
+ "eval_logps/rejected": -74.28054134368897,
+ "epoch": 0.5186452984803692,
+ "step": 500
+ },
+ {
+ "loss": 0.4276765823364258,
+ "grad_norm": 0.014744630083441734,
+ "learning_rate": 4.72538860103627e-06,
+ "entropy": 0.7855714731411717,
+ "num_tokens": 34205135.0,
+ "logits/chosen": -1.4180048717028029,
+ "logits/rejected": -1.1214338358385025,
+ "mean_token_accuracy": 0.7126064065098763,
+ "rewards/chosen": -0.82133626240171,
+ "rewards/rejected": -2.191293025066843,
+ "rewards/accuracies": 0.8175,
+ "rewards/margins": 1.3699567627965008,
+ "logps/chosen": -59.999370622634885,
+ "logps/rejected": -74.11055833339691,
+ "epoch": 0.5290182044499767,
+ "step": 510
+ },
+ {
+ "loss": 0.4480290412902832,
+ "grad_norm": 0.016974102705717087,
+ "learning_rate": 4.621761658031089e-06,
+ "entropy": 0.7657113060192206,
+ "num_tokens": 34873976.0,
+ "logits/chosen": -1.4652295046030377,
+ "logits/rejected": -1.1381464982050828,
+ "mean_token_accuracy": 0.7193653728254139,
+ "rewards/chosen": -0.8232743983055844,
+ "rewards/rejected": -2.115962935181451,
+ "rewards/accuracies": 0.79,
+ "rewards/margins": 1.2926885320199653,
+ "logps/chosen": -57.11219574928284,
+ "logps/rejected": -72.7778330230713,
+ "epoch": 0.5393911104195841,
+ "step": 520
+ },
+ {
+ "loss": 0.4185093879699707,
+ "grad_norm": 0.014205333776772022,
+ "learning_rate": 4.518134715025907e-06,
+ "entropy": 0.8013994769033161,
+ "num_tokens": 35544999.0,
+ "logits/chosen": -1.471904915902926,
+ "logits/rejected": -1.1525475558082925,
+ "mean_token_accuracy": 0.7086275420710444,
+ "rewards/chosen": -0.821920394521876,
+ "rewards/rejected": -2.3078401576905163,
+ "rewards/accuracies": 0.8275,
+ "rewards/margins": 1.4859197605680674,
+ "logps/chosen": -60.18735828638077,
+ "logps/rejected": -76.90893436431885,
+ "epoch": 0.5497640163891915,
+ "step": 530
+ },
+ {
+ "loss": 0.43764190673828124,
+ "grad_norm": 0.01859429106116295,
+ "learning_rate": 4.414507772020726e-06,
+ "entropy": 0.8030325107637327,
+ "num_tokens": 36215629.0,
+ "logits/chosen": -1.3828510403033027,
+ "logits/rejected": -1.1259861866767447,
+ "mean_token_accuracy": 0.7117785899341107,
+ "rewards/chosen": -0.8377752207776211,
+ "rewards/rejected": -2.164465318453731,
+ "rewards/accuracies": 0.8325,
+ "rewards/margins": 1.3266900933114811,
+ "logps/chosen": -59.40348263263702,
+ "logps/rejected": -73.95402544498444,
+ "epoch": 0.5601369223587989,
+ "step": 540
+ },
+ {
+ "loss": 0.43619637489318847,
+ "grad_norm": 0.022518545389175415,
+ "learning_rate": 4.310880829015544e-06,
+ "entropy": 0.8470213223638712,
+ "num_tokens": 36888235.0,
+ "logits/chosen": -1.3614448333016513,
+ "logits/rejected": -1.0777429963848568,
+ "mean_token_accuracy": 0.7116067171096802,
+ "rewards/chosen": -0.8179999309696723,
+ "rewards/rejected": -2.2714405926934886,
+ "rewards/accuracies": 0.8025,
+ "rewards/margins": 1.453440659949556,
+ "logps/chosen": -61.603306529521944,
+ "logps/rejected": -78.86535804271698,
+ "epoch": 0.5705098283284062,
+ "step": 550
+ },
+ {
+ "loss": 0.4372966766357422,
+ "grad_norm": 0.017818979918956757,
+ "learning_rate": 4.207253886010363e-06,
+ "entropy": 0.8097228137392085,
+ "num_tokens": 37560215.0,
+ "logits/chosen": -1.4065342416818991,
+ "logits/rejected": -1.1310520368013937,
+ "mean_token_accuracy": 0.7106514018215239,
+ "rewards/chosen": -0.7871383707236419,
+ "rewards/rejected": -2.1830000240239316,
+ "rewards/accuracies": 0.79375,
+ "rewards/margins": 1.3958616532059387,
+ "logps/chosen": -59.665771260261536,
+ "logps/rejected": -74.6952103304863,
+ "epoch": 0.5808827342980136,
+ "step": 560
+ },
+ {
+ "loss": 0.4129640102386475,
+ "grad_norm": 0.01815822720527649,
+ "learning_rate": 4.103626943005182e-06,
+ "entropy": 0.7890104844880989,
+ "num_tokens": 38229872.0,
+ "logits/chosen": -1.4210442868235984,
+ "logits/rejected": -1.1203699372972569,
+ "mean_token_accuracy": 0.7099655389133841,
+ "rewards/chosen": -0.8200407591336989,
+ "rewards/rejected": -2.2342929665278644,
+ "rewards/accuracies": 0.83,
+ "rewards/margins": 1.4142522051371633,
+ "logps/chosen": -59.6181519818306,
+ "logps/rejected": -74.29007930994034,
+ "epoch": 0.591255640267621,
+ "step": 570
+ },
+ {
+ "loss": 0.4248640537261963,
+ "grad_norm": 0.021521631628274918,
+ "learning_rate": 4.000000000000001e-06,
+ "entropy": 0.7830230033496628,
+ "num_tokens": 38898599.0,
+ "logits/chosen": -1.556829666838658,
+ "logits/rejected": -1.22248957027324,
+ "mean_token_accuracy": 0.7121115596592427,
+ "rewards/chosen": -0.8376483802072471,
+ "rewards/rejected": -2.30365946133883,
+ "rewards/accuracies": 0.82,
+ "rewards/margins": 1.466011079289019,
+ "logps/chosen": -57.19522080659866,
+ "logps/rejected": -74.76314610958099,
+ "epoch": 0.6016285462372284,
+ "step": 580
+ },
+ {
+ "loss": 0.46982684135437014,
+ "grad_norm": 0.0202327873557806,
+ "learning_rate": 3.896373056994819e-06,
+ "entropy": 0.789131090187293,
+ "num_tokens": 39570011.0,
+ "logits/chosen": -1.4979756501997874,
+ "logits/rejected": -1.2087421360795596,
+ "mean_token_accuracy": 0.7092419915273785,
+ "rewards/chosen": -0.9460587426692837,
+ "rewards/rejected": -2.2724328075315863,
+ "rewards/accuracies": 0.77125,
+ "rewards/margins": 1.3263740684324876,
+ "logps/chosen": -61.68762954473495,
+ "logps/rejected": -76.94917388081551,
+ "epoch": 0.6120014522068358,
+ "step": 590
+ },
+ {
+ "loss": 0.4296851634979248,
+ "grad_norm": 0.023099711164832115,
+ "learning_rate": 3.7927461139896377e-06,
+ "entropy": 0.7784710348435329,
+ "num_tokens": 40238459.0,
+ "logits/chosen": -1.5463305650925598,
+ "logits/rejected": -1.1855332843764201,
+ "mean_token_accuracy": 0.7213506529480219,
+ "rewards/chosen": -0.7897813039762969,
+ "rewards/rejected": -2.1785082579992014,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.388726954490412,
+ "logps/chosen": -55.0355699801445,
+ "logps/rejected": -74.42590669631959,
+ "epoch": 0.6223743581764432,
+ "step": 600
+ },
+ {
+ "loss": 0.4378302574157715,
+ "grad_norm": 0.017044473439455032,
+ "learning_rate": 3.6891191709844567e-06,
+ "entropy": 0.8122139454813442,
+ "num_tokens": 40907823.0,
+ "logits/chosen": -1.4782042004502953,
+ "logits/rejected": -1.1733026566264682,
+ "mean_token_accuracy": 0.7134785779751838,
+ "rewards/chosen": -0.7893356951180612,
+ "rewards/rejected": -2.1324991090265395,
+ "rewards/accuracies": 0.81125,
+ "rewards/margins": 1.3431634148815648,
+ "logps/chosen": -56.50307439565658,
+ "logps/rejected": -74.36804901599884,
+ "epoch": 0.6327472641460505,
+ "step": 610
+ },
+ {
+ "loss": 0.42770676612854003,
+ "grad_norm": 0.017237193882465363,
+ "learning_rate": 3.5854922279792748e-06,
+ "entropy": 0.7561554417069419,
+ "num_tokens": 41576629.0,
+ "logits/chosen": -1.5855324711359189,
+ "logits/rejected": -1.2464155038601519,
+ "mean_token_accuracy": 0.7195479864627123,
+ "rewards/chosen": -0.7948263869498987,
+ "rewards/rejected": -2.2129694822058084,
+ "rewards/accuracies": 0.81375,
+ "rewards/margins": 1.4181430943030864,
+ "logps/chosen": -55.47762014627457,
+ "logps/rejected": -74.27213495016098,
+ "epoch": 0.6431201701156579,
+ "step": 620
+ },
+ {
+ "eval_loss": 0.4145970344543457,
+ "eval_runtime": 312.4458,
+ "eval_samples_per_second": 3.201,
+ "eval_steps_per_second": 3.201,
+ "eval_entropy": 0.7926596267716959,
+ "eval_num_tokens": 41911681.0,
+ "eval_logits/chosen": -1.5117800904965717,
+ "eval_logits/rejected": -1.196360660912854,
+ "eval_mean_token_accuracy": 0.7107393295019865,
+ "eval_rewards/chosen": -0.7929535148476134,
+ "eval_rewards/rejected": -2.2358115541365695,
+ "eval_rewards/accuracies": 0.82,
+ "eval_rewards/margins": 1.442858038464561,
+ "eval_logps/chosen": -58.102521854400635,
+ "eval_logps/rejected": -74.9979869003296,
+ "epoch": 0.6483066231004616,
+ "step": 625
+ },
+ {
+ "loss": 0.4174999713897705,
+ "grad_norm": 0.023992126807570457,
+ "learning_rate": 3.4818652849740937e-06,
+ "entropy": 0.7907538907864364,
+ "num_tokens": 42247859.0,
+ "logits/chosen": -1.5275107018411662,
+ "logits/rejected": -1.1729716012203115,
+ "mean_token_accuracy": 0.7168286069855094,
+ "rewards/chosen": -0.785840036013833,
+ "rewards/rejected": -2.3167090034011926,
+ "rewards/accuracies": 0.82,
+ "rewards/margins": 1.5308689690474422,
+ "logps/chosen": -56.834257082939146,
+ "logps/rejected": -78.19684539794922,
+ "epoch": 0.6534930760852653,
+ "step": 630
+ },
+ {
+ "loss": 0.4136983394622803,
+ "grad_norm": 0.017943687736988068,
+ "learning_rate": 3.3782383419689123e-06,
+ "entropy": 0.782110237882007,
+ "num_tokens": 42918160.0,
+ "logits/chosen": -1.5180649119494551,
+ "logits/rejected": -1.1926738964700518,
+ "mean_token_accuracy": 0.7150443252548575,
+ "rewards/chosen": -0.8211558357491231,
+ "rewards/rejected": -2.2642588213164707,
+ "rewards/accuracies": 0.82875,
+ "rewards/margins": 1.4431029830686748,
+ "logps/chosen": -56.69160728216171,
+ "logps/rejected": -77.15469831943511,
+ "epoch": 0.6638659820548727,
+ "step": 640
+ },
+ {
+ "loss": 0.46654863357543946,
+ "grad_norm": 0.016258180141448975,
+ "learning_rate": 3.274611398963731e-06,
+ "entropy": 0.781107221354032,
+ "num_tokens": 43587843.0,
+ "logits/chosen": -1.5242385121688045,
+ "logits/rejected": -1.2121969127570629,
+ "mean_token_accuracy": 0.7117258696630597,
+ "rewards/chosen": -0.8765105128889991,
+ "rewards/rejected": -2.300915027359915,
+ "rewards/accuracies": 0.79625,
+ "rewards/margins": 1.4244045152794569,
+ "logps/chosen": -58.10146237373352,
+ "logps/rejected": -75.90745508432389,
+ "epoch": 0.6742388880244801,
+ "step": 650
+ },
+ {
+ "loss": 0.46282315254211426,
+ "grad_norm": 0.014686384238302708,
+ "learning_rate": 3.1709844559585493e-06,
+ "entropy": 0.7962630777983577,
+ "num_tokens": 44256661.0,
+ "logits/chosen": -1.4734712758844637,
+ "logits/rejected": -1.1916934080775792,
+ "mean_token_accuracy": 0.7055912931449712,
+ "rewards/chosen": -0.7513918488781929,
+ "rewards/rejected": -2.0676744230859914,
+ "rewards/accuracies": 0.8025,
+ "rewards/margins": 1.316282574729994,
+ "logps/chosen": -57.497248324155805,
+ "logps/rejected": -72.39635850191117,
+ "epoch": 0.6846117939940874,
+ "step": 660
+ },
+ {
+ "loss": 0.38728673458099366,
+ "grad_norm": 0.012905064970254898,
+ "learning_rate": 3.0673575129533683e-06,
+ "entropy": 0.7850103608728386,
+ "num_tokens": 44928208.0,
+ "logits/chosen": -1.4906549367707287,
+ "logits/rejected": -1.1785130829765196,
+ "mean_token_accuracy": 0.7167331029660999,
+ "rewards/chosen": -0.729108623168795,
+ "rewards/rejected": -2.2293507476183003,
+ "rewards/accuracies": 0.835,
+ "rewards/margins": 1.500242124721408,
+ "logps/chosen": -57.06263860464096,
+ "logps/rejected": -75.03939726114272,
+ "epoch": 0.6949846999636948,
+ "step": 670
+ },
+ {
+ "loss": 0.42326745986938474,
+ "grad_norm": 0.020321018993854523,
+ "learning_rate": 2.963730569948187e-06,
+ "entropy": 0.8228285726247122,
+ "num_tokens": 45599959.0,
+ "logits/chosen": -1.4107692209920355,
+ "logits/rejected": -1.1415338745002617,
+ "mean_token_accuracy": 0.7098798759281635,
+ "rewards/chosen": -0.7472892588960599,
+ "rewards/rejected": -2.205724095766054,
+ "rewards/accuracies": 0.82,
+ "rewards/margins": 1.4584348343219609,
+ "logps/chosen": -60.79011604785919,
+ "logps/rejected": -76.145639398098,
+ "epoch": 0.7053576059333022,
+ "step": 680
+ },
+ {
+ "loss": 0.46504626274108884,
+ "grad_norm": 0.026528460904955864,
+ "learning_rate": 2.8601036269430053e-06,
+ "entropy": 0.8374029883276671,
+ "num_tokens": 46271100.0,
+ "logits/chosen": -1.438753673313662,
+ "logits/rejected": -1.1758532082539777,
+ "mean_token_accuracy": 0.7012219382543117,
+ "rewards/chosen": -0.8035630289492474,
+ "rewards/rejected": -2.1134093309140733,
+ "rewards/accuracies": 0.78875,
+ "rewards/margins": 1.3098462954116985,
+ "logps/chosen": -60.888806674480435,
+ "logps/rejected": -75.00253082275391,
+ "epoch": 0.7157305119029096,
+ "step": 690
+ },
+ {
+ "loss": 0.3986149072647095,
+ "grad_norm": 0.014411377720534801,
+ "learning_rate": 2.7564766839378243e-06,
+ "entropy": 0.8010915271285921,
+ "num_tokens": 46940588.0,
+ "logits/chosen": -1.4958920674077623,
+ "logits/rejected": -1.1794351626729602,
+ "mean_token_accuracy": 0.7159527142904699,
+ "rewards/chosen": -0.7098511749218233,
+ "rewards/rejected": -2.1309537043143063,
+ "rewards/accuracies": 0.83,
+ "rewards/margins": 1.4211025240272284,
+ "logps/chosen": -58.82859611272812,
+ "logps/rejected": -71.86115275144577,
+ "epoch": 0.726103417872517,
+ "step": 700
+ },
+ {
+ "loss": 0.43906688690185547,
+ "grad_norm": 0.01570284552872181,
+ "learning_rate": 2.6528497409326424e-06,
+ "entropy": 0.7988389390867087,
+ "num_tokens": 47612649.0,
+ "logits/chosen": -1.5084893760106768,
+ "logits/rejected": -1.2217334834194045,
+ "mean_token_accuracy": 0.7133530013635755,
+ "rewards/chosen": -0.7729418969854669,
+ "rewards/rejected": -2.181924787040334,
+ "rewards/accuracies": 0.795,
+ "rewards/margins": 1.4089828893507366,
+ "logps/chosen": -61.10609317064285,
+ "logps/rejected": -74.83846760034561,
+ "epoch": 0.7364763238421244,
+ "step": 710
+ },
+ {
+ "loss": 0.40947885513305665,
+ "grad_norm": 0.023827839642763138,
+ "learning_rate": 2.5492227979274614e-06,
+ "entropy": 0.7832058588406653,
+ "num_tokens": 48280316.0,
+ "logits/chosen": -1.5411506649745486,
+ "logits/rejected": -1.2429037100661278,
+ "mean_token_accuracy": 0.7112616512458771,
+ "rewards/chosen": -0.6660112939180545,
+ "rewards/rejected": -2.1130318040723797,
+ "rewards/accuracies": 0.82375,
+ "rewards/margins": 1.4470205087191426,
+ "logps/chosen": -56.754196836948395,
+ "logps/rejected": -72.01071586370468,
+ "epoch": 0.7468492298117317,
+ "step": 720
+ },
+ {
+ "loss": 0.4252438545227051,
+ "grad_norm": 0.016843726858496666,
+ "learning_rate": 2.44559585492228e-06,
+ "entropy": 0.8381482209177921,
+ "num_tokens": 48952457.0,
+ "logits/chosen": -1.4216991849843164,
+ "logits/rejected": -1.1461203116135463,
+ "mean_token_accuracy": 0.7074375116080046,
+ "rewards/chosen": -0.6695640615803495,
+ "rewards/rejected": -2.0397114378174592,
+ "rewards/accuracies": 0.81125,
+ "rewards/margins": 1.3701473774015904,
+ "logps/chosen": -58.40614979505539,
+ "logps/rejected": -75.83607840061188,
+ "epoch": 0.7572221357813391,
+ "step": 730
+ },
+ {
+ "loss": 0.42221574783325194,
+ "grad_norm": 0.016754452139139175,
+ "learning_rate": 2.3419689119170984e-06,
+ "entropy": 0.8271626771151205,
+ "num_tokens": 49623039.0,
+ "logits/chosen": -1.5209721524713908,
+ "logits/rejected": -1.1751758197782376,
+ "mean_token_accuracy": 0.713498560115695,
+ "rewards/chosen": -0.693279623198905,
+ "rewards/rejected": -2.141712246098032,
+ "rewards/accuracies": 0.8225,
+ "rewards/margins": 1.448432622421533,
+ "logps/chosen": -58.120414593219756,
+ "logps/rejected": -76.37068503379822,
+ "epoch": 0.7675950417509465,
+ "step": 740
+ },
+ {
+ "loss": 0.42609853744506837,
+ "grad_norm": 0.015460499562323093,
+ "learning_rate": 2.2383419689119174e-06,
+ "entropy": 0.8402992183688912,
+ "num_tokens": 50296449.0,
+ "logits/chosen": -1.4753382490797897,
+ "logits/rejected": -1.1819095770610415,
+ "mean_token_accuracy": 0.7086446931585669,
+ "rewards/chosen": -0.756505029114196,
+ "rewards/rejected": -2.2547341962716017,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.4982291649701074,
+ "logps/chosen": -61.22059069037437,
+ "logps/rejected": -78.80186176776886,
+ "epoch": 0.7779679477205539,
+ "step": 750
+ },
+ {
+ "eval_loss": 0.411579966545105,
+ "eval_runtime": 345.2282,
+ "eval_samples_per_second": 2.897,
+ "eval_steps_per_second": 2.897,
+ "eval_entropy": 0.7931306820721366,
+ "eval_num_tokens": 50296449.0,
+ "eval_logits/chosen": -1.5620700320679246,
+ "eval_logits/rejected": -1.2442960317998197,
+ "eval_mean_token_accuracy": 0.7116911947578192,
+ "eval_rewards/chosen": -0.7726080814675516,
+ "eval_rewards/rejected": -2.2532580847712236,
+ "eval_rewards/accuracies": 0.825,
+ "eval_rewards/margins": 1.4806500013172625,
+ "eval_logps/chosen": -57.89906752586365,
+ "eval_logps/rejected": -75.1724522151947,
+ "epoch": 0.7779679477205539,
+ "step": 750
+ },
+ {
+ "loss": 0.3913722515106201,
+ "grad_norm": 0.014632239006459713,
+ "learning_rate": 2.134715025906736e-06,
+ "entropy": 0.8136140358856937,
+ "num_tokens": 50967491.0,
+ "logits/chosen": -1.562353067996462,
+ "logits/rejected": -1.2513613833271493,
+ "mean_token_accuracy": 0.7093207446113229,
+ "rewards/chosen": -0.7712609519985927,
+ "rewards/rejected": -2.253486830406473,
+ "rewards/accuracies": 0.84375,
+ "rewards/margins": 1.4822258768649772,
+ "logps/chosen": -59.60826306819916,
+ "logps/rejected": -76.35254566431045,
+ "epoch": 0.7883408536901613,
+ "step": 760
+ },
+ {
+ "loss": 0.40571393966674807,
+ "grad_norm": 0.025566810742020607,
+ "learning_rate": 2.0310880829015544e-06,
+ "entropy": 0.8285544535657391,
+ "num_tokens": 51641328.0,
+ "logits/chosen": -1.522793583175992,
+ "logits/rejected": -1.1822050205320058,
+ "mean_token_accuracy": 0.7068053354136645,
+ "rewards/chosen": -0.7923616364665214,
+ "rewards/rejected": -2.37010533751054,
+ "rewards/accuracies": 0.81375,
+ "rewards/margins": 1.5777437023073435,
+ "logps/chosen": -61.9151118016243,
+ "logps/rejected": -79.94920246601104,
+ "epoch": 0.7987137596597687,
+ "step": 770
+ },
+ {
+ "loss": 0.4391521453857422,
+ "grad_norm": 0.016789298504590988,
+ "learning_rate": 1.9274611398963734e-06,
+ "entropy": 0.7836550371508929,
+ "num_tokens": 52311665.0,
+ "logits/chosen": -1.6191126859645035,
+ "logits/rejected": -1.29179419660706,
+ "mean_token_accuracy": 0.7130378339067102,
+ "rewards/chosen": -0.8251502061705105,
+ "rewards/rejected": -2.330989454947412,
+ "rewards/accuracies": 0.81375,
+ "rewards/margins": 1.5058392491098493,
+ "logps/chosen": -58.14819228887558,
+ "logps/rejected": -76.84343997001648,
+ "epoch": 0.809086665629376,
+ "step": 780
+ },
+ {
+ "loss": 0.4155034065246582,
+ "grad_norm": 0.015209637582302094,
+ "learning_rate": 1.823834196891192e-06,
+ "entropy": 0.816227734730055,
+ "num_tokens": 52982847.0,
+ "logits/chosen": -1.552296107544077,
+ "logits/rejected": -1.2655800747433517,
+ "mean_token_accuracy": 0.7077531234174966,
+ "rewards/chosen": -0.7692844783003966,
+ "rewards/rejected": -2.2729846961359725,
+ "rewards/accuracies": 0.81625,
+ "rewards/margins": 1.5037002183427104,
+ "logps/chosen": -59.88701428174973,
+ "logps/rejected": -75.7218120956421,
+ "epoch": 0.8194595715989834,
+ "step": 790
+ },
+ {
+ "loss": 0.4396702289581299,
+ "grad_norm": 0.017707331106066704,
+ "learning_rate": 1.7202072538860104e-06,
+ "entropy": 0.8157146892453602,
+ "num_tokens": 53653414.0,
+ "logits/chosen": -1.5836949831528315,
+ "logits/rejected": -1.2884593526275128,
+ "mean_token_accuracy": 0.7132340743020177,
+ "rewards/chosen": -0.7823633211505876,
+ "rewards/rejected": -2.2281043492229764,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.4457410290511326,
+ "logps/chosen": -59.43502898097038,
+ "logps/rejected": -75.63278030633927,
+ "epoch": 0.8298324775685908,
+ "step": 800
+ },
+ {
+ "loss": 0.4166680335998535,
+ "grad_norm": 0.017773514613509178,
+ "learning_rate": 1.6165803108808292e-06,
+ "entropy": 0.797488763818983,
+ "num_tokens": 54324247.0,
+ "logits/chosen": -1.6215031533790736,
+ "logits/rejected": -1.3289979163820846,
+ "mean_token_accuracy": 0.7087218741327524,
+ "rewards/chosen": -0.761779917108106,
+ "rewards/rejected": -2.309470071569667,
+ "rewards/accuracies": 0.81125,
+ "rewards/margins": 1.5476901544816792,
+ "logps/chosen": -58.93250516653061,
+ "logps/rejected": -76.49902043104171,
+ "epoch": 0.8402053835381982,
+ "step": 810
+ },
+ {
+ "loss": 0.4239467144012451,
+ "grad_norm": 0.018757648766040802,
+ "learning_rate": 1.5129533678756477e-06,
+ "entropy": 0.8032379890448647,
+ "num_tokens": 54994016.0,
+ "logits/chosen": -1.578882165777075,
+ "logits/rejected": -1.2860967628921216,
+ "mean_token_accuracy": 0.7140155434235931,
+ "rewards/chosen": -0.7499237745841674,
+ "rewards/rejected": -2.2461700457381086,
+ "rewards/accuracies": 0.82375,
+ "rewards/margins": 1.496246272022836,
+ "logps/chosen": -58.89809783697128,
+ "logps/rejected": -74.08916418790817,
+ "epoch": 0.8505782895078056,
+ "step": 820
+ },
+ {
+ "loss": 0.419736909866333,
+ "grad_norm": 0.018885280936956406,
+ "learning_rate": 1.4093264248704663e-06,
+ "entropy": 0.7875165941729211,
+ "num_tokens": 55663818.0,
+ "logits/chosen": -1.6236698629206843,
+ "logits/rejected": -1.2788538558527598,
+ "mean_token_accuracy": 0.7192669866792858,
+ "rewards/chosen": -0.7543148858308633,
+ "rewards/rejected": -2.269310671926214,
+ "rewards/accuracies": 0.8075,
+ "rewards/margins": 1.5149957838421688,
+ "logps/chosen": -55.762867684364316,
+ "logps/rejected": -76.66288559436798,
+ "epoch": 0.860951195477413,
+ "step": 830
+ },
+ {
+ "loss": 0.4073533058166504,
+ "grad_norm": 0.01915486343204975,
+ "learning_rate": 1.3056994818652852e-06,
+ "entropy": 0.8327871753677027,
+ "num_tokens": 56334476.0,
+ "logits/chosen": -1.4996538615666455,
+ "logits/rejected": -1.237756981594407,
+ "mean_token_accuracy": 0.716468540597707,
+ "rewards/chosen": -0.7192282251862707,
+ "rewards/rejected": -2.26593919366569,
+ "rewards/accuracies": 0.82875,
+ "rewards/margins": 1.5467109655588864,
+ "logps/chosen": -59.84484759330749,
+ "logps/rejected": -76.10131199359894,
+ "epoch": 0.8713241014470204,
+ "step": 840
+ },
+ {
+ "loss": 0.42834720611572263,
+ "grad_norm": 0.01567392610013485,
+ "learning_rate": 1.2020725388601037e-06,
+ "entropy": 0.827671511786757,
+ "num_tokens": 57005010.0,
+ "logits/chosen": -1.5224089619857955,
+ "logits/rejected": -1.203097930514173,
+ "mean_token_accuracy": 0.7094048094563186,
+ "rewards/chosen": -0.8032524040471617,
+ "rewards/rejected": -2.2959909180374236,
+ "rewards/accuracies": 0.8,
+ "rewards/margins": 1.4927385161118583,
+ "logps/chosen": -60.616187605857846,
+ "logps/rejected": -76.86470454216004,
+ "epoch": 0.8816970074166278,
+ "step": 850
+ },
+ {
+ "loss": 0.4034994602203369,
+ "grad_norm": 0.014525278471410275,
+ "learning_rate": 1.0984455958549225e-06,
+ "entropy": 0.818393541239202,
+ "num_tokens": 57677536.0,
+ "logits/chosen": -1.543480139299166,
+ "logits/rejected": -1.2204454822419504,
+ "mean_token_accuracy": 0.7158707704581321,
+ "rewards/chosen": -0.7887829605976913,
+ "rewards/rejected": -2.3561961521262127,
+ "rewards/accuracies": 0.82125,
+ "rewards/margins": 1.5674131939606741,
+ "logps/chosen": -60.70202103614807,
+ "logps/rejected": -77.77336375951766,
+ "epoch": 0.8920699133862352,
+ "step": 860
+ },
+ {
+ "loss": 0.432753324508667,
+ "grad_norm": 0.02228359691798687,
+ "learning_rate": 9.94818652849741e-07,
+ "entropy": 0.8074402804669808,
+ "num_tokens": 58349497.0,
+ "logits/chosen": -1.558037182784836,
+ "logits/rejected": -1.2543498522866354,
+ "mean_token_accuracy": 0.703925465606153,
+ "rewards/chosen": -0.805554041202995,
+ "rewards/rejected": -2.2746161510888485,
+ "rewards/accuracies": 0.835,
+ "rewards/margins": 1.4690621069492773,
+ "logps/chosen": -62.29052616596222,
+ "logps/rejected": -75.3348803281784,
+ "epoch": 0.9024428193558426,
+ "step": 870
+ },
+ {
+ "eval_loss": 0.4090946316719055,
+ "eval_runtime": 375.2006,
+ "eval_samples_per_second": 2.665,
+ "eval_steps_per_second": 2.665,
+ "eval_entropy": 0.7924561413838528,
+ "eval_num_tokens": 58685036.0,
+ "eval_logits/chosen": -1.5811667035215011,
+ "eval_logits/rejected": -1.2610527832956517,
+ "eval_mean_token_accuracy": 0.7133496539741755,
+ "eval_rewards/chosen": -0.7897945828609518,
+ "eval_rewards/rejected": -2.3197074431839866,
+ "eval_rewards/accuracies": 0.828,
+ "eval_rewards/margins": 1.5299128606393932,
+ "eval_logps/chosen": -58.07093252563477,
+ "eval_logps/rejected": -75.83694574356079,
+ "epoch": 0.9076292723406463,
+ "step": 875
+ },
+ {
+ "loss": 0.39424493312835696,
+ "grad_norm": 0.013235539197921753,
+ "learning_rate": 8.911917098445596e-07,
+ "entropy": 0.7996839477901813,
+ "num_tokens": 59020756.0,
+ "logits/chosen": -1.5843618404439912,
+ "logits/rejected": -1.2211371427803261,
+ "mean_token_accuracy": 0.7160469963587821,
+ "rewards/chosen": -0.7639252483138261,
+ "rewards/rejected": -2.3575840306148166,
+ "rewards/accuracies": 0.82625,
+ "rewards/margins": 1.5936587833426892,
+ "logps/chosen": -57.04736487388611,
+ "logps/rejected": -79.49960944414138,
+ "epoch": 0.91281572532545,
+ "step": 880
+ },
+ {
+ "loss": 0.4502077102661133,
+ "grad_norm": 0.016691625118255615,
+ "learning_rate": 7.875647668393784e-07,
+ "entropy": 0.7824898976105032,
+ "num_tokens": 59691542.0,
+ "logits/chosen": -1.6007244896342394,
+ "logits/rejected": -1.2807950876104501,
+ "mean_token_accuracy": 0.7142804705537855,
+ "rewards/chosen": -0.8533335844849352,
+ "rewards/rejected": -2.263447732278146,
+ "rewards/accuracies": 0.80875,
+ "rewards/margins": 1.4101141486410051,
+ "logps/chosen": -58.45585764169693,
+ "logps/rejected": -76.72949911355973,
+ "epoch": 0.9231886312950573,
+ "step": 890
+ },
+ {
+ "loss": 0.40724778175354004,
+ "grad_norm": 0.019250568002462387,
+ "learning_rate": 6.839378238341969e-07,
+ "entropy": 0.819874558126321,
+ "num_tokens": 60363238.0,
+ "logits/chosen": -1.5555056418182593,
+ "logits/rejected": -1.2092615978992753,
+ "mean_token_accuracy": 0.7146024034544826,
+ "rewards/chosen": -0.8092134252478718,
+ "rewards/rejected": -2.3746094698365776,
+ "rewards/accuracies": 0.8225,
+ "rewards/margins": 1.565396042652428,
+ "logps/chosen": -58.4417240858078,
+ "logps/rejected": -80.7738204741478,
+ "epoch": 0.9335615372646647,
+ "step": 900
+ },
+ {
+ "loss": 0.4182878971099854,
+ "grad_norm": 0.02170160599052906,
+ "learning_rate": 5.803108808290156e-07,
+ "entropy": 0.8145798404130619,
+ "num_tokens": 61034470.0,
+ "logits/chosen": -1.5395043610571841,
+ "logits/rejected": -1.2172720667091816,
+ "mean_token_accuracy": 0.7149157860130072,
+ "rewards/chosen": -0.8415973563533043,
+ "rewards/rejected": -2.4253665239037945,
+ "rewards/accuracies": 0.81,
+ "rewards/margins": 1.5837691660691053,
+ "logps/chosen": -58.98787801384926,
+ "logps/rejected": -78.09846720218658,
+ "epoch": 0.9439344432342721,
+ "step": 910
+ },
+ {
+ "loss": 0.42937483787536623,
+ "grad_norm": 0.014422536827623844,
+ "learning_rate": 4.7668393782383424e-07,
+ "entropy": 0.7941974463209044,
+ "num_tokens": 61703303.0,
+ "logits/chosen": -1.5714764751371695,
+ "logits/rejected": -1.2391417194541616,
+ "mean_token_accuracy": 0.7113890647143125,
+ "rewards/chosen": -0.8099557375663425,
+ "rewards/rejected": -2.294640830200858,
+ "rewards/accuracies": 0.81125,
+ "rewards/margins": 1.4846850905031896,
+ "logps/chosen": -56.28238318800926,
+ "logps/rejected": -74.58795207023621,
+ "epoch": 0.9543073492038795,
+ "step": 920
+ },
+ {
+ "loss": 0.40997958183288574,
+ "grad_norm": 0.02402566932141781,
+ "learning_rate": 3.730569948186528e-07,
+ "entropy": 0.8154971726459916,
+ "num_tokens": 62375816.0,
+ "logits/chosen": -1.545892127215608,
+ "logits/rejected": -1.2600055196671498,
+ "mean_token_accuracy": 0.704818404596299,
+ "rewards/chosen": -0.7739588160630956,
+ "rewards/rejected": -2.308201268577832,
+ "rewards/accuracies": 0.8175,
+ "rewards/margins": 1.5342424516985194,
+ "logps/chosen": -60.89072738647461,
+ "logps/rejected": -77.38187520503998,
+ "epoch": 0.9646802551734869,
+ "step": 930
+ },
+ {
+ "loss": 0.4273488998413086,
+ "grad_norm": 0.016814034432172775,
+ "learning_rate": 2.694300518134715e-07,
+ "entropy": 0.81020716742496,
+ "num_tokens": 63047673.0,
+ "logits/chosen": -1.5172310235884794,
+ "logits/rejected": -1.1998523520459965,
+ "mean_token_accuracy": 0.7050479584932328,
+ "rewards/chosen": -0.8097914783160922,
+ "rewards/rejected": -2.262013399923744,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.452221922907047,
+ "logps/chosen": -60.9624414229393,
+ "logps/rejected": -75.92849832773209,
+ "epoch": 0.9750531611430943,
+ "step": 940
+ },
+ {
+ "loss": 0.4405078411102295,
+ "grad_norm": 0.02221529930830002,
+ "learning_rate": 1.6580310880829015e-07,
+ "entropy": 0.7960373500641436,
+ "num_tokens": 63718529.0,
+ "logits/chosen": -1.5261262470344286,
+ "logits/rejected": -1.2351896031266807,
+ "mean_token_accuracy": 0.7166906687803567,
+ "rewards/chosen": -0.8054070144893921,
+ "rewards/rejected": -2.2143535910593344,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.4089465801790357,
+ "logps/chosen": -58.76033858776093,
+ "logps/rejected": -75.32699841499328,
+ "epoch": 0.9854260671127016,
+ "step": 950
+ },
+ {
+ "loss": 0.4032235622406006,
+ "grad_norm": 0.015166543424129486,
+ "learning_rate": 6.217616580310881e-08,
+ "entropy": 0.8253091154253344,
+ "num_tokens": 64391541.0,
+ "logits/chosen": -1.5054467577713744,
+ "logits/rejected": -1.1940948964954847,
+ "mean_token_accuracy": 0.7102081939950585,
+ "rewards/chosen": -0.7573958630473498,
+ "rewards/rejected": -2.3041888994569306,
+ "rewards/accuracies": 0.8125,
+ "rewards/margins": 1.5467930364748463,
+ "logps/chosen": -59.873460137844084,
+ "logps/rejected": -77.67126994132995,
+ "epoch": 0.995798973082309,
+ "step": 960
+ },
+ {
+ "train_runtime": 36967.0361,
+ "train_samples_per_second": 2.086,
+ "train_steps_per_second": 0.026,
+ "total_flos": 5.2405488888805786e+17,
+ "train_loss": 0.453640728787437,
+ "entropy": 0.8554125972153952,
+ "num_tokens": 64664162.0,
+ "logits/chosen": -1.3863200845494572,
+ "logits/rejected": -1.1139205918868142,
+ "mean_token_accuracy": 0.7043190732322357,
+ "rewards/chosen": -0.8579825216625461,
+ "rewards/rejected": -2.1552414747104507,
+ "rewards/accuracies": 0.7932098765432098,
+ "rewards/margins": 1.2972589509883596,
+ "logps/chosen": -62.61760814690295,
+ "logps/rejected": -79.34557756376856,
+ "epoch": 1.0,
+ "step": 965
+ }
+ ],
+ "validation_monitor_size": 1000,
+ "validation_monitor_selection": "fixed_seed_random_without_replacement",
+ "validation_monitor_seed": 22,
+ "validation_monitor_indices": [
+ 2,
+ 10,
+ 14,
+ 21,
+ 55,
+ 63,
+ 66,
+ 69,
+ 107,
+ 110,
+ 142,
+ 144,
+ 149,
+ 150,
+ 151,
+ 152,
+ 160,
+ 163,
+ 166,
+ 167,
+ 176,
+ 181,
+ 206,
+ 207,
+ 259,
+ 267,
+ 281,
+ 295,
+ 298,
+ 306,
+ 307,
+ 317,
+ 321,
+ 331,
+ 336,
+ 341,
+ 346,
+ 349,
+ 351,
+ 352,
+ 357,
+ 358,
+ 369,
+ 370,
+ 382,
+ 386,
+ 391,
+ 401,
+ 411,
+ 412,
+ 432,
+ 437,
+ 452,
+ 462,
+ 465,
+ 488,
+ 490,
+ 491,
+ 492,
+ 494,
+ 503,
+ 504,
+ 508,
+ 528,
+ 538,
+ 540,
+ 551,
+ 553,
+ 567,
+ 586,
+ 605,
+ 606,
+ 625,
+ 627,
+ 661,
+ 663,
+ 666,
+ 677,
+ 685,
+ 690,
+ 692,
+ 704,
+ 708,
+ 714,
+ 715,
+ 727,
+ 728,
+ 733,
+ 745,
+ 752,
+ 753,
+ 755,
+ 764,
+ 773,
+ 779,
+ 783,
+ 793,
+ 796,
+ 799,
+ 803,
+ 804,
+ 805,
+ 813,
+ 816,
+ 818,
+ 823,
+ 827,
+ 829,
+ 830,
+ 837,
+ 842,
+ 845,
+ 850,
+ 853,
+ 856,
+ 889,
+ 890,
+ 905,
+ 915,
+ 924,
+ 930,
+ 939,
+ 951,
+ 988,
+ 1002,
+ 1005,
+ 1023,
+ 1029,
+ 1038,
+ 1049,
+ 1050,
+ 1054,
+ 1056,
+ 1067,
+ 1068,
+ 1079,
+ 1088,
+ 1091,
+ 1103,
+ 1114,
+ 1118,
+ 1133,
+ 1140,
+ 1144,
+ 1145,
+ 1155,
+ 1186,
+ 1195,
+ 1206,
+ 1223,
+ 1251,
+ 1252,
+ 1257,
+ 1259,
+ 1270,
+ 1271,
+ 1295,
+ 1303,
+ 1304,
+ 1305,
+ 1329,
+ 1335,
+ 1336,
+ 1343,
+ 1367,
+ 1373,
+ 1377,
+ 1403,
+ 1412,
+ 1429,
+ 1443,
+ 1457,
+ 1459,
+ 1460,
+ 1476,
+ 1483,
+ 1486,
+ 1494,
+ 1507,
+ 1510,
+ 1519,
+ 1521,
+ 1522,
+ 1545,
+ 1551,
+ 1570,
+ 1582,
+ 1593,
+ 1595,
+ 1603,
+ 1607,
+ 1614,
+ 1615,
+ 1625,
+ 1634,
+ 1639,
+ 1648,
+ 1654,
+ 1657,
+ 1662,
+ 1667,
+ 1673,
+ 1690,
+ 1704,
+ 1746,
+ 1756,
+ 1781,
+ 1783,
+ 1796,
+ 1799,
+ 1809,
+ 1814,
+ 1827,
+ 1829,
+ 1832,
+ 1844,
+ 1847,
+ 1854,
+ 1856,
+ 1857,
+ 1858,
+ 1874,
+ 1883,
+ 1889,
+ 1924,
+ 1925,
+ 1931,
+ 1932,
+ 1934,
+ 1935,
+ 1938,
+ 1950,
+ 1957,
+ 1962,
+ 1967,
+ 1975,
+ 1982,
+ 1983,
+ 1991,
+ 1998,
+ 2003,
+ 2008,
+ 2017,
+ 2021,
+ 2026,
+ 2035,
+ 2040,
+ 2050,
+ 2056,
+ 2077,
+ 2079,
+ 2082,
+ 2098,
+ 2100,
+ 2108,
+ 2121,
+ 2130,
+ 2133,
+ 2134,
+ 2138,
+ 2143,
+ 2166,
+ 2167,
+ 2180,
+ 2181,
+ 2185,
+ 2204,
+ 2205,
+ 2212,
+ 2221,
+ 2224,
+ 2226,
+ 2230,
+ 2233,
+ 2256,
+ 2261,
+ 2263,
+ 2269,
+ 2273,
+ 2290,
+ 2291,
+ 2299,
+ 2301,
+ 2321,
+ 2323,
+ 2330,
+ 2384,
+ 2401,
+ 2417,
+ 2420,
+ 2421,
+ 2424,
+ 2430,
+ 2435,
+ 2456,
+ 2488,
+ 2502,
+ 2504,
+ 2519,
+ 2527,
+ 2532,
+ 2538,
+ 2542,
+ 2553,
+ 2555,
+ 2558,
+ 2559,
+ 2562,
+ 2578,
+ 2583,
+ 2585,
+ 2590,
+ 2592,
+ 2594,
+ 2596,
+ 2600,
+ 2606,
+ 2607,
+ 2618,
+ 2630,
+ 2637,
+ 2647,
+ 2650,
+ 2657,
+ 2679,
+ 2685,
+ 2705,
+ 2706,
+ 2712,
+ 2713,
+ 2714,
+ 2727,
+ 2740,
+ 2742,
+ 2755,
+ 2780,
+ 2784,
+ 2792,
+ 2807,
+ 2808,
+ 2810,
+ 2820,
+ 2831,
+ 2839,
+ 2860,
+ 2862,
+ 2863,
+ 2866,
+ 2875,
+ 2878,
+ 2898,
+ 2905,
+ 2921,
+ 2922,
+ 2926,
+ 2930,
+ 2934,
+ 2944,
+ 2955,
+ 2957,
+ 2959,
+ 2973,
+ 2978,
+ 2998,
+ 3018,
+ 3022,
+ 3028,
+ 3031,
+ 3061,
+ 3085,
+ 3090,
+ 3104,
+ 3105,
+ 3111,
+ 3115,
+ 3121,
+ 3122,
+ 3129,
+ 3133,
+ 3135,
+ 3161,
+ 3162,
+ 3169,
+ 3173,
+ 3194,
+ 3195,
+ 3215,
+ 3225,
+ 3226,
+ 3241,
+ 3247,
+ 3250,
+ 3253,
+ 3265,
+ 3268,
+ 3293,
+ 3301,
+ 3312,
+ 3329,
+ 3352,
+ 3361,
+ 3362,
+ 3376,
+ 3396,
+ 3401,
+ 3403,
+ 3410,
+ 3419,
+ 3432,
+ 3443,
+ 3452,
+ 3467,
+ 3469,
+ 3475,
+ 3485,
+ 3503,
+ 3514,
+ 3515,
+ 3524,
+ 3528,
+ 3538,
+ 3544,
+ 3557,
+ 3560,
+ 3565,
+ 3600,
+ 3637,
+ 3642,
+ 3658,
+ 3659,
+ 3692,
+ 3693,
+ 3699,
+ 3722,
+ 3725,
+ 3728,
+ 3731,
+ 3740,
+ 3742,
+ 3762,
+ 3766,
+ 3780,
+ 3788,
+ 3794,
+ 3796,
+ 3798,
+ 3805,
+ 3817,
+ 3819,
+ 3822,
+ 3837,
+ 3839,
+ 3843,
+ 3846,
+ 3851,
+ 3854,
+ 3865,
+ 3870,
+ 3871,
+ 3884,
+ 3894,
+ 3895,
+ 3896,
+ 3907,
+ 3911,
+ 3915,
+ 3919,
+ 3920,
+ 3923,
+ 3933,
+ 3955,
+ 3967,
+ 3972,
+ 3974,
+ 3975,
+ 3984,
+ 3985,
+ 3997,
+ 4002,
+ 4010,
+ 4034,
+ 4044,
+ 4063,
+ 4073,
+ 4079,
+ 4082,
+ 4088,
+ 4121,
+ 4145,
+ 4148,
+ 4159,
+ 4161,
+ 4164,
+ 4177,
+ 4188,
+ 4199,
+ 4203,
+ 4207,
+ 4208,
+ 4213,
+ 4221,
+ 4225,
+ 4233,
+ 4241,
+ 4243,
+ 4247,
+ 4264,
+ 4274,
+ 4282,
+ 4286,
+ 4290,
+ 4308,
+ 4310,
+ 4313,
+ 4321,
+ 4324,
+ 4327,
+ 4349,
+ 4362,
+ 4370,
+ 4374,
+ 4380,
+ 4407,
+ 4409,
+ 4411,
+ 4415,
+ 4417,
+ 4418,
+ 4427,
+ 4431,
+ 4433,
+ 4451,
+ 4466,
+ 4477,
+ 4478,
+ 4479,
+ 4493,
+ 4494,
+ 4514,
+ 4527,
+ 4539,
+ 4550,
+ 4558,
+ 4568,
+ 4579,
+ 4583,
+ 4584,
+ 4586,
+ 4587,
+ 4590,
+ 4599,
+ 4602,
+ 4610,
+ 4626,
+ 4627,
+ 4630,
+ 4641,
+ 4647,
+ 4655,
+ 4656,
+ 4657,
+ 4661,
+ 4685,
+ 4714,
+ 4715,
+ 4731,
+ 4749,
+ 4752,
+ 4769,
+ 4777,
+ 4782,
+ 4791,
+ 4805,
+ 4818,
+ 4829,
+ 4833,
+ 4837,
+ 4839,
+ 4843,
+ 4871,
+ 4875,
+ 4879,
+ 4880,
+ 4885,
+ 4888,
+ 4895,
+ 4900,
+ 4923,
+ 4966,
+ 4968,
+ 4972,
+ 4989,
+ 4994,
+ 4998,
+ 5001,
+ 5013,
+ 5019,
+ 5026,
+ 5032,
+ 5034,
+ 5046,
+ 5055,
+ 5085,
+ 5086,
+ 5088,
+ 5089,
+ 5090,
+ 5095,
+ 5108,
+ 5110,
+ 5119,
+ 5120,
+ 5121,
+ 5133,
+ 5148,
+ 5154,
+ 5160,
+ 5169,
+ 5178,
+ 5222,
+ 5223,
+ 5232,
+ 5233,
+ 5240,
+ 5256,
+ 5259,
+ 5264,
+ 5271,
+ 5276,
+ 5279,
+ 5294,
+ 5300,
+ 5303,
+ 5321,
+ 5335,
+ 5337,
+ 5345,
+ 5348,
+ 5365,
+ 5373,
+ 5378,
+ 5384,
+ 5422,
+ 5442,
+ 5443,
+ 5445,
+ 5477,
+ 5485,
+ 5495,
+ 5497,
+ 5504,
+ 5526,
+ 5533,
+ 5542,
+ 5564,
+ 5570,
+ 5579,
+ 5587,
+ 5592,
+ 5608,
+ 5610,
+ 5624,
+ 5630,
+ 5638,
+ 5651,
+ 5663,
+ 5670,
+ 5675,
+ 5691,
+ 5700,
+ 5706,
+ 5707,
+ 5715,
+ 5720,
+ 5721,
+ 5722,
+ 5732,
+ 5738,
+ 5741,
+ 5769,
+ 5771,
+ 5775,
+ 5795,
+ 5796,
+ 5800,
+ 5809,
+ 5810,
+ 5815,
+ 5819,
+ 5827,
+ 5848,
+ 5849,
+ 5852,
+ 5859,
+ 5880,
+ 5884,
+ 5907,
+ 5909,
+ 5912,
+ 5919,
+ 5931,
+ 5932,
+ 5934,
+ 5941,
+ 5948,
+ 5950,
+ 5952,
+ 5953,
+ 5969,
+ 5981,
+ 6000,
+ 6003,
+ 6010,
+ 6018,
+ 6041,
+ 6065,
+ 6075,
+ 6090,
+ 6103,
+ 6106,
+ 6109,
+ 6114,
+ 6123,
+ 6131,
+ 6136,
+ 6138,
+ 6139,
+ 6164,
+ 6165,
+ 6171,
+ 6173,
+ 6180,
+ 6185,
+ 6189,
+ 6190,
+ 6221,
+ 6223,
+ 6237,
+ 6255,
+ 6262,
+ 6265,
+ 6293,
+ 6296,
+ 6305,
+ 6318,
+ 6331,
+ 6336,
+ 6340,
+ 6355,
+ 6366,
+ 6371,
+ 6396,
+ 6399,
+ 6402,
+ 6408,
+ 6432,
+ 6433,
+ 6441,
+ 6456,
+ 6465,
+ 6478,
+ 6486,
+ 6489,
+ 6507,
+ 6508,
+ 6520,
+ 6522,
+ 6528,
+ 6537,
+ 6551,
+ 6564,
+ 6589,
+ 6590,
+ 6598,
+ 6599,
+ 6611,
+ 6613,
+ 6615,
+ 6621,
+ 6634,
+ 6647,
+ 6649,
+ 6654,
+ 6676,
+ 6680,
+ 6690,
+ 6708,
+ 6729,
+ 6736,
+ 6744,
+ 6749,
+ 6756,
+ 6761,
+ 6763,
+ 6773,
+ 6788,
+ 6790,
+ 6796,
+ 6797,
+ 6811,
+ 6824,
+ 6850,
+ 6869,
+ 6871,
+ 6882,
+ 6892,
+ 6895,
+ 6906,
+ 6911,
+ 6912,
+ 6914,
+ 6927,
+ 6952,
+ 6966,
+ 6985,
+ 7001,
+ 7021,
+ 7031,
+ 7035,
+ 7038,
+ 7041,
+ 7045,
+ 7052,
+ 7057,
+ 7058,
+ 7072,
+ 7073,
+ 7076,
+ 7086,
+ 7102,
+ 7107,
+ 7109,
+ 7117,
+ 7122,
+ 7125,
+ 7166,
+ 7182,
+ 7199,
+ 7207,
+ 7212,
+ 7215,
+ 7232,
+ 7243,
+ 7246,
+ 7250,
+ 7253,
+ 7258,
+ 7263,
+ 7267,
+ 7270,
+ 7274,
+ 7280,
+ 7286,
+ 7293,
+ 7298,
+ 7302,
+ 7306,
+ 7316,
+ 7325,
+ 7326,
+ 7334,
+ 7356,
+ 7357,
+ 7363,
+ 7364,
+ 7367,
+ 7385,
+ 7392,
+ 7399,
+ 7405,
+ 7416,
+ 7420,
+ 7421,
+ 7426,
+ 7452,
+ 7476,
+ 7481,
+ 7519,
+ 7534,
+ 7542,
+ 7546,
+ 7573,
+ 7585,
+ 7601,
+ 7604,
+ 7606,
+ 7609,
+ 7629,
+ 7649,
+ 7653,
+ 7667,
+ 7682,
+ 7699,
+ 7738,
+ 7750,
+ 7786,
+ 7798,
+ 7800,
+ 7801,
+ 7805,
+ 7806,
+ 7811,
+ 7813,
+ 7832,
+ 7837,
+ 7849,
+ 7856,
+ 7876,
+ 7877,
+ 7887,
+ 7905,
+ 7916,
+ 7919,
+ 7920,
+ 7922,
+ 7923,
+ 7926,
+ 7944,
+ 7961,
+ 7966,
+ 7970,
+ 7973,
+ 7974,
+ 7976,
+ 7986,
+ 7999,
+ 8027,
+ 8028,
+ 8034,
+ 8047,
+ 8068,
+ 8074,
+ 8077,
+ 8091,
+ 8120,
+ 8122,
+ 8125,
+ 8152,
+ 8153,
+ 8164,
+ 8167,
+ 8169,
+ 8171,
+ 8177,
+ 8184,
+ 8189,
+ 8192,
+ 8196,
+ 8202,
+ 8212,
+ 8217,
+ 8231,
+ 8242,
+ 8244,
+ 8250,
+ 8256,
+ 8257,
+ 8265,
+ 8270,
+ 8274,
+ 8283,
+ 8290,
+ 8294,
+ 8301,
+ 8302,
+ 8307,
+ 8318,
+ 8326,
+ 8338,
+ 8349,
+ 8350,
+ 8353,
+ 8357,
+ 8371,
+ 8374,
+ 8377,
+ 8380,
+ 8387,
+ 8388,
+ 8396,
+ 8402,
+ 8419,
+ 8423,
+ 8428,
+ 8435,
+ 8439,
+ 8442,
+ 8444,
+ 8453,
+ 8461,
+ 8475,
+ 8481,
+ 8485,
+ 8493,
+ 8495,
+ 8498,
+ 8523,
+ 8530,
+ 8531,
+ 8562
+ ],
+ "early_stopping_patience": 3
+}
diff --git a/phase1/ablations/no_context/README.md b/phase1/ablations/no_context/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..528f703eb66e93decd20d4934b1f7d95d0698e74
--- /dev/null
+++ b/phase1/ablations/no_context/README.md
@@ -0,0 +1,80 @@
+---
+library_name: peft
+model_name: phase1-qwen-no-context-ablation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-1.5B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+license: apache-2.0
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+pipeline_tag: text-generation
+---
+
+# Phase 1 Qwen `no_context` diagnostic ablation
+
+This is the Phase 1 `no_context` diagnostic LoRA-DPO adapter fine-tuned from
+[Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct).
+The frozen training recipe and membership hashes are included in this directory.
+
+## Quick start
+
+```python
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo_id = "geyuxu/york-milestone-project-self-traing-loop-model"
+subfolder = "phase1/ablations/no_context"
+base_id = "Qwen/Qwen2.5-1.5B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(repo_id, subfolder=subfolder)
+base_model = AutoModelForCausalLM.from_pretrained(
+ base_id, torch_dtype="auto", device_map="auto"
+)
+model = PeftModel.from_pretrained(base_model, repo_id, subfolder=subfolder)
+```
+
+## Training procedure
+
+
+
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 0.29.0
+- Transformers: 5.3.0
+- Pytorch: 2.10.0
+- Datasets: 4.6.1
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+ title = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+ author = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+ year = 2023,
+ booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+ url = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+ editor = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+ title = {{TRL: Transformers Reinforcement Learning}},
+ author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+ license = {Apache-2.0},
+ url = {https://github.com/huggingface/trl},
+ year = {2020}
+}
+```
diff --git a/phase1/ablations/no_context/adapter_config.json b/phase1/ablations/no_context/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..fa47a2292cd8a23ccf1b4d101d7f7a03e3f08871
--- /dev/null
+++ b/phase1/ablations/no_context/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "k_proj",
+ "o_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
diff --git a/phase1/ablations/no_context/adapter_model.safetensors b/phase1/ablations/no_context/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6505c47843a638faa3b3f80cf032400cdf39a2b1
--- /dev/null
+++ b/phase1/ablations/no_context/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f3a8ff3c19fd3e0ff728feab4e0a73a33c71608510f37d9dd27e53b74f97523
+size 17462432
diff --git a/phase1/ablations/no_context/chat_template.jinja b/phase1/ablations/no_context/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e
--- /dev/null
+++ b/phase1/ablations/no_context/chat_template.jinja
@@ -0,0 +1,54 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- messages[0]['content'] }}
+ {%- else %}
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+ {%- endif %}
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+ {%- else %}
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {{- '<|im_start|>' + message.role }}
+ {%- if message.content %}
+ {{- '\n' + message.content }}
+ {%- endif %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- '}\n' }}
+ {%- endfor %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- message.content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}
diff --git a/phase1/ablations/no_context/tokenizer.json b/phase1/ablations/no_context/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..d73428d91463b495bc017fb6a2fb3a656646482b
--- /dev/null
+++ b/phase1/ablations/no_context/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5
+size 11422166
diff --git a/phase1/ablations/no_context/tokenizer_config.json b/phase1/ablations/no_context/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9fd0fb48e73786f54fb04e98892f5f5a0ebeebdb
--- /dev/null
+++ b/phase1/ablations/no_context/tokenizer_config.json
@@ -0,0 +1,29 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": true,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/phase1/ablations/no_context/training_args.bin b/phase1/ablations/no_context/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..4b33f87738292306dfc6d52ce5bc7be6fc8ed191
--- /dev/null
+++ b/phase1/ablations/no_context/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd47c7f20edcbf52ada253a96ebd7ddcb8045bdd2a75f52fdd0f51ad7bc0183e
+size 5841
diff --git a/phase1/ablations/no_context/training_recipe.json b/phase1/ablations/no_context/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..3566328b32f896b65ee58a4de3d44c5c533d387e
--- /dev/null
+++ b/phase1/ablations/no_context/training_recipe.json
@@ -0,0 +1,22 @@
+{
+ "format_version": 1,
+ "experiment": "confusion_ablation",
+ "ablation": "no_context",
+ "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+ "train_membership_sha256": "b497fa765223a9810ad784f31e1598edfa992a162146d6f94d9cdf4adcedd87d",
+ "validation_membership_sha256": "2f609467438cd6a01fd84b9d80c0fcd84c44767e3b2ed98f4f6a83d68a62c181",
+ "num_train_pairs": 77124,
+ "num_validation_pairs": 8563,
+ "transformation_seed": 22,
+ "validation_prompt_mode": "clean_rag",
+ "learning_rate": 1e-05,
+ "beta": 0.1,
+ "num_epochs": 1.0,
+ "seed": 22,
+ "max_length": 512,
+ "max_prompt_length": 384,
+ "gradient_accumulation_steps": 80,
+ "save_steps": 125,
+ "save_total_limit": 12,
+ "preference_pairs_format_version": 2
+}
diff --git a/phase1/ablations/no_context/training_summary.json b/phase1/ablations/no_context/training_summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..ec12d2995cbde422b4cc0067c3dc722a2a3d3bdd
--- /dev/null
+++ b/phase1/ablations/no_context/training_summary.json
@@ -0,0 +1,2925 @@
+{
+ "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+ "resolved_model": "Qwen/Qwen2.5-1.5B-Instruct",
+ "num_train_records": 77124,
+ "num_validation_records_available": 8563,
+ "training_config": {
+ "lora_rank": 16,
+ "lora_alpha": 32,
+ "learning_rate": 1e-05,
+ "num_epochs": 1.0,
+ "per_device_batch_size": 1,
+ "per_device_eval_batch_size": 1,
+ "gradient_accumulation_steps": 80,
+ "beta": 0.1,
+ "max_length": 512,
+ "max_prompt_length": 384,
+ "seed": 22,
+ "save_steps": 125,
+ "save_total_limit": 12,
+ "resume_from_checkpoint": null,
+ "precision_dtype": "torch.bfloat16",
+ "bf16": true,
+ "fp16": false,
+ "tokenizer_add_bos_token": false
+ },
+ "global_step": 965,
+ "best_metric": 0.5303567051887512,
+ "best_model_checkpoint": "outputs/confusion_ablation_phase1_repair_v3/no_context/lora/checkpoint-875",
+ "train_metrics": {
+ "train_runtime": 22000.2483,
+ "train_samples_per_second": 3.506,
+ "train_steps_per_second": 0.044,
+ "total_flos": 1.2331818072769536e+17,
+ "train_loss": 0.4788281094842624
+ },
+ "log_history": [
+ {
+ "loss": 0.6900143623352051,
+ "grad_norm": 0.012825769372284412,
+ "learning_rate": 9.906735751295338e-06,
+ "entropy": 0.9465285880398006,
+ "num_tokens": 143734.0,
+ "logits/chosen": -1.5012888478025672,
+ "logits/rejected": -1.5723651012268272,
+ "mean_token_accuracy": 0.6951504178717732,
+ "rewards/chosen": 0.008079942484078515,
+ "rewards/rejected": 0.0013256204138269822,
+ "rewards/accuracies": 0.5425,
+ "rewards/margins": 0.006754322004853748,
+ "logps/chosen": -54.978611971139905,
+ "logps/rejected": -44.00920191049576,
+ "epoch": 0.010372905969607386,
+ "step": 10
+ },
+ {
+ "loss": 0.662090539932251,
+ "grad_norm": 0.01821574941277504,
+ "learning_rate": 9.803108808290156e-06,
+ "entropy": 0.9671179948747158,
+ "num_tokens": 288007.0,
+ "logits/chosen": -1.4026643295951302,
+ "logits/rejected": -1.5102725032640183,
+ "mean_token_accuracy": 0.6902420591562987,
+ "rewards/chosen": 0.051943560895306294,
+ "rewards/rejected": -0.014707293705987468,
+ "rewards/accuracies": 0.76875,
+ "rewards/margins": 0.06665085479675327,
+ "logps/chosen": -57.24453290700912,
+ "logps/rejected": -42.05887178063393,
+ "epoch": 0.02074581193921477,
+ "step": 20
+ },
+ {
+ "loss": 0.6254077911376953,
+ "grad_norm": 0.011037914082407951,
+ "learning_rate": 9.699481865284975e-06,
+ "entropy": 1.022590966722928,
+ "num_tokens": 433590.0,
+ "logits/chosen": -1.275362931486493,
+ "logits/rejected": -1.3053425668445908,
+ "mean_token_accuracy": 0.6875665122829377,
+ "rewards/chosen": 0.104925400447828,
+ "rewards/rejected": -0.05808189030554786,
+ "rewards/accuracies": 0.785,
+ "rewards/margins": 0.16300729091250105,
+ "logps/chosen": -56.563721351623535,
+ "logps/rejected": -44.831772941350934,
+ "epoch": 0.031118717908822157,
+ "step": 30
+ },
+ {
+ "loss": 0.5835778713226318,
+ "grad_norm": 0.011659706942737103,
+ "learning_rate": 9.595854922279793e-06,
+ "entropy": 1.0491496316390112,
+ "num_tokens": 579257.0,
+ "logits/chosen": -1.103721370379413,
+ "logits/rejected": -1.1007857062801798,
+ "mean_token_accuracy": 0.6836125956382603,
+ "rewards/chosen": 0.10443844198705847,
+ "rewards/rejected": -0.17612101439032812,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 0.2805594558198936,
+ "logps/chosen": -56.31521392583847,
+ "logps/rejected": -44.80111095190048,
+ "epoch": 0.04149162387842954,
+ "step": 40
+ },
+ {
+ "loss": 0.5701297760009766,
+ "grad_norm": 0.009541669860482216,
+ "learning_rate": 9.492227979274612e-06,
+ "entropy": 1.1355412375973537,
+ "num_tokens": 724279.0,
+ "logits/chosen": -0.9254812350216316,
+ "logits/rejected": -0.9079043695626247,
+ "mean_token_accuracy": 0.6852783388644457,
+ "rewards/chosen": 0.054056347796804405,
+ "rewards/rejected": -0.30969686928990997,
+ "rewards/accuracies": 0.78,
+ "rewards/margins": 0.3637532171921339,
+ "logps/chosen": -57.41013575792313,
+ "logps/rejected": -45.747311503887175,
+ "epoch": 0.051864529848036925,
+ "step": 50
+ },
+ {
+ "loss": 0.5617631912231446,
+ "grad_norm": 0.007650681305676699,
+ "learning_rate": 9.388601036269432e-06,
+ "entropy": 1.1949225943023338,
+ "num_tokens": 868037.0,
+ "logits/chosen": -0.8281910829349749,
+ "logits/rejected": -0.7978684013816298,
+ "mean_token_accuracy": 0.6832736423797905,
+ "rewards/chosen": -0.03967508725108928,
+ "rewards/rejected": -0.47718707822641593,
+ "rewards/accuracies": 0.76625,
+ "rewards/margins": 0.4375119912764058,
+ "logps/chosen": -56.544705327749256,
+ "logps/rejected": -46.81773233771324,
+ "epoch": 0.062237435817644314,
+ "step": 60
+ },
+ {
+ "loss": 0.5275696277618408,
+ "grad_norm": 0.008909703232347965,
+ "learning_rate": 9.284974093264249e-06,
+ "entropy": 1.2164792704861611,
+ "num_tokens": 1012838.0,
+ "logits/chosen": -0.8094323563514035,
+ "logits/rejected": -0.6994889637697067,
+ "mean_token_accuracy": 0.6774187626503408,
+ "rewards/chosen": -0.13885413558671644,
+ "rewards/rejected": -0.6821774935087888,
+ "rewards/accuracies": 0.78875,
+ "rewards/margins": 0.5433233577781357,
+ "logps/chosen": -59.331356877088545,
+ "logps/rejected": -48.74847481250763,
+ "epoch": 0.0726103417872517,
+ "step": 70
+ },
+ {
+ "loss": 0.5331552982330322,
+ "grad_norm": 0.009889204055070877,
+ "learning_rate": 9.181347150259067e-06,
+ "entropy": 1.3044282801821827,
+ "num_tokens": 1157318.0,
+ "logits/chosen": -0.7322735281718121,
+ "logits/rejected": -0.641984971321019,
+ "mean_token_accuracy": 0.6695321470685304,
+ "rewards/chosen": -0.2586568782431277,
+ "rewards/rejected": -0.8568377101852093,
+ "rewards/accuracies": 0.7625,
+ "rewards/margins": 0.5981808328768239,
+ "logps/chosen": -59.761966693401334,
+ "logps/rejected": -52.77639355659485,
+ "epoch": 0.08298324775685909,
+ "step": 80
+ },
+ {
+ "loss": 0.5138424873352051,
+ "grad_norm": 0.009201128035783768,
+ "learning_rate": 9.077720207253888e-06,
+ "entropy": 1.2985483697801827,
+ "num_tokens": 1301440.0,
+ "logits/chosen": -0.7244757922082572,
+ "logits/rejected": -0.6446976643569048,
+ "mean_token_accuracy": 0.6679086892027408,
+ "rewards/chosen": -0.3228293718020723,
+ "rewards/rejected": -1.0106709606619553,
+ "rewards/accuracies": 0.78125,
+ "rewards/margins": 0.6878415882331319,
+ "logps/chosen": -61.18230092167855,
+ "logps/rejected": -53.36138055682182,
+ "epoch": 0.09335615372646647,
+ "step": 90
+ },
+ {
+ "loss": 0.5422882556915283,
+ "grad_norm": 0.011827374808490276,
+ "learning_rate": 8.974093264248706e-06,
+ "entropy": 1.3437509477324783,
+ "num_tokens": 1450582.0,
+ "logits/chosen": -0.6577579178959528,
+ "logits/rejected": -0.6052760341023756,
+ "mean_token_accuracy": 0.6652949979342521,
+ "rewards/chosen": -0.48851178389566485,
+ "rewards/rejected": -1.1487117192932055,
+ "rewards/accuracies": 0.75,
+ "rewards/margins": 0.660199935096316,
+ "logps/chosen": -66.8969352221489,
+ "logps/rejected": -59.775410163402555,
+ "epoch": 0.10372905969607385,
+ "step": 100
+ },
+ {
+ "loss": 0.4941854953765869,
+ "grad_norm": 0.009122959338128567,
+ "learning_rate": 8.870466321243523e-06,
+ "entropy": 1.3095360630284996,
+ "num_tokens": 1596102.0,
+ "logits/chosen": -0.7076684162519699,
+ "logits/rejected": -0.6176523284098799,
+ "mean_token_accuracy": 0.6634110971167684,
+ "rewards/chosen": -0.461084103367175,
+ "rewards/rejected": -1.2571547295921481,
+ "rewards/accuracies": 0.805,
+ "rewards/margins": 0.7960706273838878,
+ "logps/chosen": -62.40037791490555,
+ "logps/rejected": -55.013062043190004,
+ "epoch": 0.11410196566568125,
+ "step": 110
+ },
+ {
+ "loss": 0.5182926654815674,
+ "grad_norm": 0.009255604818463326,
+ "learning_rate": 8.766839378238343e-06,
+ "entropy": 1.3175755373714493,
+ "num_tokens": 1741007.0,
+ "logits/chosen": -0.7542329802157357,
+ "logits/rejected": -0.6569479564448296,
+ "mean_token_accuracy": 0.6686466364003718,
+ "rewards/chosen": -0.5461745312937274,
+ "rewards/rejected": -1.3049446252000052,
+ "rewards/accuracies": 0.78125,
+ "rewards/margins": 0.7587700937641785,
+ "logps/chosen": -63.756407718658444,
+ "logps/rejected": -53.10148733973503,
+ "epoch": 0.12447487163528863,
+ "step": 120
+ },
+ {
+ "eval_loss": 0.6102487444877625,
+ "eval_runtime": 187.9736,
+ "eval_samples_per_second": 5.32,
+ "eval_steps_per_second": 5.32,
+ "eval_entropy": 0.9602043167352676,
+ "eval_num_tokens": 1813585.0,
+ "eval_logits/chosen": -0.35928736165014746,
+ "eval_logits/rejected": -0.2653593227174459,
+ "eval_mean_token_accuracy": 0.71722535353899,
+ "eval_rewards/chosen": -0.11001014763419516,
+ "eval_rewards/rejected": -0.37161904421189684,
+ "eval_rewards/accuracies": 0.674,
+ "eval_rewards/margins": 0.2616088960794732,
+ "eval_logps/chosen": -51.27308830451965,
+ "eval_logps/rejected": -56.35606210327148,
+ "epoch": 0.1296613246200923,
+ "step": 125
+ },
+ {
+ "loss": 0.5195056438446045,
+ "grad_norm": 0.00852493941783905,
+ "learning_rate": 8.663212435233162e-06,
+ "entropy": 1.3568350885761902,
+ "num_tokens": 1887149.0,
+ "logits/chosen": -0.6995870647573145,
+ "logits/rejected": -0.6066075725004298,
+ "mean_token_accuracy": 0.6595153860282152,
+ "rewards/chosen": -0.5691824928086499,
+ "rewards/rejected": -1.3477147495548707,
+ "rewards/accuracies": 0.77125,
+ "rewards/margins": 0.7785322573361918,
+ "logps/chosen": -63.50056747078896,
+ "logps/rejected": -57.747277438640594,
+ "epoch": 0.13484777760489602,
+ "step": 130
+ },
+ {
+ "loss": 0.5127462863922119,
+ "grad_norm": 0.010999510996043682,
+ "learning_rate": 8.55958549222798e-06,
+ "entropy": 1.2911182015296072,
+ "num_tokens": 2030779.0,
+ "logits/chosen": -0.8425986763113174,
+ "logits/rejected": -0.6790966632280194,
+ "mean_token_accuracy": 0.6699926462769509,
+ "rewards/chosen": -0.5714914060512092,
+ "rewards/rejected": -1.3701531535839604,
+ "rewards/accuracies": 0.79,
+ "rewards/margins": 0.798661746904254,
+ "logps/chosen": -58.21274412512779,
+ "logps/rejected": -57.35458103775978,
+ "epoch": 0.1452206835745034,
+ "step": 140
+ },
+ {
+ "loss": 0.4771514892578125,
+ "grad_norm": 0.007932349108159542,
+ "learning_rate": 8.455958549222799e-06,
+ "entropy": 1.3540867683663964,
+ "num_tokens": 2177571.0,
+ "logits/chosen": -0.7752437049236788,
+ "logits/rejected": -0.6567138403536552,
+ "mean_token_accuracy": 0.6547455007676035,
+ "rewards/chosen": -0.5410610925909713,
+ "rewards/rejected": -1.4754458899376914,
+ "rewards/accuracies": 0.7975,
+ "rewards/margins": 0.9343847993016243,
+ "logps/chosen": -62.85497339010239,
+ "logps/rejected": -59.90050791025162,
+ "epoch": 0.1555935895441108,
+ "step": 150
+ },
+ {
+ "loss": 0.5065268516540528,
+ "grad_norm": 0.009417089633643627,
+ "learning_rate": 8.352331606217617e-06,
+ "entropy": 1.3703182196151464,
+ "num_tokens": 2323969.0,
+ "logits/chosen": -0.7446562811347729,
+ "logits/rejected": -0.6594706622933276,
+ "mean_token_accuracy": 0.6509130381792784,
+ "rewards/chosen": -0.6582208207750955,
+ "rewards/rejected": -1.4940199976743316,
+ "rewards/accuracies": 0.79125,
+ "rewards/margins": 0.8357991751469672,
+ "logps/chosen": -67.1081642484665,
+ "logps/rejected": -60.26060558557511,
+ "epoch": 0.16596649551371817,
+ "step": 160
+ },
+ {
+ "loss": 0.49721107482910154,
+ "grad_norm": 0.009800152853131294,
+ "learning_rate": 8.248704663212436e-06,
+ "entropy": 1.366801006840542,
+ "num_tokens": 2470885.0,
+ "logits/chosen": -0.6928292309660968,
+ "logits/rejected": -0.6209869620155596,
+ "mean_token_accuracy": 0.6535427515022456,
+ "rewards/chosen": -0.6162330767056119,
+ "rewards/rejected": -1.5209841228489678,
+ "rewards/accuracies": 0.78875,
+ "rewards/margins": 0.9047510461229831,
+ "logps/chosen": -66.02899884462357,
+ "logps/rejected": -58.60980107545853,
+ "epoch": 0.17633940148332555,
+ "step": 170
+ },
+ {
+ "loss": 0.4829860210418701,
+ "grad_norm": 0.008786365389823914,
+ "learning_rate": 8.145077720207254e-06,
+ "entropy": 1.3717705434653908,
+ "num_tokens": 2615121.0,
+ "logits/chosen": -0.6580562375952098,
+ "logits/rejected": -0.5878139433574051,
+ "mean_token_accuracy": 0.6564870945271104,
+ "rewards/chosen": -0.5556664189803269,
+ "rewards/rejected": -1.4979612402966085,
+ "rewards/accuracies": 0.7825,
+ "rewards/margins": 0.9422948203328997,
+ "logps/chosen": -62.85998333334923,
+ "logps/rejected": -57.214853674173355,
+ "epoch": 0.18671230745293294,
+ "step": 180
+ },
+ {
+ "loss": 0.4684004783630371,
+ "grad_norm": 0.009316110983490944,
+ "learning_rate": 8.041450777202073e-06,
+ "entropy": 1.4150286172702908,
+ "num_tokens": 2758874.0,
+ "logits/chosen": -0.6699597359601204,
+ "logits/rejected": -0.5473851034553693,
+ "mean_token_accuracy": 0.6443419794179499,
+ "rewards/chosen": -0.503764379483473,
+ "rewards/rejected": -1.4732603453175397,
+ "rewards/accuracies": 0.7825,
+ "rewards/margins": 0.9694959655869753,
+ "logps/chosen": -65.26039588212967,
+ "logps/rejected": -55.79923560500145,
+ "epoch": 0.19708521342254032,
+ "step": 190
+ },
+ {
+ "loss": 0.5344902038574219,
+ "grad_norm": 0.013661396689713001,
+ "learning_rate": 7.937823834196891e-06,
+ "entropy": 1.3751253792829812,
+ "num_tokens": 2902718.0,
+ "logits/chosen": -0.65461918654035,
+ "logits/rejected": -0.5536890222464702,
+ "mean_token_accuracy": 0.6590519631467759,
+ "rewards/chosen": -0.5584196869007791,
+ "rewards/rejected": -1.4234223538136574,
+ "rewards/accuracies": 0.77125,
+ "rewards/margins": 0.8650026666652412,
+ "logps/chosen": -61.13262881875038,
+ "logps/rejected": -56.61175240635872,
+ "epoch": 0.2074581193921477,
+ "step": 200
+ },
+ {
+ "loss": 0.4941723823547363,
+ "grad_norm": 0.008777705021202564,
+ "learning_rate": 7.834196891191712e-06,
+ "entropy": 1.3652805363852532,
+ "num_tokens": 3043885.0,
+ "logits/chosen": -0.7452820956668422,
+ "logits/rejected": -0.6034335554689337,
+ "mean_token_accuracy": 0.6580210606381297,
+ "rewards/chosen": -0.45352055150666276,
+ "rewards/rejected": -1.3747392913223302,
+ "rewards/accuracies": 0.78125,
+ "rewards/margins": 0.9212187370774336,
+ "logps/chosen": -58.416047328710555,
+ "logps/rejected": -53.947862002849575,
+ "epoch": 0.2178310253617551,
+ "step": 210
+ },
+ {
+ "loss": 0.47504258155822754,
+ "grad_norm": 0.009036123752593994,
+ "learning_rate": 7.730569948186528e-06,
+ "entropy": 1.4040400226414205,
+ "num_tokens": 3190499.0,
+ "logits/chosen": -0.6346807976930225,
+ "logits/rejected": -0.5184029459658583,
+ "mean_token_accuracy": 0.6466581939533352,
+ "rewards/chosen": -0.536972903214264,
+ "rewards/rejected": -1.5215354163426673,
+ "rewards/accuracies": 0.8025,
+ "rewards/margins": 0.984562511825934,
+ "logps/chosen": -62.78493340015412,
+ "logps/rejected": -58.09246099233627,
+ "epoch": 0.2282039313313625,
+ "step": 220
+ },
+ {
+ "loss": 0.45198073387146,
+ "grad_norm": 0.007527140900492668,
+ "learning_rate": 7.626943005181348e-06,
+ "entropy": 1.4020196551457047,
+ "num_tokens": 3337768.0,
+ "logits/chosen": -0.6210873045715655,
+ "logits/rejected": -0.5822921518676821,
+ "mean_token_accuracy": 0.6555538633279503,
+ "rewards/chosen": -0.4542729445986333,
+ "rewards/rejected": -1.5461742853268516,
+ "rewards/accuracies": 0.8175,
+ "rewards/margins": 1.091901341448538,
+ "logps/chosen": -67.90270529866218,
+ "logps/rejected": -58.65799897551537,
+ "epoch": 0.23857683730096987,
+ "step": 230
+ },
+ {
+ "loss": 0.4732102870941162,
+ "grad_norm": 0.016743697226047516,
+ "learning_rate": 7.523316062176167e-06,
+ "entropy": 1.408635692326352,
+ "num_tokens": 3483520.0,
+ "logits/chosen": -0.5801546989494524,
+ "logits/rejected": -0.5212051543725787,
+ "mean_token_accuracy": 0.6469839760102332,
+ "rewards/chosen": -0.5587615105054283,
+ "rewards/rejected": -1.562202037232928,
+ "rewards/accuracies": 0.8,
+ "rewards/margins": 1.003440523883328,
+ "logps/chosen": -65.64231182932853,
+ "logps/rejected": -58.95616222620011,
+ "epoch": 0.24894974327057726,
+ "step": 240
+ },
+ {
+ "loss": 0.48830270767211914,
+ "grad_norm": 0.010347820818424225,
+ "learning_rate": 7.419689119170985e-06,
+ "entropy": 1.3876401880290359,
+ "num_tokens": 3629401.0,
+ "logits/chosen": -0.6944931846154596,
+ "logits/rejected": -0.5525909304478883,
+ "mean_token_accuracy": 0.6595196689106524,
+ "rewards/chosen": -0.5708247896161629,
+ "rewards/rejected": -1.5276743739199445,
+ "rewards/accuracies": 0.7825,
+ "rewards/margins": 0.9568495863489807,
+ "logps/chosen": -65.0908039200306,
+ "logps/rejected": -59.28663974046707,
+ "epoch": 0.2593226492401846,
+ "step": 250
+ },
+ {
+ "eval_loss": 0.586716890335083,
+ "eval_runtime": 218.1079,
+ "eval_samples_per_second": 4.585,
+ "eval_steps_per_second": 4.585,
+ "eval_entropy": 0.9768514372222126,
+ "eval_num_tokens": 3629401.0,
+ "eval_logits/chosen": -0.288027065089679,
+ "eval_logits/rejected": -0.1875060971678181,
+ "eval_mean_token_accuracy": 0.715995571538806,
+ "eval_rewards/chosen": -0.13773577162688888,
+ "eval_rewards/rejected": -0.4831975232921541,
+ "eval_rewards/accuracies": 0.7,
+ "eval_rewards/margins": 0.3454617517972365,
+ "eval_logps/chosen": -51.550344535827634,
+ "eval_logps/rejected": -57.47184686470032,
+ "epoch": 0.2593226492401846,
+ "step": 250
+ },
+ {
+ "loss": 0.5234602928161621,
+ "grad_norm": 0.01417848002165556,
+ "learning_rate": 7.3160621761658035e-06,
+ "entropy": 1.4126720386464149,
+ "num_tokens": 3776772.0,
+ "logits/chosen": -0.6336535534010754,
+ "logits/rejected": -0.5590520939183056,
+ "mean_token_accuracy": 0.6559896361455322,
+ "rewards/chosen": -0.607539504897577,
+ "rewards/rejected": -1.5568346106001991,
+ "rewards/accuracies": 0.76625,
+ "rewards/margins": 0.9492951015941798,
+ "logps/chosen": -67.75836271762847,
+ "logps/rejected": -60.43963164687157,
+ "epoch": 0.26969555520979205,
+ "step": 260
+ },
+ {
+ "loss": 0.45171551704406737,
+ "grad_norm": 0.008155270479619503,
+ "learning_rate": 7.212435233160623e-06,
+ "entropy": 1.386252193665132,
+ "num_tokens": 3918594.0,
+ "logits/chosen": -0.692322886394876,
+ "logits/rejected": -0.5863808643424501,
+ "mean_token_accuracy": 0.6557753992639482,
+ "rewards/chosen": -0.4844140848268034,
+ "rewards/rejected": -1.5001980412582634,
+ "rewards/accuracies": 0.8075,
+ "rewards/margins": 1.0157839558646082,
+ "logps/chosen": -60.645376416444776,
+ "logps/rejected": -56.63234967470169,
+ "epoch": 0.28006846117939943,
+ "step": 270
+ },
+ {
+ "loss": 0.4645240306854248,
+ "grad_norm": 0.010044134221971035,
+ "learning_rate": 7.108808290155441e-06,
+ "entropy": 1.4262400729209184,
+ "num_tokens": 4064591.0,
+ "logits/chosen": -0.6778223895913023,
+ "logits/rejected": -0.5782403406452822,
+ "mean_token_accuracy": 0.6480579270608723,
+ "rewards/chosen": -0.4780267339639249,
+ "rewards/rejected": -1.5043474953982514,
+ "rewards/accuracies": 0.81,
+ "rewards/margins": 1.0263207618752495,
+ "logps/chosen": -66.15268970012664,
+ "logps/rejected": -56.97477602481842,
+ "epoch": 0.2904413671490068,
+ "step": 280
+ },
+ {
+ "loss": 0.48730764389038084,
+ "grad_norm": 0.011102184653282166,
+ "learning_rate": 7.005181347150259e-06,
+ "entropy": 1.3913073570653796,
+ "num_tokens": 4210880.0,
+ "logits/chosen": -0.6611820471311323,
+ "logits/rejected": -0.5711755343264715,
+ "mean_token_accuracy": 0.6609016039222478,
+ "rewards/chosen": -0.5330563006985176,
+ "rewards/rejected": -1.4805517839052482,
+ "rewards/accuracies": 0.79125,
+ "rewards/margins": 0.9474954811111093,
+ "logps/chosen": -64.55881326556205,
+ "logps/rejected": -58.55792887210846,
+ "epoch": 0.3008142731186142,
+ "step": 290
+ },
+ {
+ "loss": 0.4731907367706299,
+ "grad_norm": 0.008548606187105179,
+ "learning_rate": 6.9015544041450784e-06,
+ "entropy": 1.3624146432522684,
+ "num_tokens": 4356383.0,
+ "logits/chosen": -0.6928838046169237,
+ "logits/rejected": -0.6042798964782825,
+ "mean_token_accuracy": 0.6615170135349036,
+ "rewards/chosen": -0.562419148215231,
+ "rewards/rejected": -1.5397747305584197,
+ "rewards/accuracies": 0.8075,
+ "rewards/margins": 0.9773555782041512,
+ "logps/chosen": -63.553271045684816,
+ "logps/rejected": -58.97647937297821,
+ "epoch": 0.3111871790882216,
+ "step": 300
+ },
+ {
+ "loss": 0.49598259925842286,
+ "grad_norm": 0.007993742823600769,
+ "learning_rate": 6.797927461139897e-06,
+ "entropy": 1.3700548317469656,
+ "num_tokens": 4504654.0,
+ "logits/chosen": -0.644779540875669,
+ "logits/rejected": -0.5648210397573508,
+ "mean_token_accuracy": 0.6573794655874372,
+ "rewards/chosen": -0.5243248527575634,
+ "rewards/rejected": -1.5287844626943115,
+ "rewards/accuracies": 0.78625,
+ "rewards/margins": 1.00445960723795,
+ "logps/chosen": -64.00666294693947,
+ "logps/rejected": -61.50243379831314,
+ "epoch": 0.32156008505782896,
+ "step": 310
+ },
+ {
+ "loss": 0.4658979415893555,
+ "grad_norm": 0.010582108981907368,
+ "learning_rate": 6.6943005181347155e-06,
+ "entropy": 1.377428816948086,
+ "num_tokens": 4649289.0,
+ "logits/chosen": -0.6946231373321079,
+ "logits/rejected": -0.615736163527076,
+ "mean_token_accuracy": 0.6496203068085015,
+ "rewards/chosen": -0.5053178700394346,
+ "rewards/rejected": -1.5253387601411668,
+ "rewards/accuracies": 0.7975,
+ "rewards/margins": 1.020020889963489,
+ "logps/chosen": -65.05610402703286,
+ "logps/rejected": -58.5456183898449,
+ "epoch": 0.33193299102743634,
+ "step": 320
+ },
+ {
+ "loss": 0.4793886661529541,
+ "grad_norm": 0.010722431354224682,
+ "learning_rate": 6.590673575129535e-06,
+ "entropy": 1.4115223482623696,
+ "num_tokens": 4794149.0,
+ "logits/chosen": -0.640069723947673,
+ "logits/rejected": -0.5809591451997832,
+ "mean_token_accuracy": 0.6453945213742555,
+ "rewards/chosen": -0.495894172671542,
+ "rewards/rejected": -1.5554326001886511,
+ "rewards/accuracies": 0.77875,
+ "rewards/margins": 1.059538428708911,
+ "logps/chosen": -64.34471502423287,
+ "logps/rejected": -60.021323671340944,
+ "epoch": 0.3423058969970437,
+ "step": 330
+ },
+ {
+ "loss": 0.4895618438720703,
+ "grad_norm": 0.012254049070179462,
+ "learning_rate": 6.487046632124353e-06,
+ "entropy": 1.341764758527279,
+ "num_tokens": 4938961.0,
+ "logits/chosen": -0.6740448118186494,
+ "logits/rejected": -0.6176664939637678,
+ "mean_token_accuracy": 0.6506805537641048,
+ "rewards/chosen": -0.5430054054186622,
+ "rewards/rejected": -1.5190805373876355,
+ "rewards/accuracies": 0.78125,
+ "rewards/margins": 0.9760751294251532,
+ "logps/chosen": -62.98277623295784,
+ "logps/rejected": -58.627308850288394,
+ "epoch": 0.3526788029666511,
+ "step": 340
+ },
+ {
+ "loss": 0.4906949996948242,
+ "grad_norm": 0.012603401206433773,
+ "learning_rate": 6.383419689119171e-06,
+ "entropy": 1.335228986721486,
+ "num_tokens": 5085185.0,
+ "logits/chosen": -0.6529947110376427,
+ "logits/rejected": -0.6009205883950112,
+ "mean_token_accuracy": 0.6537076928000897,
+ "rewards/chosen": -0.5141149228686117,
+ "rewards/rejected": -1.5397958215233667,
+ "rewards/accuracies": 0.78375,
+ "rewards/margins": 1.0256808973429725,
+ "logps/chosen": -65.80777725458145,
+ "logps/rejected": -60.204533588886264,
+ "epoch": 0.3630517089362585,
+ "step": 350
+ },
+ {
+ "loss": 0.4906170845031738,
+ "grad_norm": 0.013215990737080574,
+ "learning_rate": 6.2797927461139905e-06,
+ "entropy": 1.2594143666606397,
+ "num_tokens": 5230359.0,
+ "logits/chosen": -0.7363216904006744,
+ "logits/rejected": -0.6437616409973009,
+ "mean_token_accuracy": 0.6658474483340978,
+ "rewards/chosen": -0.5248335571045755,
+ "rewards/rejected": -1.505687792309327,
+ "rewards/accuracies": 0.78,
+ "rewards/margins": 0.9808542363531888,
+ "logps/chosen": -61.92662429690361,
+ "logps/rejected": -57.71264513731003,
+ "epoch": 0.37342461490586587,
+ "step": 360
+ },
+ {
+ "loss": 0.47084531784057615,
+ "grad_norm": 0.010505126789212227,
+ "learning_rate": 6.176165803108809e-06,
+ "entropy": 1.3553831833507866,
+ "num_tokens": 5376488.0,
+ "logits/chosen": -0.6449173292890904,
+ "logits/rejected": -0.5551785184102145,
+ "mean_token_accuracy": 0.6537860439531505,
+ "rewards/chosen": -0.4608478151052259,
+ "rewards/rejected": -1.484516432384262,
+ "rewards/accuracies": 0.79125,
+ "rewards/margins": 1.0236686167120934,
+ "logps/chosen": -66.82504806637763,
+ "logps/rejected": -58.22365792274475,
+ "epoch": 0.38379752087547325,
+ "step": 370
+ },
+ {
+ "eval_loss": 0.5689035654067993,
+ "eval_runtime": 250.9045,
+ "eval_samples_per_second": 3.986,
+ "eval_steps_per_second": 3.986,
+ "eval_entropy": 0.9553070103861392,
+ "eval_num_tokens": 5448735.0,
+ "eval_logits/chosen": -0.2716756952684425,
+ "eval_logits/rejected": -0.16629699776825788,
+ "eval_mean_token_accuracy": 0.7177043009251356,
+ "eval_rewards/chosen": -0.11320603249309352,
+ "eval_rewards/rejected": -0.5053918305169282,
+ "eval_rewards/accuracies": 0.725,
+ "eval_rewards/margins": 0.392185798952356,
+ "eval_logps/chosen": -51.3050471496582,
+ "eval_logps/rejected": -57.69378993988037,
+ "epoch": 0.38898397386027694,
+ "step": 375
+ },
+ {
+ "loss": 0.45557408332824706,
+ "grad_norm": 0.010905127041041851,
+ "learning_rate": 6.0725388601036275e-06,
+ "entropy": 1.3120742352865635,
+ "num_tokens": 5521548.0,
+ "logits/chosen": -0.6177198584454004,
+ "logits/rejected": -0.5450247570549444,
+ "mean_token_accuracy": 0.6573154540918767,
+ "rewards/chosen": -0.3887985857592139,
+ "rewards/rejected": -1.4611027611684404,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.0723041738849133,
+ "logps/chosen": -62.481010044813154,
+ "logps/rejected": -56.344697498083114,
+ "epoch": 0.39417042684508063,
+ "step": 380
+ },
+ {
+ "loss": 0.4662487983703613,
+ "grad_norm": 0.010954899713397026,
+ "learning_rate": 5.968911917098445e-06,
+ "entropy": 1.345905083543621,
+ "num_tokens": 5667800.0,
+ "logits/chosen": -0.5844921973924755,
+ "logits/rejected": -0.48380765844974577,
+ "mean_token_accuracy": 0.6540606117062271,
+ "rewards/chosen": -0.390090519907244,
+ "rewards/rejected": -1.462948713658261,
+ "rewards/accuracies": 0.8025,
+ "rewards/margins": 1.07285819449462,
+ "logps/chosen": -64.31483766436577,
+ "logps/rejected": -59.39098523259163,
+ "epoch": 0.404543332814688,
+ "step": 390
+ },
+ {
+ "loss": 0.46701774597167967,
+ "grad_norm": 0.010829386301338673,
+ "learning_rate": 5.865284974093265e-06,
+ "entropy": 1.3651179377734661,
+ "num_tokens": 5813205.0,
+ "logits/chosen": -0.5822485740098883,
+ "logits/rejected": -0.5166833227123047,
+ "mean_token_accuracy": 0.6551306374650449,
+ "rewards/chosen": -0.35834275034227175,
+ "rewards/rejected": -1.365874043785443,
+ "rewards/accuracies": 0.78875,
+ "rewards/margins": 1.0075312922801822,
+ "logps/chosen": -63.65349508166313,
+ "logps/rejected": -58.2333509349823,
+ "epoch": 0.4149162387842954,
+ "step": 400
+ },
+ {
+ "loss": 0.46943206787109376,
+ "grad_norm": 0.012473917566239834,
+ "learning_rate": 5.761658031088083e-06,
+ "entropy": 1.3225441289972515,
+ "num_tokens": 5959181.0,
+ "logits/chosen": -0.6183900032329287,
+ "logits/rejected": -0.5336649034548987,
+ "mean_token_accuracy": 0.6569803632609547,
+ "rewards/chosen": -0.35909413290792147,
+ "rewards/rejected": -1.3612432872604223,
+ "rewards/accuracies": 0.795,
+ "rewards/margins": 1.002149153780192,
+ "logps/chosen": -62.85942430019379,
+ "logps/rejected": -57.029898535013196,
+ "epoch": 0.4252891447539028,
+ "step": 410
+ },
+ {
+ "loss": 0.43849844932556153,
+ "grad_norm": 0.008844634518027306,
+ "learning_rate": 5.658031088082902e-06,
+ "entropy": 1.3128622455336154,
+ "num_tokens": 6104961.0,
+ "logits/chosen": -0.6210691174054768,
+ "logits/rejected": -0.576289690726631,
+ "mean_token_accuracy": 0.6577657607197761,
+ "rewards/chosen": -0.3327711314160842,
+ "rewards/rejected": -1.4455869932868517,
+ "rewards/accuracies": 0.825,
+ "rewards/margins": 1.1128158613247796,
+ "logps/chosen": -64.98676935911179,
+ "logps/rejected": -57.15970268011093,
+ "epoch": 0.4356620507235102,
+ "step": 420
+ },
+ {
+ "loss": 0.43562884330749513,
+ "grad_norm": 0.00888186227530241,
+ "learning_rate": 5.554404145077721e-06,
+ "entropy": 1.3373534345161169,
+ "num_tokens": 6251772.0,
+ "logits/chosen": -0.5417993838933208,
+ "logits/rejected": -0.4885455063213445,
+ "mean_token_accuracy": 0.6521896417625248,
+ "rewards/chosen": -0.39902110468392493,
+ "rewards/rejected": -1.5569669758284,
+ "rewards/accuracies": 0.83625,
+ "rewards/margins": 1.1579458705382422,
+ "logps/chosen": -63.90601393699646,
+ "logps/rejected": -58.24345481872559,
+ "epoch": 0.4460349566931176,
+ "step": 430
+ },
+ {
+ "loss": 0.4367820262908936,
+ "grad_norm": 0.011835568584501743,
+ "learning_rate": 5.4507772020725395e-06,
+ "entropy": 1.302687416060362,
+ "num_tokens": 6397498.0,
+ "logits/chosen": -0.6515899833172267,
+ "logits/rejected": -0.5672783053215388,
+ "mean_token_accuracy": 0.6643678575474768,
+ "rewards/chosen": -0.4223745361570036,
+ "rewards/rejected": -1.5821643100841902,
+ "rewards/accuracies": 0.8225,
+ "rewards/margins": 1.1597897751070558,
+ "logps/chosen": -62.087783161401745,
+ "logps/rejected": -58.09040701627731,
+ "epoch": 0.456407862662725,
+ "step": 440
+ },
+ {
+ "loss": 0.4615288257598877,
+ "grad_norm": 0.011570082977414131,
+ "learning_rate": 5.347150259067357e-06,
+ "entropy": 1.3379024799168109,
+ "num_tokens": 6544292.0,
+ "logits/chosen": -0.6416285881723627,
+ "logits/rejected": -0.5313172360798328,
+ "mean_token_accuracy": 0.6538208884559572,
+ "rewards/chosen": -0.43649624643381685,
+ "rewards/rejected": -1.6286552884728007,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.192159042903222,
+ "logps/chosen": -62.45707392811775,
+ "logps/rejected": -60.70825082540512,
+ "epoch": 0.46678076863233237,
+ "step": 450
+ },
+ {
+ "loss": 0.4522082328796387,
+ "grad_norm": 0.01289224810898304,
+ "learning_rate": 5.243523316062177e-06,
+ "entropy": 1.2857942930911668,
+ "num_tokens": 6688703.0,
+ "logits/chosen": -0.6984393716749847,
+ "logits/rejected": -0.616657341014624,
+ "mean_token_accuracy": 0.6569575572758913,
+ "rewards/chosen": -0.4177262162156694,
+ "rewards/rejected": -1.5946978869343729,
+ "rewards/accuracies": 0.82125,
+ "rewards/margins": 1.1769716703612358,
+ "logps/chosen": -62.88765250682831,
+ "logps/rejected": -57.941710426807404,
+ "epoch": 0.47715367460193975,
+ "step": 460
+ },
+ {
+ "loss": 0.4684887886047363,
+ "grad_norm": 0.013750020414590836,
+ "learning_rate": 5.139896373056995e-06,
+ "entropy": 1.2873771674977617,
+ "num_tokens": 6831976.0,
+ "logits/chosen": -0.7360928165548556,
+ "logits/rejected": -0.6532006934035273,
+ "mean_token_accuracy": 0.649969874471426,
+ "rewards/chosen": -0.45782311129412845,
+ "rewards/rejected": -1.5659245843085228,
+ "rewards/accuracies": 0.7975,
+ "rewards/margins": 1.1081014727987348,
+ "logps/chosen": -62.05344919681549,
+ "logps/rejected": -57.16824733495712,
+ "epoch": 0.48752658057154713,
+ "step": 470
+ },
+ {
+ "loss": 0.4470407962799072,
+ "grad_norm": 0.010775277391076088,
+ "learning_rate": 5.036269430051814e-06,
+ "entropy": 1.3075012436602265,
+ "num_tokens": 6979562.0,
+ "logits/chosen": -0.6950100893212269,
+ "logits/rejected": -0.6174856653398093,
+ "mean_token_accuracy": 0.6546831333450973,
+ "rewards/chosen": -0.4612816582483902,
+ "rewards/rejected": -1.6060521737975069,
+ "rewards/accuracies": 0.79625,
+ "rewards/margins": 1.1447705142386257,
+ "logps/chosen": -65.90533972740174,
+ "logps/rejected": -60.80634157061577,
+ "epoch": 0.4978994865411545,
+ "step": 480
+ },
+ {
+ "loss": 0.44890718460083007,
+ "grad_norm": 0.010780111886560917,
+ "learning_rate": 4.932642487046633e-06,
+ "entropy": 1.2887511976994575,
+ "num_tokens": 7126641.0,
+ "logits/chosen": -0.6761976126203127,
+ "logits/rejected": -0.6278953113861633,
+ "mean_token_accuracy": 0.6634333984181285,
+ "rewards/chosen": -0.4675208572049451,
+ "rewards/rejected": -1.6545327439898392,
+ "rewards/accuracies": 0.82375,
+ "rewards/margins": 1.1870118879154326,
+ "logps/chosen": -63.26052482247353,
+ "logps/rejected": -61.286433795690535,
+ "epoch": 0.5082723925107618,
+ "step": 490
+ },
+ {
+ "loss": 0.4814422607421875,
+ "grad_norm": 0.012935275211930275,
+ "learning_rate": 4.829015544041451e-06,
+ "entropy": 1.27828567199409,
+ "num_tokens": 7271511.0,
+ "logits/chosen": -0.7259033669003568,
+ "logits/rejected": -0.6351857639096569,
+ "mean_token_accuracy": 0.6566927696019411,
+ "rewards/chosen": -0.43330920343832985,
+ "rewards/rejected": -1.5484318724216428,
+ "rewards/accuracies": 0.78875,
+ "rewards/margins": 1.1151226695766672,
+ "logps/chosen": -61.74719344258308,
+ "logps/rejected": -59.84465143203735,
+ "epoch": 0.5186452984803692,
+ "step": 500
+ },
+ {
+ "eval_loss": 0.5487673282623291,
+ "eval_runtime": 282.9922,
+ "eval_samples_per_second": 3.534,
+ "eval_steps_per_second": 3.534,
+ "eval_entropy": 0.9428232955671847,
+ "eval_num_tokens": 7271511.0,
+ "eval_logits/chosen": -0.28059689932805515,
+ "eval_logits/rejected": -0.16151944836492374,
+ "eval_mean_token_accuracy": 0.7168366620391607,
+ "eval_rewards/chosen": -0.1563246784962248,
+ "eval_rewards/rejected": -0.6193268298726762,
+ "eval_rewards/accuracies": 0.754,
+ "eval_rewards/margins": 0.4630021521952003,
+ "eval_logps/chosen": -51.73623360443115,
+ "eval_logps/rejected": -58.83313990974426,
+ "epoch": 0.5186452984803692,
+ "step": 500
+ },
+ {
+ "loss": 0.4656700134277344,
+ "grad_norm": 0.01303118746727705,
+ "learning_rate": 4.72538860103627e-06,
+ "entropy": 1.2692697253311054,
+ "num_tokens": 7417281.0,
+ "logits/chosen": -0.6733742115379525,
+ "logits/rejected": -0.6193994454969072,
+ "mean_token_accuracy": 0.6586294612661004,
+ "rewards/chosen": -0.43541748865740376,
+ "rewards/rejected": -1.5032758536882467,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.0678583633713423,
+ "logps/chosen": -64.05021148204804,
+ "logps/rejected": -56.13307309627533,
+ "epoch": 0.5290182044499767,
+ "step": 510
+ },
+ {
+ "loss": 0.4864551067352295,
+ "grad_norm": 0.01242531556636095,
+ "learning_rate": 4.621761658031089e-06,
+ "entropy": 1.2640815615979955,
+ "num_tokens": 7560432.0,
+ "logits/chosen": -0.7002353169810469,
+ "logits/rejected": -0.6140356327426939,
+ "mean_token_accuracy": 0.6589579802565276,
+ "rewards/chosen": -0.38904498486765077,
+ "rewards/rejected": -1.4170939752570848,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 1.0280489912256598,
+ "logps/chosen": -60.270782203674315,
+ "logps/rejected": -56.58300987482071,
+ "epoch": 0.5393911104195841,
+ "step": 520
+ },
+ {
+ "loss": 0.4569260597229004,
+ "grad_norm": 0.01068835612386465,
+ "learning_rate": 4.518134715025907e-06,
+ "entropy": 1.3159204521216452,
+ "num_tokens": 7705977.0,
+ "logits/chosen": -0.6491517350356284,
+ "logits/rejected": -0.5585772213969517,
+ "mean_token_accuracy": 0.6497207802906633,
+ "rewards/chosen": -0.3849831897905824,
+ "rewards/rejected": -1.5199620288111328,
+ "rewards/accuracies": 0.82125,
+ "rewards/margins": 1.1349788387073203,
+ "logps/chosen": -63.44159636735916,
+ "logps/rejected": -59.03181514501571,
+ "epoch": 0.5497640163891915,
+ "step": 530
+ },
+ {
+ "loss": 0.4587429046630859,
+ "grad_norm": 0.013803384266793728,
+ "learning_rate": 4.414507772020726e-06,
+ "entropy": 1.291237823315896,
+ "num_tokens": 7852079.0,
+ "logits/chosen": -0.6414628461310453,
+ "logits/rejected": -0.5613248039563115,
+ "mean_token_accuracy": 0.6623153394274414,
+ "rewards/chosen": -0.4346328362337226,
+ "rewards/rejected": -1.540268263059552,
+ "rewards/accuracies": 0.8125,
+ "rewards/margins": 1.1056354277441278,
+ "logps/chosen": -62.11032348752022,
+ "logps/rejected": -58.996895933151244,
+ "epoch": 0.5601369223587989,
+ "step": 540
+ },
+ {
+ "loss": 0.46028594970703124,
+ "grad_norm": 0.013468707911670208,
+ "learning_rate": 4.310880829015544e-06,
+ "entropy": 1.3394903557561337,
+ "num_tokens": 7999115.0,
+ "logits/chosen": -0.5828356510219183,
+ "logits/rejected": -0.5372745959336506,
+ "mean_token_accuracy": 0.6568853073753417,
+ "rewards/chosen": -0.4290946354267362,
+ "rewards/rejected": -1.5929214715841227,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.1638268361473456,
+ "logps/chosen": -65.59857477068901,
+ "logps/rejected": -62.133251576423646,
+ "epoch": 0.5705098283284062,
+ "step": 550
+ },
+ {
+ "loss": 0.45781307220458983,
+ "grad_norm": 0.014243297278881073,
+ "learning_rate": 4.207253886010363e-06,
+ "entropy": 1.2652439445722847,
+ "num_tokens": 8145639.0,
+ "logits/chosen": -0.6962334313624986,
+ "logits/rejected": -0.6396616762561939,
+ "mean_token_accuracy": 0.6539523831009865,
+ "rewards/chosen": -0.42706657521594027,
+ "rewards/rejected": -1.5893809160019736,
+ "rewards/accuracies": 0.795,
+ "rewards/margins": 1.16231434000656,
+ "logps/chosen": -63.68221395015716,
+ "logps/rejected": -58.812913880348205,
+ "epoch": 0.5808827342980136,
+ "step": 560
+ },
+ {
+ "loss": 0.45180602073669435,
+ "grad_norm": 0.011750273406505585,
+ "learning_rate": 4.103626943005182e-06,
+ "entropy": 1.271689679301344,
+ "num_tokens": 8289932.0,
+ "logits/chosen": -0.6709009276653929,
+ "logits/rejected": -0.6110994225113662,
+ "mean_token_accuracy": 0.6528454353101552,
+ "rewards/chosen": -0.43575992634396243,
+ "rewards/rejected": -1.544044768281892,
+ "rewards/accuracies": 0.81875,
+ "rewards/margins": 1.1082848401460796,
+ "logps/chosen": -63.166086630821226,
+ "logps/rejected": -56.468742752075194,
+ "epoch": 0.591255640267621,
+ "step": 570
+ },
+ {
+ "loss": 0.46036481857299805,
+ "grad_norm": 0.014452456496655941,
+ "learning_rate": 4.000000000000001e-06,
+ "entropy": 1.254093002313748,
+ "num_tokens": 8433751.0,
+ "logits/chosen": -0.7479031936089587,
+ "logits/rejected": -0.660857903954597,
+ "mean_token_accuracy": 0.66091203879565,
+ "rewards/chosen": -0.4369000616581616,
+ "rewards/rejected": -1.5926436452555937,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.1557435841672123,
+ "logps/chosen": -60.516501158475876,
+ "logps/rejected": -57.770808889865876,
+ "epoch": 0.6016285462372284,
+ "step": 580
+ },
+ {
+ "loss": 0.4958089828491211,
+ "grad_norm": 0.014941485598683357,
+ "learning_rate": 3.896373056994819e-06,
+ "entropy": 1.2861195527855307,
+ "num_tokens": 8580235.0,
+ "logits/chosen": -0.7134623334720516,
+ "logits/rejected": -0.6337572470141326,
+ "mean_token_accuracy": 0.6624758186563849,
+ "rewards/chosen": -0.48507625065831234,
+ "rewards/rejected": -1.5025616684707348,
+ "rewards/accuracies": 0.7675,
+ "rewards/margins": 1.0174854172300547,
+ "logps/chosen": -64.41009007811546,
+ "logps/rejected": -59.815929347276686,
+ "epoch": 0.6120014522068358,
+ "step": 590
+ },
+ {
+ "loss": 0.47800679206848146,
+ "grad_norm": 0.01430908776819706,
+ "learning_rate": 3.7927461139896377e-06,
+ "entropy": 1.2737568323127926,
+ "num_tokens": 8723363.0,
+ "logits/chosen": -0.6718185966589912,
+ "logits/rejected": -0.5801828286442322,
+ "mean_token_accuracy": 0.6622481289878488,
+ "rewards/chosen": -0.38251684666481195,
+ "rewards/rejected": -1.4414349760363712,
+ "rewards/accuracies": 0.78375,
+ "rewards/margins": 1.0589181298250332,
+ "logps/chosen": -59.11278188347816,
+ "logps/rejected": -57.41570061087609,
+ "epoch": 0.6223743581764432,
+ "step": 600
+ },
+ {
+ "loss": 0.4607038974761963,
+ "grad_norm": 0.016786962747573853,
+ "learning_rate": 3.6891191709844567e-06,
+ "entropy": 1.3031947114598006,
+ "num_tokens": 8867843.0,
+ "logits/chosen": -0.660966853554005,
+ "logits/rejected": -0.582685814465534,
+ "mean_token_accuracy": 0.6579678988829255,
+ "rewards/chosen": -0.3749055393272283,
+ "rewards/rejected": -1.4482067039191133,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.0733011648245157,
+ "logps/chosen": -59.77094477295876,
+ "logps/rejected": -58.26709199547768,
+ "epoch": 0.6327472641460505,
+ "step": 610
+ },
+ {
+ "loss": 0.4435013771057129,
+ "grad_norm": 0.013243520632386208,
+ "learning_rate": 3.5854922279792748e-06,
+ "entropy": 1.2442782195843756,
+ "num_tokens": 9012429.0,
+ "logits/chosen": -0.7155245964218712,
+ "logits/rejected": -0.62228141826014,
+ "mean_token_accuracy": 0.6603258750028909,
+ "rewards/chosen": -0.35220212864573114,
+ "rewards/rejected": -1.49272357024136,
+ "rewards/accuracies": 0.8175,
+ "rewards/margins": 1.1405214420426637,
+ "logps/chosen": -58.541049842834475,
+ "logps/rejected": -57.54325157284737,
+ "epoch": 0.6431201701156579,
+ "step": 620
+ },
+ {
+ "eval_loss": 0.5402435660362244,
+ "eval_runtime": 293.8352,
+ "eval_samples_per_second": 3.403,
+ "eval_steps_per_second": 3.403,
+ "eval_entropy": 0.9412421704679728,
+ "eval_num_tokens": 9084893.0,
+ "eval_logits/chosen": -0.2831247920504241,
+ "eval_logits/rejected": -0.15805534941173774,
+ "eval_mean_token_accuracy": 0.7173487603366375,
+ "eval_rewards/chosen": -0.163662012138695,
+ "eval_rewards/rejected": -0.6573172951535962,
+ "eval_rewards/accuracies": 0.763,
+ "eval_rewards/margins": 0.49365528268739584,
+ "eval_logps/chosen": -51.8096069393158,
+ "eval_logps/rejected": -59.21304455375672,
+ "epoch": 0.6483066231004616,
+ "step": 625
+ },
+ {
+ "loss": 0.45146970748901366,
+ "grad_norm": 0.011234515346586704,
+ "learning_rate": 3.4818652849740937e-06,
+ "entropy": 1.2889114275900646,
+ "num_tokens": 9158703.0,
+ "logits/chosen": -0.6542699654733424,
+ "logits/rejected": -0.5822428945211472,
+ "mean_token_accuracy": 0.6584817282296718,
+ "rewards/chosen": -0.3719235458994808,
+ "rewards/rejected": -1.5630993095623853,
+ "rewards/accuracies": 0.8,
+ "rewards/margins": 1.1911757623543964,
+ "logps/chosen": -61.62524257659912,
+ "logps/rejected": -59.77060217618942,
+ "epoch": 0.6534930760852653,
+ "step": 630
+ },
+ {
+ "loss": 0.45473732948303225,
+ "grad_norm": 0.01427957508713007,
+ "learning_rate": 3.3782383419689123e-06,
+ "entropy": 1.270301983555546,
+ "num_tokens": 9303628.0,
+ "logits/chosen": -0.6727831000063272,
+ "logits/rejected": -0.5958556125497929,
+ "mean_token_accuracy": 0.6592282411269843,
+ "rewards/chosen": -0.39630664114709363,
+ "rewards/rejected": -1.5085867938393493,
+ "rewards/accuracies": 0.81125,
+ "rewards/margins": 1.112280152433086,
+ "logps/chosen": -60.25154410600662,
+ "logps/rejected": -59.81609448671341,
+ "epoch": 0.6638659820548727,
+ "step": 640
+ },
+ {
+ "loss": 0.4724769115447998,
+ "grad_norm": 0.011104694567620754,
+ "learning_rate": 3.274611398963731e-06,
+ "entropy": 1.2825965376850217,
+ "num_tokens": 9447859.0,
+ "logits/chosen": -0.6886272934207637,
+ "logits/rejected": -0.6191571248326155,
+ "mean_token_accuracy": 0.657224724534899,
+ "rewards/chosen": -0.39751313600787397,
+ "rewards/rejected": -1.501351127484304,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.1038379945326597,
+ "logps/chosen": -61.231552537679676,
+ "logps/rejected": -57.24312862277031,
+ "epoch": 0.6742388880244801,
+ "step": 650
+ },
+ {
+ "loss": 0.48994994163513184,
+ "grad_norm": 0.015749365091323853,
+ "learning_rate": 3.1709844559585493e-06,
+ "entropy": 1.2718282664287834,
+ "num_tokens": 9591275.0,
+ "logits/chosen": -0.7000141689252796,
+ "logits/rejected": -0.6344551697912493,
+ "mean_token_accuracy": 0.6556407183595002,
+ "rewards/chosen": -0.3735031143521701,
+ "rewards/rejected": -1.4077299095626221,
+ "rewards/accuracies": 0.8075,
+ "rewards/margins": 1.034226797488518,
+ "logps/chosen": -60.32739723324776,
+ "logps/rejected": -56.30893276691437,
+ "epoch": 0.6846117939940874,
+ "step": 660
+ },
+ {
+ "loss": 0.42113280296325684,
+ "grad_norm": 0.011937125585973263,
+ "learning_rate": 3.0673575129533683e-06,
+ "entropy": 1.27422906415537,
+ "num_tokens": 9737336.0,
+ "logits/chosen": -0.6675869420516058,
+ "logits/rejected": -0.6171588368781001,
+ "mean_token_accuracy": 0.6570126633532345,
+ "rewards/chosen": -0.3044447978468088,
+ "rewards/rejected": -1.5031647271365,
+ "rewards/accuracies": 0.8175,
+ "rewards/margins": 1.1987199306860565,
+ "logps/chosen": -62.614556882381436,
+ "logps/rejected": -57.67475826740265,
+ "epoch": 0.6949846999636948,
+ "step": 670
+ },
+ {
+ "loss": 0.4509533405303955,
+ "grad_norm": 0.014425016939640045,
+ "learning_rate": 2.963730569948187e-06,
+ "entropy": 1.2917946016066708,
+ "num_tokens": 9884203.0,
+ "logits/chosen": -0.6209272877296821,
+ "logits/rejected": -0.5547300822758102,
+ "mean_token_accuracy": 0.6614484623633325,
+ "rewards/chosen": -0.34687376230140216,
+ "rewards/rejected": -1.5009351192926987,
+ "rewards/accuracies": 0.8025,
+ "rewards/margins": 1.1540613523963839,
+ "logps/chosen": -63.675368639230726,
+ "logps/rejected": -59.546725879907605,
+ "epoch": 0.7053576059333022,
+ "step": 680
+ },
+ {
+ "loss": 0.49210338592529296,
+ "grad_norm": 0.01453041099011898,
+ "learning_rate": 2.8601036269430053e-06,
+ "entropy": 1.318970390278846,
+ "num_tokens": 10030490.0,
+ "logits/chosen": -0.6129511188439006,
+ "logits/rejected": -0.5470711603742071,
+ "mean_token_accuracy": 0.6580428531672805,
+ "rewards/chosen": -0.35959291283600125,
+ "rewards/rejected": -1.394242706346704,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.0346497943252324,
+ "logps/chosen": -64.7718941605091,
+ "logps/rejected": -57.55288832068443,
+ "epoch": 0.7157305119029096,
+ "step": 690
+ },
+ {
+ "loss": 0.4248403549194336,
+ "grad_norm": 0.012130402959883213,
+ "learning_rate": 2.7564766839378243e-06,
+ "entropy": 1.2868076485674829,
+ "num_tokens": 10175134.0,
+ "logits/chosen": -0.6680388671827403,
+ "logits/rejected": -0.5829757325025796,
+ "mean_token_accuracy": 0.6597682436835021,
+ "rewards/chosen": -0.30255757274717326,
+ "rewards/rejected": -1.4632275151461362,
+ "rewards/accuracies": 0.82875,
+ "rewards/margins": 1.1606699449336155,
+ "logps/chosen": -62.52647110342979,
+ "logps/rejected": -54.934954075813295,
+ "epoch": 0.726103417872517,
+ "step": 700
+ },
+ {
+ "loss": 0.46398053169250486,
+ "grad_norm": 0.010899660177528858,
+ "learning_rate": 2.6528497409326424e-06,
+ "entropy": 1.3012964005907997,
+ "num_tokens": 10321605.0,
+ "logits/chosen": -0.6793537159810091,
+ "logits/rejected": -0.5945389351521791,
+ "mean_token_accuracy": 0.6582902568951249,
+ "rewards/chosen": -0.3586843095816721,
+ "rewards/rejected": -1.5040888605307554,
+ "rewards/accuracies": 0.78625,
+ "rewards/margins": 1.1454045504983514,
+ "logps/chosen": -66.0418261361122,
+ "logps/rejected": -57.98255445957184,
+ "epoch": 0.7364763238421244,
+ "step": 710
+ },
+ {
+ "loss": 0.45627450942993164,
+ "grad_norm": 0.015107187442481518,
+ "learning_rate": 2.5492227979274614e-06,
+ "entropy": 1.2992314287554472,
+ "num_tokens": 10464696.0,
+ "logits/chosen": -0.6788668626078029,
+ "logits/rejected": -0.598369878754869,
+ "mean_token_accuracy": 0.6590372899640351,
+ "rewards/chosen": -0.32335921899306413,
+ "rewards/rejected": -1.4580287650012178,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.1346695464709773,
+ "logps/chosen": -60.56386103153229,
+ "logps/rejected": -55.05920248985291,
+ "epoch": 0.7468492298117317,
+ "step": 720
+ },
+ {
+ "loss": 0.45744032859802247,
+ "grad_norm": 0.015074568800628185,
+ "learning_rate": 2.44559585492228e-06,
+ "entropy": 1.3187106101540849,
+ "num_tokens": 10611681.0,
+ "logits/chosen": -0.6193487567657021,
+ "logits/rejected": -0.5510364978587164,
+ "mean_token_accuracy": 0.6565000848285854,
+ "rewards/chosen": -0.3346571850046166,
+ "rewards/rejected": -1.5139837610156974,
+ "rewards/accuracies": 0.7975,
+ "rewards/margins": 1.1793265779595823,
+ "logps/chosen": -62.09045646429062,
+ "logps/rejected": -61.43362274646759,
+ "epoch": 0.7572221357813391,
+ "step": 730
+ },
+ {
+ "loss": 0.44286222457885743,
+ "grad_norm": 0.01326123159378767,
+ "learning_rate": 2.3419689119170984e-06,
+ "entropy": 1.327558269179426,
+ "num_tokens": 10756633.0,
+ "logits/chosen": -0.6719882830268841,
+ "logits/rejected": -0.5616402227001511,
+ "mean_token_accuracy": 0.6571272361278534,
+ "rewards/chosen": -0.33738715873330877,
+ "rewards/rejected": -1.5023427059937966,
+ "rewards/accuracies": 0.82125,
+ "rewards/margins": 1.1649555454589426,
+ "logps/chosen": -62.31321774363518,
+ "logps/rejected": -59.3358825302124,
+ "epoch": 0.7675950417509465,
+ "step": 740
+ },
+ {
+ "loss": 0.4650571823120117,
+ "grad_norm": 0.0126119963824749,
+ "learning_rate": 2.2383419689119174e-06,
+ "entropy": 1.29965307561215,
+ "num_tokens": 10905175.0,
+ "logits/chosen": -0.6533030424492793,
+ "logits/rejected": -0.5875300805405916,
+ "mean_token_accuracy": 0.6576582338660956,
+ "rewards/chosen": -0.40247532632221467,
+ "rewards/rejected": -1.5875308242870414,
+ "rewards/accuracies": 0.79,
+ "rewards/margins": 1.1850554993841798,
+ "logps/chosen": -65.17352761864662,
+ "logps/rejected": -62.21493496775627,
+ "epoch": 0.7779679477205539,
+ "step": 750
+ },
+ {
+ "eval_loss": 0.5359359383583069,
+ "eval_runtime": 320.8935,
+ "eval_samples_per_second": 3.116,
+ "eval_steps_per_second": 3.116,
+ "eval_entropy": 0.9404631896130741,
+ "eval_num_tokens": 10905175.0,
+ "eval_logits/chosen": -0.2688203306854898,
+ "eval_logits/rejected": -0.1410600388922386,
+ "eval_mean_token_accuracy": 0.7175761694908143,
+ "eval_rewards/chosen": -0.1531370894421125,
+ "eval_rewards/rejected": -0.6597482364820425,
+ "eval_rewards/accuracies": 0.764,
+ "eval_rewards/margins": 0.5066111467555166,
+ "eval_logps/chosen": -51.70435771179199,
+ "eval_logps/rejected": -59.23735397338867,
+ "epoch": 0.7779679477205539,
+ "step": 750
+ },
+ {
+ "loss": 0.44441781044006345,
+ "grad_norm": 0.01466372236609459,
+ "learning_rate": 2.134715025906736e-06,
+ "entropy": 1.28976634433493,
+ "num_tokens": 11051293.0,
+ "logits/chosen": -0.7179963626211149,
+ "logits/rejected": -0.6301743637240872,
+ "mean_token_accuracy": 0.6519320147298276,
+ "rewards/chosen": -0.35903158226523374,
+ "rewards/rejected": -1.4595573616991169,
+ "rewards/accuracies": 0.805,
+ "rewards/margins": 1.100525778569281,
+ "logps/chosen": -62.626072100400926,
+ "logps/rejected": -58.10015594720841,
+ "epoch": 0.7883408536901613,
+ "step": 760
+ },
+ {
+ "loss": 0.4427034854888916,
+ "grad_norm": 0.012671389617025852,
+ "learning_rate": 2.0310880829015544e-06,
+ "entropy": 1.3044293180131354,
+ "num_tokens": 11200144.0,
+ "logits/chosen": -0.6538815830450716,
+ "logits/rejected": -0.5728917452531731,
+ "mean_token_accuracy": 0.6632946115918458,
+ "rewards/chosen": -0.37740983954085094,
+ "rewards/rejected": -1.6017498846648959,
+ "rewards/accuracies": 0.82375,
+ "rewards/margins": 1.2243400452192872,
+ "logps/chosen": -65.87450021147728,
+ "logps/rejected": -62.50144905328751,
+ "epoch": 0.7987137596597687,
+ "step": 770
+ },
+ {
+ "loss": 0.47789411544799804,
+ "grad_norm": 0.014209717512130737,
+ "learning_rate": 1.9274611398963734e-06,
+ "entropy": 1.2642307026335038,
+ "num_tokens": 11345001.0,
+ "logits/chosen": -0.7598325245824121,
+ "logits/rejected": -0.6345409980160763,
+ "mean_token_accuracy": 0.6561008535698056,
+ "rewards/chosen": -0.4101038860027188,
+ "rewards/rejected": -1.5055885665896493,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 1.09548467958346,
+ "logps/chosen": -60.939151479005815,
+ "logps/rejected": -59.33771441936493,
+ "epoch": 0.809086665629376,
+ "step": 780
+ },
+ {
+ "loss": 0.45084495544433595,
+ "grad_norm": 0.01159591879695654,
+ "learning_rate": 1.823834196891192e-06,
+ "entropy": 1.2825998100219294,
+ "num_tokens": 11490957.0,
+ "logits/chosen": -0.6798757077356125,
+ "logits/rejected": -0.5974159600128078,
+ "mean_token_accuracy": 0.6551897264830768,
+ "rewards/chosen": -0.37878839314620566,
+ "rewards/rejected": -1.565106635761622,
+ "rewards/accuracies": 0.81125,
+ "rewards/margins": 1.1863182406220585,
+ "logps/chosen": -62.11761864900589,
+ "logps/rejected": -58.61645050764084,
+ "epoch": 0.8194595715989834,
+ "step": 790
+ },
+ {
+ "loss": 0.4757333278656006,
+ "grad_norm": 0.01929500699043274,
+ "learning_rate": 1.7202072538860104e-06,
+ "entropy": 1.3137290544318967,
+ "num_tokens": 11635952.0,
+ "logits/chosen": -0.7116650574710064,
+ "logits/rejected": -0.6434065148170602,
+ "mean_token_accuracy": 0.6526090941950679,
+ "rewards/chosen": -0.39931287897430595,
+ "rewards/rejected": -1.51923489038134,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.119922012281604,
+ "logps/chosen": -63.77980515360832,
+ "logps/rejected": -58.51232698440552,
+ "epoch": 0.8298324775685908,
+ "step": 800
+ },
+ {
+ "loss": 0.4533553600311279,
+ "grad_norm": 0.011488179676234722,
+ "learning_rate": 1.6165803108808292e-06,
+ "entropy": 1.2781752744410186,
+ "num_tokens": 11781419.0,
+ "logits/chosen": -0.7322858283680784,
+ "logits/rejected": -0.6704393794115492,
+ "mean_token_accuracy": 0.6515787079930305,
+ "rewards/chosen": -0.3753399283361523,
+ "rewards/rejected": -1.5520857451565098,
+ "rewards/accuracies": 0.805,
+ "rewards/margins": 1.1767458136426285,
+ "logps/chosen": -62.6512784576416,
+ "logps/rejected": -59.1226301574707,
+ "epoch": 0.8402053835381982,
+ "step": 810
+ },
+ {
+ "loss": 0.459058952331543,
+ "grad_norm": 0.010664924047887325,
+ "learning_rate": 1.5129533678756477e-06,
+ "entropy": 1.2839164751721546,
+ "num_tokens": 11926234.0,
+ "logits/chosen": -0.668114556411545,
+ "logits/rejected": -0.6314948882239511,
+ "mean_token_accuracy": 0.6545133054628969,
+ "rewards/chosen": -0.3676813648158713,
+ "rewards/rejected": -1.5132245452463393,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.145543181069661,
+ "logps/chosen": -63.80598108768463,
+ "logps/rejected": -57.3190500497818,
+ "epoch": 0.8505782895078056,
+ "step": 820
+ },
+ {
+ "loss": 0.45433964729309084,
+ "grad_norm": 0.013902204111218452,
+ "learning_rate": 1.4093264248704663e-06,
+ "entropy": 1.2906162818288431,
+ "num_tokens": 12071032.0,
+ "logits/chosen": -0.6819364086266573,
+ "logits/rejected": -0.5817802480567199,
+ "mean_token_accuracy": 0.6650253617018461,
+ "rewards/chosen": -0.3574298277559137,
+ "rewards/rejected": -1.5355529067799216,
+ "rewards/accuracies": 0.82125,
+ "rewards/margins": 1.1781230779876932,
+ "logps/chosen": -59.579396767616274,
+ "logps/rejected": -58.291543385982514,
+ "epoch": 0.860951195477413,
+ "step": 830
+ },
+ {
+ "loss": 0.43956761360168456,
+ "grad_norm": 0.017366426065564156,
+ "learning_rate": 1.3056994818652852e-06,
+ "entropy": 1.3188584124017506,
+ "num_tokens": 12216650.0,
+ "logits/chosen": -0.6219026821907607,
+ "logits/rejected": -0.5560462281839198,
+ "mean_token_accuracy": 0.6576781215891242,
+ "rewards/chosen": -0.3260511834644603,
+ "rewards/rejected": -1.558354403564008,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.2323032197169959,
+ "logps/chosen": -64.06994863152504,
+ "logps/rejected": -58.97870760083199,
+ "epoch": 0.8713241014470204,
+ "step": 840
+ },
+ {
+ "loss": 0.46459689140319826,
+ "grad_norm": 0.012618121691048145,
+ "learning_rate": 1.2020725388601037e-06,
+ "entropy": 1.315548148807138,
+ "num_tokens": 12362070.0,
+ "logits/chosen": -0.6251195627372872,
+ "logits/rejected": -0.5604575453557197,
+ "mean_token_accuracy": 0.6604597151651979,
+ "rewards/chosen": -0.3555148102169187,
+ "rewards/rejected": -1.5313407544395887,
+ "rewards/accuracies": 0.79625,
+ "rewards/margins": 1.1758259430155158,
+ "logps/chosen": -62.78060804247856,
+ "logps/rejected": -58.929887549877165,
+ "epoch": 0.8816970074166278,
+ "step": 850
+ },
+ {
+ "loss": 0.44550557136535646,
+ "grad_norm": 0.013532118871808052,
+ "learning_rate": 1.0984455958549225e-06,
+ "entropy": 1.3122669545235113,
+ "num_tokens": 12509338.0,
+ "logits/chosen": -0.6126990686398949,
+ "logits/rejected": -0.5539581104426821,
+ "mean_token_accuracy": 0.6569988044165075,
+ "rewards/chosen": -0.38766197150049264,
+ "rewards/rejected": -1.5939512548525818,
+ "rewards/accuracies": 0.81875,
+ "rewards/margins": 1.2062892844853923,
+ "logps/chosen": -65.91424354076385,
+ "logps/rejected": -59.84309137582779,
+ "epoch": 0.8920699133862352,
+ "step": 860
+ },
+ {
+ "loss": 0.44367341995239257,
+ "grad_norm": 0.014622218906879425,
+ "learning_rate": 9.94818652849741e-07,
+ "entropy": 1.29490221851971,
+ "num_tokens": 12656013.0,
+ "logits/chosen": -0.6367152223575993,
+ "logits/rejected": -0.5619754576335303,
+ "mean_token_accuracy": 0.6582811014540494,
+ "rewards/chosen": -0.3575555875984719,
+ "rewards/rejected": -1.485135663910769,
+ "rewards/accuracies": 0.81625,
+ "rewards/margins": 1.1275800754828378,
+ "logps/chosen": -64.74992589473725,
+ "logps/rejected": -58.28154501080513,
+ "epoch": 0.9024428193558426,
+ "step": 870
+ },
+ {
+ "eval_loss": 0.5303567051887512,
+ "eval_runtime": 351.9964,
+ "eval_samples_per_second": 2.841,
+ "eval_steps_per_second": 2.841,
+ "eval_entropy": 0.9427960855923593,
+ "eval_num_tokens": 12728578.0,
+ "eval_logits/chosen": -0.26863109543646074,
+ "eval_logits/rejected": -0.13725877180195847,
+ "eval_mean_token_accuracy": 0.7159041211903096,
+ "eval_rewards/chosen": -0.17772461017644672,
+ "eval_rewards/rejected": -0.7113549838000909,
+ "eval_rewards/accuracies": 0.763,
+ "eval_rewards/margins": 0.5336303729685024,
+ "eval_logps/chosen": -51.95023292160034,
+ "eval_logps/rejected": -59.75342144203186,
+ "epoch": 0.9076292723406463,
+ "step": 875
+ },
+ {
+ "loss": 0.42188777923583987,
+ "grad_norm": 0.014083835296332836,
+ "learning_rate": 8.911917098445596e-07,
+ "entropy": 1.2842787204217165,
+ "num_tokens": 12801884.0,
+ "logits/chosen": -0.6543839594717639,
+ "logits/rejected": -0.5576565805204583,
+ "mean_token_accuracy": 0.6647658421378583,
+ "rewards/chosen": -0.32246251756288985,
+ "rewards/rejected": -1.561580713846779,
+ "rewards/accuracies": 0.825,
+ "rewards/margins": 1.2391181947570293,
+ "logps/chosen": -60.62016844153404,
+ "logps/rejected": -60.981985919475555,
+ "epoch": 0.91281572532545,
+ "step": 880
+ },
+ {
+ "loss": 0.4861095905303955,
+ "grad_norm": 0.01509900763630867,
+ "learning_rate": 7.875647668393784e-07,
+ "entropy": 1.2736156480619685,
+ "num_tokens": 12946912.0,
+ "logits/chosen": -0.7128123134940787,
+ "logits/rejected": -0.6054497727959169,
+ "mean_token_accuracy": 0.6572731367498637,
+ "rewards/chosen": -0.4675049069095985,
+ "rewards/rejected": -1.5612919216800947,
+ "rewards/accuracies": 0.7825,
+ "rewards/margins": 1.0937870144005866,
+ "logps/chosen": -61.985627712011336,
+ "logps/rejected": -60.579013855457305,
+ "epoch": 0.9231886312950573,
+ "step": 890
+ },
+ {
+ "loss": 0.43419761657714845,
+ "grad_norm": 0.014685068279504776,
+ "learning_rate": 6.839378238341969e-07,
+ "entropy": 1.2819190438790247,
+ "num_tokens": 13093558.0,
+ "logits/chosen": -0.6502587498062717,
+ "logits/rejected": -0.5614254849582602,
+ "mean_token_accuracy": 0.6592135391384363,
+ "rewards/chosen": -0.36472693517142035,
+ "rewards/rejected": -1.6288802374559601,
+ "rewards/accuracies": 0.82,
+ "rewards/margins": 1.2641532999722402,
+ "logps/chosen": -61.57865165352821,
+ "logps/rejected": -62.50596989512444,
+ "epoch": 0.9335615372646647,
+ "step": 900
+ },
+ {
+ "loss": 0.4454813003540039,
+ "grad_norm": 0.011209873482584953,
+ "learning_rate": 5.803108808290156e-07,
+ "entropy": 1.3184868184709921,
+ "num_tokens": 13239740.0,
+ "logits/chosen": -0.6909169377714659,
+ "logits/rejected": -0.601211010607464,
+ "mean_token_accuracy": 0.6575440725311636,
+ "rewards/chosen": -0.36972430324880406,
+ "rewards/rejected": -1.6006858524744165,
+ "rewards/accuracies": 0.82875,
+ "rewards/margins": 1.2309615502599627,
+ "logps/chosen": -63.300528687238696,
+ "logps/rejected": -60.001617946624755,
+ "epoch": 0.9439344432342721,
+ "step": 910
+ },
+ {
+ "loss": 0.4595484733581543,
+ "grad_norm": 0.010628749616444111,
+ "learning_rate": 4.7668393782383424e-07,
+ "entropy": 1.271923498995602,
+ "num_tokens": 13383045.0,
+ "logits/chosen": -0.7072264696856618,
+ "logits/rejected": -0.6024060889326301,
+ "mean_token_accuracy": 0.6569805011339486,
+ "rewards/chosen": -0.3887278968687133,
+ "rewards/rejected": -1.5501526693382766,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.1614247707859613,
+ "logps/chosen": -60.07761764883995,
+ "logps/rejected": -56.95813281774521,
+ "epoch": 0.9543073492038795,
+ "step": 920
+ },
+ {
+ "loss": 0.4529257297515869,
+ "grad_norm": 0.014489009976387024,
+ "learning_rate": 3.730569948186528e-07,
+ "entropy": 1.2988391564786435,
+ "num_tokens": 13530054.0,
+ "logits/chosen": -0.7166704098766242,
+ "logits/rejected": -0.6650401865166429,
+ "mean_token_accuracy": 0.6520283976197243,
+ "rewards/chosen": -0.4043681784943692,
+ "rewards/rejected": -1.5586467561577593,
+ "rewards/accuracies": 0.81875,
+ "rewards/margins": 1.1542785764392465,
+ "logps/chosen": -64.28691901445389,
+ "logps/rejected": -60.51436370611191,
+ "epoch": 0.9646802551734869,
+ "step": 930
+ },
+ {
+ "loss": 0.4493688106536865,
+ "grad_norm": 0.0122548658400774,
+ "learning_rate": 2.694300518134715e-07,
+ "entropy": 1.293265828914009,
+ "num_tokens": 13676771.0,
+ "logits/chosen": -0.638114123144128,
+ "logits/rejected": -0.6033606771543887,
+ "mean_token_accuracy": 0.6584486217796802,
+ "rewards/chosen": -0.353517619818158,
+ "rewards/rejected": -1.4791943001397885,
+ "rewards/accuracies": 0.82625,
+ "rewards/margins": 1.1256766823492945,
+ "logps/chosen": -63.997545466423034,
+ "logps/rejected": -58.525367016792295,
+ "epoch": 0.9750531611430943,
+ "step": 940
+ },
+ {
+ "loss": 0.46651930809020997,
+ "grad_norm": 0.01745172217488289,
+ "learning_rate": 1.6580310880829015e-07,
+ "entropy": 1.2602451098989695,
+ "num_tokens": 13822091.0,
+ "logits/chosen": -0.7271677677940491,
+ "logits/rejected": -0.6540065888305867,
+ "mean_token_accuracy": 0.6601362200826406,
+ "rewards/chosen": -0.40040454823116306,
+ "rewards/rejected": -1.5421940996023478,
+ "rewards/accuracies": 0.80125,
+ "rewards/margins": 1.141789550515823,
+ "logps/chosen": -61.94225021719932,
+ "logps/rejected": -58.28781437397003,
+ "epoch": 0.9854260671127016,
+ "step": 950
+ },
+ {
+ "loss": 0.46761231422424315,
+ "grad_norm": 0.012348760850727558,
+ "learning_rate": 6.217616580310881e-08,
+ "entropy": 1.2971244535176083,
+ "num_tokens": 13969713.0,
+ "logits/chosen": -0.6472266813048246,
+ "logits/rejected": -0.5813531516986534,
+ "mean_token_accuracy": 0.6541070010513067,
+ "rewards/chosen": -0.4221547889366411,
+ "rewards/rejected": -1.5586294095951598,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 1.1364746230933815,
+ "logps/chosen": -64.67578131318092,
+ "logps/rejected": -59.52407863378525,
+ "epoch": 0.995798973082309,
+ "step": 960
+ },
+ {
+ "train_runtime": 22000.2483,
+ "train_samples_per_second": 3.506,
+ "train_steps_per_second": 0.044,
+ "total_flos": 1.2331818072769536e+17,
+ "train_loss": 0.4788281094842624,
+ "entropy": 1.3463219977616343,
+ "num_tokens": 14029916.0,
+ "logits/chosen": -0.5587557057517026,
+ "logits/rejected": -0.5043508288937856,
+ "mean_token_accuracy": 0.6551804588072829,
+ "rewards/chosen": -0.42263167897738046,
+ "rewards/rejected": -1.5862010600171799,
+ "rewards/accuracies": 0.8148148148148148,
+ "rewards/margins": 1.163569376240542,
+ "logps/chosen": -66.20575960182849,
+ "logps/rejected": -63.83122500666865,
+ "epoch": 1.0,
+ "step": 965
+ }
+ ],
+ "validation_monitor_size": 1000,
+ "validation_monitor_selection": "fixed_seed_random_without_replacement",
+ "validation_monitor_seed": 22,
+ "validation_monitor_indices": [
+ 2,
+ 10,
+ 14,
+ 21,
+ 55,
+ 63,
+ 66,
+ 69,
+ 107,
+ 110,
+ 142,
+ 144,
+ 149,
+ 150,
+ 151,
+ 152,
+ 160,
+ 163,
+ 166,
+ 167,
+ 176,
+ 181,
+ 206,
+ 207,
+ 259,
+ 267,
+ 281,
+ 295,
+ 298,
+ 306,
+ 307,
+ 317,
+ 321,
+ 331,
+ 336,
+ 341,
+ 346,
+ 349,
+ 351,
+ 352,
+ 357,
+ 358,
+ 369,
+ 370,
+ 382,
+ 386,
+ 391,
+ 401,
+ 411,
+ 412,
+ 432,
+ 437,
+ 452,
+ 462,
+ 465,
+ 488,
+ 490,
+ 491,
+ 492,
+ 494,
+ 503,
+ 504,
+ 508,
+ 528,
+ 538,
+ 540,
+ 551,
+ 553,
+ 567,
+ 586,
+ 605,
+ 606,
+ 625,
+ 627,
+ 661,
+ 663,
+ 666,
+ 677,
+ 685,
+ 690,
+ 692,
+ 704,
+ 708,
+ 714,
+ 715,
+ 727,
+ 728,
+ 733,
+ 745,
+ 752,
+ 753,
+ 755,
+ 764,
+ 773,
+ 779,
+ 783,
+ 793,
+ 796,
+ 799,
+ 803,
+ 804,
+ 805,
+ 813,
+ 816,
+ 818,
+ 823,
+ 827,
+ 829,
+ 830,
+ 837,
+ 842,
+ 845,
+ 850,
+ 853,
+ 856,
+ 889,
+ 890,
+ 905,
+ 915,
+ 924,
+ 930,
+ 939,
+ 951,
+ 988,
+ 1002,
+ 1005,
+ 1023,
+ 1029,
+ 1038,
+ 1049,
+ 1050,
+ 1054,
+ 1056,
+ 1067,
+ 1068,
+ 1079,
+ 1088,
+ 1091,
+ 1103,
+ 1114,
+ 1118,
+ 1133,
+ 1140,
+ 1144,
+ 1145,
+ 1155,
+ 1186,
+ 1195,
+ 1206,
+ 1223,
+ 1251,
+ 1252,
+ 1257,
+ 1259,
+ 1270,
+ 1271,
+ 1295,
+ 1303,
+ 1304,
+ 1305,
+ 1329,
+ 1335,
+ 1336,
+ 1343,
+ 1367,
+ 1373,
+ 1377,
+ 1403,
+ 1412,
+ 1429,
+ 1443,
+ 1457,
+ 1459,
+ 1460,
+ 1476,
+ 1483,
+ 1486,
+ 1494,
+ 1507,
+ 1510,
+ 1519,
+ 1521,
+ 1522,
+ 1545,
+ 1551,
+ 1570,
+ 1582,
+ 1593,
+ 1595,
+ 1603,
+ 1607,
+ 1614,
+ 1615,
+ 1625,
+ 1634,
+ 1639,
+ 1648,
+ 1654,
+ 1657,
+ 1662,
+ 1667,
+ 1673,
+ 1690,
+ 1704,
+ 1746,
+ 1756,
+ 1781,
+ 1783,
+ 1796,
+ 1799,
+ 1809,
+ 1814,
+ 1827,
+ 1829,
+ 1832,
+ 1844,
+ 1847,
+ 1854,
+ 1856,
+ 1857,
+ 1858,
+ 1874,
+ 1883,
+ 1889,
+ 1924,
+ 1925,
+ 1931,
+ 1932,
+ 1934,
+ 1935,
+ 1938,
+ 1950,
+ 1957,
+ 1962,
+ 1967,
+ 1975,
+ 1982,
+ 1983,
+ 1991,
+ 1998,
+ 2003,
+ 2008,
+ 2017,
+ 2021,
+ 2026,
+ 2035,
+ 2040,
+ 2050,
+ 2056,
+ 2077,
+ 2079,
+ 2082,
+ 2098,
+ 2100,
+ 2108,
+ 2121,
+ 2130,
+ 2133,
+ 2134,
+ 2138,
+ 2143,
+ 2166,
+ 2167,
+ 2180,
+ 2181,
+ 2185,
+ 2204,
+ 2205,
+ 2212,
+ 2221,
+ 2224,
+ 2226,
+ 2230,
+ 2233,
+ 2256,
+ 2261,
+ 2263,
+ 2269,
+ 2273,
+ 2290,
+ 2291,
+ 2299,
+ 2301,
+ 2321,
+ 2323,
+ 2330,
+ 2384,
+ 2401,
+ 2417,
+ 2420,
+ 2421,
+ 2424,
+ 2430,
+ 2435,
+ 2456,
+ 2488,
+ 2502,
+ 2504,
+ 2519,
+ 2527,
+ 2532,
+ 2538,
+ 2542,
+ 2553,
+ 2555,
+ 2558,
+ 2559,
+ 2562,
+ 2578,
+ 2583,
+ 2585,
+ 2590,
+ 2592,
+ 2594,
+ 2596,
+ 2600,
+ 2606,
+ 2607,
+ 2618,
+ 2630,
+ 2637,
+ 2647,
+ 2650,
+ 2657,
+ 2679,
+ 2685,
+ 2705,
+ 2706,
+ 2712,
+ 2713,
+ 2714,
+ 2727,
+ 2740,
+ 2742,
+ 2755,
+ 2780,
+ 2784,
+ 2792,
+ 2807,
+ 2808,
+ 2810,
+ 2820,
+ 2831,
+ 2839,
+ 2860,
+ 2862,
+ 2863,
+ 2866,
+ 2875,
+ 2878,
+ 2898,
+ 2905,
+ 2921,
+ 2922,
+ 2926,
+ 2930,
+ 2934,
+ 2944,
+ 2955,
+ 2957,
+ 2959,
+ 2973,
+ 2978,
+ 2998,
+ 3018,
+ 3022,
+ 3028,
+ 3031,
+ 3061,
+ 3085,
+ 3090,
+ 3104,
+ 3105,
+ 3111,
+ 3115,
+ 3121,
+ 3122,
+ 3129,
+ 3133,
+ 3135,
+ 3161,
+ 3162,
+ 3169,
+ 3173,
+ 3194,
+ 3195,
+ 3215,
+ 3225,
+ 3226,
+ 3241,
+ 3247,
+ 3250,
+ 3253,
+ 3265,
+ 3268,
+ 3293,
+ 3301,
+ 3312,
+ 3329,
+ 3352,
+ 3361,
+ 3362,
+ 3376,
+ 3396,
+ 3401,
+ 3403,
+ 3410,
+ 3419,
+ 3432,
+ 3443,
+ 3452,
+ 3467,
+ 3469,
+ 3475,
+ 3485,
+ 3503,
+ 3514,
+ 3515,
+ 3524,
+ 3528,
+ 3538,
+ 3544,
+ 3557,
+ 3560,
+ 3565,
+ 3600,
+ 3637,
+ 3642,
+ 3658,
+ 3659,
+ 3692,
+ 3693,
+ 3699,
+ 3722,
+ 3725,
+ 3728,
+ 3731,
+ 3740,
+ 3742,
+ 3762,
+ 3766,
+ 3780,
+ 3788,
+ 3794,
+ 3796,
+ 3798,
+ 3805,
+ 3817,
+ 3819,
+ 3822,
+ 3837,
+ 3839,
+ 3843,
+ 3846,
+ 3851,
+ 3854,
+ 3865,
+ 3870,
+ 3871,
+ 3884,
+ 3894,
+ 3895,
+ 3896,
+ 3907,
+ 3911,
+ 3915,
+ 3919,
+ 3920,
+ 3923,
+ 3933,
+ 3955,
+ 3967,
+ 3972,
+ 3974,
+ 3975,
+ 3984,
+ 3985,
+ 3997,
+ 4002,
+ 4010,
+ 4034,
+ 4044,
+ 4063,
+ 4073,
+ 4079,
+ 4082,
+ 4088,
+ 4121,
+ 4145,
+ 4148,
+ 4159,
+ 4161,
+ 4164,
+ 4177,
+ 4188,
+ 4199,
+ 4203,
+ 4207,
+ 4208,
+ 4213,
+ 4221,
+ 4225,
+ 4233,
+ 4241,
+ 4243,
+ 4247,
+ 4264,
+ 4274,
+ 4282,
+ 4286,
+ 4290,
+ 4308,
+ 4310,
+ 4313,
+ 4321,
+ 4324,
+ 4327,
+ 4349,
+ 4362,
+ 4370,
+ 4374,
+ 4380,
+ 4407,
+ 4409,
+ 4411,
+ 4415,
+ 4417,
+ 4418,
+ 4427,
+ 4431,
+ 4433,
+ 4451,
+ 4466,
+ 4477,
+ 4478,
+ 4479,
+ 4493,
+ 4494,
+ 4514,
+ 4527,
+ 4539,
+ 4550,
+ 4558,
+ 4568,
+ 4579,
+ 4583,
+ 4584,
+ 4586,
+ 4587,
+ 4590,
+ 4599,
+ 4602,
+ 4610,
+ 4626,
+ 4627,
+ 4630,
+ 4641,
+ 4647,
+ 4655,
+ 4656,
+ 4657,
+ 4661,
+ 4685,
+ 4714,
+ 4715,
+ 4731,
+ 4749,
+ 4752,
+ 4769,
+ 4777,
+ 4782,
+ 4791,
+ 4805,
+ 4818,
+ 4829,
+ 4833,
+ 4837,
+ 4839,
+ 4843,
+ 4871,
+ 4875,
+ 4879,
+ 4880,
+ 4885,
+ 4888,
+ 4895,
+ 4900,
+ 4923,
+ 4966,
+ 4968,
+ 4972,
+ 4989,
+ 4994,
+ 4998,
+ 5001,
+ 5013,
+ 5019,
+ 5026,
+ 5032,
+ 5034,
+ 5046,
+ 5055,
+ 5085,
+ 5086,
+ 5088,
+ 5089,
+ 5090,
+ 5095,
+ 5108,
+ 5110,
+ 5119,
+ 5120,
+ 5121,
+ 5133,
+ 5148,
+ 5154,
+ 5160,
+ 5169,
+ 5178,
+ 5222,
+ 5223,
+ 5232,
+ 5233,
+ 5240,
+ 5256,
+ 5259,
+ 5264,
+ 5271,
+ 5276,
+ 5279,
+ 5294,
+ 5300,
+ 5303,
+ 5321,
+ 5335,
+ 5337,
+ 5345,
+ 5348,
+ 5365,
+ 5373,
+ 5378,
+ 5384,
+ 5422,
+ 5442,
+ 5443,
+ 5445,
+ 5477,
+ 5485,
+ 5495,
+ 5497,
+ 5504,
+ 5526,
+ 5533,
+ 5542,
+ 5564,
+ 5570,
+ 5579,
+ 5587,
+ 5592,
+ 5608,
+ 5610,
+ 5624,
+ 5630,
+ 5638,
+ 5651,
+ 5663,
+ 5670,
+ 5675,
+ 5691,
+ 5700,
+ 5706,
+ 5707,
+ 5715,
+ 5720,
+ 5721,
+ 5722,
+ 5732,
+ 5738,
+ 5741,
+ 5769,
+ 5771,
+ 5775,
+ 5795,
+ 5796,
+ 5800,
+ 5809,
+ 5810,
+ 5815,
+ 5819,
+ 5827,
+ 5848,
+ 5849,
+ 5852,
+ 5859,
+ 5880,
+ 5884,
+ 5907,
+ 5909,
+ 5912,
+ 5919,
+ 5931,
+ 5932,
+ 5934,
+ 5941,
+ 5948,
+ 5950,
+ 5952,
+ 5953,
+ 5969,
+ 5981,
+ 6000,
+ 6003,
+ 6010,
+ 6018,
+ 6041,
+ 6065,
+ 6075,
+ 6090,
+ 6103,
+ 6106,
+ 6109,
+ 6114,
+ 6123,
+ 6131,
+ 6136,
+ 6138,
+ 6139,
+ 6164,
+ 6165,
+ 6171,
+ 6173,
+ 6180,
+ 6185,
+ 6189,
+ 6190,
+ 6221,
+ 6223,
+ 6237,
+ 6255,
+ 6262,
+ 6265,
+ 6293,
+ 6296,
+ 6305,
+ 6318,
+ 6331,
+ 6336,
+ 6340,
+ 6355,
+ 6366,
+ 6371,
+ 6396,
+ 6399,
+ 6402,
+ 6408,
+ 6432,
+ 6433,
+ 6441,
+ 6456,
+ 6465,
+ 6478,
+ 6486,
+ 6489,
+ 6507,
+ 6508,
+ 6520,
+ 6522,
+ 6528,
+ 6537,
+ 6551,
+ 6564,
+ 6589,
+ 6590,
+ 6598,
+ 6599,
+ 6611,
+ 6613,
+ 6615,
+ 6621,
+ 6634,
+ 6647,
+ 6649,
+ 6654,
+ 6676,
+ 6680,
+ 6690,
+ 6708,
+ 6729,
+ 6736,
+ 6744,
+ 6749,
+ 6756,
+ 6761,
+ 6763,
+ 6773,
+ 6788,
+ 6790,
+ 6796,
+ 6797,
+ 6811,
+ 6824,
+ 6850,
+ 6869,
+ 6871,
+ 6882,
+ 6892,
+ 6895,
+ 6906,
+ 6911,
+ 6912,
+ 6914,
+ 6927,
+ 6952,
+ 6966,
+ 6985,
+ 7001,
+ 7021,
+ 7031,
+ 7035,
+ 7038,
+ 7041,
+ 7045,
+ 7052,
+ 7057,
+ 7058,
+ 7072,
+ 7073,
+ 7076,
+ 7086,
+ 7102,
+ 7107,
+ 7109,
+ 7117,
+ 7122,
+ 7125,
+ 7166,
+ 7182,
+ 7199,
+ 7207,
+ 7212,
+ 7215,
+ 7232,
+ 7243,
+ 7246,
+ 7250,
+ 7253,
+ 7258,
+ 7263,
+ 7267,
+ 7270,
+ 7274,
+ 7280,
+ 7286,
+ 7293,
+ 7298,
+ 7302,
+ 7306,
+ 7316,
+ 7325,
+ 7326,
+ 7334,
+ 7356,
+ 7357,
+ 7363,
+ 7364,
+ 7367,
+ 7385,
+ 7392,
+ 7399,
+ 7405,
+ 7416,
+ 7420,
+ 7421,
+ 7426,
+ 7452,
+ 7476,
+ 7481,
+ 7519,
+ 7534,
+ 7542,
+ 7546,
+ 7573,
+ 7585,
+ 7601,
+ 7604,
+ 7606,
+ 7609,
+ 7629,
+ 7649,
+ 7653,
+ 7667,
+ 7682,
+ 7699,
+ 7738,
+ 7750,
+ 7786,
+ 7798,
+ 7800,
+ 7801,
+ 7805,
+ 7806,
+ 7811,
+ 7813,
+ 7832,
+ 7837,
+ 7849,
+ 7856,
+ 7876,
+ 7877,
+ 7887,
+ 7905,
+ 7916,
+ 7919,
+ 7920,
+ 7922,
+ 7923,
+ 7926,
+ 7944,
+ 7961,
+ 7966,
+ 7970,
+ 7973,
+ 7974,
+ 7976,
+ 7986,
+ 7999,
+ 8027,
+ 8028,
+ 8034,
+ 8047,
+ 8068,
+ 8074,
+ 8077,
+ 8091,
+ 8120,
+ 8122,
+ 8125,
+ 8152,
+ 8153,
+ 8164,
+ 8167,
+ 8169,
+ 8171,
+ 8177,
+ 8184,
+ 8189,
+ 8192,
+ 8196,
+ 8202,
+ 8212,
+ 8217,
+ 8231,
+ 8242,
+ 8244,
+ 8250,
+ 8256,
+ 8257,
+ 8265,
+ 8270,
+ 8274,
+ 8283,
+ 8290,
+ 8294,
+ 8301,
+ 8302,
+ 8307,
+ 8318,
+ 8326,
+ 8338,
+ 8349,
+ 8350,
+ 8353,
+ 8357,
+ 8371,
+ 8374,
+ 8377,
+ 8380,
+ 8387,
+ 8388,
+ 8396,
+ 8402,
+ 8419,
+ 8423,
+ 8428,
+ 8435,
+ 8439,
+ 8442,
+ 8444,
+ 8453,
+ 8461,
+ 8475,
+ 8481,
+ 8485,
+ 8493,
+ 8495,
+ 8498,
+ 8523,
+ 8530,
+ 8531,
+ 8562
+ ],
+ "early_stopping_patience": 3
+}
diff --git a/phase1/ablations/random_labels/README.md b/phase1/ablations/random_labels/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..1efc18014b07a8e67b5608e63d45c896cd4338d0
--- /dev/null
+++ b/phase1/ablations/random_labels/README.md
@@ -0,0 +1,80 @@
+---
+library_name: peft
+model_name: phase1-qwen-random-labels-ablation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-1.5B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+license: apache-2.0
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+pipeline_tag: text-generation
+---
+
+# Phase 1 Qwen `random_labels` diagnostic ablation
+
+This is the Phase 1 `random_labels` diagnostic LoRA-DPO adapter fine-tuned from
+[Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct).
+The frozen training recipe and membership hashes are included in this directory.
+
+## Quick start
+
+```python
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo_id = "geyuxu/york-milestone-project-self-traing-loop-model"
+subfolder = "phase1/ablations/random_labels"
+base_id = "Qwen/Qwen2.5-1.5B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(repo_id, subfolder=subfolder)
+base_model = AutoModelForCausalLM.from_pretrained(
+ base_id, torch_dtype="auto", device_map="auto"
+)
+model = PeftModel.from_pretrained(base_model, repo_id, subfolder=subfolder)
+```
+
+## Training procedure
+
+
+
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 0.29.0
+- Transformers: 5.3.0
+- Pytorch: 2.10.0
+- Datasets: 4.6.1
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+ title = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+ author = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+ year = 2023,
+ booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+ url = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+ editor = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+ title = {{TRL: Transformers Reinforcement Learning}},
+ author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+ license = {Apache-2.0},
+ url = {https://github.com/huggingface/trl},
+ year = {2020}
+}
+```
diff --git a/phase1/ablations/random_labels/adapter_config.json b/phase1/ablations/random_labels/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..fa47a2292cd8a23ccf1b4d101d7f7a03e3f08871
--- /dev/null
+++ b/phase1/ablations/random_labels/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "k_proj",
+ "o_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
diff --git a/phase1/ablations/random_labels/adapter_model.safetensors b/phase1/ablations/random_labels/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2efb4cb851f476f06a9dbdfcb09606c751e760db
--- /dev/null
+++ b/phase1/ablations/random_labels/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bebded9e10f63603bfd9d6a135dda87c8bcac25b4a43b365e2b1ce882bf4c93
+size 17462432
diff --git a/phase1/ablations/random_labels/chat_template.jinja b/phase1/ablations/random_labels/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e
--- /dev/null
+++ b/phase1/ablations/random_labels/chat_template.jinja
@@ -0,0 +1,54 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- messages[0]['content'] }}
+ {%- else %}
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+ {%- endif %}
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+ {%- else %}
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {{- '<|im_start|>' + message.role }}
+ {%- if message.content %}
+ {{- '\n' + message.content }}
+ {%- endif %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- '}\n' }}
+ {%- endfor %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- message.content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}
diff --git a/phase1/ablations/random_labels/tokenizer.json b/phase1/ablations/random_labels/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..d73428d91463b495bc017fb6a2fb3a656646482b
--- /dev/null
+++ b/phase1/ablations/random_labels/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5
+size 11422166
diff --git a/phase1/ablations/random_labels/tokenizer_config.json b/phase1/ablations/random_labels/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9fd0fb48e73786f54fb04e98892f5f5a0ebeebdb
--- /dev/null
+++ b/phase1/ablations/random_labels/tokenizer_config.json
@@ -0,0 +1,29 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": true,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/phase1/ablations/random_labels/training_args.bin b/phase1/ablations/random_labels/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..869847f6912bd9cca4a1fca95e3be6af6c50b8ed
--- /dev/null
+++ b/phase1/ablations/random_labels/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad84a00370d0c977fcd38a1fce033b130494f3fffd4a3c0cbc44a9dd43c0bdbf
+size 5841
diff --git a/phase1/ablations/random_labels/training_recipe.json b/phase1/ablations/random_labels/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..1f033e0b98e0786f8e7cb044d632591f1bd9ae3f
--- /dev/null
+++ b/phase1/ablations/random_labels/training_recipe.json
@@ -0,0 +1,22 @@
+{
+ "format_version": 1,
+ "experiment": "confusion_ablation",
+ "ablation": "random_labels",
+ "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+ "train_membership_sha256": "b497fa765223a9810ad784f31e1598edfa992a162146d6f94d9cdf4adcedd87d",
+ "validation_membership_sha256": "2f609467438cd6a01fd84b9d80c0fcd84c44767e3b2ed98f4f6a83d68a62c181",
+ "num_train_pairs": 77124,
+ "num_validation_pairs": 8563,
+ "transformation_seed": 22,
+ "validation_prompt_mode": "clean_rag",
+ "learning_rate": 1e-05,
+ "beta": 0.1,
+ "num_epochs": 1.0,
+ "seed": 22,
+ "max_length": 512,
+ "max_prompt_length": 384,
+ "gradient_accumulation_steps": 80,
+ "save_steps": 125,
+ "save_total_limit": 12,
+ "preference_pairs_format_version": 2
+}
diff --git a/phase1/ablations/random_labels/training_summary.json b/phase1/ablations/random_labels/training_summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..7a6f701cf3b55a1aeb940fd42a01bd02d5026018
--- /dev/null
+++ b/phase1/ablations/random_labels/training_summary.json
@@ -0,0 +1,2517 @@
+{
+ "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+ "resolved_model": "Qwen/Qwen2.5-1.5B-Instruct",
+ "num_train_records": 77124,
+ "num_validation_records_available": 8563,
+ "training_config": {
+ "lora_rank": 16,
+ "lora_alpha": 32,
+ "learning_rate": 1e-05,
+ "num_epochs": 1.0,
+ "per_device_batch_size": 1,
+ "per_device_eval_batch_size": 1,
+ "gradient_accumulation_steps": 80,
+ "beta": 0.1,
+ "max_length": 512,
+ "max_prompt_length": 384,
+ "seed": 22,
+ "save_steps": 125,
+ "save_total_limit": 12,
+ "resume_from_checkpoint": null,
+ "precision_dtype": "torch.bfloat16",
+ "bf16": true,
+ "fp16": false,
+ "tokenizer_add_bos_token": false
+ },
+ "global_step": 750,
+ "best_metric": 0.6905523538589478,
+ "best_model_checkpoint": "outputs/confusion_ablation_phase1_repair_v3/random_labels/lora/checkpoint-375",
+ "train_metrics": {
+ "train_runtime": 27561.1083,
+ "train_samples_per_second": 2.798,
+ "train_steps_per_second": 0.035,
+ "total_flos": 4.075839588886364e+17,
+ "train_loss": 0.6933243910471598
+ },
+ "log_history": [
+ {
+ "loss": 0.6934387683868408,
+ "grad_norm": 0.009266729466617107,
+ "learning_rate": 9.906735751295338e-06,
+ "entropy": 0.8080450683785603,
+ "num_tokens": 669528.0,
+ "logits/chosen": -0.9213643590964665,
+ "logits/rejected": -0.9683298173272368,
+ "mean_token_accuracy": 0.7228609627112746,
+ "rewards/chosen": 0.0007450232512769616,
+ "rewards/rejected": 0.001045633094436198,
+ "rewards/accuracies": 0.44,
+ "rewards/margins": -0.0003006098522382672,
+ "logps/chosen": -51.68650775671005,
+ "logps/rejected": -50.719243061542514,
+ "epoch": 0.010372905969607386,
+ "step": 10
+ },
+ {
+ "loss": 0.6930126190185547,
+ "grad_norm": 0.009303063154220581,
+ "learning_rate": 9.803108808290156e-06,
+ "entropy": 0.7915201940061525,
+ "num_tokens": 1338759.0,
+ "logits/chosen": -0.9846059676325559,
+ "logits/rejected": -0.9714732527595322,
+ "mean_token_accuracy": 0.7229587784409524,
+ "rewards/chosen": -0.00012686825164792025,
+ "rewards/rejected": -0.0007469337083875871,
+ "rewards/accuracies": 0.48625,
+ "rewards/margins": 0.0006200655033171643,
+ "logps/chosen": -50.57927699804306,
+ "logps/rejected": -51.70512897491455,
+ "epoch": 0.02074581193921477,
+ "step": 20
+ },
+ {
+ "loss": 0.693855619430542,
+ "grad_norm": 0.009770886041224003,
+ "learning_rate": 9.699481865284975e-06,
+ "entropy": 0.8051797628332861,
+ "num_tokens": 2009918.0,
+ "logits/chosen": -1.0153866272103405,
+ "logits/rejected": -0.9948517174473516,
+ "mean_token_accuracy": 0.7255033986549825,
+ "rewards/chosen": 0.000551955679097773,
+ "rewards/rejected": 0.0015872831797685194,
+ "rewards/accuracies": 0.495,
+ "rewards/margins": -0.0010353274924636935,
+ "logps/chosen": -51.339062130451204,
+ "logps/rejected": -52.78074060201645,
+ "epoch": 0.031118717908822157,
+ "step": 30
+ },
+ {
+ "loss": 0.69316725730896,
+ "grad_norm": 0.008510654792189598,
+ "learning_rate": 9.595854922279793e-06,
+ "entropy": 0.7799565396877006,
+ "num_tokens": 2680367.0,
+ "logits/chosen": -1.0400407581404876,
+ "logits/rejected": -0.9926833103096355,
+ "mean_token_accuracy": 0.7316536229476333,
+ "rewards/chosen": 0.0013948245377036982,
+ "rewards/rejected": 0.001106869012758125,
+ "rewards/accuracies": 0.4775,
+ "rewards/margins": 0.0002879555685285595,
+ "logps/chosen": -50.836872832775114,
+ "logps/rejected": -52.290282304286954,
+ "epoch": 0.04149162387842954,
+ "step": 40
+ },
+ {
+ "loss": 0.6937148094177246,
+ "grad_norm": 0.00940113328397274,
+ "learning_rate": 9.492227979274612e-06,
+ "entropy": 0.784734519557096,
+ "num_tokens": 3350695.0,
+ "logits/chosen": -1.0085622581752787,
+ "logits/rejected": -0.9717813104276766,
+ "mean_token_accuracy": 0.7277172536216676,
+ "rewards/chosen": 0.0007200987445082774,
+ "rewards/rejected": 0.001504770267456479,
+ "rewards/accuracies": 0.495,
+ "rewards/margins": -0.0007846715349296573,
+ "logps/chosen": -50.88885704755783,
+ "logps/rejected": -51.595236620903016,
+ "epoch": 0.051864529848036925,
+ "step": 50
+ },
+ {
+ "loss": 0.6925368785858155,
+ "grad_norm": 0.01000447291880846,
+ "learning_rate": 9.388601036269432e-06,
+ "entropy": 0.7792391402181238,
+ "num_tokens": 4019867.0,
+ "logits/chosen": -0.989283063151833,
+ "logits/rejected": -0.9758414097701983,
+ "mean_token_accuracy": 0.7253596894256771,
+ "rewards/chosen": 0.00419369634595455,
+ "rewards/rejected": 0.00261573103512319,
+ "rewards/accuracies": 0.515,
+ "rewards/margins": 0.0015779653389472515,
+ "logps/chosen": -50.12135502576828,
+ "logps/rejected": -49.72630759477615,
+ "epoch": 0.062237435817644314,
+ "step": 60
+ },
+ {
+ "loss": 0.6928633213043213,
+ "grad_norm": 0.008523018099367619,
+ "learning_rate": 9.284974093264249e-06,
+ "entropy": 0.7859808096196502,
+ "num_tokens": 4689830.0,
+ "logits/chosen": -0.9542620254422428,
+ "logits/rejected": -0.9723457128166635,
+ "mean_token_accuracy": 0.7191485194861889,
+ "rewards/chosen": 0.006607790327220755,
+ "rewards/rejected": 0.005695816857155478,
+ "rewards/accuracies": 0.50625,
+ "rewards/margins": 0.0009119734441992478,
+ "logps/chosen": -52.69678068399429,
+ "logps/rejected": -49.40951271772385,
+ "epoch": 0.0726103417872517,
+ "step": 70
+ },
+ {
+ "loss": 0.6931606769561768,
+ "grad_norm": 0.009254093281924725,
+ "learning_rate": 9.181347150259067e-06,
+ "entropy": 0.8076216962421313,
+ "num_tokens": 5359790.0,
+ "logits/chosen": -0.9293808590029637,
+ "logits/rejected": -0.9847294370574589,
+ "mean_token_accuracy": 0.7200651096645743,
+ "rewards/chosen": 0.011739555572351037,
+ "rewards/rejected": 0.011363767318175632,
+ "rewards/accuracies": 0.4875,
+ "rewards/margins": 0.00037578820330963936,
+ "logps/chosen": -51.38067405223846,
+ "logps/rejected": -50.72847024202347,
+ "epoch": 0.08298324775685909,
+ "step": 80
+ },
+ {
+ "loss": 0.6942155361175537,
+ "grad_norm": 0.008851570077240467,
+ "learning_rate": 9.077720207253888e-06,
+ "entropy": 0.8039566927566193,
+ "num_tokens": 6029642.0,
+ "logits/chosen": -0.9478755777728227,
+ "logits/rejected": -0.9763882057133852,
+ "mean_token_accuracy": 0.7233742567896843,
+ "rewards/chosen": 0.01409353660361603,
+ "rewards/rejected": 0.015802892130523674,
+ "rewards/accuracies": 0.48375,
+ "rewards/margins": -0.001709355540224351,
+ "logps/chosen": -51.33848252058029,
+ "logps/rejected": -52.321811312437056,
+ "epoch": 0.09335615372646647,
+ "step": 90
+ },
+ {
+ "loss": 0.6936118602752686,
+ "grad_norm": 0.009475240483880043,
+ "learning_rate": 8.974093264248706e-06,
+ "entropy": 0.8324551365058869,
+ "num_tokens": 6703750.0,
+ "logits/chosen": -0.9270073968773294,
+ "logits/rejected": -0.9359207155608903,
+ "mean_token_accuracy": 0.7229399124160409,
+ "rewards/chosen": 0.014994417404241176,
+ "rewards/rejected": 0.015462831489760163,
+ "rewards/accuracies": 0.49625,
+ "rewards/margins": -0.0004684141512552742,
+ "logps/chosen": -55.6720170044899,
+ "logps/rejected": -56.016610951423644,
+ "epoch": 0.10372905969607385,
+ "step": 100
+ },
+ {
+ "loss": 0.6934245586395263,
+ "grad_norm": 0.008504034951329231,
+ "learning_rate": 8.870466321243523e-06,
+ "entropy": 0.7861935871373862,
+ "num_tokens": 7374932.0,
+ "logits/chosen": -0.965592457466306,
+ "logits/rejected": -0.9543162182972067,
+ "mean_token_accuracy": 0.7248496604338288,
+ "rewards/chosen": 0.016234664240764685,
+ "rewards/rejected": 0.016324117717140468,
+ "rewards/accuracies": 0.495,
+ "rewards/margins": -8.945348919951357e-05,
+ "logps/chosen": -50.975947844982144,
+ "logps/rejected": -51.70058757305145,
+ "epoch": 0.11410196566568125,
+ "step": 110
+ },
+ {
+ "loss": 0.6940177917480469,
+ "grad_norm": 0.009204786270856857,
+ "learning_rate": 8.766839378238343e-06,
+ "entropy": 0.7727768113883212,
+ "num_tokens": 8045025.0,
+ "logits/chosen": -1.03487960348445,
+ "logits/rejected": -0.9815775502270385,
+ "mean_token_accuracy": 0.7311673213727772,
+ "rewards/chosen": 0.01172756811563886,
+ "rewards/rejected": 0.01306509348746431,
+ "rewards/accuracies": 0.47875,
+ "rewards/margins": -0.0013375253186222835,
+ "logps/chosen": -49.14816425204277,
+ "logps/rejected": -51.98841401100159,
+ "epoch": 0.12447487163528863,
+ "step": 120
+ },
+ {
+ "eval_loss": 0.6955941319465637,
+ "eval_runtime": 196.5871,
+ "eval_samples_per_second": 5.087,
+ "eval_steps_per_second": 5.087,
+ "eval_entropy": 0.8034458611719311,
+ "eval_num_tokens": 8380297.0,
+ "eval_logits/chosen": -0.991135582788011,
+ "eval_logits/rejected": -0.950536454501934,
+ "eval_mean_token_accuracy": 0.7337288541048765,
+ "eval_rewards/chosen": 0.008104163538169359,
+ "eval_rewards/rejected": 0.01258701845750079,
+ "eval_rewards/accuracies": 0.437,
+ "eval_rewards/margins": -0.0044828549473313615,
+ "eval_logps/chosen": -50.09194520950317,
+ "eval_logps/rejected": -52.51400153923035,
+ "epoch": 0.1296613246200923,
+ "step": 125
+ },
+ {
+ "loss": 0.6930018901824951,
+ "grad_norm": 0.00838699284940958,
+ "learning_rate": 8.663212435233162e-06,
+ "entropy": 0.8142909701773897,
+ "num_tokens": 8716283.0,
+ "logits/chosen": -0.971143901841604,
+ "logits/rejected": -1.0017587172299427,
+ "mean_token_accuracy": 0.729599024951458,
+ "rewards/chosen": 0.009170935571445398,
+ "rewards/rejected": 0.008475723866073964,
+ "rewards/accuracies": 0.52375,
+ "rewards/margins": 0.0006952117068431108,
+ "logps/chosen": -51.31078794002533,
+ "logps/rejected": -53.1062014913559,
+ "epoch": 0.13484777760489602,
+ "step": 130
+ },
+ {
+ "loss": 0.6927727699279785,
+ "grad_norm": 0.009330841712653637,
+ "learning_rate": 8.55958549222798e-06,
+ "entropy": 0.7786517775012181,
+ "num_tokens": 9385323.0,
+ "logits/chosen": -1.0336392573752233,
+ "logits/rejected": -1.019804302735747,
+ "mean_token_accuracy": 0.7317706294171512,
+ "rewards/chosen": 0.004249347767993186,
+ "rewards/rejected": 0.0031492145435004205,
+ "rewards/accuracies": 0.51875,
+ "rewards/margins": 0.001100133234722307,
+ "logps/chosen": -48.82712723731995,
+ "logps/rejected": -49.81842919588089,
+ "epoch": 0.1452206835745034,
+ "step": 140
+ },
+ {
+ "loss": 0.6928228378295899,
+ "grad_norm": 0.009226374328136444,
+ "learning_rate": 8.455958549222799e-06,
+ "entropy": 0.7707585748913698,
+ "num_tokens": 10057097.0,
+ "logits/chosen": -1.0021628340641857,
+ "logits/rejected": -1.0272578388562763,
+ "mean_token_accuracy": 0.7250917989574373,
+ "rewards/chosen": 0.0021179335123679265,
+ "rewards/rejected": 0.0010407016340150221,
+ "rewards/accuracies": 0.51875,
+ "rewards/margins": 0.0010772318854651531,
+ "logps/chosen": -50.82353290557862,
+ "logps/rejected": -51.57363982439041,
+ "epoch": 0.1555935895441108,
+ "step": 150
+ },
+ {
+ "loss": 0.6935010433197022,
+ "grad_norm": 0.008547156117856503,
+ "learning_rate": 8.352331606217617e-06,
+ "entropy": 0.8095202751364559,
+ "num_tokens": 10728653.0,
+ "logits/chosen": -0.9613099360765643,
+ "logits/rejected": -0.9753560247735856,
+ "mean_token_accuracy": 0.726740376483649,
+ "rewards/chosen": 0.003875704376255271,
+ "rewards/rejected": 0.004182930276788284,
+ "rewards/accuracies": 0.48375,
+ "rewards/margins": -0.00030722589199285724,
+ "logps/chosen": -53.913278470039366,
+ "logps/rejected": -53.74137356996536,
+ "epoch": 0.16596649551371817,
+ "step": 160
+ },
+ {
+ "loss": 0.6931381702423096,
+ "grad_norm": 0.008811748586595058,
+ "learning_rate": 8.248704663212436e-06,
+ "entropy": 0.8115301547432319,
+ "num_tokens": 11400551.0,
+ "logits/chosen": -0.9568430709542813,
+ "logits/rejected": -0.9675442470577922,
+ "mean_token_accuracy": 0.7244884933158755,
+ "rewards/chosen": 0.003357977192756607,
+ "rewards/rejected": 0.0028972053575580504,
+ "rewards/accuracies": 0.4925,
+ "rewards/margins": 0.00046077182138105856,
+ "logps/chosen": -53.2519864654541,
+ "logps/rejected": -52.62679670572281,
+ "epoch": 0.17633940148332555,
+ "step": 170
+ },
+ {
+ "loss": 0.6925437927246094,
+ "grad_norm": 0.00891165155917406,
+ "learning_rate": 8.145077720207254e-06,
+ "entropy": 0.79428493458312,
+ "num_tokens": 12070305.0,
+ "logits/chosen": -0.9719879220022611,
+ "logits/rejected": -0.9580149375927722,
+ "mean_token_accuracy": 0.7303908407129347,
+ "rewards/chosen": 0.002974712664896515,
+ "rewards/rejected": 0.0013871385961903115,
+ "rewards/accuracies": 0.5275,
+ "rewards/margins": 0.0015875740901537937,
+ "logps/chosen": -52.03131418466568,
+ "logps/rejected": -49.467778403759006,
+ "epoch": 0.18671230745293294,
+ "step": 180
+ },
+ {
+ "loss": 0.6946820259094239,
+ "grad_norm": 0.011048010550439358,
+ "learning_rate": 8.041450777202073e-06,
+ "entropy": 0.801795727997087,
+ "num_tokens": 12738966.0,
+ "logits/chosen": -0.9541037205291851,
+ "logits/rejected": -0.9584797920583412,
+ "mean_token_accuracy": 0.7201638549566269,
+ "rewards/chosen": 0.004079287395195053,
+ "rewards/rejected": 0.006725511867425666,
+ "rewards/accuracies": 0.47,
+ "rewards/margins": -0.0026462244295544224,
+ "logps/chosen": -51.030084483623504,
+ "logps/rejected": -52.38975264072418,
+ "epoch": 0.19708521342254032,
+ "step": 190
+ },
+ {
+ "loss": 0.6930304050445557,
+ "grad_norm": 0.009633849374949932,
+ "learning_rate": 7.937823834196891e-06,
+ "entropy": 0.7963748186221347,
+ "num_tokens": 13408002.0,
+ "logits/chosen": -0.9894887990110749,
+ "logits/rejected": -0.9980301170848317,
+ "mean_token_accuracy": 0.7193645982630551,
+ "rewards/chosen": 0.0047459728431940105,
+ "rewards/rejected": 0.004120458228130701,
+ "rewards/accuracies": 0.48625,
+ "rewards/margins": 0.000625514651183039,
+ "logps/chosen": -50.538159133195876,
+ "logps/rejected": -50.48108030319214,
+ "epoch": 0.2074581193921477,
+ "step": 200
+ },
+ {
+ "loss": 0.6936575412750244,
+ "grad_norm": 0.008548871614038944,
+ "learning_rate": 7.834196891191712e-06,
+ "entropy": 0.7598572969343513,
+ "num_tokens": 14074115.0,
+ "logits/chosen": -1.045821409259606,
+ "logits/rejected": -0.999601151613033,
+ "mean_token_accuracy": 0.7357131129689515,
+ "rewards/chosen": 0.005255431280912717,
+ "rewards/rejected": 0.005940479632897677,
+ "rewards/accuracies": 0.47,
+ "rewards/margins": -0.0006850483406742569,
+ "logps/chosen": -46.40404602527619,
+ "logps/rejected": -50.13248352050781,
+ "epoch": 0.2178310253617551,
+ "step": 210
+ },
+ {
+ "loss": 0.6938835144042969,
+ "grad_norm": 0.009261125698685646,
+ "learning_rate": 7.730569948186528e-06,
+ "entropy": 0.767732237551827,
+ "num_tokens": 14745903.0,
+ "logits/chosen": -1.0030631508707841,
+ "logits/rejected": -1.0114240661200429,
+ "mean_token_accuracy": 0.7297109466791153,
+ "rewards/chosen": 0.004763001757364691,
+ "rewards/rejected": 0.005841993642383727,
+ "rewards/accuracies": 0.49125,
+ "rewards/margins": -0.0010789919237868162,
+ "logps/chosen": -51.14548614025116,
+ "logps/rejected": -50.2862161898613,
+ "epoch": 0.2282039313313625,
+ "step": 220
+ },
+ {
+ "loss": 0.6946812152862549,
+ "grad_norm": 0.009788029827177525,
+ "learning_rate": 7.626943005181348e-06,
+ "entropy": 0.8289732250571251,
+ "num_tokens": 15418570.0,
+ "logits/chosen": -0.9488809894056489,
+ "logits/rejected": -1.0089717702518306,
+ "mean_token_accuracy": 0.7215882153064013,
+ "rewards/chosen": 0.006585003775003315,
+ "rewards/rejected": 0.009181454044451129,
+ "rewards/accuracies": 0.47625,
+ "rewards/margins": -0.0025964502768329113,
+ "logps/chosen": -56.4708499455452,
+ "logps/rejected": -53.11574334859848,
+ "epoch": 0.23857683730096987,
+ "step": 230
+ },
+ {
+ "loss": 0.6926135063171387,
+ "grad_norm": 0.009469996206462383,
+ "learning_rate": 7.523316062176167e-06,
+ "entropy": 0.8170388517202809,
+ "num_tokens": 16089276.0,
+ "logits/chosen": -0.9783852286633136,
+ "logits/rejected": -0.9603607895142418,
+ "mean_token_accuracy": 0.7241931633278728,
+ "rewards/chosen": 0.00405766910489433,
+ "rewards/rejected": 0.0026022558279692022,
+ "rewards/accuracies": 0.5175,
+ "rewards/margins": 0.00145541326062812,
+ "logps/chosen": -51.54288928151131,
+ "logps/rejected": -54.08886151790619,
+ "epoch": 0.24894974327057726,
+ "step": 240
+ },
+ {
+ "loss": 0.6937024593353271,
+ "grad_norm": 0.009117631241679192,
+ "learning_rate": 7.419689119170985e-06,
+ "entropy": 0.8002800923399627,
+ "num_tokens": 16760081.0,
+ "logits/chosen": -1.0175496722318613,
+ "logits/rejected": -0.9702955326900312,
+ "mean_token_accuracy": 0.7222046358138323,
+ "rewards/chosen": 0.0016811337291983363,
+ "rewards/rejected": 0.0023915858741713693,
+ "rewards/accuracies": 0.50625,
+ "rewards/margins": -0.0007104521640576422,
+ "logps/chosen": -51.49065291881561,
+ "logps/rejected": -54.3223114323616,
+ "epoch": 0.2593226492401846,
+ "step": 250
+ },
+ {
+ "eval_loss": 0.6968227624893188,
+ "eval_runtime": 223.7572,
+ "eval_samples_per_second": 4.469,
+ "eval_steps_per_second": 4.469,
+ "eval_entropy": 0.8013349784091115,
+ "eval_num_tokens": 16760081.0,
+ "eval_logits/chosen": -1.0127526828657614,
+ "eval_logits/rejected": -0.9702886589714581,
+ "eval_mean_token_accuracy": 0.7333163165301084,
+ "eval_rewards/chosen": 0.0008883264788382803,
+ "eval_rewards/rejected": 0.007837165784684316,
+ "eval_rewards/accuracies": 0.415,
+ "eval_rewards/margins": -0.00694883929831849,
+ "eval_logps/chosen": -50.164103578567506,
+ "eval_logps/rejected": -52.56150006484985,
+ "epoch": 0.2593226492401846,
+ "step": 250
+ },
+ {
+ "loss": 0.6938982009887695,
+ "grad_norm": 0.008658971637487411,
+ "learning_rate": 7.3160621761658035e-06,
+ "entropy": 0.8275363711640239,
+ "num_tokens": 17432770.0,
+ "logits/chosen": -0.9894071595486238,
+ "logits/rejected": -1.021770131425495,
+ "mean_token_accuracy": 0.7267817178182304,
+ "rewards/chosen": -6.325603737082019e-05,
+ "rewards/rejected": 0.0009481332914026552,
+ "rewards/accuracies": 0.5,
+ "rewards/margins": -0.0010113894138703472,
+ "logps/chosen": -53.769077570438384,
+ "logps/rejected": -54.760430970191955,
+ "epoch": 0.26969555520979205,
+ "step": 260
+ },
+ {
+ "loss": 0.6941106796264649,
+ "grad_norm": 0.009195917285978794,
+ "learning_rate": 7.212435233160623e-06,
+ "entropy": 0.8036660263128579,
+ "num_tokens": 18100438.0,
+ "logits/chosen": -1.0066855113664344,
+ "logits/rejected": -1.0124089541250572,
+ "mean_token_accuracy": 0.717040218450129,
+ "rewards/chosen": -0.003151719864085862,
+ "rewards/rejected": -0.00164005833167721,
+ "rewards/accuracies": 0.475,
+ "rewards/margins": -0.001511661496660963,
+ "logps/chosen": -49.700445964336396,
+ "logps/rejected": -50.435976998806,
+ "epoch": 0.28006846117939943,
+ "step": 270
+ },
+ {
+ "loss": 0.6933230876922607,
+ "grad_norm": 0.009418987669050694,
+ "learning_rate": 7.108808290155441e-06,
+ "entropy": 0.8158398796850815,
+ "num_tokens": 18772061.0,
+ "logits/chosen": -1.0278496914643969,
+ "logits/rejected": -1.0390610888345118,
+ "mean_token_accuracy": 0.724490509070456,
+ "rewards/chosen": -0.005928723971054524,
+ "rewards/rejected": -0.006031536482905722,
+ "rewards/accuracies": 0.52,
+ "rewards/margins": 0.0001028125207812991,
+ "logps/chosen": -53.07417652249336,
+ "logps/rejected": -52.73169189453125,
+ "epoch": 0.2904413671490068,
+ "step": 280
+ },
+ {
+ "loss": 0.6924946308135986,
+ "grad_norm": 0.008996859192848206,
+ "learning_rate": 7.005181347150259e-06,
+ "entropy": 0.801404341221787,
+ "num_tokens": 19443506.0,
+ "logits/chosen": -1.020471675081665,
+ "logits/rejected": -1.0060470756374977,
+ "mean_token_accuracy": 0.7225255416892469,
+ "rewards/chosen": -0.008651466769783837,
+ "rewards/rejected": -0.010357174093505818,
+ "rewards/accuracies": 0.5175,
+ "rewards/margins": 0.0017057072926036198,
+ "logps/chosen": -52.63102213382721,
+ "logps/rejected": -53.120495743751526,
+ "epoch": 0.3008142731186142,
+ "step": 290
+ },
+ {
+ "loss": 0.6929198741912842,
+ "grad_norm": 0.00904047954827547,
+ "learning_rate": 6.9015544041450784e-06,
+ "entropy": 0.7905970112606883,
+ "num_tokens": 20114567.0,
+ "logits/chosen": -1.0243293848557635,
+ "logits/rejected": -1.058803992605315,
+ "mean_token_accuracy": 0.7341359755769372,
+ "rewards/chosen": -0.012200816824647518,
+ "rewards/rejected": -0.01307745838902747,
+ "rewards/accuracies": 0.52125,
+ "rewards/margins": 0.000876641578397539,
+ "logps/chosen": -52.0853961968422,
+ "logps/rejected": -52.80914256095886,
+ "epoch": 0.3111871790882216,
+ "step": 300
+ },
+ {
+ "loss": 0.6928537845611572,
+ "grad_norm": 0.009816721081733704,
+ "learning_rate": 6.797927461139897e-06,
+ "entropy": 0.80872525641229,
+ "num_tokens": 20787766.0,
+ "logits/chosen": -0.9839863753234047,
+ "logits/rejected": -1.0009211278188388,
+ "mean_token_accuracy": 0.724884872585535,
+ "rewards/chosen": -0.012630892752328008,
+ "rewards/rejected": -0.01370205567994617,
+ "rewards/accuracies": 0.51125,
+ "rewards/margins": 0.0010711629761499353,
+ "logps/chosen": -54.78661850452423,
+ "logps/rejected": -53.24911024570465,
+ "epoch": 0.32156008505782896,
+ "step": 310
+ },
+ {
+ "loss": 0.6942753314971923,
+ "grad_norm": 0.009577946737408638,
+ "learning_rate": 6.6943005181347155e-06,
+ "entropy": 0.8015230012265966,
+ "num_tokens": 21457559.0,
+ "logits/chosen": -0.9992799805599515,
+ "logits/rejected": -1.0047482614113592,
+ "mean_token_accuracy": 0.7206378514692188,
+ "rewards/chosen": -0.015205535748583543,
+ "rewards/rejected": -0.013390423674327395,
+ "rewards/accuracies": 0.49625,
+ "rewards/margins": -0.0018151119937829209,
+ "logps/chosen": -52.25986159801483,
+ "logps/rejected": -53.13925377726555,
+ "epoch": 0.33193299102743634,
+ "step": 320
+ },
+ {
+ "loss": 0.6924636840820313,
+ "grad_norm": 0.009727108292281628,
+ "learning_rate": 6.590673575129535e-06,
+ "entropy": 0.8345451738173142,
+ "num_tokens": 22128277.0,
+ "logits/chosen": -0.971072161782628,
+ "logits/rejected": -0.9826513089025171,
+ "mean_token_accuracy": 0.7212898347340524,
+ "rewards/chosen": -0.016975908809049544,
+ "rewards/rejected": -0.018840813715414698,
+ "rewards/accuracies": 0.51625,
+ "rewards/margins": 0.001864904925096198,
+ "logps/chosen": -52.514592845439914,
+ "logps/rejected": -54.03662874937058,
+ "epoch": 0.3423058969970437,
+ "step": 330
+ },
+ {
+ "loss": 0.6921487808227539,
+ "grad_norm": 0.009848830290138721,
+ "learning_rate": 6.487046632124353e-06,
+ "entropy": 0.8036045160889626,
+ "num_tokens": 22798429.0,
+ "logits/chosen": -0.986366250980096,
+ "logits/rejected": -1.0395464497059017,
+ "mean_token_accuracy": 0.7169532440230251,
+ "rewards/chosen": -0.017634948804889063,
+ "rewards/rejected": -0.02014962821162044,
+ "rewards/accuracies": 0.54,
+ "rewards/margins": 0.002514679412379337,
+ "logps/chosen": -53.712123572826385,
+ "logps/rejected": -50.68536148548126,
+ "epoch": 0.3526788029666511,
+ "step": 340
+ },
+ {
+ "loss": 0.6948633670806885,
+ "grad_norm": 0.009547681547701359,
+ "learning_rate": 6.383419689119171e-06,
+ "entropy": 0.7954194891778752,
+ "num_tokens": 23470239.0,
+ "logits/chosen": -1.0014035524600693,
+ "logits/rejected": -0.9692854696785687,
+ "mean_token_accuracy": 0.7198419809341431,
+ "rewards/chosen": -0.019529370157788436,
+ "rewards/rejected": -0.016619607036818706,
+ "rewards/accuracies": 0.5,
+ "rewards/margins": -0.002909763151255902,
+ "logps/chosen": -54.53537731409073,
+ "logps/rejected": -52.964653131961825,
+ "epoch": 0.3630517089362585,
+ "step": 350
+ },
+ {
+ "loss": 0.6930148601531982,
+ "grad_norm": 0.00898800976574421,
+ "learning_rate": 6.2797927461139905e-06,
+ "entropy": 0.7794413399184122,
+ "num_tokens": 24140239.0,
+ "logits/chosen": -1.0375333631207078,
+ "logits/rejected": -1.0100693145671888,
+ "mean_token_accuracy": 0.737273293249309,
+ "rewards/chosen": -0.016578698544473695,
+ "rewards/rejected": -0.017291930461324226,
+ "rewards/accuracies": 0.4975,
+ "rewards/margins": 0.000713231952759088,
+ "logps/chosen": -49.863780752420425,
+ "logps/rejected": -52.97674039840698,
+ "epoch": 0.37342461490586587,
+ "step": 360
+ },
+ {
+ "loss": 0.6929784297943116,
+ "grad_norm": 0.008984371088445187,
+ "learning_rate": 6.176165803108809e-06,
+ "entropy": 0.8129843348357827,
+ "num_tokens": 24812178.0,
+ "logits/chosen": -0.9893493454110454,
+ "logits/rejected": -0.9891808390055474,
+ "mean_token_accuracy": 0.727164792958647,
+ "rewards/chosen": -0.018017661249964477,
+ "rewards/rejected": -0.01882471950921399,
+ "rewards/accuracies": 0.4975,
+ "rewards/margins": 0.0008070582516302239,
+ "logps/chosen": -53.014821940660475,
+ "logps/rejected": -54.58460282325745,
+ "epoch": 0.38379752087547325,
+ "step": 370
+ },
+ {
+ "eval_loss": 0.6905523538589478,
+ "eval_runtime": 245.3265,
+ "eval_samples_per_second": 4.076,
+ "eval_steps_per_second": 4.076,
+ "eval_entropy": 0.7969645112715662,
+ "eval_num_tokens": 25147105.0,
+ "eval_logits/chosen": -1.0253711017755525,
+ "eval_logits/rejected": -0.9797990125942502,
+ "eval_mean_token_accuracy": 0.7333250712603331,
+ "eval_rewards/chosen": -0.015263017903128458,
+ "eval_rewards/rejected": -0.020903651809865552,
+ "eval_rewards/accuracies": 0.55,
+ "eval_rewards/margins": 0.005640633857896319,
+ "eval_logps/chosen": -50.32561701965332,
+ "eval_logps/rejected": -52.84890823554993,
+ "epoch": 0.38898397386027694,
+ "step": 375
+ },
+ {
+ "loss": 0.6925588607788086,
+ "grad_norm": 0.009255084209144115,
+ "learning_rate": 6.0725388601036275e-06,
+ "entropy": 0.7894979556603358,
+ "num_tokens": 25482532.0,
+ "logits/chosen": -1.0074889789440618,
+ "logits/rejected": -0.9709891010763303,
+ "mean_token_accuracy": 0.7268249686434864,
+ "rewards/chosen": -0.017496899721906517,
+ "rewards/rejected": -0.019117686107287854,
+ "rewards/accuracies": 0.52625,
+ "rewards/margins": 0.001620786436178605,
+ "logps/chosen": -51.8191835963726,
+ "logps/rejected": -51.592100315093994,
+ "epoch": 0.39417042684508063,
+ "step": 380
+ },
+ {
+ "loss": 0.6923740386962891,
+ "grad_norm": 0.009324838407337666,
+ "learning_rate": 5.968911917098445e-06,
+ "entropy": 0.8094338296726346,
+ "num_tokens": 26154266.0,
+ "logits/chosen": -0.9966710708958524,
+ "logits/rejected": -0.9879505808785741,
+ "mean_token_accuracy": 0.7231129581481218,
+ "rewards/chosen": -0.017760099685847307,
+ "rewards/rejected": -0.019829784269199992,
+ "rewards/accuracies": 0.51875,
+ "rewards/margins": 0.0020696846699866,
+ "logps/chosen": -53.66926085233688,
+ "logps/rejected": -54.18102238893509,
+ "epoch": 0.404543332814688,
+ "step": 390
+ },
+ {
+ "loss": 0.6929931163787841,
+ "grad_norm": 0.010886781848967075,
+ "learning_rate": 5.865284974093265e-06,
+ "entropy": 0.8130081182997674,
+ "num_tokens": 26825159.0,
+ "logits/chosen": -1.026189590043526,
+ "logits/rejected": -0.980070075731585,
+ "mean_token_accuracy": 0.7226042114198208,
+ "rewards/chosen": -0.019220582297640475,
+ "rewards/rejected": -0.020057831880958474,
+ "rewards/accuracies": 0.48125,
+ "rewards/margins": 0.0008372495489311404,
+ "logps/chosen": -52.43204548358917,
+ "logps/rejected": -54.36065078496933,
+ "epoch": 0.4149162387842954,
+ "step": 400
+ },
+ {
+ "loss": 0.693073320388794,
+ "grad_norm": 0.009156073443591595,
+ "learning_rate": 5.761658031088083e-06,
+ "entropy": 0.7959862072952092,
+ "num_tokens": 27495663.0,
+ "logits/chosen": -1.0194205629554802,
+ "logits/rejected": -1.008366009739435,
+ "mean_token_accuracy": 0.7243296534568071,
+ "rewards/chosen": -0.01809768541224912,
+ "rewards/rejected": -0.01875489959079914,
+ "rewards/accuracies": 0.5025,
+ "rewards/margins": 0.0006572141727883718,
+ "logps/chosen": -52.169967291355135,
+ "logps/rejected": -53.42250164747238,
+ "epoch": 0.4252891447539028,
+ "step": 410
+ },
+ {
+ "loss": 0.6924574375152588,
+ "grad_norm": 0.010202614590525627,
+ "learning_rate": 5.658031088082902e-06,
+ "entropy": 0.7918436706252396,
+ "num_tokens": 28167389.0,
+ "logits/chosen": -1.0232014742112978,
+ "logits/rejected": -1.0440852867532966,
+ "mean_token_accuracy": 0.7258787673525512,
+ "rewards/chosen": -0.015947307585583416,
+ "rewards/rejected": -0.017838909620567733,
+ "rewards/accuracies": 0.5075,
+ "rewards/margins": 0.0018916020590404514,
+ "logps/chosen": -54.433519496917725,
+ "logps/rejected": -53.2144430232048,
+ "epoch": 0.4356620507235102,
+ "step": 420
+ },
+ {
+ "loss": 0.6926542282104492,
+ "grad_norm": 0.01011913362890482,
+ "learning_rate": 5.554404145077721e-06,
+ "entropy": 0.8117843008134514,
+ "num_tokens": 28839388.0,
+ "logits/chosen": -0.9718024988912555,
+ "logits/rejected": -0.9786808761441761,
+ "mean_token_accuracy": 0.7248365879803896,
+ "rewards/chosen": -0.014674390776453947,
+ "rewards/rejected": -0.016148133582082665,
+ "rewards/accuracies": 0.52875,
+ "rewards/margins": 0.001473742745729396,
+ "logps/chosen": -51.187984328269955,
+ "logps/rejected": -54.15149886369705,
+ "epoch": 0.4460349566931176,
+ "step": 430
+ },
+ {
+ "loss": 0.6937287330627442,
+ "grad_norm": 0.008680191822350025,
+ "learning_rate": 5.4507772020725395e-06,
+ "entropy": 0.7826772296521812,
+ "num_tokens": 29510096.0,
+ "logits/chosen": -1.038226736136573,
+ "logits/rejected": -1.0398130834010744,
+ "mean_token_accuracy": 0.7318821278214455,
+ "rewards/chosen": -0.01869894893682158,
+ "rewards/rejected": -0.01806240452982365,
+ "rewards/accuracies": 0.47625,
+ "rewards/margins": -0.0006365444045513868,
+ "logps/chosen": -51.81987749576569,
+ "logps/rejected": -51.031341784000396,
+ "epoch": 0.456407862662725,
+ "step": 440
+ },
+ {
+ "loss": 0.6937658309936523,
+ "grad_norm": 0.009057857096195221,
+ "learning_rate": 5.347150259067357e-06,
+ "entropy": 0.7959510098583996,
+ "num_tokens": 30181792.0,
+ "logits/chosen": -1.0239556820922306,
+ "logits/rejected": -1.0394439887711346,
+ "mean_token_accuracy": 0.7198793402686715,
+ "rewards/chosen": -0.01936899814491426,
+ "rewards/rejected": -0.01868944700636348,
+ "rewards/accuracies": 0.50375,
+ "rewards/margins": -0.0006795510534720961,
+ "logps/chosen": -54.870300602912906,
+ "logps/rejected": -50.76141629695892,
+ "epoch": 0.46678076863233237,
+ "step": 450
+ },
+ {
+ "loss": 0.694075345993042,
+ "grad_norm": 0.009986916556954384,
+ "learning_rate": 5.243523316062177e-06,
+ "entropy": 0.7800668972823769,
+ "num_tokens": 30851409.0,
+ "logits/chosen": -1.0086595854613267,
+ "logits/rejected": -1.0357231874675175,
+ "mean_token_accuracy": 0.7262966002896428,
+ "rewards/chosen": -0.020605784776303152,
+ "rewards/rejected": -0.019295487125091312,
+ "rewards/accuracies": 0.48125,
+ "rewards/margins": -0.0013102977239759638,
+ "logps/chosen": -53.48096129179001,
+ "logps/rejected": -51.299415925741194,
+ "epoch": 0.47715367460193975,
+ "step": 460
+ },
+ {
+ "loss": 0.6933310985565185,
+ "grad_norm": 0.007825742475688457,
+ "learning_rate": 5.139896373056995e-06,
+ "entropy": 0.7754787660343573,
+ "num_tokens": 31519564.0,
+ "logits/chosen": -1.0726965671758735,
+ "logits/rejected": -1.003480362234256,
+ "mean_token_accuracy": 0.7201942896470428,
+ "rewards/chosen": -0.017236333386230172,
+ "rewards/rejected": -0.017384318440863352,
+ "rewards/accuracies": 0.4825,
+ "rewards/margins": 0.00014798504006648726,
+ "logps/chosen": -49.58583438396454,
+ "logps/rejected": -51.69670748949051,
+ "epoch": 0.48752658057154713,
+ "step": 470
+ },
+ {
+ "loss": 0.6922287940979004,
+ "grad_norm": 0.008367015048861504,
+ "learning_rate": 5.036269430051814e-06,
+ "entropy": 0.8279160013142973,
+ "num_tokens": 32192056.0,
+ "logits/chosen": -0.9712868713229077,
+ "logits/rejected": -0.9765040762093227,
+ "mean_token_accuracy": 0.7188576187193394,
+ "rewards/chosen": -0.01717338605103805,
+ "rewards/rejected": -0.019571030664633327,
+ "rewards/accuracies": 0.51125,
+ "rewards/margins": 0.0023976446047163334,
+ "logps/chosen": -54.479641914367676,
+ "logps/rejected": -54.63456130981445,
+ "epoch": 0.4978994865411545,
+ "step": 480
+ },
+ {
+ "loss": 0.6939034461975098,
+ "grad_norm": 0.008626270107924938,
+ "learning_rate": 4.932642487046633e-06,
+ "entropy": 0.8010151171591133,
+ "num_tokens": 32864077.0,
+ "logits/chosen": -1.0169706789174378,
+ "logits/rejected": -1.0165859789377836,
+ "mean_token_accuracy": 0.7208031569048763,
+ "rewards/chosen": -0.01987447723532,
+ "rewards/rejected": -0.018864784324341598,
+ "rewards/accuracies": 0.495,
+ "rewards/margins": -0.0010096929259452735,
+ "logps/chosen": -53.33563009738922,
+ "logps/rejected": -53.87229782342911,
+ "epoch": 0.5082723925107618,
+ "step": 490
+ },
+ {
+ "loss": 0.6938543319702148,
+ "grad_norm": 0.010269825346767902,
+ "learning_rate": 4.829015544041451e-06,
+ "entropy": 0.7891198094375431,
+ "num_tokens": 33534279.0,
+ "logits/chosen": -1.033360492470276,
+ "logits/rejected": -1.0354731196495317,
+ "mean_token_accuracy": 0.7333813977241516,
+ "rewards/chosen": -0.019480555365971666,
+ "rewards/rejected": -0.01867740810344003,
+ "rewards/accuracies": 0.51,
+ "rewards/margins": -0.0008031472803850193,
+ "logps/chosen": -52.630670845508575,
+ "logps/rejected": -50.910942976474765,
+ "epoch": 0.5186452984803692,
+ "step": 500
+ },
+ {
+ "eval_loss": 0.6942050457000732,
+ "eval_runtime": 276.9162,
+ "eval_samples_per_second": 3.611,
+ "eval_steps_per_second": 3.611,
+ "eval_entropy": 0.7935083331502975,
+ "eval_num_tokens": 33534279.0,
+ "eval_logits/chosen": -1.0600360631491812,
+ "eval_logits/rejected": -1.0166348149416253,
+ "eval_mean_token_accuracy": 0.7330459494143724,
+ "eval_rewards/chosen": -0.021802856784763208,
+ "eval_rewards/rejected": -0.02036227696077913,
+ "eval_rewards/accuracies": 0.477,
+ "eval_rewards/margins": -0.0014405797821236774,
+ "eval_logps/chosen": -50.39101540756226,
+ "eval_logps/rejected": -52.84349448776245,
+ "epoch": 0.5186452984803692,
+ "step": 500
+ },
+ {
+ "loss": 0.6938997745513916,
+ "grad_norm": 0.00896107591688633,
+ "learning_rate": 4.72538860103627e-06,
+ "entropy": 0.7882147227507085,
+ "num_tokens": 34205135.0,
+ "logits/chosen": -1.0076570653558603,
+ "logits/rejected": -1.0416575961228518,
+ "mean_token_accuracy": 0.7137932823970914,
+ "rewards/chosen": -0.018858550680542974,
+ "rewards/rejected": -0.017997891291797713,
+ "rewards/accuracies": 0.495,
+ "rewards/margins": -0.0008606594161392423,
+ "logps/chosen": -54.12554481744766,
+ "logps/rejected": -50.22665623903274,
+ "epoch": 0.5290182044499767,
+ "step": 510
+ },
+ {
+ "loss": 0.6940505504608154,
+ "grad_norm": 0.009844881482422352,
+ "learning_rate": 4.621761658031089e-06,
+ "entropy": 0.7678585767536424,
+ "num_tokens": 34873976.0,
+ "logits/chosen": -1.0794209366681549,
+ "logits/rejected": -1.0414292466399215,
+ "mean_token_accuracy": 0.7310516293905676,
+ "rewards/chosen": -0.013213514556548489,
+ "rewards/rejected": -0.011983713133254241,
+ "rewards/accuracies": 0.515,
+ "rewards/margins": -0.0012298014553380199,
+ "logps/chosen": -51.02845492362976,
+ "logps/rejected": -49.72117327451706,
+ "epoch": 0.5393911104195841,
+ "step": 520
+ },
+ {
+ "loss": 0.6937035083770752,
+ "grad_norm": 0.009832088835537434,
+ "learning_rate": 4.518134715025907e-06,
+ "entropy": 0.8068192195380106,
+ "num_tokens": 35544999.0,
+ "logits/chosen": -1.0203095909710327,
+ "logits/rejected": -1.0544799987360511,
+ "mean_token_accuracy": 0.7208610328473151,
+ "rewards/chosen": -0.013758049447855001,
+ "rewards/rejected": -0.013221281499590986,
+ "rewards/accuracies": 0.50375,
+ "rewards/margins": -0.0005367679687333293,
+ "logps/chosen": -53.891741864681244,
+ "logps/rejected": -52.176739025115964,
+ "epoch": 0.5497640163891915,
+ "step": 530
+ },
+ {
+ "loss": 0.693358039855957,
+ "grad_norm": 0.009179627522826195,
+ "learning_rate": 4.414507772020726e-06,
+ "entropy": 0.8059323144424707,
+ "num_tokens": 36215629.0,
+ "logits/chosen": -1.0271287046641675,
+ "logits/rejected": -1.0058586337462303,
+ "mean_token_accuracy": 0.7286789271980524,
+ "rewards/chosen": -0.01192244978853978,
+ "rewards/rejected": -0.011959875774573447,
+ "rewards/accuracies": 0.495,
+ "rewards/margins": 3.742602792044636e-05,
+ "logps/chosen": -50.73393731594086,
+ "logps/rejected": -52.839989113807675,
+ "epoch": 0.5601369223587989,
+ "step": 540
+ },
+ {
+ "loss": 0.693217134475708,
+ "grad_norm": 0.008687409572303295,
+ "learning_rate": 4.310880829015544e-06,
+ "entropy": 0.8455263479612768,
+ "num_tokens": 36888235.0,
+ "logits/chosen": -0.9577911917232584,
+ "logits/rejected": -0.9647921425538288,
+ "mean_token_accuracy": 0.7241826535388828,
+ "rewards/chosen": -0.012230233296829738,
+ "rewards/rejected": -0.012580791678369679,
+ "rewards/accuracies": 0.52,
+ "rewards/margins": 0.00035055841028224676,
+ "logps/chosen": -53.09883669376373,
+ "logps/rejected": -56.72353342294693,
+ "epoch": 0.5705098283284062,
+ "step": 550
+ },
+ {
+ "loss": 0.6929863929748535,
+ "grad_norm": 0.008931291289627552,
+ "learning_rate": 4.207253886010363e-06,
+ "entropy": 0.8086647934932262,
+ "num_tokens": 37560215.0,
+ "logits/chosen": -1.0313529810131257,
+ "logits/rejected": -1.0182036791475924,
+ "mean_token_accuracy": 0.7183289608359337,
+ "rewards/chosen": -0.013141918864030232,
+ "rewards/rejected": -0.013932020409274628,
+ "rewards/accuracies": 0.505,
+ "rewards/margins": 0.0007901015544848633,
+ "logps/chosen": -52.67770826101303,
+ "logps/rejected": -52.252629327774045,
+ "epoch": 0.5808827342980136,
+ "step": 560
+ },
+ {
+ "loss": 0.6919141292572022,
+ "grad_norm": 0.00928713008761406,
+ "learning_rate": 4.103626943005182e-06,
+ "entropy": 0.795804328629747,
+ "num_tokens": 38229872.0,
+ "logits/chosen": -1.0115969525038464,
+ "logits/rejected": -0.9389554266112111,
+ "mean_token_accuracy": 0.7316008464246988,
+ "rewards/chosen": -0.011151957947622576,
+ "rewards/rejected": -0.014132183314768555,
+ "rewards/accuracies": 0.54375,
+ "rewards/margins": 0.0029802253525849664,
+ "logps/chosen": -50.29158478021622,
+ "logps/rejected": -53.326151065826416,
+ "epoch": 0.591255640267621,
+ "step": 570
+ },
+ {
+ "loss": 0.6936386585235595,
+ "grad_norm": 0.009219864383339882,
+ "learning_rate": 4.000000000000001e-06,
+ "entropy": 0.7849282765295357,
+ "num_tokens": 38898599.0,
+ "logits/chosen": -1.0318535424181245,
+ "logits/rejected": -1.0693508828519998,
+ "mean_token_accuracy": 0.7279249535314739,
+ "rewards/chosen": -0.015236615104731755,
+ "rewards/rejected": -0.01477163603422241,
+ "rewards/accuracies": 0.49375,
+ "rewards/margins": -0.00046497906170770873,
+ "logps/chosen": -50.606445105075835,
+ "logps/rejected": -50.23892628192902,
+ "epoch": 0.6016285462372284,
+ "step": 580
+ },
+ {
+ "loss": 0.6924667358398438,
+ "grad_norm": 0.009871033020317554,
+ "learning_rate": 3.896373056994819e-06,
+ "entropy": 0.7915234410995617,
+ "num_tokens": 39570011.0,
+ "logits/chosen": -1.0704431655924542,
+ "logits/rejected": -1.0262311271415063,
+ "mean_token_accuracy": 0.7341923769563437,
+ "rewards/chosen": -0.013985364213192497,
+ "rewards/rejected": -0.015847310858209767,
+ "rewards/accuracies": 0.53625,
+ "rewards/margins": 0.0018619466816016939,
+ "logps/chosen": -53.37289155721665,
+ "logps/rejected": -53.37732348442078,
+ "epoch": 0.6120014522068358,
+ "step": 590
+ },
+ {
+ "loss": 0.6934967041015625,
+ "grad_norm": 0.008916029706597328,
+ "learning_rate": 3.7927461139896377e-06,
+ "entropy": 0.7781280868733301,
+ "num_tokens": 40238459.0,
+ "logits/chosen": -1.031845585303488,
+ "logits/rejected": -1.0679663812358249,
+ "mean_token_accuracy": 0.7267250791564583,
+ "rewards/chosen": -0.015354626149126034,
+ "rewards/rejected": -0.015155612989574366,
+ "rewards/accuracies": 0.495,
+ "rewards/margins": -0.0001990132535865996,
+ "logps/chosen": -50.91896806716919,
+ "logps/rejected": -49.1647160077095,
+ "epoch": 0.6223743581764432,
+ "step": 600
+ },
+ {
+ "loss": 0.6943734169006348,
+ "grad_norm": 0.009726111777126789,
+ "learning_rate": 3.6891191709844567e-06,
+ "entropy": 0.8024784850515425,
+ "num_tokens": 40907823.0,
+ "logits/chosen": -1.0033099905488874,
+ "logits/rejected": -1.004673033873931,
+ "mean_token_accuracy": 0.7266425576806068,
+ "rewards/chosen": -0.016839503781302483,
+ "rewards/rejected": -0.014924811632938316,
+ "rewards/accuracies": 0.485,
+ "rewards/margins": -0.0019146921102219494,
+ "logps/chosen": -51.37045606851578,
+ "logps/rejected": -50.59996291279793,
+ "epoch": 0.6327472641460505,
+ "step": 610
+ },
+ {
+ "loss": 0.693054485321045,
+ "grad_norm": 0.00853179581463337,
+ "learning_rate": 3.5854922279792748e-06,
+ "entropy": 0.7575920634204522,
+ "num_tokens": 41576629.0,
+ "logits/chosen": -1.0128204565402743,
+ "logits/rejected": -1.0291723665089934,
+ "mean_token_accuracy": 0.7279130771569907,
+ "rewards/chosen": -0.014212547984803337,
+ "rewards/rejected": -0.014947609209768871,
+ "rewards/accuracies": 0.48875,
+ "rewards/margins": 0.000735061179366312,
+ "logps/chosen": -50.57350071668625,
+ "logps/rejected": -49.389897744655606,
+ "epoch": 0.6431201701156579,
+ "step": 620
+ },
+ {
+ "eval_loss": 0.6942050457000732,
+ "eval_runtime": 307.8984,
+ "eval_samples_per_second": 3.248,
+ "eval_steps_per_second": 3.248,
+ "eval_entropy": 0.7959122954644263,
+ "eval_num_tokens": 41911681.0,
+ "eval_logits/chosen": -1.048361439070786,
+ "eval_logits/rejected": -1.0055296550561397,
+ "eval_mean_token_accuracy": 0.7335138544887304,
+ "eval_rewards/chosen": -0.014208579457705128,
+ "eval_rewards/rejected": -0.01261782132985536,
+ "eval_rewards/accuracies": 0.478,
+ "eval_rewards/margins": -0.0015907581194769592,
+ "eval_logps/chosen": -50.31507263565064,
+ "eval_logps/rejected": -52.76604993247986,
+ "epoch": 0.6483066231004616,
+ "step": 625
+ },
+ {
+ "loss": 0.6939579486846924,
+ "grad_norm": 0.009789089672267437,
+ "learning_rate": 3.4818652849740937e-06,
+ "entropy": 0.7933417669357732,
+ "num_tokens": 42247859.0,
+ "logits/chosen": -1.0118922806884323,
+ "logits/rejected": -1.0382741999872491,
+ "mean_token_accuracy": 0.726150699108839,
+ "rewards/chosen": -0.014976464999999734,
+ "rewards/rejected": -0.013932963377121722,
+ "rewards/accuracies": 0.47375,
+ "rewards/margins": -0.0010435017172130757,
+ "logps/chosen": -53.83100793600082,
+ "logps/rejected": -50.46369902849197,
+ "epoch": 0.6534930760852653,
+ "step": 630
+ },
+ {
+ "loss": 0.694810152053833,
+ "grad_norm": 0.009261916391551495,
+ "learning_rate": 3.3782383419689123e-06,
+ "entropy": 0.7883505333820358,
+ "num_tokens": 42918160.0,
+ "logits/chosen": -1.017383539174631,
+ "logits/rejected": -1.054998851475991,
+ "mean_token_accuracy": 0.7168397939577699,
+ "rewards/chosen": -0.015089196973789569,
+ "rewards/rejected": -0.012242867652641962,
+ "rewards/accuracies": 0.46,
+ "rewards/margins": -0.0028463293392269407,
+ "logps/chosen": -54.35950309753418,
+ "logps/rejected": -48.9059769487381,
+ "epoch": 0.6638659820548727,
+ "step": 640
+ },
+ {
+ "loss": 0.6925888061523438,
+ "grad_norm": 0.00910037662833929,
+ "learning_rate": 3.274611398963731e-06,
+ "entropy": 0.790731361717917,
+ "num_tokens": 43587843.0,
+ "logits/chosen": -0.9965395541483734,
+ "logits/rejected": -1.0179589161213116,
+ "mean_token_accuracy": 0.7306452417373657,
+ "rewards/chosen": -0.010891549229872908,
+ "rewards/rejected": -0.012513885581938666,
+ "rewards/accuracies": 0.52875,
+ "rewards/margins": 0.0016223363954850356,
+ "logps/chosen": -51.07387138366699,
+ "logps/rejected": -51.394845466613766,
+ "epoch": 0.6742388880244801,
+ "step": 650
+ },
+ {
+ "loss": 0.693898057937622,
+ "grad_norm": 0.009298712946474552,
+ "learning_rate": 3.1709844559585493e-06,
+ "entropy": 0.7892762251244858,
+ "num_tokens": 44256661.0,
+ "logits/chosen": -1.040483162193093,
+ "logits/rejected": -1.0218257141848732,
+ "mean_token_accuracy": 0.7231168592534959,
+ "rewards/chosen": -0.012846070201399017,
+ "rewards/rejected": -0.01177774412741428,
+ "rewards/accuracies": 0.495,
+ "rewards/margins": -0.001068326040294778,
+ "logps/chosen": -50.02721215367317,
+ "logps/rejected": -51.92197064161301,
+ "epoch": 0.6846117939940874,
+ "step": 660
+ },
+ {
+ "loss": 0.6929262161254883,
+ "grad_norm": 0.009983985684812069,
+ "learning_rate": 3.0673575129533683e-06,
+ "entropy": 0.7727112902654335,
+ "num_tokens": 44928208.0,
+ "logits/chosen": -1.033872497474533,
+ "logits/rejected": -1.0355189189593426,
+ "mean_token_accuracy": 0.7299729858152568,
+ "rewards/chosen": -0.012007646745373676,
+ "rewards/rejected": -0.012911729082511557,
+ "rewards/accuracies": 0.49625,
+ "rewards/margins": 0.0009040823050418112,
+ "logps/chosen": -52.13899112701416,
+ "logps/rejected": -50.62764532446861,
+ "epoch": 0.6949846999636948,
+ "step": 670
+ },
+ {
+ "loss": 0.6946879863739014,
+ "grad_norm": 0.009130201302468777,
+ "learning_rate": 2.963730569948187e-06,
+ "entropy": 0.8138119697524234,
+ "num_tokens": 45599959.0,
+ "logits/chosen": -0.966092846678868,
+ "logits/rejected": -0.9774829642341187,
+ "mean_token_accuracy": 0.7230793483182788,
+ "rewards/chosen": -0.015864108831710837,
+ "rewards/rejected": -0.013314598076967742,
+ "rewards/accuracies": 0.4675,
+ "rewards/margins": -0.0025495107232927695,
+ "logps/chosen": -54.92970582008362,
+ "logps/rejected": -52.76770358800888,
+ "epoch": 0.7053576059333022,
+ "step": 680
+ },
+ {
+ "loss": 0.6929837703704834,
+ "grad_norm": 0.009250026196241379,
+ "learning_rate": 2.8601036269430053e-06,
+ "entropy": 0.82899127332028,
+ "num_tokens": 46271100.0,
+ "logits/chosen": -0.9898684929018607,
+ "logits/rejected": -0.9973996765394122,
+ "mean_token_accuracy": 0.719442187603563,
+ "rewards/chosen": -0.010820029947699367,
+ "rewards/rejected": -0.01162173981451815,
+ "rewards/accuracies": 0.515,
+ "rewards/margins": 0.0008017099273274653,
+ "logps/chosen": -51.98621440887451,
+ "logps/rejected": -54.959817726612094,
+ "epoch": 0.7157305119029096,
+ "step": 690
+ },
+ {
+ "loss": 0.6941079139709473,
+ "grad_norm": 0.009462058544158936,
+ "learning_rate": 2.7564766839378243e-06,
+ "entropy": 0.7970602755853906,
+ "num_tokens": 46940588.0,
+ "logits/chosen": -0.990150519364322,
+ "logits/rejected": -0.9981860405217275,
+ "mean_token_accuracy": 0.7297776956856251,
+ "rewards/chosen": -0.012557482681352213,
+ "rewards/rejected": -0.011030125059482998,
+ "rewards/accuracies": 0.51,
+ "rewards/margins": -0.0015273576516483445,
+ "logps/chosen": -49.37967163324356,
+ "logps/rejected": -53.13790496587753,
+ "epoch": 0.726103417872517,
+ "step": 700
+ },
+ {
+ "loss": 0.6923154830932617,
+ "grad_norm": 0.008459668606519699,
+ "learning_rate": 2.6528497409326424e-06,
+ "entropy": 0.7973380678170361,
+ "num_tokens": 47612649.0,
+ "logits/chosen": -1.0290440507413436,
+ "logits/rejected": -1.0325854925972269,
+ "mean_token_accuracy": 0.7238865295890718,
+ "rewards/chosen": -0.012139474128680376,
+ "rewards/rejected": -0.014260805117913602,
+ "rewards/accuracies": 0.52875,
+ "rewards/margins": 0.0021213308708684054,
+ "logps/chosen": -52.259123842716214,
+ "logps/rejected": -54.400773532390595,
+ "epoch": 0.7364763238421244,
+ "step": 710
+ },
+ {
+ "loss": 0.6927196025848389,
+ "grad_norm": 0.008741075173020363,
+ "learning_rate": 2.5492227979274614e-06,
+ "entropy": 0.776990000186488,
+ "num_tokens": 48280316.0,
+ "logits/chosen": -1.02981366786685,
+ "logits/rejected": -1.062369960988718,
+ "mean_token_accuracy": 0.728738241456449,
+ "rewards/chosen": -0.0141699135041722,
+ "rewards/rejected": -0.01543546461060032,
+ "rewards/accuracies": 0.52,
+ "rewards/margins": 0.0012655511016782838,
+ "logps/chosen": -50.75450169801712,
+ "logps/rejected": -50.51603441476822,
+ "epoch": 0.7468492298117317,
+ "step": 720
+ },
+ {
+ "loss": 0.6939631938934326,
+ "grad_norm": 0.008915440179407597,
+ "learning_rate": 2.44559585492228e-06,
+ "entropy": 0.8170583094470203,
+ "num_tokens": 48952457.0,
+ "logits/chosen": -1.003780851996667,
+ "logits/rejected": -0.9864155673137232,
+ "mean_token_accuracy": 0.7165420038998127,
+ "rewards/chosen": -0.013467036513036419,
+ "rewards/rejected": -0.012332614251897666,
+ "rewards/accuracies": 0.495,
+ "rewards/margins": -0.0011344222621119116,
+ "logps/chosen": -54.76776822566986,
+ "logps/rejected": -52.639701968431474,
+ "epoch": 0.7572221357813391,
+ "step": 730
+ },
+ {
+ "loss": 0.6925458908081055,
+ "grad_norm": 0.01004419568926096,
+ "learning_rate": 2.3419689119170984e-06,
+ "entropy": 0.8180013949144631,
+ "num_tokens": 49623039.0,
+ "logits/chosen": -1.0169417811830719,
+ "logits/rejected": -1.0213671464834864,
+ "mean_token_accuracy": 0.7187614095583558,
+ "rewards/chosen": -0.012766179290275658,
+ "rewards/rejected": -0.014406545395818284,
+ "rewards/accuracies": 0.52625,
+ "rewards/margins": 0.001640366151041235,
+ "logps/chosen": -52.5052701497078,
+ "logps/rejected": -53.90763850927353,
+ "epoch": 0.7675950417509465,
+ "step": 740
+ },
+ {
+ "loss": 0.6922706127166748,
+ "grad_norm": 0.009859760291874409,
+ "learning_rate": 2.2383419689119174e-06,
+ "entropy": 0.8285305345850066,
+ "num_tokens": 50296449.0,
+ "logits/chosen": -0.9801905178679585,
+ "logits/rejected": -0.9738778555663805,
+ "mean_token_accuracy": 0.723729298338294,
+ "rewards/chosen": -0.009565252497509391,
+ "rewards/rejected": -0.01175256536917459,
+ "rewards/accuracies": 0.5175,
+ "rewards/margins": 0.0021873128600418566,
+ "logps/chosen": -54.38914824962616,
+ "logps/rejected": -55.734090762138365,
+ "epoch": 0.7779679477205539,
+ "step": 750
+ },
+ {
+ "eval_loss": 0.6926036477088928,
+ "eval_runtime": 339.6814,
+ "eval_samples_per_second": 2.944,
+ "eval_steps_per_second": 2.944,
+ "eval_entropy": 0.7968713178224862,
+ "eval_num_tokens": 50296449.0,
+ "eval_logits/chosen": -1.0377698264720323,
+ "eval_logits/rejected": -0.9943464753298086,
+ "eval_mean_token_accuracy": 0.7334770380109549,
+ "eval_rewards/chosen": -0.012114850055493434,
+ "eval_rewards/rejected": -0.013621491047078962,
+ "eval_rewards/accuracies": 0.504,
+ "eval_rewards/margins": 0.0015066410247527528,
+ "eval_logps/chosen": -50.294135341644285,
+ "eval_logps/rejected": -52.77608662986756,
+ "epoch": 0.7779679477205539,
+ "step": 750
+ },
+ {
+ "train_runtime": 27561.1083,
+ "train_samples_per_second": 2.798,
+ "train_steps_per_second": 0.035,
+ "total_flos": 4.075839588886364e+17,
+ "train_loss": 0.6933243910471598,
+ "epoch": 0.7779679477205539,
+ "step": 750
+ }
+ ],
+ "validation_monitor_size": 1000,
+ "validation_monitor_selection": "fixed_seed_random_without_replacement",
+ "validation_monitor_seed": 22,
+ "validation_monitor_indices": [
+ 2,
+ 10,
+ 14,
+ 21,
+ 55,
+ 63,
+ 66,
+ 69,
+ 107,
+ 110,
+ 142,
+ 144,
+ 149,
+ 150,
+ 151,
+ 152,
+ 160,
+ 163,
+ 166,
+ 167,
+ 176,
+ 181,
+ 206,
+ 207,
+ 259,
+ 267,
+ 281,
+ 295,
+ 298,
+ 306,
+ 307,
+ 317,
+ 321,
+ 331,
+ 336,
+ 341,
+ 346,
+ 349,
+ 351,
+ 352,
+ 357,
+ 358,
+ 369,
+ 370,
+ 382,
+ 386,
+ 391,
+ 401,
+ 411,
+ 412,
+ 432,
+ 437,
+ 452,
+ 462,
+ 465,
+ 488,
+ 490,
+ 491,
+ 492,
+ 494,
+ 503,
+ 504,
+ 508,
+ 528,
+ 538,
+ 540,
+ 551,
+ 553,
+ 567,
+ 586,
+ 605,
+ 606,
+ 625,
+ 627,
+ 661,
+ 663,
+ 666,
+ 677,
+ 685,
+ 690,
+ 692,
+ 704,
+ 708,
+ 714,
+ 715,
+ 727,
+ 728,
+ 733,
+ 745,
+ 752,
+ 753,
+ 755,
+ 764,
+ 773,
+ 779,
+ 783,
+ 793,
+ 796,
+ 799,
+ 803,
+ 804,
+ 805,
+ 813,
+ 816,
+ 818,
+ 823,
+ 827,
+ 829,
+ 830,
+ 837,
+ 842,
+ 845,
+ 850,
+ 853,
+ 856,
+ 889,
+ 890,
+ 905,
+ 915,
+ 924,
+ 930,
+ 939,
+ 951,
+ 988,
+ 1002,
+ 1005,
+ 1023,
+ 1029,
+ 1038,
+ 1049,
+ 1050,
+ 1054,
+ 1056,
+ 1067,
+ 1068,
+ 1079,
+ 1088,
+ 1091,
+ 1103,
+ 1114,
+ 1118,
+ 1133,
+ 1140,
+ 1144,
+ 1145,
+ 1155,
+ 1186,
+ 1195,
+ 1206,
+ 1223,
+ 1251,
+ 1252,
+ 1257,
+ 1259,
+ 1270,
+ 1271,
+ 1295,
+ 1303,
+ 1304,
+ 1305,
+ 1329,
+ 1335,
+ 1336,
+ 1343,
+ 1367,
+ 1373,
+ 1377,
+ 1403,
+ 1412,
+ 1429,
+ 1443,
+ 1457,
+ 1459,
+ 1460,
+ 1476,
+ 1483,
+ 1486,
+ 1494,
+ 1507,
+ 1510,
+ 1519,
+ 1521,
+ 1522,
+ 1545,
+ 1551,
+ 1570,
+ 1582,
+ 1593,
+ 1595,
+ 1603,
+ 1607,
+ 1614,
+ 1615,
+ 1625,
+ 1634,
+ 1639,
+ 1648,
+ 1654,
+ 1657,
+ 1662,
+ 1667,
+ 1673,
+ 1690,
+ 1704,
+ 1746,
+ 1756,
+ 1781,
+ 1783,
+ 1796,
+ 1799,
+ 1809,
+ 1814,
+ 1827,
+ 1829,
+ 1832,
+ 1844,
+ 1847,
+ 1854,
+ 1856,
+ 1857,
+ 1858,
+ 1874,
+ 1883,
+ 1889,
+ 1924,
+ 1925,
+ 1931,
+ 1932,
+ 1934,
+ 1935,
+ 1938,
+ 1950,
+ 1957,
+ 1962,
+ 1967,
+ 1975,
+ 1982,
+ 1983,
+ 1991,
+ 1998,
+ 2003,
+ 2008,
+ 2017,
+ 2021,
+ 2026,
+ 2035,
+ 2040,
+ 2050,
+ 2056,
+ 2077,
+ 2079,
+ 2082,
+ 2098,
+ 2100,
+ 2108,
+ 2121,
+ 2130,
+ 2133,
+ 2134,
+ 2138,
+ 2143,
+ 2166,
+ 2167,
+ 2180,
+ 2181,
+ 2185,
+ 2204,
+ 2205,
+ 2212,
+ 2221,
+ 2224,
+ 2226,
+ 2230,
+ 2233,
+ 2256,
+ 2261,
+ 2263,
+ 2269,
+ 2273,
+ 2290,
+ 2291,
+ 2299,
+ 2301,
+ 2321,
+ 2323,
+ 2330,
+ 2384,
+ 2401,
+ 2417,
+ 2420,
+ 2421,
+ 2424,
+ 2430,
+ 2435,
+ 2456,
+ 2488,
+ 2502,
+ 2504,
+ 2519,
+ 2527,
+ 2532,
+ 2538,
+ 2542,
+ 2553,
+ 2555,
+ 2558,
+ 2559,
+ 2562,
+ 2578,
+ 2583,
+ 2585,
+ 2590,
+ 2592,
+ 2594,
+ 2596,
+ 2600,
+ 2606,
+ 2607,
+ 2618,
+ 2630,
+ 2637,
+ 2647,
+ 2650,
+ 2657,
+ 2679,
+ 2685,
+ 2705,
+ 2706,
+ 2712,
+ 2713,
+ 2714,
+ 2727,
+ 2740,
+ 2742,
+ 2755,
+ 2780,
+ 2784,
+ 2792,
+ 2807,
+ 2808,
+ 2810,
+ 2820,
+ 2831,
+ 2839,
+ 2860,
+ 2862,
+ 2863,
+ 2866,
+ 2875,
+ 2878,
+ 2898,
+ 2905,
+ 2921,
+ 2922,
+ 2926,
+ 2930,
+ 2934,
+ 2944,
+ 2955,
+ 2957,
+ 2959,
+ 2973,
+ 2978,
+ 2998,
+ 3018,
+ 3022,
+ 3028,
+ 3031,
+ 3061,
+ 3085,
+ 3090,
+ 3104,
+ 3105,
+ 3111,
+ 3115,
+ 3121,
+ 3122,
+ 3129,
+ 3133,
+ 3135,
+ 3161,
+ 3162,
+ 3169,
+ 3173,
+ 3194,
+ 3195,
+ 3215,
+ 3225,
+ 3226,
+ 3241,
+ 3247,
+ 3250,
+ 3253,
+ 3265,
+ 3268,
+ 3293,
+ 3301,
+ 3312,
+ 3329,
+ 3352,
+ 3361,
+ 3362,
+ 3376,
+ 3396,
+ 3401,
+ 3403,
+ 3410,
+ 3419,
+ 3432,
+ 3443,
+ 3452,
+ 3467,
+ 3469,
+ 3475,
+ 3485,
+ 3503,
+ 3514,
+ 3515,
+ 3524,
+ 3528,
+ 3538,
+ 3544,
+ 3557,
+ 3560,
+ 3565,
+ 3600,
+ 3637,
+ 3642,
+ 3658,
+ 3659,
+ 3692,
+ 3693,
+ 3699,
+ 3722,
+ 3725,
+ 3728,
+ 3731,
+ 3740,
+ 3742,
+ 3762,
+ 3766,
+ 3780,
+ 3788,
+ 3794,
+ 3796,
+ 3798,
+ 3805,
+ 3817,
+ 3819,
+ 3822,
+ 3837,
+ 3839,
+ 3843,
+ 3846,
+ 3851,
+ 3854,
+ 3865,
+ 3870,
+ 3871,
+ 3884,
+ 3894,
+ 3895,
+ 3896,
+ 3907,
+ 3911,
+ 3915,
+ 3919,
+ 3920,
+ 3923,
+ 3933,
+ 3955,
+ 3967,
+ 3972,
+ 3974,
+ 3975,
+ 3984,
+ 3985,
+ 3997,
+ 4002,
+ 4010,
+ 4034,
+ 4044,
+ 4063,
+ 4073,
+ 4079,
+ 4082,
+ 4088,
+ 4121,
+ 4145,
+ 4148,
+ 4159,
+ 4161,
+ 4164,
+ 4177,
+ 4188,
+ 4199,
+ 4203,
+ 4207,
+ 4208,
+ 4213,
+ 4221,
+ 4225,
+ 4233,
+ 4241,
+ 4243,
+ 4247,
+ 4264,
+ 4274,
+ 4282,
+ 4286,
+ 4290,
+ 4308,
+ 4310,
+ 4313,
+ 4321,
+ 4324,
+ 4327,
+ 4349,
+ 4362,
+ 4370,
+ 4374,
+ 4380,
+ 4407,
+ 4409,
+ 4411,
+ 4415,
+ 4417,
+ 4418,
+ 4427,
+ 4431,
+ 4433,
+ 4451,
+ 4466,
+ 4477,
+ 4478,
+ 4479,
+ 4493,
+ 4494,
+ 4514,
+ 4527,
+ 4539,
+ 4550,
+ 4558,
+ 4568,
+ 4579,
+ 4583,
+ 4584,
+ 4586,
+ 4587,
+ 4590,
+ 4599,
+ 4602,
+ 4610,
+ 4626,
+ 4627,
+ 4630,
+ 4641,
+ 4647,
+ 4655,
+ 4656,
+ 4657,
+ 4661,
+ 4685,
+ 4714,
+ 4715,
+ 4731,
+ 4749,
+ 4752,
+ 4769,
+ 4777,
+ 4782,
+ 4791,
+ 4805,
+ 4818,
+ 4829,
+ 4833,
+ 4837,
+ 4839,
+ 4843,
+ 4871,
+ 4875,
+ 4879,
+ 4880,
+ 4885,
+ 4888,
+ 4895,
+ 4900,
+ 4923,
+ 4966,
+ 4968,
+ 4972,
+ 4989,
+ 4994,
+ 4998,
+ 5001,
+ 5013,
+ 5019,
+ 5026,
+ 5032,
+ 5034,
+ 5046,
+ 5055,
+ 5085,
+ 5086,
+ 5088,
+ 5089,
+ 5090,
+ 5095,
+ 5108,
+ 5110,
+ 5119,
+ 5120,
+ 5121,
+ 5133,
+ 5148,
+ 5154,
+ 5160,
+ 5169,
+ 5178,
+ 5222,
+ 5223,
+ 5232,
+ 5233,
+ 5240,
+ 5256,
+ 5259,
+ 5264,
+ 5271,
+ 5276,
+ 5279,
+ 5294,
+ 5300,
+ 5303,
+ 5321,
+ 5335,
+ 5337,
+ 5345,
+ 5348,
+ 5365,
+ 5373,
+ 5378,
+ 5384,
+ 5422,
+ 5442,
+ 5443,
+ 5445,
+ 5477,
+ 5485,
+ 5495,
+ 5497,
+ 5504,
+ 5526,
+ 5533,
+ 5542,
+ 5564,
+ 5570,
+ 5579,
+ 5587,
+ 5592,
+ 5608,
+ 5610,
+ 5624,
+ 5630,
+ 5638,
+ 5651,
+ 5663,
+ 5670,
+ 5675,
+ 5691,
+ 5700,
+ 5706,
+ 5707,
+ 5715,
+ 5720,
+ 5721,
+ 5722,
+ 5732,
+ 5738,
+ 5741,
+ 5769,
+ 5771,
+ 5775,
+ 5795,
+ 5796,
+ 5800,
+ 5809,
+ 5810,
+ 5815,
+ 5819,
+ 5827,
+ 5848,
+ 5849,
+ 5852,
+ 5859,
+ 5880,
+ 5884,
+ 5907,
+ 5909,
+ 5912,
+ 5919,
+ 5931,
+ 5932,
+ 5934,
+ 5941,
+ 5948,
+ 5950,
+ 5952,
+ 5953,
+ 5969,
+ 5981,
+ 6000,
+ 6003,
+ 6010,
+ 6018,
+ 6041,
+ 6065,
+ 6075,
+ 6090,
+ 6103,
+ 6106,
+ 6109,
+ 6114,
+ 6123,
+ 6131,
+ 6136,
+ 6138,
+ 6139,
+ 6164,
+ 6165,
+ 6171,
+ 6173,
+ 6180,
+ 6185,
+ 6189,
+ 6190,
+ 6221,
+ 6223,
+ 6237,
+ 6255,
+ 6262,
+ 6265,
+ 6293,
+ 6296,
+ 6305,
+ 6318,
+ 6331,
+ 6336,
+ 6340,
+ 6355,
+ 6366,
+ 6371,
+ 6396,
+ 6399,
+ 6402,
+ 6408,
+ 6432,
+ 6433,
+ 6441,
+ 6456,
+ 6465,
+ 6478,
+ 6486,
+ 6489,
+ 6507,
+ 6508,
+ 6520,
+ 6522,
+ 6528,
+ 6537,
+ 6551,
+ 6564,
+ 6589,
+ 6590,
+ 6598,
+ 6599,
+ 6611,
+ 6613,
+ 6615,
+ 6621,
+ 6634,
+ 6647,
+ 6649,
+ 6654,
+ 6676,
+ 6680,
+ 6690,
+ 6708,
+ 6729,
+ 6736,
+ 6744,
+ 6749,
+ 6756,
+ 6761,
+ 6763,
+ 6773,
+ 6788,
+ 6790,
+ 6796,
+ 6797,
+ 6811,
+ 6824,
+ 6850,
+ 6869,
+ 6871,
+ 6882,
+ 6892,
+ 6895,
+ 6906,
+ 6911,
+ 6912,
+ 6914,
+ 6927,
+ 6952,
+ 6966,
+ 6985,
+ 7001,
+ 7021,
+ 7031,
+ 7035,
+ 7038,
+ 7041,
+ 7045,
+ 7052,
+ 7057,
+ 7058,
+ 7072,
+ 7073,
+ 7076,
+ 7086,
+ 7102,
+ 7107,
+ 7109,
+ 7117,
+ 7122,
+ 7125,
+ 7166,
+ 7182,
+ 7199,
+ 7207,
+ 7212,
+ 7215,
+ 7232,
+ 7243,
+ 7246,
+ 7250,
+ 7253,
+ 7258,
+ 7263,
+ 7267,
+ 7270,
+ 7274,
+ 7280,
+ 7286,
+ 7293,
+ 7298,
+ 7302,
+ 7306,
+ 7316,
+ 7325,
+ 7326,
+ 7334,
+ 7356,
+ 7357,
+ 7363,
+ 7364,
+ 7367,
+ 7385,
+ 7392,
+ 7399,
+ 7405,
+ 7416,
+ 7420,
+ 7421,
+ 7426,
+ 7452,
+ 7476,
+ 7481,
+ 7519,
+ 7534,
+ 7542,
+ 7546,
+ 7573,
+ 7585,
+ 7601,
+ 7604,
+ 7606,
+ 7609,
+ 7629,
+ 7649,
+ 7653,
+ 7667,
+ 7682,
+ 7699,
+ 7738,
+ 7750,
+ 7786,
+ 7798,
+ 7800,
+ 7801,
+ 7805,
+ 7806,
+ 7811,
+ 7813,
+ 7832,
+ 7837,
+ 7849,
+ 7856,
+ 7876,
+ 7877,
+ 7887,
+ 7905,
+ 7916,
+ 7919,
+ 7920,
+ 7922,
+ 7923,
+ 7926,
+ 7944,
+ 7961,
+ 7966,
+ 7970,
+ 7973,
+ 7974,
+ 7976,
+ 7986,
+ 7999,
+ 8027,
+ 8028,
+ 8034,
+ 8047,
+ 8068,
+ 8074,
+ 8077,
+ 8091,
+ 8120,
+ 8122,
+ 8125,
+ 8152,
+ 8153,
+ 8164,
+ 8167,
+ 8169,
+ 8171,
+ 8177,
+ 8184,
+ 8189,
+ 8192,
+ 8196,
+ 8202,
+ 8212,
+ 8217,
+ 8231,
+ 8242,
+ 8244,
+ 8250,
+ 8256,
+ 8257,
+ 8265,
+ 8270,
+ 8274,
+ 8283,
+ 8290,
+ 8294,
+ 8301,
+ 8302,
+ 8307,
+ 8318,
+ 8326,
+ 8338,
+ 8349,
+ 8350,
+ 8353,
+ 8357,
+ 8371,
+ 8374,
+ 8377,
+ 8380,
+ 8387,
+ 8388,
+ 8396,
+ 8402,
+ 8419,
+ 8423,
+ 8428,
+ 8435,
+ 8439,
+ 8442,
+ 8444,
+ 8453,
+ 8461,
+ 8475,
+ 8481,
+ 8485,
+ 8493,
+ 8495,
+ 8498,
+ 8523,
+ 8530,
+ 8531,
+ 8562
+ ],
+ "early_stopping_patience": 3
+}
diff --git a/phase1/ablations/shuffled_documents/README.md b/phase1/ablations/shuffled_documents/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..cb985f2e1d45eff1244420af9883541cab0cfcc2
--- /dev/null
+++ b/phase1/ablations/shuffled_documents/README.md
@@ -0,0 +1,81 @@
+---
+library_name: peft
+model_name: phase1-qwen-shuffled-documents-ablation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-1.5B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+license: apache-2.0
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+pipeline_tag: text-generation
+---
+
+# Phase 1 Qwen `shuffled_documents` diagnostic ablation
+
+This is the Phase 1 `shuffled_documents` diagnostic LoRA-DPO adapter fine-tuned
+from
+[Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct).
+The frozen training recipe and membership hashes are included in this directory.
+
+## Quick start
+
+```python
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo_id = "geyuxu/york-milestone-project-self-traing-loop-model"
+subfolder = "phase1/ablations/shuffled_documents"
+base_id = "Qwen/Qwen2.5-1.5B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(repo_id, subfolder=subfolder)
+base_model = AutoModelForCausalLM.from_pretrained(
+ base_id, torch_dtype="auto", device_map="auto"
+)
+model = PeftModel.from_pretrained(base_model, repo_id, subfolder=subfolder)
+```
+
+## Training procedure
+
+
+
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 0.29.0
+- Transformers: 5.3.0
+- Pytorch: 2.10.0
+- Datasets: 4.6.1
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+ title = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+ author = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+ year = 2023,
+ booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+ url = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+ editor = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+ title = {{TRL: Transformers Reinforcement Learning}},
+ author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+ license = {Apache-2.0},
+ url = {https://github.com/huggingface/trl},
+ year = {2020}
+}
+```
diff --git a/phase1/ablations/shuffled_documents/adapter_config.json b/phase1/ablations/shuffled_documents/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..fa47a2292cd8a23ccf1b4d101d7f7a03e3f08871
--- /dev/null
+++ b/phase1/ablations/shuffled_documents/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "k_proj",
+ "o_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
diff --git a/phase1/ablations/shuffled_documents/adapter_model.safetensors b/phase1/ablations/shuffled_documents/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b90ad15ec043f3443c08e93e294db5f4a33d5d5a
--- /dev/null
+++ b/phase1/ablations/shuffled_documents/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9d8b830ce2a3b583eb54eab7470a7b3acb9c163dee812de0313c454462de064
+size 17462432
diff --git a/phase1/ablations/shuffled_documents/chat_template.jinja b/phase1/ablations/shuffled_documents/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e
--- /dev/null
+++ b/phase1/ablations/shuffled_documents/chat_template.jinja
@@ -0,0 +1,54 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- messages[0]['content'] }}
+ {%- else %}
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+ {%- endif %}
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+ {%- else %}
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {{- '<|im_start|>' + message.role }}
+ {%- if message.content %}
+ {{- '\n' + message.content }}
+ {%- endif %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- '}\n' }}
+ {%- endfor %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- message.content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}
diff --git a/phase1/ablations/shuffled_documents/tokenizer.json b/phase1/ablations/shuffled_documents/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..d73428d91463b495bc017fb6a2fb3a656646482b
--- /dev/null
+++ b/phase1/ablations/shuffled_documents/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5
+size 11422166
diff --git a/phase1/ablations/shuffled_documents/tokenizer_config.json b/phase1/ablations/shuffled_documents/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9fd0fb48e73786f54fb04e98892f5f5a0ebeebdb
--- /dev/null
+++ b/phase1/ablations/shuffled_documents/tokenizer_config.json
@@ -0,0 +1,29 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": true,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/phase1/ablations/shuffled_documents/training_args.bin b/phase1/ablations/shuffled_documents/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..271cc8aab7c863188d6fb6c04e10b5c3e1c56d81
--- /dev/null
+++ b/phase1/ablations/shuffled_documents/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eaedd299a54d4c58a5c10cd9e379b80d44bbfd7922ccd578d5731f3e7adeb7cb
+size 5841
diff --git a/phase1/ablations/shuffled_documents/training_recipe.json b/phase1/ablations/shuffled_documents/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..b135fecfbffc39053205ee997ce6a0527b333fa0
--- /dev/null
+++ b/phase1/ablations/shuffled_documents/training_recipe.json
@@ -0,0 +1,22 @@
+{
+ "format_version": 1,
+ "experiment": "confusion_ablation",
+ "ablation": "shuffled_documents",
+ "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+ "train_membership_sha256": "b497fa765223a9810ad784f31e1598edfa992a162146d6f94d9cdf4adcedd87d",
+ "validation_membership_sha256": "2f609467438cd6a01fd84b9d80c0fcd84c44767e3b2ed98f4f6a83d68a62c181",
+ "num_train_pairs": 77124,
+ "num_validation_pairs": 8563,
+ "transformation_seed": 22,
+ "validation_prompt_mode": "clean_rag",
+ "learning_rate": 1e-05,
+ "beta": 0.1,
+ "num_epochs": 1.0,
+ "seed": 22,
+ "max_length": 512,
+ "max_prompt_length": 384,
+ "gradient_accumulation_steps": 80,
+ "save_steps": 125,
+ "save_total_limit": 12,
+ "preference_pairs_format_version": 2
+}
diff --git a/phase1/ablations/shuffled_documents/training_summary.json b/phase1/ablations/shuffled_documents/training_summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..d29f62030e38741316b3eb5171cd09541aef6157
--- /dev/null
+++ b/phase1/ablations/shuffled_documents/training_summary.json
@@ -0,0 +1,2925 @@
+{
+ "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+ "resolved_model": "Qwen/Qwen2.5-1.5B-Instruct",
+ "num_train_records": 77124,
+ "num_validation_records_available": 8563,
+ "training_config": {
+ "lora_rank": 16,
+ "lora_alpha": 32,
+ "learning_rate": 1e-05,
+ "num_epochs": 1.0,
+ "per_device_batch_size": 1,
+ "per_device_eval_batch_size": 1,
+ "gradient_accumulation_steps": 80,
+ "beta": 0.1,
+ "max_length": 512,
+ "max_prompt_length": 384,
+ "seed": 22,
+ "save_steps": 125,
+ "save_total_limit": 12,
+ "resume_from_checkpoint": null,
+ "precision_dtype": "torch.bfloat16",
+ "bf16": true,
+ "fp16": false,
+ "tokenizer_add_bos_token": false
+ },
+ "global_step": 965,
+ "best_metric": 0.5229986906051636,
+ "best_model_checkpoint": "outputs/confusion_ablation_phase1_repair_v3/shuffled_documents/lora/checkpoint-875",
+ "train_metrics": {
+ "train_runtime": 37547.325,
+ "train_samples_per_second": 2.054,
+ "train_steps_per_second": 0.026,
+ "total_flos": 5.240658898225705e+17,
+ "train_loss": 0.4893577980871645
+ },
+ "log_history": [
+ {
+ "loss": 0.6884657859802246,
+ "grad_norm": 0.012298274785280228,
+ "learning_rate": 9.906735751295338e-06,
+ "entropy": 1.105509125497192,
+ "num_tokens": 669280.0,
+ "logits/chosen": -0.6872332618784401,
+ "logits/rejected": -0.756335103700859,
+ "mean_token_accuracy": 0.6783600400015711,
+ "rewards/chosen": 0.006566623165663259,
+ "rewards/rejected": -0.0032956102347088743,
+ "rewards/accuracies": 0.54875,
+ "rewards/margins": 0.009862233377571101,
+ "logps/chosen": -55.77936493992806,
+ "logps/rejected": -47.51172333955765,
+ "epoch": 0.010372905969607386,
+ "step": 10
+ },
+ {
+ "loss": 0.6606436729431152,
+ "grad_norm": 0.016955673694610596,
+ "learning_rate": 9.803108808290156e-06,
+ "entropy": 1.133750858772546,
+ "num_tokens": 1338649.0,
+ "logits/chosen": -0.5871208397589887,
+ "logits/rejected": -0.6698573597018519,
+ "mean_token_accuracy": 0.6718795161508023,
+ "rewards/chosen": 0.04518332487099542,
+ "rewards/rejected": -0.02475843264875948,
+ "rewards/accuracies": 0.77625,
+ "rewards/margins": 0.06994175776082556,
+ "logps/chosen": -57.90504644036293,
+ "logps/rejected": -46.12085329890251,
+ "epoch": 0.02074581193921477,
+ "step": 20
+ },
+ {
+ "loss": 0.6296379089355468,
+ "grad_norm": 0.009446840733289719,
+ "learning_rate": 9.699481865284975e-06,
+ "entropy": 1.1883827408123762,
+ "num_tokens": 2009418.0,
+ "logits/chosen": -0.5129337609376847,
+ "logits/rejected": -0.521814939463297,
+ "mean_token_accuracy": 0.6750773158855736,
+ "rewards/chosen": 0.07124414303190861,
+ "rewards/rejected": -0.07998447065116351,
+ "rewards/accuracies": 0.7475,
+ "rewards/margins": 0.15122861398209353,
+ "logps/chosen": -57.641008596420285,
+ "logps/rejected": -48.77114168405533,
+ "epoch": 0.031118717908822157,
+ "step": 30
+ },
+ {
+ "loss": 0.598163366317749,
+ "grad_norm": 0.00930850487202406,
+ "learning_rate": 9.595854922279793e-06,
+ "entropy": 1.2363800938241183,
+ "num_tokens": 2680351.0,
+ "logits/chosen": -0.39490034436465016,
+ "logits/rejected": -0.38446442411270937,
+ "mean_token_accuracy": 0.6727914265915751,
+ "rewards/chosen": 0.06864972691008006,
+ "rewards/rejected": -0.17466060707674388,
+ "rewards/accuracies": 0.7575,
+ "rewards/margins": 0.2433103342913091,
+ "logps/chosen": -57.28514166235924,
+ "logps/rejected": -48.75534886360168,
+ "epoch": 0.04149162387842954,
+ "step": 40
+ },
+ {
+ "loss": 0.579415225982666,
+ "grad_norm": 0.008170416578650475,
+ "learning_rate": 9.492227979274612e-06,
+ "entropy": 1.3126574665866793,
+ "num_tokens": 3350707.0,
+ "logits/chosen": -0.29500715260247096,
+ "logits/rejected": -0.24820786208910245,
+ "mean_token_accuracy": 0.6680559937469661,
+ "rewards/chosen": 0.030048758659595477,
+ "rewards/rejected": -0.3040865566948378,
+ "rewards/accuracies": 0.76875,
+ "rewards/margins": 0.33413531533209606,
+ "logps/chosen": -58.29518891453743,
+ "logps/rejected": -49.01092249631882,
+ "epoch": 0.051864529848036925,
+ "step": 50
+ },
+ {
+ "loss": 0.5702046871185302,
+ "grad_norm": 0.008693543262779713,
+ "learning_rate": 9.388601036269432e-06,
+ "entropy": 1.3737784252408891,
+ "num_tokens": 4019963.0,
+ "logits/chosen": -0.21998480819560648,
+ "logits/rejected": -0.16028713959825802,
+ "mean_token_accuracy": 0.668321382869035,
+ "rewards/chosen": -0.01997856437286828,
+ "rewards/rejected": -0.4219406557866023,
+ "rewards/accuracies": 0.765,
+ "rewards/margins": 0.40196209065266886,
+ "logps/chosen": -57.433870965242384,
+ "logps/rejected": -49.84653505563736,
+ "epoch": 0.062237435817644314,
+ "step": 60
+ },
+ {
+ "loss": 0.5322866916656495,
+ "grad_norm": 0.008744485676288605,
+ "learning_rate": 9.284974093264249e-06,
+ "entropy": 1.3998539651371538,
+ "num_tokens": 4689814.0,
+ "logits/chosen": -0.20973319955981945,
+ "logits/rejected": -0.06686862488846866,
+ "mean_token_accuracy": 0.6616723272390663,
+ "rewards/chosen": -0.06465568452780644,
+ "rewards/rejected": -0.5979045833898999,
+ "rewards/accuracies": 0.7625,
+ "rewards/margins": 0.5332488995540189,
+ "logps/chosen": -59.22309771895409,
+ "logps/rejected": -51.44588413834572,
+ "epoch": 0.0726103417872517,
+ "step": 70
+ },
+ {
+ "loss": 0.5353285789489746,
+ "grad_norm": 0.008123241364955902,
+ "learning_rate": 9.181347150259067e-06,
+ "entropy": 1.505200822763145,
+ "num_tokens": 5359764.0,
+ "logits/chosen": -0.1390746882185333,
+ "logits/rejected": -0.0019247237053158573,
+ "mean_token_accuracy": 0.6578724461607635,
+ "rewards/chosen": -0.18390433938111528,
+ "rewards/rejected": -0.7765666933602188,
+ "rewards/accuracies": 0.77,
+ "rewards/margins": 0.5926623552781529,
+ "logps/chosen": -60.01300884962082,
+ "logps/rejected": -55.53570630192757,
+ "epoch": 0.08298324775685909,
+ "step": 80
+ },
+ {
+ "loss": 0.5359155654907226,
+ "grad_norm": 0.009612024761736393,
+ "learning_rate": 9.077720207253888e-06,
+ "entropy": 1.5023025350272656,
+ "num_tokens": 6029676.0,
+ "logits/chosen": -0.11111740139892143,
+ "logits/rejected": 0.01945645116999368,
+ "mean_token_accuracy": 0.65636579496786,
+ "rewards/chosen": -0.2407170122365642,
+ "rewards/rejected": -0.9039892829529709,
+ "rewards/accuracies": 0.77,
+ "rewards/margins": 0.6632722699642182,
+ "logps/chosen": -61.11234235405922,
+ "logps/rejected": -55.99775354385376,
+ "epoch": 0.09335615372646647,
+ "step": 90
+ },
+ {
+ "loss": 0.5361231803894043,
+ "grad_norm": 0.009299149736762047,
+ "learning_rate": 8.974093264248706e-06,
+ "entropy": 1.551793462112546,
+ "num_tokens": 6703908.0,
+ "logits/chosen": -0.0496634218575803,
+ "logits/rejected": 0.05476262310486179,
+ "mean_token_accuracy": 0.6553104593977332,
+ "rewards/chosen": -0.3408397056261765,
+ "rewards/rejected": -1.0588691204045608,
+ "rewards/accuracies": 0.7475,
+ "rewards/margins": 0.7180294132977724,
+ "logps/chosen": -66.09697647929191,
+ "logps/rejected": -62.22113917827606,
+ "epoch": 0.10372905969607385,
+ "step": 100
+ },
+ {
+ "loss": 0.5009335041046142,
+ "grad_norm": 0.011182536371052265,
+ "learning_rate": 8.870466321243523e-06,
+ "entropy": 1.5229294698499143,
+ "num_tokens": 7374540.0,
+ "logits/chosen": -0.09139268708523057,
+ "logits/rejected": 0.04862301456245882,
+ "mean_token_accuracy": 0.6561858785152436,
+ "rewards/chosen": -0.32845730935261597,
+ "rewards/rejected": -1.1125860015151556,
+ "rewards/accuracies": 0.8,
+ "rewards/margins": 0.7841286932909861,
+ "logps/chosen": -61.53394140005112,
+ "logps/rejected": -57.48781939744949,
+ "epoch": 0.11410196566568125,
+ "step": 110
+ },
+ {
+ "loss": 0.526285982131958,
+ "grad_norm": 0.007051755208522081,
+ "learning_rate": 8.766839378238343e-06,
+ "entropy": 1.5233578365854918,
+ "num_tokens": 8044495.0,
+ "logits/chosen": -0.15768700395031163,
+ "logits/rejected": -0.013001435805658218,
+ "mean_token_accuracy": 0.6615354198031127,
+ "rewards/chosen": -0.3864951348638078,
+ "rewards/rejected": -1.1354628334190056,
+ "rewards/accuracies": 0.76625,
+ "rewards/margins": 0.7489676963654347,
+ "logps/chosen": -62.9133579492569,
+ "logps/rejected": -55.22580579996109,
+ "epoch": 0.12447487163528863,
+ "step": 120
+ },
+ {
+ "eval_loss": 0.5760948657989502,
+ "eval_runtime": 189.325,
+ "eval_samples_per_second": 5.282,
+ "eval_steps_per_second": 5.282,
+ "eval_entropy": 1.1003064152523876,
+ "eval_num_tokens": 8379955.0,
+ "eval_logits/chosen": -0.20125681515889032,
+ "eval_logits/rejected": -0.04611639937520787,
+ "eval_mean_token_accuracy": 0.6978467227965593,
+ "eval_rewards/chosen": -0.518214108169428,
+ "eval_rewards/rejected": -1.0567502626019996,
+ "eval_rewards/accuracies": 0.671,
+ "eval_rewards/margins": 0.5385361527912319,
+ "eval_logps/chosen": -55.35512784194946,
+ "eval_logps/rejected": -63.207374160766605,
+ "epoch": 0.1296613246200923,
+ "step": 125
+ },
+ {
+ "loss": 0.527292537689209,
+ "grad_norm": 0.008284475654363632,
+ "learning_rate": 8.663212435233162e-06,
+ "entropy": 1.5593797556497158,
+ "num_tokens": 8715927.0,
+ "logits/chosen": -0.13633529239582157,
+ "logits/rejected": 0.00036253381978516077,
+ "mean_token_accuracy": 0.6549849599599838,
+ "rewards/chosen": -0.385840279032418,
+ "rewards/rejected": -1.156078426001768,
+ "rewards/accuracies": 0.77125,
+ "rewards/margins": 0.7702381464419886,
+ "logps/chosen": -62.51519522190094,
+ "logps/rejected": -59.316818933486935,
+ "epoch": 0.13484777760489602,
+ "step": 130
+ },
+ {
+ "loss": 0.5253304481506348,
+ "grad_norm": 0.00900308322161436,
+ "learning_rate": 8.55958549222798e-06,
+ "entropy": 1.4882201609387993,
+ "num_tokens": 9384889.0,
+ "logits/chosen": -0.2849882547966558,
+ "logits/rejected": -0.11130883394408073,
+ "mean_token_accuracy": 0.6634942902065813,
+ "rewards/chosen": -0.3596310636661656,
+ "rewards/rejected": -1.091110472352011,
+ "rewards/accuracies": 0.76625,
+ "rewards/margins": 0.7314794071600772,
+ "logps/chosen": -57.09630239605904,
+ "logps/rejected": -58.20026801586151,
+ "epoch": 0.1452206835745034,
+ "step": 140
+ },
+ {
+ "loss": 0.49091219902038574,
+ "grad_norm": 0.010285602882504463,
+ "learning_rate": 8.455958549222799e-06,
+ "entropy": 1.5471849937550723,
+ "num_tokens": 10056547.0,
+ "logits/chosen": -0.2356386486926287,
+ "logits/rejected": -0.09653027008020171,
+ "mean_token_accuracy": 0.6451748718414456,
+ "rewards/chosen": -0.29642743543074174,
+ "rewards/rejected": -1.1533646477002184,
+ "rewards/accuracies": 0.7975,
+ "rewards/margins": 0.8569372114818543,
+ "logps/chosen": -61.48972049832344,
+ "logps/rejected": -60.38522361636162,
+ "epoch": 0.1555935895441108,
+ "step": 150
+ },
+ {
+ "loss": 0.5228443622589112,
+ "grad_norm": 0.01041741855442524,
+ "learning_rate": 8.352331606217617e-06,
+ "entropy": 1.5723768063262105,
+ "num_tokens": 10728371.0,
+ "logits/chosen": -0.22289431271854238,
+ "logits/rejected": -0.11860647859510084,
+ "mean_token_accuracy": 0.6467062424123288,
+ "rewards/chosen": -0.40625758489735747,
+ "rewards/rejected": -1.1412722651916556,
+ "rewards/accuracies": 0.765,
+ "rewards/margins": 0.7350146814109757,
+ "logps/chosen": -65.39053342223167,
+ "logps/rejected": -60.2540340256691,
+ "epoch": 0.16596649551371817,
+ "step": 160
+ },
+ {
+ "loss": 0.5066161155700684,
+ "grad_norm": 0.0107549037784338,
+ "learning_rate": 8.248704663212436e-06,
+ "entropy": 1.5730788687244057,
+ "num_tokens": 11400691.0,
+ "logits/chosen": -0.1826010631193764,
+ "logits/rejected": -0.0979360531869457,
+ "mean_token_accuracy": 0.6487311513535678,
+ "rewards/chosen": -0.366861000080653,
+ "rewards/rejected": -1.1764580160021432,
+ "rewards/accuracies": 0.785,
+ "rewards/margins": 0.8095970169454813,
+ "logps/chosen": -63.89318405985832,
+ "logps/rejected": -58.80061838746071,
+ "epoch": 0.17633940148332555,
+ "step": 170
+ },
+ {
+ "loss": 0.49001736640930177,
+ "grad_norm": 0.009108452126383781,
+ "learning_rate": 8.145077720207254e-06,
+ "entropy": 1.5708052035421134,
+ "num_tokens": 12070033.0,
+ "logits/chosen": -0.19381916063920684,
+ "logits/rejected": -0.1097196340552256,
+ "mean_token_accuracy": 0.6498123442195356,
+ "rewards/chosen": -0.3718085140203402,
+ "rewards/rejected": -1.2254445453887457,
+ "rewards/accuracies": 0.79125,
+ "rewards/margins": 0.8536360321566463,
+ "logps/chosen": -61.2718432867527,
+ "logps/rejected": -58.15766977310181,
+ "epoch": 0.18671230745293294,
+ "step": 180
+ },
+ {
+ "loss": 0.485823917388916,
+ "grad_norm": 0.008646439760923386,
+ "learning_rate": 8.041450777202073e-06,
+ "entropy": 1.619616899229586,
+ "num_tokens": 12738738.0,
+ "logits/chosen": -0.23694109852957265,
+ "logits/rejected": -0.10073302225284902,
+ "mean_token_accuracy": 0.6411672846786678,
+ "rewards/chosen": -0.35420428678007737,
+ "rewards/rejected": -1.2305188758441363,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 0.8763145874883048,
+ "logps/chosen": -64.08159978985786,
+ "logps/rejected": -57.158839713335034,
+ "epoch": 0.19708521342254032,
+ "step": 190
+ },
+ {
+ "loss": 0.5416598796844483,
+ "grad_norm": 0.011695838533341885,
+ "learning_rate": 7.937823834196891e-06,
+ "entropy": 1.5905032951384783,
+ "num_tokens": 13408076.0,
+ "logits/chosen": -0.19543056621701266,
+ "logits/rejected": -0.0932328823177738,
+ "mean_token_accuracy": 0.6534733981452883,
+ "rewards/chosen": -0.43409986807382667,
+ "rewards/rejected": -1.2331794095533406,
+ "rewards/accuracies": 0.75,
+ "rewards/margins": 0.7990795407560654,
+ "logps/chosen": -60.66096189260483,
+ "logps/rejected": -58.47315075159073,
+ "epoch": 0.2074581193921477,
+ "step": 200
+ },
+ {
+ "loss": 0.4877790451049805,
+ "grad_norm": 0.007835142314434052,
+ "learning_rate": 7.834196891191712e-06,
+ "entropy": 1.57742129791528,
+ "num_tokens": 14074371.0,
+ "logits/chosen": -0.3145613949504757,
+ "logits/rejected": -0.17554936279826738,
+ "mean_token_accuracy": 0.6423946806136519,
+ "rewards/chosen": -0.33800375567599983,
+ "rewards/rejected": -1.2202126982567278,
+ "rewards/accuracies": 0.795,
+ "rewards/margins": 0.8822089441586286,
+ "logps/chosen": -57.953816633224484,
+ "logps/rejected": -55.68584906101227,
+ "epoch": 0.2178310253617551,
+ "step": 210
+ },
+ {
+ "loss": 0.49164819717407227,
+ "grad_norm": 0.0103975310921669,
+ "learning_rate": 7.730569948186528e-06,
+ "entropy": 1.6296498909033834,
+ "num_tokens": 14745565.0,
+ "logits/chosen": -0.22528820916902675,
+ "logits/rejected": -0.1081294187855141,
+ "mean_token_accuracy": 0.6335770518891514,
+ "rewards/chosen": -0.49201680309750373,
+ "rewards/rejected": -1.3906073757648119,
+ "rewards/accuracies": 0.77875,
+ "rewards/margins": 0.8985905739758163,
+ "logps/chosen": -63.02672209978104,
+ "logps/rejected": -60.92768079042435,
+ "epoch": 0.2282039313313625,
+ "step": 220
+ },
+ {
+ "loss": 0.46550302505493163,
+ "grad_norm": 0.008888382464647293,
+ "learning_rate": 7.626943005181348e-06,
+ "entropy": 1.6522049311921,
+ "num_tokens": 15418258.0,
+ "logits/chosen": -0.19608391304632053,
+ "logits/rejected": -0.13757200366329492,
+ "mean_token_accuracy": 0.6412607514113188,
+ "rewards/chosen": -0.4089041862002341,
+ "rewards/rejected": -1.4265998849637982,
+ "rewards/accuracies": 0.81625,
+ "rewards/margins": 1.0176956978812814,
+ "logps/chosen": -67.77835583925247,
+ "logps/rejected": -61.52449678421021,
+ "epoch": 0.23857683730096987,
+ "step": 230
+ },
+ {
+ "loss": 0.4947453498840332,
+ "grad_norm": 0.013031491078436375,
+ "learning_rate": 7.523316062176167e-06,
+ "entropy": 1.645399712510407,
+ "num_tokens": 16089178.0,
+ "logits/chosen": -0.17655915871864677,
+ "logits/rejected": -0.09432907682600761,
+ "mean_token_accuracy": 0.633196273688227,
+ "rewards/chosen": -0.4906138309863218,
+ "rewards/rejected": -1.4527097501110984,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 0.9620959201268852,
+ "logps/chosen": -65.66129240393639,
+ "logps/rejected": -61.8218787586689,
+ "epoch": 0.24894974327057726,
+ "step": 240
+ },
+ {
+ "loss": 0.4975076675415039,
+ "grad_norm": 0.010897196829319,
+ "learning_rate": 7.419689119170985e-06,
+ "entropy": 1.6312594626657664,
+ "num_tokens": 16760085.0,
+ "logits/chosen": -0.2438774302066993,
+ "logits/rejected": -0.11363609492659117,
+ "mean_token_accuracy": 0.6422833159938455,
+ "rewards/chosen": -0.47743636105085896,
+ "rewards/rejected": -1.39387952313984,
+ "rewards/accuracies": 0.78,
+ "rewards/margins": 0.9164431605814025,
+ "logps/chosen": -64.82744197368622,
+ "logps/rejected": -61.472064785957336,
+ "epoch": 0.2593226492401846,
+ "step": 250
+ },
+ {
+ "eval_loss": 0.5714118480682373,
+ "eval_runtime": 218.9609,
+ "eval_samples_per_second": 4.567,
+ "eval_steps_per_second": 4.567,
+ "eval_entropy": 1.1695467286109924,
+ "eval_num_tokens": 16760085.0,
+ "eval_logits/chosen": -0.33755278870089783,
+ "eval_logits/rejected": -0.1903600640166493,
+ "eval_mean_token_accuracy": 0.6928319974392653,
+ "eval_rewards/chosen": -0.5571019815959735,
+ "eval_rewards/rejected": -1.1734315770440735,
+ "eval_rewards/accuracies": 0.681,
+ "eval_rewards/margins": 0.6163295963965356,
+ "eval_logps/chosen": -55.744006565094,
+ "eval_logps/rejected": -64.37418729972839,
+ "epoch": 0.2593226492401846,
+ "step": 250
+ },
+ {
+ "loss": 0.5368201732635498,
+ "grad_norm": 0.014366215094923973,
+ "learning_rate": 7.3160621761658035e-06,
+ "entropy": 1.6642464812565594,
+ "num_tokens": 17432944.0,
+ "logits/chosen": -0.21442803743027763,
+ "logits/rejected": -0.1533687336947489,
+ "mean_token_accuracy": 0.6383416177332402,
+ "rewards/chosen": -0.521161526012147,
+ "rewards/rejected": -1.3946754023598624,
+ "rewards/accuracies": 0.75125,
+ "rewards/margins": 0.873513876660727,
+ "logps/chosen": -67.29541030287743,
+ "logps/rejected": -62.28271156668663,
+ "epoch": 0.26969555520979205,
+ "step": 260
+ },
+ {
+ "loss": 0.46097960472106936,
+ "grad_norm": 0.008764931000769138,
+ "learning_rate": 7.212435233160623e-06,
+ "entropy": 1.6307499624509365,
+ "num_tokens": 18100344.0,
+ "logits/chosen": -0.2942488690036322,
+ "logits/rejected": -0.205049856105403,
+ "mean_token_accuracy": 0.63437733290717,
+ "rewards/chosen": -0.40293989607162073,
+ "rewards/rejected": -1.3633229219498388,
+ "rewards/accuracies": 0.82,
+ "rewards/margins": 0.9603830263298004,
+ "logps/chosen": -60.72848893165588,
+ "logps/rejected": -58.830401865243914,
+ "epoch": 0.28006846117939943,
+ "step": 270
+ },
+ {
+ "loss": 0.4768073558807373,
+ "grad_norm": 0.013310693204402924,
+ "learning_rate": 7.108808290155441e-06,
+ "entropy": 1.683792082052678,
+ "num_tokens": 18771815.0,
+ "logits/chosen": -0.2853975746818681,
+ "logits/rejected": -0.19509288131538574,
+ "mean_token_accuracy": 0.6286856073141098,
+ "rewards/chosen": -0.45493435901131307,
+ "rewards/rejected": -1.4305240448995027,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 0.9755896841548384,
+ "logps/chosen": -66.43298780918121,
+ "logps/rejected": -60.264468108415606,
+ "epoch": 0.2904413671490068,
+ "step": 280
+ },
+ {
+ "loss": 0.49738154411315916,
+ "grad_norm": 0.011530010960996151,
+ "learning_rate": 7.005181347150259e-06,
+ "entropy": 1.6574138733558357,
+ "num_tokens": 19443478.0,
+ "logits/chosen": -0.25113046885999535,
+ "logits/rejected": -0.19086733946274176,
+ "mean_token_accuracy": 0.6388111670315265,
+ "rewards/chosen": -0.5135602542446577,
+ "rewards/rejected": -1.4373351091743098,
+ "rewards/accuracies": 0.7875,
+ "rewards/margins": 0.9237748555000871,
+ "logps/chosen": -64.76606289744377,
+ "logps/rejected": -61.73117418050766,
+ "epoch": 0.3008142731186142,
+ "step": 290
+ },
+ {
+ "loss": 0.4747192859649658,
+ "grad_norm": 0.01024437416344881,
+ "learning_rate": 6.9015544041450784e-06,
+ "entropy": 1.6370413560792805,
+ "num_tokens": 20114539.0,
+ "logits/chosen": -0.2804376500769946,
+ "logits/rejected": -0.20980492008240428,
+ "mean_token_accuracy": 0.6386570621281862,
+ "rewards/chosen": -0.5331722995669407,
+ "rewards/rejected": -1.5079922395838365,
+ "rewards/accuracies": 0.79625,
+ "rewards/margins": 0.9748199374601245,
+ "logps/chosen": -64.08541687965393,
+ "logps/rejected": -62.5286465549469,
+ "epoch": 0.3111871790882216,
+ "step": 300
+ },
+ {
+ "loss": 0.5026699542999268,
+ "grad_norm": 0.008775296621024609,
+ "learning_rate": 6.797927461139897e-06,
+ "entropy": 1.6209908311348409,
+ "num_tokens": 20787816.0,
+ "logits/chosen": -0.25151910251321974,
+ "logits/rejected": -0.21123922330556838,
+ "mean_token_accuracy": 0.637009305190295,
+ "rewards/chosen": -0.5232165511250787,
+ "rewards/rejected": -1.528597271647377,
+ "rewards/accuracies": 0.77875,
+ "rewards/margins": 1.0053807196952402,
+ "logps/chosen": -64.69553521633148,
+ "logps/rejected": -65.09379201054573,
+ "epoch": 0.32156008505782896,
+ "step": 310
+ },
+ {
+ "loss": 0.4780869483947754,
+ "grad_norm": 0.011877980083227158,
+ "learning_rate": 6.6943005181347155e-06,
+ "entropy": 1.6530776448501274,
+ "num_tokens": 21457261.0,
+ "logits/chosen": -0.3040290445166409,
+ "logits/rejected": -0.2582128301280066,
+ "mean_token_accuracy": 0.6281850132159889,
+ "rewards/chosen": -0.5049963393166399,
+ "rewards/rejected": -1.5010533356002997,
+ "rewards/accuracies": 0.7975,
+ "rewards/margins": 0.9960569961182774,
+ "logps/chosen": -65.50288674592971,
+ "logps/rejected": -62.25389522314072,
+ "epoch": 0.33193299102743634,
+ "step": 320
+ },
+ {
+ "loss": 0.4790470123291016,
+ "grad_norm": 0.011483334004878998,
+ "learning_rate": 6.590673575129535e-06,
+ "entropy": 1.7004518122132868,
+ "num_tokens": 22127813.0,
+ "logits/chosen": -0.2665928630702493,
+ "logits/rejected": -0.21917896340725312,
+ "mean_token_accuracy": 0.6241211013868451,
+ "rewards/chosen": -0.5605986861829296,
+ "rewards/rejected": -1.6507765721459873,
+ "rewards/accuracies": 0.7875,
+ "rewards/margins": 1.0901778864115477,
+ "logps/chosen": -65.43279960155488,
+ "logps/rejected": -64.90576170921325,
+ "epoch": 0.3423058969970437,
+ "step": 330
+ },
+ {
+ "loss": 0.49932498931884767,
+ "grad_norm": 0.011705898679792881,
+ "learning_rate": 6.487046632124353e-06,
+ "entropy": 1.627526972265914,
+ "num_tokens": 22798029.0,
+ "logits/chosen": -0.2497662070999108,
+ "logits/rejected": -0.1943012572657782,
+ "mean_token_accuracy": 0.629542014990002,
+ "rewards/chosen": -0.5459633090042917,
+ "rewards/rejected": -1.536823488854352,
+ "rewards/accuracies": 0.78125,
+ "rewards/margins": 0.9908601802797057,
+ "logps/chosen": -63.95577235102653,
+ "logps/rejected": -62.38620036482811,
+ "epoch": 0.3526788029666511,
+ "step": 340
+ },
+ {
+ "loss": 0.5073616027832031,
+ "grad_norm": 0.010271189734339714,
+ "learning_rate": 6.383419689119171e-06,
+ "entropy": 1.6555507829785348,
+ "num_tokens": 23469695.0,
+ "logits/chosen": -0.12372383655417052,
+ "logits/rejected": -0.06133092001205623,
+ "mean_token_accuracy": 0.6365941292047501,
+ "rewards/chosen": -0.45281789854270754,
+ "rewards/rejected": -1.4345268653615495,
+ "rewards/accuracies": 0.76625,
+ "rewards/margins": 0.9817089692596346,
+ "logps/chosen": -65.98795211911201,
+ "logps/rejected": -63.176999225616456,
+ "epoch": 0.3630517089362585,
+ "step": 350
+ },
+ {
+ "loss": 0.49364256858825684,
+ "grad_norm": 0.011196079663932323,
+ "learning_rate": 6.2797927461139905e-06,
+ "entropy": 1.5875802629441023,
+ "num_tokens": 24140119.0,
+ "logits/chosen": -0.14808247634584407,
+ "logits/rejected": -0.05311759222309109,
+ "mean_token_accuracy": 0.6542789761908352,
+ "rewards/chosen": -0.42567219032192954,
+ "rewards/rejected": -1.4186390993191162,
+ "rewards/accuracies": 0.78125,
+ "rewards/margins": 0.9929669088963419,
+ "logps/chosen": -61.54896705150604,
+ "logps/rejected": -60.50213465809822,
+ "epoch": 0.37342461490586587,
+ "step": 360
+ },
+ {
+ "loss": 0.483473539352417,
+ "grad_norm": 0.00855876225978136,
+ "learning_rate": 6.176165803108809e-06,
+ "entropy": 1.6722470651892944,
+ "num_tokens": 24811816.0,
+ "logits/chosen": -0.10432099490996176,
+ "logits/rejected": -0.016737991147920683,
+ "mean_token_accuracy": 0.6325697888806462,
+ "rewards/chosen": -0.4370591178828181,
+ "rewards/rejected": -1.4563098769000498,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.0192507596686482,
+ "logps/chosen": -66.96794133782387,
+ "logps/rejected": -61.84955484747886,
+ "epoch": 0.38379752087547325,
+ "step": 370
+ },
+ {
+ "eval_loss": 0.5579664707183838,
+ "eval_runtime": 259.8032,
+ "eval_samples_per_second": 3.849,
+ "eval_steps_per_second": 3.849,
+ "eval_entropy": 1.1737834954187274,
+ "eval_num_tokens": 25146613.0,
+ "eval_logits/chosen": -0.24876413873978034,
+ "eval_logits/rejected": -0.10992595889452522,
+ "eval_mean_token_accuracy": 0.6986516900211572,
+ "eval_rewards/chosen": -0.46025879815377996,
+ "eval_rewards/rejected": -1.1139206278724596,
+ "eval_rewards/accuracies": 0.69,
+ "eval_rewards/margins": 0.653661830753088,
+ "eval_logps/chosen": -54.77557474708557,
+ "eval_logps/rejected": -63.77907782173157,
+ "epoch": 0.38898397386027694,
+ "step": 375
+ },
+ {
+ "loss": 0.4502089977264404,
+ "grad_norm": 0.010930197313427925,
+ "learning_rate": 6.0725388601036275e-06,
+ "entropy": 1.6199329065252095,
+ "num_tokens": 25482244.0,
+ "logits/chosen": -0.08688961392902067,
+ "logits/rejected": -0.02652044551893199,
+ "mean_token_accuracy": 0.6438922439515591,
+ "rewards/chosen": -0.3519340772558644,
+ "rewards/rejected": -1.4421476342028472,
+ "rewards/accuracies": 0.82,
+ "rewards/margins": 1.0902135530067607,
+ "logps/chosen": -62.673097778558734,
+ "logps/rejected": -59.82661822557449,
+ "epoch": 0.39417042684508063,
+ "step": 380
+ },
+ {
+ "loss": 0.47233095169067385,
+ "grad_norm": 0.010069029405713081,
+ "learning_rate": 5.968911917098445e-06,
+ "entropy": 1.6362504732236267,
+ "num_tokens": 26153516.0,
+ "logits/chosen": -0.117927543996971,
+ "logits/rejected": -0.020026116136071335,
+ "mean_token_accuracy": 0.6393325614184141,
+ "rewards/chosen": -0.3878356325280038,
+ "rewards/rejected": -1.425175780861173,
+ "rewards/accuracies": 0.7975,
+ "rewards/margins": 1.0373401497886516,
+ "logps/chosen": -65.00698279976845,
+ "logps/rejected": -63.11618382692337,
+ "epoch": 0.404543332814688,
+ "step": 390
+ },
+ {
+ "loss": 0.47489471435546876,
+ "grad_norm": 0.009898822754621506,
+ "learning_rate": 5.865284974093265e-06,
+ "entropy": 1.6706327036954463,
+ "num_tokens": 26824251.0,
+ "logits/chosen": -0.10839314775215383,
+ "logits/rejected": -0.034963929941317536,
+ "mean_token_accuracy": 0.6374997748341411,
+ "rewards/chosen": -0.4009263012804513,
+ "rewards/rejected": -1.3926532304068677,
+ "rewards/accuracies": 0.8,
+ "rewards/margins": 0.9917269288050011,
+ "logps/chosen": -64.74012166261673,
+ "logps/rejected": -61.97128087759018,
+ "epoch": 0.4149162387842954,
+ "step": 400
+ },
+ {
+ "loss": 0.4803769111633301,
+ "grad_norm": 0.010940629988908768,
+ "learning_rate": 5.761658031088083e-06,
+ "entropy": 1.63126095501706,
+ "num_tokens": 27495141.0,
+ "logits/chosen": -0.11447524275234765,
+ "logits/rejected": -0.049791539508999005,
+ "mean_token_accuracy": 0.6397510032169521,
+ "rewards/chosen": -0.40976859070429783,
+ "rewards/rejected": -1.408572825542651,
+ "rewards/accuracies": 0.79625,
+ "rewards/margins": 0.9988042342383414,
+ "logps/chosen": -64.12175590753556,
+ "logps/rejected": -61.26593731760979,
+ "epoch": 0.4252891447539028,
+ "step": 410
+ },
+ {
+ "loss": 0.4590421676635742,
+ "grad_norm": 0.009904208593070507,
+ "learning_rate": 5.658031088082902e-06,
+ "entropy": 1.6269367651827633,
+ "num_tokens": 28166897.0,
+ "logits/chosen": -0.12299671549070275,
+ "logits/rejected": -0.07870207017521408,
+ "mean_token_accuracy": 0.639423014242202,
+ "rewards/chosen": -0.42271745148642365,
+ "rewards/rejected": -1.5163998405314123,
+ "rewards/accuracies": 0.80875,
+ "rewards/margins": 1.0936823888216167,
+ "logps/chosen": -66.27873115181923,
+ "logps/rejected": -61.664743056297304,
+ "epoch": 0.4356620507235102,
+ "step": 420
+ },
+ {
+ "loss": 0.44393315315246584,
+ "grad_norm": 0.007425240706652403,
+ "learning_rate": 5.554404145077721e-06,
+ "entropy": 1.654088821541518,
+ "num_tokens": 28838688.0,
+ "logits/chosen": -0.046875228210155474,
+ "logits/rejected": 0.02169555469420267,
+ "mean_token_accuracy": 0.633773233667016,
+ "rewards/chosen": -0.4147428805350137,
+ "rewards/rejected": -1.5705322344205341,
+ "rewards/accuracies": 0.82875,
+ "rewards/margins": 1.1557893524505198,
+ "logps/chosen": -64.6313831782341,
+ "logps/rejected": -62.269921224117276,
+ "epoch": 0.4460349566931176,
+ "step": 430
+ },
+ {
+ "loss": 0.45639653205871583,
+ "grad_norm": 0.01209650281816721,
+ "learning_rate": 5.4507772020725395e-06,
+ "entropy": 1.616792434314266,
+ "num_tokens": 29509172.0,
+ "logits/chosen": -0.11781563740683831,
+ "logits/rejected": -0.0357408626073251,
+ "mean_token_accuracy": 0.6438933240342886,
+ "rewards/chosen": -0.4713032754930464,
+ "rewards/rejected": -1.576865721391514,
+ "rewards/accuracies": 0.8075,
+ "rewards/margins": 1.105562445949763,
+ "logps/chosen": -63.11039539933205,
+ "logps/rejected": -62.069657690525055,
+ "epoch": 0.456407862662725,
+ "step": 440
+ },
+ {
+ "loss": 0.4802379131317139,
+ "grad_norm": 0.012935981154441833,
+ "learning_rate": 5.347150259067357e-06,
+ "entropy": 1.6445236429013312,
+ "num_tokens": 30180698.0,
+ "logits/chosen": -0.12016731428543326,
+ "logits/rejected": -0.02180240765374299,
+ "mean_token_accuracy": 0.6402219843491912,
+ "rewards/chosen": -0.48889325076772366,
+ "rewards/rejected": -1.6065474805486155,
+ "rewards/accuracies": 0.79125,
+ "rewards/margins": 1.1176542268134653,
+ "logps/chosen": -63.898983215093615,
+ "logps/rejected": -64.41106705546379,
+ "epoch": 0.46678076863233237,
+ "step": 450
+ },
+ {
+ "loss": 0.45149784088134765,
+ "grad_norm": 0.010028840973973274,
+ "learning_rate": 5.243523316062177e-06,
+ "entropy": 1.5921511804498731,
+ "num_tokens": 30850391.0,
+ "logits/chosen": -0.137444507891216,
+ "logits/rejected": -0.0779091628359406,
+ "mean_token_accuracy": 0.6352657766267658,
+ "rewards/chosen": -0.4104501232576513,
+ "rewards/rejected": -1.53232252900023,
+ "rewards/accuracies": 0.82,
+ "rewards/margins": 1.1218724092375487,
+ "logps/chosen": -63.677142459154126,
+ "logps/rejected": -61.06894833087921,
+ "epoch": 0.47715367460193975,
+ "step": 460
+ },
+ {
+ "loss": 0.48416786193847655,
+ "grad_norm": 0.017470093443989754,
+ "learning_rate": 5.139896373056995e-06,
+ "entropy": 1.6218253993801772,
+ "num_tokens": 31518644.0,
+ "logits/chosen": -0.17963024617659684,
+ "logits/rejected": -0.11590594375515367,
+ "mean_token_accuracy": 0.6306483455188573,
+ "rewards/chosen": -0.48125705970713173,
+ "rewards/rejected": -1.5042318812455777,
+ "rewards/accuracies": 0.78875,
+ "rewards/margins": 1.022974822570104,
+ "logps/chosen": -62.842620354890826,
+ "logps/rejected": -60.41043012142181,
+ "epoch": 0.48752658057154713,
+ "step": 470
+ },
+ {
+ "loss": 0.4613227844238281,
+ "grad_norm": 0.011262920685112476,
+ "learning_rate": 5.036269430051814e-06,
+ "entropy": 1.6439712375216187,
+ "num_tokens": 32191270.0,
+ "logits/chosen": -0.13840929670441426,
+ "logits/rejected": -0.06270617111823595,
+ "mean_token_accuracy": 0.6358192806318402,
+ "rewards/chosen": -0.5173802295615314,
+ "rewards/rejected": -1.5994968451908789,
+ "rewards/accuracies": 0.78875,
+ "rewards/margins": 1.0821166158560664,
+ "logps/chosen": -67.11039644002915,
+ "logps/rejected": -64.66348296999931,
+ "epoch": 0.4978994865411545,
+ "step": 480
+ },
+ {
+ "loss": 0.4716683864593506,
+ "grad_norm": 0.010376542806625366,
+ "learning_rate": 4.932642487046633e-06,
+ "entropy": 1.624296410465613,
+ "num_tokens": 32863739.0,
+ "logits/chosen": -0.12371006824984962,
+ "logits/rejected": -0.07942390242049989,
+ "mean_token_accuracy": 0.6453884858079255,
+ "rewards/chosen": -0.5051976552634733,
+ "rewards/rejected": -1.5973712733900174,
+ "rewards/accuracies": 0.79625,
+ "rewards/margins": 1.09217361707706,
+ "logps/chosen": -64.34702114582062,
+ "logps/rejected": -64.62907078266144,
+ "epoch": 0.5082723925107618,
+ "step": 490
+ },
+ {
+ "loss": 0.49039602279663086,
+ "grad_norm": 0.009508524090051651,
+ "learning_rate": 4.829015544041451e-06,
+ "entropy": 1.620068084243685,
+ "num_tokens": 33534123.0,
+ "logits/chosen": -0.14151537366366368,
+ "logits/rejected": -0.06308392833608244,
+ "mean_token_accuracy": 0.6416816004179418,
+ "rewards/chosen": -0.45566922199541293,
+ "rewards/rejected": -1.5016374166216702,
+ "rewards/accuracies": 0.77125,
+ "rewards/margins": 1.0459681928623468,
+ "logps/chosen": -62.64698366522789,
+ "logps/rejected": -63.112683627605435,
+ "epoch": 0.5186452984803692,
+ "step": 500
+ },
+ {
+ "eval_loss": 0.5447177886962891,
+ "eval_runtime": 282.5727,
+ "eval_samples_per_second": 3.539,
+ "eval_steps_per_second": 3.539,
+ "eval_entropy": 1.1561195281520487,
+ "eval_num_tokens": 33534123.0,
+ "eval_logits/chosen": -0.2889595885969217,
+ "eval_logits/rejected": -0.14876548266875028,
+ "eval_mean_token_accuracy": 0.6995251678973436,
+ "eval_rewards/chosen": -0.4679097114284523,
+ "eval_rewards/rejected": -1.1751951004834846,
+ "eval_rewards/accuracies": 0.718,
+ "eval_rewards/margins": 0.7072853878363967,
+ "eval_logps/chosen": -54.85208387184143,
+ "eval_logps/rejected": -64.39182249259949,
+ "epoch": 0.5186452984803692,
+ "step": 500
+ },
+ {
+ "loss": 0.47274036407470704,
+ "grad_norm": 0.011393043212592602,
+ "learning_rate": 4.72538860103627e-06,
+ "entropy": 1.6078121318202465,
+ "num_tokens": 34205019.0,
+ "logits/chosen": -0.11068593414216622,
+ "logits/rejected": -0.05753107212707231,
+ "mean_token_accuracy": 0.6421808904409408,
+ "rewards/chosen": -0.4764720965552806,
+ "rewards/rejected": -1.5107146011234727,
+ "rewards/accuracies": 0.81125,
+ "rewards/margins": 1.034242505049333,
+ "logps/chosen": -65.03627744436264,
+ "logps/rejected": -60.319275975227356,
+ "epoch": 0.5290182044499767,
+ "step": 510
+ },
+ {
+ "loss": 0.49851455688476565,
+ "grad_norm": 0.009646100923418999,
+ "learning_rate": 4.621761658031089e-06,
+ "entropy": 1.5942507219407707,
+ "num_tokens": 34873534.0,
+ "logits/chosen": -0.1425558861577052,
+ "logits/rejected": -0.08471900949010301,
+ "mean_token_accuracy": 0.6420148031599819,
+ "rewards/chosen": -0.4576815483915561,
+ "rewards/rejected": -1.4324994718033122,
+ "rewards/accuracies": 0.78,
+ "rewards/margins": 0.9748179239872843,
+ "logps/chosen": -61.40950953125954,
+ "logps/rejected": -60.10699124455452,
+ "epoch": 0.5393911104195841,
+ "step": 520
+ },
+ {
+ "loss": 0.48189678192138674,
+ "grad_norm": 0.009592341259121895,
+ "learning_rate": 4.518134715025907e-06,
+ "entropy": 1.6446389424987138,
+ "num_tokens": 35544511.0,
+ "logits/chosen": -0.13110756052267608,
+ "logits/rejected": -0.06357341075451498,
+ "mean_token_accuracy": 0.632728576976806,
+ "rewards/chosen": -0.48985899652365333,
+ "rewards/rejected": -1.5666234930680367,
+ "rewards/accuracies": 0.805,
+ "rewards/margins": 1.0767644950002433,
+ "logps/chosen": -65.08468866825103,
+ "logps/rejected": -63.443355305194856,
+ "epoch": 0.5497640163891915,
+ "step": 530
+ },
+ {
+ "loss": 0.468688440322876,
+ "grad_norm": 0.013844112865626812,
+ "learning_rate": 4.414507772020726e-06,
+ "entropy": 1.6056221339013428,
+ "num_tokens": 36215333.0,
+ "logits/chosen": -0.14999839608893362,
+ "logits/rejected": -0.09044282285613432,
+ "mean_token_accuracy": 0.6412966704368591,
+ "rewards/chosen": -0.547224923124304,
+ "rewards/rejected": -1.6111413582094247,
+ "rewards/accuracies": 0.80125,
+ "rewards/margins": 1.063916433537379,
+ "logps/chosen": -63.49809960961342,
+ "logps/rejected": -63.379293674230574,
+ "epoch": 0.5601369223587989,
+ "step": 540
+ },
+ {
+ "loss": 0.47130799293518066,
+ "grad_norm": 0.013357527554035187,
+ "learning_rate": 4.310880829015544e-06,
+ "entropy": 1.6576832227222622,
+ "num_tokens": 36887745.0,
+ "logits/chosen": -0.09151633492208026,
+ "logits/rejected": -0.05831491488755683,
+ "mean_token_accuracy": 0.6373074753023684,
+ "rewards/chosen": -0.5282300677981402,
+ "rewards/rejected": -1.6641296455645351,
+ "rewards/accuracies": 0.8125,
+ "rewards/margins": 1.1358995781932026,
+ "logps/chosen": -67.07182881236076,
+ "logps/rejected": -66.767291328907,
+ "epoch": 0.5705098283284062,
+ "step": 550
+ },
+ {
+ "loss": 0.47469630241394045,
+ "grad_norm": 0.010529081337153912,
+ "learning_rate": 4.207253886010363e-06,
+ "entropy": 1.5956833818554879,
+ "num_tokens": 37559399.0,
+ "logits/chosen": -0.15354086541936426,
+ "logits/rejected": -0.1337727985771215,
+ "mean_token_accuracy": 0.6338519185595214,
+ "rewards/chosen": -0.5099609697687992,
+ "rewards/rejected": -1.6287131954543292,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.11875222671777,
+ "logps/chosen": -65.30963395237923,
+ "logps/rejected": -62.971670610904695,
+ "epoch": 0.5808827342980136,
+ "step": 560
+ },
+ {
+ "loss": 0.46091413497924805,
+ "grad_norm": 0.010177496820688248,
+ "learning_rate": 4.103626943005182e-06,
+ "entropy": 1.5913864125311374,
+ "num_tokens": 38229010.0,
+ "logits/chosen": -0.13525813478847795,
+ "logits/rejected": -0.09249211440964598,
+ "mean_token_accuracy": 0.6336300628073513,
+ "rewards/chosen": -0.4981531216715939,
+ "rewards/rejected": -1.5639994585248496,
+ "rewards/accuracies": 0.81,
+ "rewards/margins": 1.0658463343884796,
+ "logps/chosen": -64.47193114757538,
+ "logps/rejected": -60.892896996736525,
+ "epoch": 0.591255640267621,
+ "step": 570
+ },
+ {
+ "loss": 0.4695730686187744,
+ "grad_norm": 0.011662053875625134,
+ "learning_rate": 4.000000000000001e-06,
+ "entropy": 1.5622135086357594,
+ "num_tokens": 38898199.0,
+ "logits/chosen": -0.2250718619124712,
+ "logits/rejected": -0.1731223735468456,
+ "mean_token_accuracy": 0.6406736394017934,
+ "rewards/chosen": -0.46840151986456474,
+ "rewards/rejected": -1.5864200799472747,
+ "rewards/accuracies": 0.79625,
+ "rewards/margins": 1.1180185582535342,
+ "logps/chosen": -61.685848199129104,
+ "logps/rejected": -61.54680736422539,
+ "epoch": 0.6016285462372284,
+ "step": 580
+ },
+ {
+ "loss": 0.5114735126495361,
+ "grad_norm": 0.012209425680339336,
+ "learning_rate": 3.896373056994819e-06,
+ "entropy": 1.5807212091330438,
+ "num_tokens": 39570071.0,
+ "logits/chosen": -0.20266442113231448,
+ "logits/rejected": -0.17378155213538463,
+ "mean_token_accuracy": 0.6373632573708892,
+ "rewards/chosen": -0.5869286243220995,
+ "rewards/rejected": -1.5809136723290431,
+ "rewards/accuracies": 0.76625,
+ "rewards/margins": 0.993985049307812,
+ "logps/chosen": -66.22412416219711,
+ "logps/rejected": -64.40389844059945,
+ "epoch": 0.6120014522068358,
+ "step": 590
+ },
+ {
+ "loss": 0.4908578395843506,
+ "grad_norm": 0.012631360441446304,
+ "learning_rate": 3.7927461139896377e-06,
+ "entropy": 1.5845268149208278,
+ "num_tokens": 40238339.0,
+ "logits/chosen": -0.1760518834317504,
+ "logits/rejected": -0.12661257104922638,
+ "mean_token_accuracy": 0.6428114072233438,
+ "rewards/chosen": -0.47756292890640906,
+ "rewards/rejected": -1.5143917541232077,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 1.036828825349221,
+ "logps/chosen": -60.982233242988585,
+ "logps/rejected": -61.829094297885895,
+ "epoch": 0.6223743581764432,
+ "step": 600
+ },
+ {
+ "loss": 0.4755990505218506,
+ "grad_norm": 0.011146514676511288,
+ "learning_rate": 3.6891191709844567e-06,
+ "entropy": 1.616417101584375,
+ "num_tokens": 40908073.0,
+ "logits/chosen": -0.15273173400238405,
+ "logits/rejected": -0.10939495965890679,
+ "mean_token_accuracy": 0.6361582010798156,
+ "rewards/chosen": -0.46836249336316543,
+ "rewards/rejected": -1.530430060197832,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.0620675668818875,
+ "logps/chosen": -61.67246505379677,
+ "logps/rejected": -62.54117285490036,
+ "epoch": 0.6327472641460505,
+ "step": 610
+ },
+ {
+ "loss": 0.4618831157684326,
+ "grad_norm": 0.010837269015610218,
+ "learning_rate": 3.5854922279792748e-06,
+ "entropy": 1.5439357980620116,
+ "num_tokens": 41577227.0,
+ "logits/chosen": -0.18130013224530928,
+ "logits/rejected": -0.11479609439593057,
+ "mean_token_accuracy": 0.6398758164048195,
+ "rewards/chosen": -0.4230602414139139,
+ "rewards/rejected": -1.5293280950934423,
+ "rewards/accuracies": 0.805,
+ "rewards/margins": 1.1062678526248784,
+ "logps/chosen": -60.17235267043114,
+ "logps/rejected": -61.54575915336609,
+ "epoch": 0.6431201701156579,
+ "step": 620
+ },
+ {
+ "eval_loss": 0.5342973470687866,
+ "eval_runtime": 313.9899,
+ "eval_samples_per_second": 3.185,
+ "eval_steps_per_second": 3.185,
+ "eval_entropy": 1.1306908284947277,
+ "eval_num_tokens": 41912429.0,
+ "eval_logits/chosen": -0.3345527509936572,
+ "eval_logits/rejected": -0.19592343447135618,
+ "eval_mean_token_accuracy": 0.6978758063018322,
+ "eval_rewards/chosen": -0.5062940621431917,
+ "eval_rewards/rejected": -1.2705660811300623,
+ "eval_rewards/accuracies": 0.729,
+ "eval_rewards/margins": 0.7642720182482153,
+ "eval_logps/chosen": -55.23592737579346,
+ "eval_logps/rejected": -65.34553232955933,
+ "epoch": 0.6483066231004616,
+ "step": 625
+ },
+ {
+ "loss": 0.4688296318054199,
+ "grad_norm": 0.012612175196409225,
+ "learning_rate": 3.4818652849740937e-06,
+ "entropy": 1.5918685279786586,
+ "num_tokens": 42248355.0,
+ "logits/chosen": -0.13449424627890724,
+ "logits/rejected": -0.08865577303502642,
+ "mean_token_accuracy": 0.6395252890512347,
+ "rewards/chosen": -0.45570429358987896,
+ "rewards/rejected": -1.5667857097371598,
+ "rewards/accuracies": 0.79625,
+ "rewards/margins": 1.1110814131516964,
+ "logps/chosen": -63.0096686398983,
+ "logps/rejected": -64.22273498296738,
+ "epoch": 0.6534930760852653,
+ "step": 630
+ },
+ {
+ "loss": 0.45305862426757815,
+ "grad_norm": 0.013145489618182182,
+ "learning_rate": 3.3782383419689123e-06,
+ "entropy": 1.5790020452067255,
+ "num_tokens": 42918958.0,
+ "logits/chosen": -0.12399229798184251,
+ "logits/rejected": -0.0790595345566506,
+ "mean_token_accuracy": 0.638872587159276,
+ "rewards/chosen": -0.498632894383627,
+ "rewards/rejected": -1.6101470832383347,
+ "rewards/accuracies": 0.81,
+ "rewards/margins": 1.111514187841676,
+ "logps/chosen": -61.910882869958876,
+ "logps/rejected": -64.29858126163482,
+ "epoch": 0.6638659820548727,
+ "step": 640
+ },
+ {
+ "loss": 0.5027210235595703,
+ "grad_norm": 0.010981565341353416,
+ "learning_rate": 3.274611398963731e-06,
+ "entropy": 1.5813618222996593,
+ "num_tokens": 43588637.0,
+ "logits/chosen": -0.17636409657708957,
+ "logits/rejected": -0.13223095230259532,
+ "mean_token_accuracy": 0.6379721114598215,
+ "rewards/chosen": -0.4963178468355909,
+ "rewards/rejected": -1.5267454569012626,
+ "rewards/accuracies": 0.78125,
+ "rewards/margins": 1.0304276071023195,
+ "logps/chosen": -62.9744295334816,
+ "logps/rejected": -61.37450294733048,
+ "epoch": 0.6742388880244801,
+ "step": 650
+ },
+ {
+ "loss": 0.5052100658416748,
+ "grad_norm": 0.00917180348187685,
+ "learning_rate": 3.1709844559585493e-06,
+ "entropy": 1.5768494224920868,
+ "num_tokens": 44257795.0,
+ "logits/chosen": -0.1306501266924166,
+ "logits/rejected": -0.110707865062261,
+ "mean_token_accuracy": 0.6388484319858253,
+ "rewards/chosen": -0.43548518963740207,
+ "rewards/rejected": -1.4259171091011376,
+ "rewards/accuracies": 0.795,
+ "rewards/margins": 0.9904319175425917,
+ "logps/chosen": -61.97638823986053,
+ "logps/rejected": -60.160553097724915,
+ "epoch": 0.6846117939940874,
+ "step": 660
+ },
+ {
+ "loss": 0.4341450214385986,
+ "grad_norm": 0.01135904062539339,
+ "learning_rate": 3.0673575129533683e-06,
+ "entropy": 1.5525631321314721,
+ "num_tokens": 44929114.0,
+ "logits/chosen": -0.14121290833212632,
+ "logits/rejected": -0.12193397964955385,
+ "mean_token_accuracy": 0.6406230745092034,
+ "rewards/chosen": -0.3947988249294576,
+ "rewards/rejected": -1.5214268560019264,
+ "rewards/accuracies": 0.805,
+ "rewards/margins": 1.1266280310088768,
+ "logps/chosen": -64.25430464625359,
+ "logps/rejected": -61.65177075624466,
+ "epoch": 0.6949846999636948,
+ "step": 670
+ },
+ {
+ "loss": 0.48023161888122556,
+ "grad_norm": 0.012425442226231098,
+ "learning_rate": 2.963730569948187e-06,
+ "entropy": 1.569240757385269,
+ "num_tokens": 45601053.0,
+ "logits/chosen": -0.10826928815748788,
+ "logits/rejected": -0.06131814862870058,
+ "mean_token_accuracy": 0.6415985404700041,
+ "rewards/chosen": -0.39569982240791435,
+ "rewards/rejected": -1.4680290185977356,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.0723291979916394,
+ "logps/chosen": -64.85109462738038,
+ "logps/rejected": -62.83043103218078,
+ "epoch": 0.7053576059333022,
+ "step": 680
+ },
+ {
+ "loss": 0.5059038162231445,
+ "grad_norm": 0.012226400896906853,
+ "learning_rate": 2.8601036269430053e-06,
+ "entropy": 1.5985452976450325,
+ "num_tokens": 46272416.0,
+ "logits/chosen": -0.07408260970012749,
+ "logits/rejected": -0.044598548594781996,
+ "mean_token_accuracy": 0.6376935013197362,
+ "rewards/chosen": -0.43009026095212904,
+ "rewards/rejected": -1.4046015239062035,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 0.9745112636708655,
+ "logps/chosen": -66.05170575976372,
+ "logps/rejected": -61.49258904218674,
+ "epoch": 0.7157305119029096,
+ "step": 690
+ },
+ {
+ "loss": 0.44657320976257325,
+ "grad_norm": 0.01066418457776308,
+ "learning_rate": 2.7564766839378243e-06,
+ "entropy": 1.555307752629742,
+ "num_tokens": 46942192.0,
+ "logits/chosen": -0.1133366585618524,
+ "logits/rejected": -0.08814206115702024,
+ "mean_token_accuracy": 0.6415997881442308,
+ "rewards/chosen": -0.38704808527943896,
+ "rewards/rejected": -1.4645700649346691,
+ "rewards/accuracies": 0.81625,
+ "rewards/margins": 1.0775219783838839,
+ "logps/chosen": -64.02207235455514,
+ "logps/rejected": -58.637507121562955,
+ "epoch": 0.726103417872517,
+ "step": 700
+ },
+ {
+ "loss": 0.4767612934112549,
+ "grad_norm": 0.011431827209889889,
+ "learning_rate": 2.6528497409326424e-06,
+ "entropy": 1.5519457180798053,
+ "num_tokens": 47613801.0,
+ "logits/chosen": -0.1855044761632082,
+ "logits/rejected": -0.16003899451371845,
+ "mean_token_accuracy": 0.6355127618275582,
+ "rewards/chosen": -0.4479449386331544,
+ "rewards/rejected": -1.5271121441631113,
+ "rewards/accuracies": 0.7825,
+ "rewards/margins": 1.0791672053001822,
+ "logps/chosen": -67.1749314057827,
+ "logps/rejected": -62.050304319858554,
+ "epoch": 0.7364763238421244,
+ "step": 710
+ },
+ {
+ "loss": 0.46796340942382814,
+ "grad_norm": 0.014987263828516006,
+ "learning_rate": 2.5492227979274614e-06,
+ "entropy": 1.5421490759123118,
+ "num_tokens": 48282514.0,
+ "logits/chosen": -0.21524596853619962,
+ "logits/rejected": -0.1803562152249967,
+ "mean_token_accuracy": 0.6399628999084235,
+ "rewards/chosen": -0.4025335052545415,
+ "rewards/rejected": -1.4638134895112307,
+ "rewards/accuracies": 0.81125,
+ "rewards/margins": 1.0612799843633547,
+ "logps/chosen": -62.34168538808822,
+ "logps/rejected": -58.95266834259033,
+ "epoch": 0.7468492298117317,
+ "step": 720
+ },
+ {
+ "loss": 0.46932258605957033,
+ "grad_norm": 0.015692368149757385,
+ "learning_rate": 2.44559585492228e-06,
+ "entropy": 1.5745291117485614,
+ "num_tokens": 48954943.0,
+ "logits/chosen": -0.1285935002288144,
+ "logits/rejected": -0.09589593946946354,
+ "mean_token_accuracy": 0.6359935710392892,
+ "rewards/chosen": -0.43215721267159096,
+ "rewards/rejected": -1.5352867500041611,
+ "rewards/accuracies": 0.8,
+ "rewards/margins": 1.1031295386189595,
+ "logps/chosen": -64.10728573679924,
+ "logps/rejected": -65.76713324308395,
+ "epoch": 0.7572221357813391,
+ "step": 730
+ },
+ {
+ "loss": 0.45352988243103026,
+ "grad_norm": 0.016576601192355156,
+ "learning_rate": 2.3419689119170984e-06,
+ "entropy": 1.5855714071169495,
+ "num_tokens": 49625089.0,
+ "logits/chosen": -0.18068304366710994,
+ "logits/rejected": -0.1021847750997751,
+ "mean_token_accuracy": 0.6383709345012903,
+ "rewards/chosen": -0.40431298587100173,
+ "rewards/rejected": -1.5056831875129137,
+ "rewards/accuracies": 0.82125,
+ "rewards/margins": 1.1013702021841891,
+ "logps/chosen": -63.80637561678886,
+ "logps/rejected": -63.55216552257538,
+ "epoch": 0.7675950417509465,
+ "step": 740
+ },
+ {
+ "loss": 0.47931923866271975,
+ "grad_norm": 0.011076811701059341,
+ "learning_rate": 2.2383419689119174e-06,
+ "entropy": 1.5709061060287057,
+ "num_tokens": 50298779.0,
+ "logits/chosen": -0.12009490934399182,
+ "logits/rejected": -0.08929539108746075,
+ "mean_token_accuracy": 0.639037843607366,
+ "rewards/chosen": -0.49217503207488333,
+ "rewards/rejected": -1.6097049169614912,
+ "rewards/accuracies": 0.78875,
+ "rewards/margins": 1.1175298846373334,
+ "logps/chosen": -66.4990734577179,
+ "logps/rejected": -66.20608123540879,
+ "epoch": 0.7779679477205539,
+ "step": 750
+ },
+ {
+ "eval_loss": 0.5270404815673828,
+ "eval_runtime": 345.9274,
+ "eval_samples_per_second": 2.891,
+ "eval_steps_per_second": 2.891,
+ "eval_entropy": 1.110700085580349,
+ "eval_num_tokens": 50298779.0,
+ "eval_logits/chosen": -0.32906322941516086,
+ "eval_logits/rejected": -0.18860008122397576,
+ "eval_mean_token_accuracy": 0.7015585756152868,
+ "eval_rewards/chosen": -0.4489634925366845,
+ "eval_rewards/rejected": -1.2056301589403302,
+ "eval_rewards/accuracies": 0.741,
+ "eval_rewards/margins": 0.7566666670814156,
+ "eval_logps/chosen": -54.662621686935424,
+ "eval_logps/rejected": -64.69617311096191,
+ "epoch": 0.7779679477205539,
+ "step": 750
+ },
+ {
+ "loss": 0.4540732383728027,
+ "grad_norm": 0.010880699381232262,
+ "learning_rate": 2.134715025906736e-06,
+ "entropy": 1.5645483719184994,
+ "num_tokens": 50969763.0,
+ "logits/chosen": -0.16403584506517532,
+ "logits/rejected": -0.12529662964612162,
+ "mean_token_accuracy": 0.6381968346890062,
+ "rewards/chosen": -0.4469367304503976,
+ "rewards/rejected": -1.5024195825937203,
+ "rewards/accuracies": 0.79625,
+ "rewards/margins": 1.0554828522354365,
+ "logps/chosen": -64.17523063778877,
+ "logps/rejected": -62.25719483613968,
+ "epoch": 0.7883408536901613,
+ "step": 760
+ },
+ {
+ "loss": 0.4533174991607666,
+ "grad_norm": 0.013751443475484848,
+ "learning_rate": 2.0310880829015544e-06,
+ "entropy": 1.572054564883001,
+ "num_tokens": 51644082.0,
+ "logits/chosen": -0.12553626378190955,
+ "logits/rejected": -0.08071406955215293,
+ "mean_token_accuracy": 0.6449453581683338,
+ "rewards/chosen": -0.47219557088945296,
+ "rewards/rejected": -1.6429482020807336,
+ "rewards/accuracies": 0.81125,
+ "rewards/margins": 1.1707526312023402,
+ "logps/chosen": -67.36336461186409,
+ "logps/rejected": -66.28742809772491,
+ "epoch": 0.7987137596597687,
+ "step": 770
+ },
+ {
+ "loss": 0.4908913135528564,
+ "grad_norm": 0.012934792786836624,
+ "learning_rate": 1.9274611398963734e-06,
+ "entropy": 1.552621606560424,
+ "num_tokens": 52314131.0,
+ "logits/chosen": -0.18498517361282538,
+ "logits/rejected": -0.11354406717633747,
+ "mean_token_accuracy": 0.6353732559084893,
+ "rewards/chosen": -0.5131413355581753,
+ "rewards/rejected": -1.5564261709357379,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 1.0432848367001861,
+ "logps/chosen": -63.0331081187725,
+ "logps/rejected": -63.343865761756895,
+ "epoch": 0.809086665629376,
+ "step": 780
+ },
+ {
+ "loss": 0.4565439701080322,
+ "grad_norm": 0.009556037373840809,
+ "learning_rate": 1.823834196891192e-06,
+ "entropy": 1.5621476396266372,
+ "num_tokens": 52985085.0,
+ "logits/chosen": -0.15417287925592482,
+ "logits/rejected": -0.1223619992398392,
+ "mean_token_accuracy": 0.6369738419540226,
+ "rewards/chosen": -0.4733358434267575,
+ "rewards/rejected": -1.6106882282317383,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.137352383164689,
+ "logps/chosen": -63.7558537530899,
+ "logps/rejected": -62.9910383605957,
+ "epoch": 0.8194595715989834,
+ "step": 790
+ },
+ {
+ "loss": 0.4741868495941162,
+ "grad_norm": 0.016206873580813408,
+ "learning_rate": 1.7202072538860104e-06,
+ "entropy": 1.601190996048972,
+ "num_tokens": 53655618.0,
+ "logits/chosen": -0.17460177579155492,
+ "logits/rejected": -0.13103846312794093,
+ "mean_token_accuracy": 0.630397092960775,
+ "rewards/chosen": -0.505800349440251,
+ "rewards/rejected": -1.5950146451813634,
+ "rewards/accuracies": 0.795,
+ "rewards/margins": 1.0892142936307936,
+ "logps/chosen": -65.44515601754189,
+ "logps/rejected": -63.24514980912208,
+ "epoch": 0.8298324775685908,
+ "step": 800
+ },
+ {
+ "loss": 0.4649956703186035,
+ "grad_norm": 0.010627168230712414,
+ "learning_rate": 1.6165803108808292e-06,
+ "entropy": 1.555380341531709,
+ "num_tokens": 54325923.0,
+ "logits/chosen": -0.19398651192502125,
+ "logits/rejected": -0.15082703285229165,
+ "mean_token_accuracy": 0.6328423308022321,
+ "rewards/chosen": -0.4637420481389563,
+ "rewards/rejected": -1.595540246453602,
+ "rewards/accuracies": 0.7975,
+ "rewards/margins": 1.1317981974454596,
+ "logps/chosen": -64.03305829644204,
+ "logps/rejected": -63.18673894524574,
+ "epoch": 0.8402053835381982,
+ "step": 810
+ },
+ {
+ "loss": 0.47422428131103517,
+ "grad_norm": 0.009889532811939716,
+ "learning_rate": 1.5129533678756477e-06,
+ "entropy": 1.5673230712581425,
+ "num_tokens": 54996048.0,
+ "logits/chosen": -0.126372156575865,
+ "logits/rejected": -0.1181616730672691,
+ "mean_token_accuracy": 0.6365260764025151,
+ "rewards/chosen": -0.4596140426142665,
+ "rewards/rejected": -1.5445942938700319,
+ "rewards/accuracies": 0.79125,
+ "rewards/margins": 1.0849802498798817,
+ "logps/chosen": -64.76353208780289,
+ "logps/rejected": -61.268287732601166,
+ "epoch": 0.8505782895078056,
+ "step": 820
+ },
+ {
+ "loss": 0.46468091011047363,
+ "grad_norm": 0.012822868302464485,
+ "learning_rate": 1.4093264248704663e-06,
+ "entropy": 1.5624252317007632,
+ "num_tokens": 55665592.0,
+ "logits/chosen": -0.17290901349535917,
+ "logits/rejected": -0.09302096610554733,
+ "mean_token_accuracy": 0.6442977831698954,
+ "rewards/chosen": -0.4371060668613791,
+ "rewards/rejected": -1.536119586258028,
+ "rewards/accuracies": 0.8,
+ "rewards/margins": 1.0990135185187682,
+ "logps/chosen": -61.236400594711306,
+ "logps/rejected": -62.05081979513168,
+ "epoch": 0.860951195477413,
+ "step": 830
+ },
+ {
+ "loss": 0.45096325874328613,
+ "grad_norm": 0.013951211236417294,
+ "learning_rate": 1.3056994818652852e-06,
+ "entropy": 1.5971369505673647,
+ "num_tokens": 56336442.0,
+ "logits/chosen": -0.10429545347666931,
+ "logits/rejected": -0.06649558410107631,
+ "mean_token_accuracy": 0.6393469601497054,
+ "rewards/chosen": -0.40763613226081363,
+ "rewards/rejected": -1.5621690288920944,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.154532897588797,
+ "logps/chosen": -65.28316462039948,
+ "logps/rejected": -62.96997811794281,
+ "epoch": 0.8713241014470204,
+ "step": 840
+ },
+ {
+ "loss": 0.47904314994812014,
+ "grad_norm": 0.00990319438278675,
+ "learning_rate": 1.2020725388601037e-06,
+ "entropy": 1.5907638165587559,
+ "num_tokens": 57007058.0,
+ "logits/chosen": -0.09286406420063983,
+ "logits/rejected": -0.07470467026337152,
+ "mean_token_accuracy": 0.6382128950580954,
+ "rewards/chosen": -0.4365648794851586,
+ "rewards/rejected": -1.510114096890611,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.0735492193046958,
+ "logps/chosen": -64.26915530443192,
+ "logps/rejected": -62.46030176877976,
+ "epoch": 0.8816970074166278,
+ "step": 850
+ },
+ {
+ "loss": 0.44410099983215334,
+ "grad_norm": 0.01345039252191782,
+ "learning_rate": 1.0984455958549225e-06,
+ "entropy": 1.592563019664958,
+ "num_tokens": 57679494.0,
+ "logits/chosen": -0.0886190660607926,
+ "logits/rejected": -0.07358264838387342,
+ "mean_token_accuracy": 0.6425007794424892,
+ "rewards/chosen": -0.4673090093833434,
+ "rewards/rejected": -1.6629120723775122,
+ "rewards/accuracies": 0.81875,
+ "rewards/margins": 1.1956030650436877,
+ "logps/chosen": -67.38507036685944,
+ "logps/rejected": -64.36338531255723,
+ "epoch": 0.8920699133862352,
+ "step": 860
+ },
+ {
+ "loss": 0.4594274044036865,
+ "grad_norm": 0.013964708894491196,
+ "learning_rate": 9.94818652849741e-07,
+ "entropy": 1.5719784983014689,
+ "num_tokens": 58351583.0,
+ "logits/chosen": -0.12534822828100103,
+ "logits/rejected": -0.08890342052546357,
+ "mean_token_accuracy": 0.6392641539499163,
+ "rewards/chosen": -0.4524850499225431,
+ "rewards/rejected": -1.519640732365424,
+ "rewards/accuracies": 0.81,
+ "rewards/margins": 1.067155679371208,
+ "logps/chosen": -66.18104577779769,
+ "logps/rejected": -62.2332259953022,
+ "epoch": 0.9024428193558426,
+ "step": 870
+ },
+ {
+ "eval_loss": 0.5229986906051636,
+ "eval_runtime": 381.7545,
+ "eval_samples_per_second": 2.619,
+ "eval_steps_per_second": 2.619,
+ "eval_entropy": 1.1186311850771309,
+ "eval_num_tokens": 58687050.0,
+ "eval_logits/chosen": -0.3429043124152476,
+ "eval_logits/rejected": -0.19782168371902933,
+ "eval_mean_token_accuracy": 0.6990677160173655,
+ "eval_rewards/chosen": -0.4997570053608506,
+ "eval_rewards/rejected": -1.3087458543572574,
+ "eval_rewards/accuracies": 0.742,
+ "eval_rewards/margins": 0.8089888453334569,
+ "eval_logps/chosen": -55.1705568113327,
+ "eval_logps/rejected": -65.72733002090455,
+ "epoch": 0.9076292723406463,
+ "step": 875
+ },
+ {
+ "loss": 0.443557596206665,
+ "grad_norm": 0.012585914693772793,
+ "learning_rate": 8.911917098445596e-07,
+ "entropy": 1.5672686619590968,
+ "num_tokens": 59023046.0,
+ "logits/chosen": -0.12382831435841032,
+ "logits/rejected": -0.0645244036783407,
+ "mean_token_accuracy": 0.6483317786641419,
+ "rewards/chosen": -0.4306466249283403,
+ "rewards/rejected": -1.5827327115193475,
+ "rewards/accuracies": 0.81875,
+ "rewards/margins": 1.152086088140495,
+ "logps/chosen": -62.15216760993004,
+ "logps/rejected": -64.93114773035049,
+ "epoch": 0.91281572532545,
+ "step": 880
+ },
+ {
+ "loss": 0.48163304328918455,
+ "grad_norm": 0.016623329371213913,
+ "learning_rate": 7.875647668393784e-07,
+ "entropy": 1.5462013640487566,
+ "num_tokens": 59693896.0,
+ "logits/chosen": -0.18203885317446325,
+ "logits/rejected": -0.13588479034259882,
+ "mean_token_accuracy": 0.6381713828258216,
+ "rewards/chosen": -0.521100371115972,
+ "rewards/rejected": -1.621375085162581,
+ "rewards/accuracies": 0.79,
+ "rewards/margins": 1.1002747156540864,
+ "logps/chosen": -63.47919486403465,
+ "logps/rejected": -64.74247067213058,
+ "epoch": 0.9231886312950573,
+ "step": 890
+ },
+ {
+ "loss": 0.44486308097839355,
+ "grad_norm": 0.013689976185560226,
+ "learning_rate": 6.839378238341969e-07,
+ "entropy": 1.5661965133296325,
+ "num_tokens": 60365892.0,
+ "logits/chosen": -0.11118840329620845,
+ "logits/rejected": -0.07329385257054667,
+ "mean_token_accuracy": 0.6402364380471408,
+ "rewards/chosen": -0.4615006167604588,
+ "rewards/rejected": -1.6645225426967953,
+ "rewards/accuracies": 0.805,
+ "rewards/margins": 1.2030219237180426,
+ "logps/chosen": -63.24172857046128,
+ "logps/rejected": -66.57125444769859,
+ "epoch": 0.9335615372646647,
+ "step": 900
+ },
+ {
+ "loss": 0.4535379886627197,
+ "grad_norm": 0.010069926269352436,
+ "learning_rate": 5.803108808290156e-07,
+ "entropy": 1.6097470640204847,
+ "num_tokens": 61036862.0,
+ "logits/chosen": -0.13289301790319819,
+ "logits/rejected": -0.0735025375326134,
+ "mean_token_accuracy": 0.6403341065347194,
+ "rewards/chosen": -0.4923504410259193,
+ "rewards/rejected": -1.6649721652292646,
+ "rewards/accuracies": 0.81625,
+ "rewards/margins": 1.1726217230875045,
+ "logps/chosen": -65.2402667415142,
+ "logps/rejected": -64.35793403387069,
+ "epoch": 0.9439344432342721,
+ "step": 910
+ },
+ {
+ "loss": 0.47214512825012206,
+ "grad_norm": 0.012374049052596092,
+ "learning_rate": 4.7668393782383424e-07,
+ "entropy": 1.554362382190302,
+ "num_tokens": 61705337.0,
+ "logits/chosen": -0.1705622702929202,
+ "logits/rejected": -0.11171698211418946,
+ "mean_token_accuracy": 0.6395000527519733,
+ "rewards/chosen": -0.48797103923920077,
+ "rewards/rejected": -1.6139281645056327,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.1259571241028608,
+ "logps/chosen": -61.68648973822594,
+ "logps/rejected": -61.576703943014145,
+ "epoch": 0.9543073492038795,
+ "step": 920
+ },
+ {
+ "loss": 0.45867042541503905,
+ "grad_norm": 0.01340759638696909,
+ "learning_rate": 3.730569948186528e-07,
+ "entropy": 1.5967881159111856,
+ "num_tokens": 62377312.0,
+ "logits/chosen": -0.12794651993287004,
+ "logits/rejected": -0.11582026918000818,
+ "mean_token_accuracy": 0.6418261045776308,
+ "rewards/chosen": -0.47417242905015883,
+ "rewards/rejected": -1.6198302093641177,
+ "rewards/accuracies": 0.79625,
+ "rewards/margins": 1.1456577759655193,
+ "logps/chosen": -65.74660252451896,
+ "logps/rejected": -64.64849405288696,
+ "epoch": 0.9646802551734869,
+ "step": 930
+ },
+ {
+ "loss": 0.4571826934814453,
+ "grad_norm": 0.013874146156013012,
+ "learning_rate": 2.694300518134715e-07,
+ "entropy": 1.5732189149875193,
+ "num_tokens": 63049225.0,
+ "logits/chosen": -0.0920138380328118,
+ "logits/rejected": -0.08312331933128955,
+ "mean_token_accuracy": 0.6384654704108834,
+ "rewards/chosen": -0.4673209265776677,
+ "rewards/rejected": -1.5875896779643517,
+ "rewards/accuracies": 0.81875,
+ "rewards/margins": 1.1202687494084238,
+ "logps/chosen": -65.47650277853012,
+ "logps/rejected": -62.935032539367675,
+ "epoch": 0.9750531611430943,
+ "step": 940
+ },
+ {
+ "loss": 0.46979718208312987,
+ "grad_norm": 0.014985826797783375,
+ "learning_rate": 1.6580310880829015e-07,
+ "entropy": 1.5484525384940206,
+ "num_tokens": 63720031.0,
+ "logits/chosen": -0.15286090245971876,
+ "logits/rejected": -0.1204497936021734,
+ "mean_token_accuracy": 0.6424062415957451,
+ "rewards/chosen": -0.49533468891182564,
+ "rewards/rejected": -1.60940544168232,
+ "rewards/accuracies": 0.79,
+ "rewards/margins": 1.114070752542466,
+ "logps/chosen": -63.495964756011965,
+ "logps/rejected": -62.70811087369919,
+ "epoch": 0.9854260671127016,
+ "step": 950
+ },
+ {
+ "loss": 0.4802603244781494,
+ "grad_norm": 0.011967616155743599,
+ "learning_rate": 6.217616580310881e-08,
+ "entropy": 1.5840327659342437,
+ "num_tokens": 64392923.0,
+ "logits/chosen": -0.10475629294980437,
+ "logits/rejected": -0.07119824883517484,
+ "mean_token_accuracy": 0.6352237542532384,
+ "rewards/chosen": -0.5007370743073989,
+ "rewards/rejected": -1.6135246429864492,
+ "rewards/accuracies": 0.7825,
+ "rewards/margins": 1.112787566203624,
+ "logps/chosen": -66.47015105843543,
+ "logps/rejected": -63.966525557041166,
+ "epoch": 0.995798973082309,
+ "step": 960
+ },
+ {
+ "train_runtime": 37547.325,
+ "train_samples_per_second": 2.054,
+ "train_steps_per_second": 0.026,
+ "total_flos": 5.240658898225705e+17,
+ "train_loss": 0.4893577980871645,
+ "entropy": 1.6188829190753125,
+ "num_tokens": 64665552.0,
+ "logits/chosen": -0.007985222363138011,
+ "logits/rejected": 0.00036492390230720645,
+ "mean_token_accuracy": 0.6398491414240849,
+ "rewards/chosen": -0.5084118424838604,
+ "rewards/rejected": -1.617123565151152,
+ "rewards/accuracies": 0.7932098765432098,
+ "rewards/margins": 1.1087117217435145,
+ "logps/chosen": -67.4950445699103,
+ "logps/rejected": -67.93077748498798,
+ "epoch": 1.0,
+ "step": 965
+ }
+ ],
+ "validation_monitor_size": 1000,
+ "validation_monitor_selection": "fixed_seed_random_without_replacement",
+ "validation_monitor_seed": 22,
+ "validation_monitor_indices": [
+ 2,
+ 10,
+ 14,
+ 21,
+ 55,
+ 63,
+ 66,
+ 69,
+ 107,
+ 110,
+ 142,
+ 144,
+ 149,
+ 150,
+ 151,
+ 152,
+ 160,
+ 163,
+ 166,
+ 167,
+ 176,
+ 181,
+ 206,
+ 207,
+ 259,
+ 267,
+ 281,
+ 295,
+ 298,
+ 306,
+ 307,
+ 317,
+ 321,
+ 331,
+ 336,
+ 341,
+ 346,
+ 349,
+ 351,
+ 352,
+ 357,
+ 358,
+ 369,
+ 370,
+ 382,
+ 386,
+ 391,
+ 401,
+ 411,
+ 412,
+ 432,
+ 437,
+ 452,
+ 462,
+ 465,
+ 488,
+ 490,
+ 491,
+ 492,
+ 494,
+ 503,
+ 504,
+ 508,
+ 528,
+ 538,
+ 540,
+ 551,
+ 553,
+ 567,
+ 586,
+ 605,
+ 606,
+ 625,
+ 627,
+ 661,
+ 663,
+ 666,
+ 677,
+ 685,
+ 690,
+ 692,
+ 704,
+ 708,
+ 714,
+ 715,
+ 727,
+ 728,
+ 733,
+ 745,
+ 752,
+ 753,
+ 755,
+ 764,
+ 773,
+ 779,
+ 783,
+ 793,
+ 796,
+ 799,
+ 803,
+ 804,
+ 805,
+ 813,
+ 816,
+ 818,
+ 823,
+ 827,
+ 829,
+ 830,
+ 837,
+ 842,
+ 845,
+ 850,
+ 853,
+ 856,
+ 889,
+ 890,
+ 905,
+ 915,
+ 924,
+ 930,
+ 939,
+ 951,
+ 988,
+ 1002,
+ 1005,
+ 1023,
+ 1029,
+ 1038,
+ 1049,
+ 1050,
+ 1054,
+ 1056,
+ 1067,
+ 1068,
+ 1079,
+ 1088,
+ 1091,
+ 1103,
+ 1114,
+ 1118,
+ 1133,
+ 1140,
+ 1144,
+ 1145,
+ 1155,
+ 1186,
+ 1195,
+ 1206,
+ 1223,
+ 1251,
+ 1252,
+ 1257,
+ 1259,
+ 1270,
+ 1271,
+ 1295,
+ 1303,
+ 1304,
+ 1305,
+ 1329,
+ 1335,
+ 1336,
+ 1343,
+ 1367,
+ 1373,
+ 1377,
+ 1403,
+ 1412,
+ 1429,
+ 1443,
+ 1457,
+ 1459,
+ 1460,
+ 1476,
+ 1483,
+ 1486,
+ 1494,
+ 1507,
+ 1510,
+ 1519,
+ 1521,
+ 1522,
+ 1545,
+ 1551,
+ 1570,
+ 1582,
+ 1593,
+ 1595,
+ 1603,
+ 1607,
+ 1614,
+ 1615,
+ 1625,
+ 1634,
+ 1639,
+ 1648,
+ 1654,
+ 1657,
+ 1662,
+ 1667,
+ 1673,
+ 1690,
+ 1704,
+ 1746,
+ 1756,
+ 1781,
+ 1783,
+ 1796,
+ 1799,
+ 1809,
+ 1814,
+ 1827,
+ 1829,
+ 1832,
+ 1844,
+ 1847,
+ 1854,
+ 1856,
+ 1857,
+ 1858,
+ 1874,
+ 1883,
+ 1889,
+ 1924,
+ 1925,
+ 1931,
+ 1932,
+ 1934,
+ 1935,
+ 1938,
+ 1950,
+ 1957,
+ 1962,
+ 1967,
+ 1975,
+ 1982,
+ 1983,
+ 1991,
+ 1998,
+ 2003,
+ 2008,
+ 2017,
+ 2021,
+ 2026,
+ 2035,
+ 2040,
+ 2050,
+ 2056,
+ 2077,
+ 2079,
+ 2082,
+ 2098,
+ 2100,
+ 2108,
+ 2121,
+ 2130,
+ 2133,
+ 2134,
+ 2138,
+ 2143,
+ 2166,
+ 2167,
+ 2180,
+ 2181,
+ 2185,
+ 2204,
+ 2205,
+ 2212,
+ 2221,
+ 2224,
+ 2226,
+ 2230,
+ 2233,
+ 2256,
+ 2261,
+ 2263,
+ 2269,
+ 2273,
+ 2290,
+ 2291,
+ 2299,
+ 2301,
+ 2321,
+ 2323,
+ 2330,
+ 2384,
+ 2401,
+ 2417,
+ 2420,
+ 2421,
+ 2424,
+ 2430,
+ 2435,
+ 2456,
+ 2488,
+ 2502,
+ 2504,
+ 2519,
+ 2527,
+ 2532,
+ 2538,
+ 2542,
+ 2553,
+ 2555,
+ 2558,
+ 2559,
+ 2562,
+ 2578,
+ 2583,
+ 2585,
+ 2590,
+ 2592,
+ 2594,
+ 2596,
+ 2600,
+ 2606,
+ 2607,
+ 2618,
+ 2630,
+ 2637,
+ 2647,
+ 2650,
+ 2657,
+ 2679,
+ 2685,
+ 2705,
+ 2706,
+ 2712,
+ 2713,
+ 2714,
+ 2727,
+ 2740,
+ 2742,
+ 2755,
+ 2780,
+ 2784,
+ 2792,
+ 2807,
+ 2808,
+ 2810,
+ 2820,
+ 2831,
+ 2839,
+ 2860,
+ 2862,
+ 2863,
+ 2866,
+ 2875,
+ 2878,
+ 2898,
+ 2905,
+ 2921,
+ 2922,
+ 2926,
+ 2930,
+ 2934,
+ 2944,
+ 2955,
+ 2957,
+ 2959,
+ 2973,
+ 2978,
+ 2998,
+ 3018,
+ 3022,
+ 3028,
+ 3031,
+ 3061,
+ 3085,
+ 3090,
+ 3104,
+ 3105,
+ 3111,
+ 3115,
+ 3121,
+ 3122,
+ 3129,
+ 3133,
+ 3135,
+ 3161,
+ 3162,
+ 3169,
+ 3173,
+ 3194,
+ 3195,
+ 3215,
+ 3225,
+ 3226,
+ 3241,
+ 3247,
+ 3250,
+ 3253,
+ 3265,
+ 3268,
+ 3293,
+ 3301,
+ 3312,
+ 3329,
+ 3352,
+ 3361,
+ 3362,
+ 3376,
+ 3396,
+ 3401,
+ 3403,
+ 3410,
+ 3419,
+ 3432,
+ 3443,
+ 3452,
+ 3467,
+ 3469,
+ 3475,
+ 3485,
+ 3503,
+ 3514,
+ 3515,
+ 3524,
+ 3528,
+ 3538,
+ 3544,
+ 3557,
+ 3560,
+ 3565,
+ 3600,
+ 3637,
+ 3642,
+ 3658,
+ 3659,
+ 3692,
+ 3693,
+ 3699,
+ 3722,
+ 3725,
+ 3728,
+ 3731,
+ 3740,
+ 3742,
+ 3762,
+ 3766,
+ 3780,
+ 3788,
+ 3794,
+ 3796,
+ 3798,
+ 3805,
+ 3817,
+ 3819,
+ 3822,
+ 3837,
+ 3839,
+ 3843,
+ 3846,
+ 3851,
+ 3854,
+ 3865,
+ 3870,
+ 3871,
+ 3884,
+ 3894,
+ 3895,
+ 3896,
+ 3907,
+ 3911,
+ 3915,
+ 3919,
+ 3920,
+ 3923,
+ 3933,
+ 3955,
+ 3967,
+ 3972,
+ 3974,
+ 3975,
+ 3984,
+ 3985,
+ 3997,
+ 4002,
+ 4010,
+ 4034,
+ 4044,
+ 4063,
+ 4073,
+ 4079,
+ 4082,
+ 4088,
+ 4121,
+ 4145,
+ 4148,
+ 4159,
+ 4161,
+ 4164,
+ 4177,
+ 4188,
+ 4199,
+ 4203,
+ 4207,
+ 4208,
+ 4213,
+ 4221,
+ 4225,
+ 4233,
+ 4241,
+ 4243,
+ 4247,
+ 4264,
+ 4274,
+ 4282,
+ 4286,
+ 4290,
+ 4308,
+ 4310,
+ 4313,
+ 4321,
+ 4324,
+ 4327,
+ 4349,
+ 4362,
+ 4370,
+ 4374,
+ 4380,
+ 4407,
+ 4409,
+ 4411,
+ 4415,
+ 4417,
+ 4418,
+ 4427,
+ 4431,
+ 4433,
+ 4451,
+ 4466,
+ 4477,
+ 4478,
+ 4479,
+ 4493,
+ 4494,
+ 4514,
+ 4527,
+ 4539,
+ 4550,
+ 4558,
+ 4568,
+ 4579,
+ 4583,
+ 4584,
+ 4586,
+ 4587,
+ 4590,
+ 4599,
+ 4602,
+ 4610,
+ 4626,
+ 4627,
+ 4630,
+ 4641,
+ 4647,
+ 4655,
+ 4656,
+ 4657,
+ 4661,
+ 4685,
+ 4714,
+ 4715,
+ 4731,
+ 4749,
+ 4752,
+ 4769,
+ 4777,
+ 4782,
+ 4791,
+ 4805,
+ 4818,
+ 4829,
+ 4833,
+ 4837,
+ 4839,
+ 4843,
+ 4871,
+ 4875,
+ 4879,
+ 4880,
+ 4885,
+ 4888,
+ 4895,
+ 4900,
+ 4923,
+ 4966,
+ 4968,
+ 4972,
+ 4989,
+ 4994,
+ 4998,
+ 5001,
+ 5013,
+ 5019,
+ 5026,
+ 5032,
+ 5034,
+ 5046,
+ 5055,
+ 5085,
+ 5086,
+ 5088,
+ 5089,
+ 5090,
+ 5095,
+ 5108,
+ 5110,
+ 5119,
+ 5120,
+ 5121,
+ 5133,
+ 5148,
+ 5154,
+ 5160,
+ 5169,
+ 5178,
+ 5222,
+ 5223,
+ 5232,
+ 5233,
+ 5240,
+ 5256,
+ 5259,
+ 5264,
+ 5271,
+ 5276,
+ 5279,
+ 5294,
+ 5300,
+ 5303,
+ 5321,
+ 5335,
+ 5337,
+ 5345,
+ 5348,
+ 5365,
+ 5373,
+ 5378,
+ 5384,
+ 5422,
+ 5442,
+ 5443,
+ 5445,
+ 5477,
+ 5485,
+ 5495,
+ 5497,
+ 5504,
+ 5526,
+ 5533,
+ 5542,
+ 5564,
+ 5570,
+ 5579,
+ 5587,
+ 5592,
+ 5608,
+ 5610,
+ 5624,
+ 5630,
+ 5638,
+ 5651,
+ 5663,
+ 5670,
+ 5675,
+ 5691,
+ 5700,
+ 5706,
+ 5707,
+ 5715,
+ 5720,
+ 5721,
+ 5722,
+ 5732,
+ 5738,
+ 5741,
+ 5769,
+ 5771,
+ 5775,
+ 5795,
+ 5796,
+ 5800,
+ 5809,
+ 5810,
+ 5815,
+ 5819,
+ 5827,
+ 5848,
+ 5849,
+ 5852,
+ 5859,
+ 5880,
+ 5884,
+ 5907,
+ 5909,
+ 5912,
+ 5919,
+ 5931,
+ 5932,
+ 5934,
+ 5941,
+ 5948,
+ 5950,
+ 5952,
+ 5953,
+ 5969,
+ 5981,
+ 6000,
+ 6003,
+ 6010,
+ 6018,
+ 6041,
+ 6065,
+ 6075,
+ 6090,
+ 6103,
+ 6106,
+ 6109,
+ 6114,
+ 6123,
+ 6131,
+ 6136,
+ 6138,
+ 6139,
+ 6164,
+ 6165,
+ 6171,
+ 6173,
+ 6180,
+ 6185,
+ 6189,
+ 6190,
+ 6221,
+ 6223,
+ 6237,
+ 6255,
+ 6262,
+ 6265,
+ 6293,
+ 6296,
+ 6305,
+ 6318,
+ 6331,
+ 6336,
+ 6340,
+ 6355,
+ 6366,
+ 6371,
+ 6396,
+ 6399,
+ 6402,
+ 6408,
+ 6432,
+ 6433,
+ 6441,
+ 6456,
+ 6465,
+ 6478,
+ 6486,
+ 6489,
+ 6507,
+ 6508,
+ 6520,
+ 6522,
+ 6528,
+ 6537,
+ 6551,
+ 6564,
+ 6589,
+ 6590,
+ 6598,
+ 6599,
+ 6611,
+ 6613,
+ 6615,
+ 6621,
+ 6634,
+ 6647,
+ 6649,
+ 6654,
+ 6676,
+ 6680,
+ 6690,
+ 6708,
+ 6729,
+ 6736,
+ 6744,
+ 6749,
+ 6756,
+ 6761,
+ 6763,
+ 6773,
+ 6788,
+ 6790,
+ 6796,
+ 6797,
+ 6811,
+ 6824,
+ 6850,
+ 6869,
+ 6871,
+ 6882,
+ 6892,
+ 6895,
+ 6906,
+ 6911,
+ 6912,
+ 6914,
+ 6927,
+ 6952,
+ 6966,
+ 6985,
+ 7001,
+ 7021,
+ 7031,
+ 7035,
+ 7038,
+ 7041,
+ 7045,
+ 7052,
+ 7057,
+ 7058,
+ 7072,
+ 7073,
+ 7076,
+ 7086,
+ 7102,
+ 7107,
+ 7109,
+ 7117,
+ 7122,
+ 7125,
+ 7166,
+ 7182,
+ 7199,
+ 7207,
+ 7212,
+ 7215,
+ 7232,
+ 7243,
+ 7246,
+ 7250,
+ 7253,
+ 7258,
+ 7263,
+ 7267,
+ 7270,
+ 7274,
+ 7280,
+ 7286,
+ 7293,
+ 7298,
+ 7302,
+ 7306,
+ 7316,
+ 7325,
+ 7326,
+ 7334,
+ 7356,
+ 7357,
+ 7363,
+ 7364,
+ 7367,
+ 7385,
+ 7392,
+ 7399,
+ 7405,
+ 7416,
+ 7420,
+ 7421,
+ 7426,
+ 7452,
+ 7476,
+ 7481,
+ 7519,
+ 7534,
+ 7542,
+ 7546,
+ 7573,
+ 7585,
+ 7601,
+ 7604,
+ 7606,
+ 7609,
+ 7629,
+ 7649,
+ 7653,
+ 7667,
+ 7682,
+ 7699,
+ 7738,
+ 7750,
+ 7786,
+ 7798,
+ 7800,
+ 7801,
+ 7805,
+ 7806,
+ 7811,
+ 7813,
+ 7832,
+ 7837,
+ 7849,
+ 7856,
+ 7876,
+ 7877,
+ 7887,
+ 7905,
+ 7916,
+ 7919,
+ 7920,
+ 7922,
+ 7923,
+ 7926,
+ 7944,
+ 7961,
+ 7966,
+ 7970,
+ 7973,
+ 7974,
+ 7976,
+ 7986,
+ 7999,
+ 8027,
+ 8028,
+ 8034,
+ 8047,
+ 8068,
+ 8074,
+ 8077,
+ 8091,
+ 8120,
+ 8122,
+ 8125,
+ 8152,
+ 8153,
+ 8164,
+ 8167,
+ 8169,
+ 8171,
+ 8177,
+ 8184,
+ 8189,
+ 8192,
+ 8196,
+ 8202,
+ 8212,
+ 8217,
+ 8231,
+ 8242,
+ 8244,
+ 8250,
+ 8256,
+ 8257,
+ 8265,
+ 8270,
+ 8274,
+ 8283,
+ 8290,
+ 8294,
+ 8301,
+ 8302,
+ 8307,
+ 8318,
+ 8326,
+ 8338,
+ 8349,
+ 8350,
+ 8353,
+ 8357,
+ 8371,
+ 8374,
+ 8377,
+ 8380,
+ 8387,
+ 8388,
+ 8396,
+ 8402,
+ 8419,
+ 8423,
+ 8428,
+ 8435,
+ 8439,
+ 8442,
+ 8444,
+ 8453,
+ 8461,
+ 8475,
+ 8481,
+ 8485,
+ 8493,
+ 8495,
+ 8498,
+ 8523,
+ 8530,
+ 8531,
+ 8562
+ ],
+ "early_stopping_patience": 3
+}
diff --git a/phase1/llama-dpo-v3/README.md b/phase1/llama-dpo-v3/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..bbf7198e1dd125a154033276ba2bf5684956b804
--- /dev/null
+++ b/phase1/llama-dpo-v3/README.md
@@ -0,0 +1,82 @@
+---
+library_name: peft
+model_name: phase1-llama-dpo-v3
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+license: llama3.2
+base_model: meta-llama/Llama-3.2-1B-Instruct
+pipeline_tag: text-generation
+---
+
+# Phase 1 Llama cross-family DPO adapter
+
+This is the Phase 1 cross-family LoRA-DPO adapter fine-tuned from
+[Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct)
+with 99,986 preference pairs. Access to the gated base model and acceptance of its
+license are required. It has been trained using
+[TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo_id = "geyuxu/york-milestone-project-self-traing-loop-model"
+subfolder = "phase1/llama-dpo-v3"
+base_id = "meta-llama/Llama-3.2-1B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(repo_id, subfolder=subfolder)
+base_model = AutoModelForCausalLM.from_pretrained(
+ base_id, torch_dtype="auto", device_map="auto"
+)
+model = PeftModel.from_pretrained(base_model, repo_id, subfolder=subfolder)
+```
+
+## Training procedure
+
+
+
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 0.29.0
+- Transformers: 5.3.0
+- Pytorch: 2.10.0
+- Datasets: 4.6.1
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+ title = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+ author = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+ year = 2023,
+ booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+ url = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+ editor = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+ title = {{TRL: Transformers Reinforcement Learning}},
+ author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+ license = {Apache-2.0},
+ url = {https://github.com/huggingface/trl},
+ year = {2020}
+}
+```
diff --git a/phase1/llama-dpo-v3/adapter_config.json b/phase1/llama-dpo-v3/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..525da8319752287f2fcefcad959afda346b5c3b7
--- /dev/null
+++ b/phase1/llama-dpo-v3/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "o_proj",
+ "k_proj",
+ "v_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
diff --git a/phase1/llama-dpo-v3/adapter_model.safetensors b/phase1/llama-dpo-v3/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..4f68992b10ba69a7b23455131cbac723b3b64b6e
--- /dev/null
+++ b/phase1/llama-dpo-v3/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2633d2be48f838d5c4404786ca9baa38ff2b0ed66d2fa5d7d11eb74fe63c178d
+size 13648488
diff --git a/phase1/llama-dpo-v3/chat_template.jinja b/phase1/llama-dpo-v3/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/phase1/llama-dpo-v3/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/phase1/llama-dpo-v3/tokenizer.json b/phase1/llama-dpo-v3/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..c81be5af09de8a32108b33764591b09234d26a0e
--- /dev/null
+++ b/phase1/llama-dpo-v3/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b20d148821d1e209454ef0d54a1d2bb1ff12cd365a0ff5391d96010e897b9644
+size 17208988
diff --git a/phase1/llama-dpo-v3/tokenizer_config.json b/phase1/llama-dpo-v3/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..95a7a465203072f48c7cd80dfd78f77c661c8c9a
--- /dev/null
+++ b/phase1/llama-dpo-v3/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": true,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/phase1/llama-dpo-v3/training_args.bin b/phase1/llama-dpo-v3/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..1f42dd7747e3865e86ef93b8dbb088965a1c2dd0
--- /dev/null
+++ b/phase1/llama-dpo-v3/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fced63a90802dbed1aa84b5b0863e967f730e08882f30d6a2cec2f76954c2675
+size 5841
diff --git a/phase1/llama-dpo-v3/training_recipe.json b/phase1/llama-dpo-v3/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..10aef2a59c786436b236d2a472be28b228fded8c
--- /dev/null
+++ b/phase1/llama-dpo-v3/training_recipe.json
@@ -0,0 +1,20 @@
+{
+ "format_version": 1,
+ "experiment": "cross_family_preference_transfer",
+ "model_name": "meta-llama/Llama-3.2-1B-Instruct",
+ "train_membership_sha256": "e105dbe9f9e652f095541415523cdcda51cc9e8d12d3ba4ca03e29e40296324f",
+ "validation_membership_sha256": "a46c1c0d0a6c3c444a3ac940322e42a2f1be5ec8e5f5f729a002e7484742551b",
+ "num_train_pairs": 99986,
+ "num_validation_pairs": 11110,
+ "learning_rate": 1e-05,
+ "beta": 0.1,
+ "num_epochs": 1.0,
+ "seed": 22,
+ "max_length": 512,
+ "max_prompt_length": 384,
+ "gradient_accumulation_steps": 80,
+ "save_steps": 125,
+ "save_total_limit": 12,
+ "preference_pairs_format_version": 2,
+ "dpo_prompt_mode": "rag_prompt_shared"
+}
diff --git a/phase1/llama-dpo-v3/training_summary.json b/phase1/llama-dpo-v3/training_summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..e3f2dc33ff250770ab21d2aebe3203cd8f40abcb
--- /dev/null
+++ b/phase1/llama-dpo-v3/training_summary.json
@@ -0,0 +1,3493 @@
+{
+ "model_name": "meta-llama/Llama-3.2-1B-Instruct",
+ "resolved_model": "meta-llama/Llama-3.2-1B-Instruct",
+ "num_train_records": 99986,
+ "num_validation_records_available": 11110,
+ "training_config": {
+ "lora_rank": 16,
+ "lora_alpha": 32,
+ "learning_rate": 1e-05,
+ "num_epochs": 1.0,
+ "per_device_batch_size": 1,
+ "per_device_eval_batch_size": 1,
+ "gradient_accumulation_steps": 80,
+ "beta": 0.1,
+ "max_length": 512,
+ "max_prompt_length": 384,
+ "seed": 22,
+ "save_steps": 125,
+ "save_total_limit": 12,
+ "resume_from_checkpoint": null,
+ "precision_dtype": "torch.bfloat16",
+ "bf16": true,
+ "fp16": false,
+ "tokenizer_add_bos_token": false
+ },
+ "global_step": 1250,
+ "best_metric": 0.3831380307674408,
+ "best_model_checkpoint": "outputs/model_generalization_llama32_1b_phase1_repair_v3/lora/checkpoint-1250",
+ "train_metrics": {
+ "train_runtime": 37881.4288,
+ "train_samples_per_second": 2.639,
+ "train_steps_per_second": 0.033,
+ "total_flos": 5.0312011545074074e+17,
+ "train_loss": 0.44697874088287354
+ },
+ "log_history": [
+ {
+ "loss": 0.6876621723175049,
+ "grad_norm": 0.02085641585290432,
+ "learning_rate": 9.928e-06,
+ "entropy": 0.8953838606365025,
+ "num_tokens": 669186.0,
+ "logits/chosen": 1.7523135298926353,
+ "logits/rejected": 1.641919029812606,
+ "mean_token_accuracy": 0.6771072404831648,
+ "rewards/chosen": 0.010034057162379213,
+ "rewards/rejected": -0.001445179752987542,
+ "rewards/accuracies": 0.53625,
+ "rewards/margins": 0.01147923693381017,
+ "logps/chosen": -63.841570150852206,
+ "logps/rejected": -68.64502122402192,
+ "epoch": 0.008001120156821955,
+ "step": 10
+ },
+ {
+ "loss": 0.6597066879272461,
+ "grad_norm": 0.018694201484322548,
+ "learning_rate": 9.848000000000001e-06,
+ "entropy": 0.8793668389227242,
+ "num_tokens": 1338260.0,
+ "logits/chosen": 1.7622254303912428,
+ "logits/rejected": 1.6655854483602721,
+ "mean_token_accuracy": 0.6876659346744418,
+ "rewards/chosen": 0.028637834031824242,
+ "rewards/rejected": -0.04387104604453271,
+ "rewards/accuracies": 0.76625,
+ "rewards/margins": 0.07250888019130798,
+ "logps/chosen": -60.44827336549759,
+ "logps/rejected": -70.3050590801239,
+ "epoch": 0.01600224031364391,
+ "step": 20
+ },
+ {
+ "loss": 0.6248809337615967,
+ "grad_norm": 0.01813693717122078,
+ "learning_rate": 9.768e-06,
+ "entropy": 0.8689308112114668,
+ "num_tokens": 2006531.0,
+ "logits/chosen": 1.7339595036742759,
+ "logits/rejected": 1.6745694846655177,
+ "mean_token_accuracy": 0.6879833539575338,
+ "rewards/chosen": -0.005004954708329024,
+ "rewards/rejected": -0.1686695772564417,
+ "rewards/accuracies": 0.73875,
+ "rewards/margins": 0.16366462260280967,
+ "logps/chosen": -59.31012097835541,
+ "logps/rejected": -70.12172441601753,
+ "epoch": 0.024003360470465866,
+ "step": 30
+ },
+ {
+ "loss": 0.611721420288086,
+ "grad_norm": 0.01976812072098255,
+ "learning_rate": 9.688000000000001e-06,
+ "entropy": 0.8979608153458685,
+ "num_tokens": 2678018.0,
+ "logits/chosen": 1.6847163705957298,
+ "logits/rejected": 1.6548631404733476,
+ "mean_token_accuracy": 0.6856286776997149,
+ "rewards/chosen": -0.09582840940453025,
+ "rewards/rejected": -0.32414788081659934,
+ "rewards/accuracies": 0.7175,
+ "rewards/margins": 0.22831947083352133,
+ "logps/chosen": -66.34679408550262,
+ "logps/rejected": -73.70246529817581,
+ "epoch": 0.03200448062728782,
+ "step": 40
+ },
+ {
+ "loss": 0.5515981197357178,
+ "grad_norm": 0.019012274220585823,
+ "learning_rate": 9.608e-06,
+ "entropy": 0.8629842744022608,
+ "num_tokens": 3345909.0,
+ "logits/chosen": 1.672130467668529,
+ "logits/rejected": 1.6777180424532323,
+ "mean_token_accuracy": 0.6851089650392532,
+ "rewards/chosen": -0.13558172254211967,
+ "rewards/rejected": -0.5490579532997799,
+ "rewards/accuracies": 0.75625,
+ "rewards/margins": 0.41347622983856125,
+ "logps/chosen": -61.12240834712982,
+ "logps/rejected": -73.76463801145553,
+ "epoch": 0.04000560078410977,
+ "step": 50
+ },
+ {
+ "loss": 0.5451503753662109,
+ "grad_norm": 0.018058450892567635,
+ "learning_rate": 9.528000000000001e-06,
+ "entropy": 0.872456860463135,
+ "num_tokens": 4016757.0,
+ "logits/chosen": 1.6254275480609275,
+ "logits/rejected": 1.6532141976649157,
+ "mean_token_accuracy": 0.6732470593973994,
+ "rewards/chosen": -0.3387448867566127,
+ "rewards/rejected": -0.8533535525448679,
+ "rewards/accuracies": 0.74875,
+ "rewards/margins": 0.5146086666500196,
+ "logps/chosen": -65.87585063457489,
+ "logps/rejected": -81.09233409166336,
+ "epoch": 0.04800672094093173,
+ "step": 60
+ },
+ {
+ "loss": 0.5362163543701172,
+ "grad_norm": 0.020387593656778336,
+ "learning_rate": 9.448e-06,
+ "entropy": 0.8608407187415287,
+ "num_tokens": 4686751.0,
+ "logits/chosen": 1.5602627965932958,
+ "logits/rejected": 1.5906360590500324,
+ "mean_token_accuracy": 0.6706002498418093,
+ "rewards/chosen": -0.4640727364263876,
+ "rewards/rejected": -1.0529923798827803,
+ "rewards/accuracies": 0.74625,
+ "rewards/margins": 0.5889196432288736,
+ "logps/chosen": -70.29113697052001,
+ "logps/rejected": -80.11449808835984,
+ "epoch": 0.05600784109775368,
+ "step": 70
+ },
+ {
+ "loss": 0.5358813762664795,
+ "grad_norm": 0.023884067311882973,
+ "learning_rate": 9.368e-06,
+ "entropy": 0.8284324945218395,
+ "num_tokens": 5357166.0,
+ "logits/chosen": 1.4795679082575908,
+ "logits/rejected": 1.5384505947213227,
+ "mean_token_accuracy": 0.6774465151503682,
+ "rewards/chosen": -0.5764407913558535,
+ "rewards/rejected": -1.2395925800641998,
+ "rewards/accuracies": 0.75375,
+ "rewards/margins": 0.6631517899245956,
+ "logps/chosen": -69.7327395427227,
+ "logps/rejected": -80.65829716205597,
+ "epoch": 0.06400896125457564,
+ "step": 80
+ },
+ {
+ "loss": 0.5180981636047364,
+ "grad_norm": 0.022150974720716476,
+ "learning_rate": 9.288e-06,
+ "entropy": 0.8348123076790944,
+ "num_tokens": 6023639.0,
+ "logits/chosen": 1.4908146018031239,
+ "logits/rejected": 1.5654357319194836,
+ "mean_token_accuracy": 0.6684274521842599,
+ "rewards/chosen": -0.5458557065276546,
+ "rewards/rejected": -1.2516879384859931,
+ "rewards/accuracies": 0.74375,
+ "rewards/margins": 0.7058322310540825,
+ "logps/chosen": -65.40104442834854,
+ "logps/rejected": -78.71251199007034,
+ "epoch": 0.0720100814113976,
+ "step": 90
+ },
+ {
+ "loss": 0.5002727508544922,
+ "grad_norm": 0.02316654473543167,
+ "learning_rate": 9.208e-06,
+ "entropy": 0.8209165922319517,
+ "num_tokens": 6694942.0,
+ "logits/chosen": 1.504576485953224,
+ "logits/rejected": 1.5631756865086541,
+ "mean_token_accuracy": 0.6788686821609735,
+ "rewards/chosen": -0.6265445317371268,
+ "rewards/rejected": -1.4296979560064937,
+ "rewards/accuracies": 0.7725,
+ "rewards/margins": 0.8031534244865179,
+ "logps/chosen": -67.86892468690873,
+ "logps/rejected": -87.28859835624695,
+ "epoch": 0.08001120156821954,
+ "step": 100
+ },
+ {
+ "loss": 0.4797528743743896,
+ "grad_norm": 0.025832341983914375,
+ "learning_rate": 9.128e-06,
+ "entropy": 0.839785308111459,
+ "num_tokens": 7366337.0,
+ "logits/chosen": 1.436498547320632,
+ "logits/rejected": 1.5516475294483854,
+ "mean_token_accuracy": 0.6673610024340451,
+ "rewards/chosen": -0.6058229519365704,
+ "rewards/rejected": -1.4726398780666203,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 0.8668169262981974,
+ "logps/chosen": -70.91443045377731,
+ "logps/rejected": -86.69431616544723,
+ "epoch": 0.0880123217250415,
+ "step": 110
+ },
+ {
+ "loss": 0.5151589870452881,
+ "grad_norm": 0.024567700922489166,
+ "learning_rate": 9.048e-06,
+ "entropy": 0.8108099042752293,
+ "num_tokens": 8034830.0,
+ "logits/chosen": 1.415350063930816,
+ "logits/rejected": 1.530800795235843,
+ "mean_token_accuracy": 0.670044612903148,
+ "rewards/chosen": -0.6920958398148287,
+ "rewards/rejected": -1.5167533266616373,
+ "rewards/accuracies": 0.755,
+ "rewards/margins": 0.8246574836131185,
+ "logps/chosen": -69.44658513307571,
+ "logps/rejected": -82.48484308958054,
+ "epoch": 0.09601344188186346,
+ "step": 120
+ },
+ {
+ "eval_loss": 0.4768791198730469,
+ "eval_runtime": 127.5854,
+ "eval_samples_per_second": 7.838,
+ "eval_steps_per_second": 7.838,
+ "eval_entropy": 0.8030976068265736,
+ "eval_num_tokens": 8371601.0,
+ "eval_logits/chosen": 1.41053144572057,
+ "eval_logits/rejected": 1.5178837893711545,
+ "eval_mean_token_accuracy": 0.6734412264749408,
+ "eval_rewards/chosen": -0.6052715343544259,
+ "eval_rewards/rejected": -1.5583781674216153,
+ "eval_rewards/accuracies": 0.793,
+ "eval_rewards/margins": 0.9531066331192851,
+ "eval_logps/chosen": -66.51480191421508,
+ "eval_logps/rejected": -84.00160307693481,
+ "epoch": 0.10001400196027443,
+ "step": 125
+ },
+ {
+ "loss": 0.48606295585632325,
+ "grad_norm": 0.019938811659812927,
+ "learning_rate": 8.968000000000001e-06,
+ "entropy": 0.8076416058605537,
+ "num_tokens": 8706435.0,
+ "logits/chosen": 1.421707210036048,
+ "logits/rejected": 1.5092911593902412,
+ "mean_token_accuracy": 0.6668024481087923,
+ "rewards/chosen": -0.6526209685721551,
+ "rewards/rejected": -1.5515566976579793,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 0.8989357261685654,
+ "logps/chosen": -73.65435603499412,
+ "logps/rejected": -86.77437145233154,
+ "epoch": 0.10401456203868542,
+ "step": 130
+ },
+ {
+ "loss": 0.4929227352142334,
+ "grad_norm": 0.024309106171131134,
+ "learning_rate": 8.888e-06,
+ "entropy": 0.7990755480783992,
+ "num_tokens": 9375004.0,
+ "logits/chosen": 1.3674875151235775,
+ "logits/rejected": 1.5141029199823328,
+ "mean_token_accuracy": 0.6754287776350975,
+ "rewards/chosen": -0.5831789968456724,
+ "rewards/rejected": -1.5065973946775193,
+ "rewards/accuracies": 0.77125,
+ "rewards/margins": 0.9234183955239132,
+ "logps/chosen": -66.23368896722793,
+ "logps/rejected": -84.38780091047288,
+ "epoch": 0.11201568219550737,
+ "step": 140
+ },
+ {
+ "loss": 0.45296435356140136,
+ "grad_norm": 0.03157660365104675,
+ "learning_rate": 8.808000000000001e-06,
+ "entropy": 0.7914871014188976,
+ "num_tokens": 10045658.0,
+ "logits/chosen": 1.3009003627722433,
+ "logits/rejected": 1.4217221088726517,
+ "mean_token_accuracy": 0.6674710770510137,
+ "rewards/chosen": -0.680510274215012,
+ "rewards/rejected": -1.7216353922217968,
+ "rewards/accuracies": 0.79375,
+ "rewards/margins": 1.0411251192959026,
+ "logps/chosen": -69.23368401288987,
+ "logps/rejected": -90.06080745935441,
+ "epoch": 0.12001680235232932,
+ "step": 150
+ },
+ {
+ "loss": 0.44395751953125,
+ "grad_norm": 0.023377399891614914,
+ "learning_rate": 8.728e-06,
+ "entropy": 0.7299869983480312,
+ "num_tokens": 10712570.0,
+ "logits/chosen": 1.1509836349559144,
+ "logits/rejected": 1.3356629677769771,
+ "mean_token_accuracy": 0.6707272659242153,
+ "rewards/chosen": -0.7615989098043064,
+ "rewards/rejected": -1.8726644670779207,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.111065554342931,
+ "logps/chosen": -65.57545948505401,
+ "logps/rejected": -86.87440029740334,
+ "epoch": 0.12801792250915128,
+ "step": 160
+ },
+ {
+ "loss": 0.5061419486999512,
+ "grad_norm": 0.030593544244766235,
+ "learning_rate": 8.648000000000001e-06,
+ "entropy": 0.7306625125929713,
+ "num_tokens": 11378520.0,
+ "logits/chosen": 1.1232699992990192,
+ "logits/rejected": 1.274977415558953,
+ "mean_token_accuracy": 0.6708245670981705,
+ "rewards/chosen": -0.8173911243397742,
+ "rewards/rejected": -1.8643126943940296,
+ "rewards/accuracies": 0.76125,
+ "rewards/margins": 1.0469215692440048,
+ "logps/chosen": -66.93890744447708,
+ "logps/rejected": -85.03745621204376,
+ "epoch": 0.13601904266597323,
+ "step": 170
+ },
+ {
+ "loss": 0.4606666088104248,
+ "grad_norm": 0.02202947810292244,
+ "learning_rate": 8.568e-06,
+ "entropy": 0.7190171673579607,
+ "num_tokens": 12046037.0,
+ "logits/chosen": 1.104012429231747,
+ "logits/rejected": 1.2739071112220537,
+ "mean_token_accuracy": 0.6792438442073763,
+ "rewards/chosen": -0.701165832249535,
+ "rewards/rejected": -1.7531834567774787,
+ "rewards/accuracies": 0.80125,
+ "rewards/margins": 1.052017622818239,
+ "logps/chosen": -65.97542519807816,
+ "logps/rejected": -85.26719992280006,
+ "epoch": 0.1440201628227952,
+ "step": 180
+ },
+ {
+ "loss": 0.48903474807739256,
+ "grad_norm": 0.022225037217140198,
+ "learning_rate": 8.488e-06,
+ "entropy": 0.7176039846445201,
+ "num_tokens": 12715377.0,
+ "logits/chosen": 1.0541427551802536,
+ "logits/rejected": 1.213088437282036,
+ "mean_token_accuracy": 0.6683101067692041,
+ "rewards/chosen": -0.7613639336443885,
+ "rewards/rejected": -1.7644094955532637,
+ "rewards/accuracies": 0.77875,
+ "rewards/margins": 1.0030455615464597,
+ "logps/chosen": -69.61218286514283,
+ "logps/rejected": -85.76036041498185,
+ "epoch": 0.15202128297961714,
+ "step": 190
+ },
+ {
+ "loss": 0.4629779815673828,
+ "grad_norm": 0.028611615300178528,
+ "learning_rate": 8.408e-06,
+ "entropy": 0.7528939384757541,
+ "num_tokens": 13385367.0,
+ "logits/chosen": 1.0996369249238103,
+ "logits/rejected": 1.2262926881032539,
+ "mean_token_accuracy": 0.6614944562129677,
+ "rewards/chosen": -0.696452237009944,
+ "rewards/rejected": -1.7153711370468954,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.0189188991859555,
+ "logps/chosen": -72.45389879226684,
+ "logps/rejected": -85.92664870500565,
+ "epoch": 0.1600224031364391,
+ "step": 200
+ },
+ {
+ "loss": 0.4567533016204834,
+ "grad_norm": 0.028668882325291634,
+ "learning_rate": 8.328e-06,
+ "entropy": 0.7630001913988963,
+ "num_tokens": 14056267.0,
+ "logits/chosen": 1.097843614731083,
+ "logits/rejected": 1.2666853049820384,
+ "mean_token_accuracy": 0.6626137829013169,
+ "rewards/chosen": -0.6983124747333932,
+ "rewards/rejected": -1.8107990216754843,
+ "rewards/accuracies": 0.79,
+ "rewards/margins": 1.1124865488195792,
+ "logps/chosen": -71.39139920949935,
+ "logps/rejected": -89.45441583156585,
+ "epoch": 0.16802352329326106,
+ "step": 210
+ },
+ {
+ "loss": 0.4515247821807861,
+ "grad_norm": 0.026669377461075783,
+ "learning_rate": 8.248e-06,
+ "entropy": 0.7527169055410196,
+ "num_tokens": 14725614.0,
+ "logits/chosen": 1.0385593041977932,
+ "logits/rejected": 1.190695984160462,
+ "mean_token_accuracy": 0.6638811925798654,
+ "rewards/chosen": -0.6806083437831694,
+ "rewards/rejected": -1.7687839797040215,
+ "rewards/accuracies": 0.79375,
+ "rewards/margins": 1.088175632879138,
+ "logps/chosen": -69.61510648488998,
+ "logps/rejected": -88.65261955261231,
+ "epoch": 0.176024643450083,
+ "step": 220
+ },
+ {
+ "loss": 0.4524831771850586,
+ "grad_norm": 0.03151402622461319,
+ "learning_rate": 8.168e-06,
+ "entropy": 0.73724962518143,
+ "num_tokens": 15392501.0,
+ "logits/chosen": 1.0519082065874066,
+ "logits/rejected": 1.1749508200290943,
+ "mean_token_accuracy": 0.6576838022284209,
+ "rewards/chosen": -0.7482118729477224,
+ "rewards/rejected": -1.906323363717529,
+ "rewards/accuracies": 0.79625,
+ "rewards/margins": 1.1581114920042455,
+ "logps/chosen": -70.1697038435936,
+ "logps/rejected": -85.81395731210709,
+ "epoch": 0.18402576360690498,
+ "step": 230
+ },
+ {
+ "loss": 0.4577242374420166,
+ "grad_norm": 0.02352779172360897,
+ "learning_rate": 8.088e-06,
+ "entropy": 0.7625965754780918,
+ "num_tokens": 16059851.0,
+ "logits/chosen": 1.042450346162851,
+ "logits/rejected": 1.1952004922839772,
+ "mean_token_accuracy": 0.6561587981320918,
+ "rewards/chosen": -0.7152940149759525,
+ "rewards/rejected": -1.8993994699225005,
+ "rewards/accuracies": 0.79,
+ "rewards/margins": 1.184105452466756,
+ "logps/chosen": -72.37042717456818,
+ "logps/rejected": -85.09895798921585,
+ "epoch": 0.19202688376372692,
+ "step": 240
+ },
+ {
+ "loss": 0.4437845230102539,
+ "grad_norm": 0.030115000903606415,
+ "learning_rate": 8.008e-06,
+ "entropy": 0.7543669298232999,
+ "num_tokens": 16728461.0,
+ "logits/chosen": 1.0735585108444008,
+ "logits/rejected": 1.2861681065582564,
+ "mean_token_accuracy": 0.6635138799995184,
+ "rewards/chosen": -0.7473218186138547,
+ "rewards/rejected": -2.009459927038406,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.2621381114050747,
+ "logps/chosen": -68.2476191997528,
+ "logps/rejected": -89.38162505626678,
+ "epoch": 0.20002800392054887,
+ "step": 250
+ },
+ {
+ "eval_loss": 0.4355338215827942,
+ "eval_runtime": 156.8228,
+ "eval_samples_per_second": 6.377,
+ "eval_steps_per_second": 6.377,
+ "eval_entropy": 0.7431049114335329,
+ "eval_num_tokens": 16728461.0,
+ "eval_logits/chosen": 1.0260521991178062,
+ "eval_logits/rejected": 1.2071183657217026,
+ "eval_mean_token_accuracy": 0.6690842649564147,
+ "eval_rewards/chosen": -0.7681690972906071,
+ "eval_rewards/rejected": -2.0975164352003484,
+ "eval_rewards/accuracies": 0.816,
+ "eval_rewards/margins": 1.3293473376147449,
+ "eval_logps/chosen": -68.14377753639221,
+ "eval_logps/rejected": -89.3929857006073,
+ "epoch": 0.20002800392054887,
+ "step": 250
+ },
+ {
+ "loss": 0.4492465019226074,
+ "grad_norm": 0.03398913890123367,
+ "learning_rate": 7.928e-06,
+ "entropy": 0.7092570207372774,
+ "num_tokens": 17395008.0,
+ "logits/chosen": 0.9581539016344277,
+ "logits/rejected": 1.159804104755373,
+ "mean_token_accuracy": 0.6754050040617585,
+ "rewards/chosen": -0.7407069788262015,
+ "rewards/rejected": -2.03097447785025,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.2902674995921553,
+ "logps/chosen": -66.1459436917305,
+ "logps/rejected": -86.54546817541123,
+ "epoch": 0.20802912407737084,
+ "step": 260
+ },
+ {
+ "loss": 0.44979472160339357,
+ "grad_norm": 0.024289635941386223,
+ "learning_rate": 7.848000000000002e-06,
+ "entropy": 0.7417988654365764,
+ "num_tokens": 18063939.0,
+ "logits/chosen": 0.9882898531664387,
+ "logits/rejected": 1.1797352032413542,
+ "mean_token_accuracy": 0.6598335723392665,
+ "rewards/chosen": -0.7044147013814654,
+ "rewards/rejected": -1.9355830125192006,
+ "rewards/accuracies": 0.785,
+ "rewards/margins": 1.2311683113407343,
+ "logps/chosen": -67.67734955310821,
+ "logps/rejected": -89.65328444957733,
+ "epoch": 0.2160302442341928,
+ "step": 270
+ },
+ {
+ "loss": 0.46891136169433595,
+ "grad_norm": 0.027483809739351273,
+ "learning_rate": 7.768e-06,
+ "entropy": 0.7476156710305805,
+ "num_tokens": 18733318.0,
+ "logits/chosen": 0.9810655611779316,
+ "logits/rejected": 1.1440915421776412,
+ "mean_token_accuracy": 0.6648328566737473,
+ "rewards/chosen": -0.7664122937600769,
+ "rewards/rejected": -2.021782549621712,
+ "rewards/accuracies": 0.775,
+ "rewards/margins": 1.255370253128931,
+ "logps/chosen": -70.15842220425606,
+ "logps/rejected": -89.9700861620903,
+ "epoch": 0.22403136439101473,
+ "step": 280
+ },
+ {
+ "loss": 0.44730210304260254,
+ "grad_norm": 0.02446960099041462,
+ "learning_rate": 7.688000000000002e-06,
+ "entropy": 0.7660960466810502,
+ "num_tokens": 19400332.0,
+ "logits/chosen": 1.0521407140469692,
+ "logits/rejected": 1.244036548145974,
+ "mean_token_accuracy": 0.6655525470338762,
+ "rewards/chosen": -0.6997575274415431,
+ "rewards/rejected": -1.9546332472981884,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.2548757187998854,
+ "logps/chosen": -68.3457584643364,
+ "logps/rejected": -89.20595526456833,
+ "epoch": 0.2320324845478367,
+ "step": 290
+ },
+ {
+ "loss": 0.4688117504119873,
+ "grad_norm": 0.025852346792817116,
+ "learning_rate": 7.608000000000001e-06,
+ "entropy": 0.7853818394022528,
+ "num_tokens": 20069869.0,
+ "logits/chosen": 1.0582436553956536,
+ "logits/rejected": 1.249718604996073,
+ "mean_token_accuracy": 0.6564654364995658,
+ "rewards/chosen": -0.7000699595443439,
+ "rewards/rejected": -1.8295867117367743,
+ "rewards/accuracies": 0.805,
+ "rewards/margins": 1.1295167520851828,
+ "logps/chosen": -72.92780254602432,
+ "logps/rejected": -87.27597820997238,
+ "epoch": 0.24003360470465865,
+ "step": 300
+ },
+ {
+ "loss": 0.44611387252807616,
+ "grad_norm": 0.02276112511754036,
+ "learning_rate": 7.528000000000001e-06,
+ "entropy": 0.7570507207023911,
+ "num_tokens": 20738870.0,
+ "logits/chosen": 1.0553903994206508,
+ "logits/rejected": 1.2092241120963385,
+ "mean_token_accuracy": 0.6636787656508386,
+ "rewards/chosen": -0.640383563094947,
+ "rewards/rejected": -1.8241450248898763,
+ "rewards/accuracies": 0.795,
+ "rewards/margins": 1.1837614625599235,
+ "logps/chosen": -68.32228486299515,
+ "logps/rejected": -87.3179730963707,
+ "epoch": 0.24803472486148062,
+ "step": 310
+ },
+ {
+ "loss": 0.45958242416381834,
+ "grad_norm": 0.03167266771197319,
+ "learning_rate": 7.4480000000000005e-06,
+ "entropy": 0.7608478293952067,
+ "num_tokens": 21409595.0,
+ "logits/chosen": 1.0073344990829305,
+ "logits/rejected": 1.1633892473218177,
+ "mean_token_accuracy": 0.6613082770071924,
+ "rewards/chosen": -0.748872835691378,
+ "rewards/rejected": -1.9699224924319423,
+ "rewards/accuracies": 0.78875,
+ "rewards/margins": 1.22104965666309,
+ "logps/chosen": -72.77693994998931,
+ "logps/rejected": -88.67001478910446,
+ "epoch": 0.25603584501830257,
+ "step": 320
+ },
+ {
+ "loss": 0.41903347969055177,
+ "grad_norm": 0.027006885036826134,
+ "learning_rate": 7.3680000000000004e-06,
+ "entropy": 0.7372902630781755,
+ "num_tokens": 22077531.0,
+ "logits/chosen": 0.930071076548155,
+ "logits/rejected": 1.1433182541651794,
+ "mean_token_accuracy": 0.6762370486371219,
+ "rewards/chosen": -0.6912996655895404,
+ "rewards/rejected": -2.0490244489716134,
+ "rewards/accuracies": 0.81625,
+ "rewards/margins": 1.3577247859165071,
+ "logps/chosen": -65.67593628168106,
+ "logps/rejected": -89.9126684165001,
+ "epoch": 0.26403696517512454,
+ "step": 330
+ },
+ {
+ "loss": 0.43892674446105956,
+ "grad_norm": 0.028983909636735916,
+ "learning_rate": 7.288e-06,
+ "entropy": 0.7466868895856896,
+ "num_tokens": 22747051.0,
+ "logits/chosen": 0.8528140235280446,
+ "logits/rejected": 1.0567630613910928,
+ "mean_token_accuracy": 0.6699943310581148,
+ "rewards/chosen": -0.790548151754756,
+ "rewards/rejected": -2.1185401497303245,
+ "rewards/accuracies": 0.78875,
+ "rewards/margins": 1.3279919968137983,
+ "logps/chosen": -70.40046869158745,
+ "logps/rejected": -90.60527857899666,
+ "epoch": 0.27203808533194646,
+ "step": 340
+ },
+ {
+ "loss": 0.48100433349609373,
+ "grad_norm": 0.033078648149967194,
+ "learning_rate": 7.208e-06,
+ "entropy": 0.7587742877868004,
+ "num_tokens": 23416620.0,
+ "logits/chosen": 0.8100388799304219,
+ "logits/rejected": 0.967934954130305,
+ "mean_token_accuracy": 0.6637939850240946,
+ "rewards/chosen": -0.7917629819798457,
+ "rewards/rejected": -2.035186970634677,
+ "rewards/accuracies": 0.77875,
+ "rewards/margins": 1.2434239887745935,
+ "logps/chosen": -71.57891977548599,
+ "logps/rejected": -91.39597170114517,
+ "epoch": 0.28003920548876843,
+ "step": 350
+ },
+ {
+ "loss": 0.41542987823486327,
+ "grad_norm": 0.029060667380690575,
+ "learning_rate": 7.128e-06,
+ "entropy": 0.7580543871223927,
+ "num_tokens": 24085679.0,
+ "logits/chosen": 0.8433804797774842,
+ "logits/rejected": 1.0650731621603458,
+ "mean_token_accuracy": 0.6673501774854959,
+ "rewards/chosen": -0.64634647201583,
+ "rewards/rejected": -2.018033398252446,
+ "rewards/accuracies": 0.81125,
+ "rewards/margins": 1.371686919608619,
+ "logps/chosen": -69.87940875649453,
+ "logps/rejected": -89.1765217113495,
+ "epoch": 0.2880403256455904,
+ "step": 360
+ },
+ {
+ "loss": 0.4648158550262451,
+ "grad_norm": 0.03573554754257202,
+ "learning_rate": 7.048e-06,
+ "entropy": 0.7539360517368187,
+ "num_tokens": 24754967.0,
+ "logits/chosen": 0.9021167366765691,
+ "logits/rejected": 1.0135648914099702,
+ "mean_token_accuracy": 0.6625216764211654,
+ "rewards/chosen": -0.7168381907340882,
+ "rewards/rejected": -1.8988150142121594,
+ "rewards/accuracies": 0.79,
+ "rewards/margins": 1.181976824013982,
+ "logps/chosen": -69.49502610325813,
+ "logps/rejected": -86.62321401238441,
+ "epoch": 0.2960414458024123,
+ "step": 370
+ },
+ {
+ "eval_loss": 0.4164125919342041,
+ "eval_runtime": 192.3097,
+ "eval_samples_per_second": 5.2,
+ "eval_steps_per_second": 5.2,
+ "eval_entropy": 0.7576178104039282,
+ "eval_num_tokens": 25087583.0,
+ "eval_logits/chosen": 0.8733557614100357,
+ "eval_logits/rejected": 1.0646610536239536,
+ "eval_mean_token_accuracy": 0.6653144072294235,
+ "eval_rewards/chosen": -0.6746753788966453,
+ "eval_rewards/rejected": -2.1017218775587097,
+ "eval_rewards/accuracies": 0.824,
+ "eval_rewards/margins": 1.4270464973822237,
+ "eval_logps/chosen": -67.20884035587311,
+ "eval_logps/rejected": -89.43504008293152,
+ "epoch": 0.3000420058808233,
+ "step": 375
+ },
+ {
+ "loss": 0.4322528839111328,
+ "grad_norm": 0.029773183166980743,
+ "learning_rate": 6.968e-06,
+ "entropy": 0.768126052528387,
+ "num_tokens": 25423331.0,
+ "logits/chosen": 0.908505152727753,
+ "logits/rejected": 1.0774095909087906,
+ "mean_token_accuracy": 0.6575940113514662,
+ "rewards/chosen": -0.6992590621344607,
+ "rewards/rejected": -2.1102835578612575,
+ "rewards/accuracies": 0.82,
+ "rewards/margins": 1.4110244943667203,
+ "logps/chosen": -69.7000712609291,
+ "logps/rejected": -89.70367893457413,
+ "epoch": 0.3040425659592343,
+ "step": 380
+ },
+ {
+ "loss": 0.46060609817504883,
+ "grad_norm": 0.03156392648816109,
+ "learning_rate": 6.888e-06,
+ "entropy": 0.7681930413638475,
+ "num_tokens": 26093424.0,
+ "logits/chosen": 0.9001171337749253,
+ "logits/rejected": 1.0952869662687845,
+ "mean_token_accuracy": 0.658901194408536,
+ "rewards/chosen": -0.7717651543853935,
+ "rewards/rejected": -2.1153567287203625,
+ "rewards/accuracies": 0.78125,
+ "rewards/margins": 1.3435915715340525,
+ "logps/chosen": -71.94703212022782,
+ "logps/rejected": -92.13618421554565,
+ "epoch": 0.31204368611605626,
+ "step": 390
+ },
+ {
+ "loss": 0.4654114246368408,
+ "grad_norm": 0.03953603282570839,
+ "learning_rate": 6.808e-06,
+ "entropy": 0.7949102269229479,
+ "num_tokens": 26763102.0,
+ "logits/chosen": 0.8984706428720675,
+ "logits/rejected": 1.075903093245547,
+ "mean_token_accuracy": 0.657767103202641,
+ "rewards/chosen": -0.6850485583826957,
+ "rewards/rejected": -1.8486362679247395,
+ "rewards/accuracies": 0.79375,
+ "rewards/margins": 1.16358771039173,
+ "logps/chosen": -69.50252655863761,
+ "logps/rejected": -88.11836134672166,
+ "epoch": 0.3200448062728782,
+ "step": 400
+ },
+ {
+ "loss": 0.45608930587768554,
+ "grad_norm": 0.032714180648326874,
+ "learning_rate": 6.728e-06,
+ "entropy": 0.7676817936728184,
+ "num_tokens": 27431927.0,
+ "logits/chosen": 0.8882831961356138,
+ "logits/rejected": 1.0850572574945536,
+ "mean_token_accuracy": 0.6676110390387475,
+ "rewards/chosen": -0.6412556826127548,
+ "rewards/rejected": -1.954114315220504,
+ "rewards/accuracies": 0.78875,
+ "rewards/margins": 1.312858630893752,
+ "logps/chosen": -68.36359276890755,
+ "logps/rejected": -87.32514333963394,
+ "epoch": 0.32804592642970015,
+ "step": 410
+ },
+ {
+ "loss": 0.4094356060028076,
+ "grad_norm": 0.02800224907696247,
+ "learning_rate": 6.648e-06,
+ "entropy": 0.7408479347568937,
+ "num_tokens": 28100578.0,
+ "logits/chosen": 0.8408725598306885,
+ "logits/rejected": 1.0481387106436133,
+ "mean_token_accuracy": 0.6668380026239902,
+ "rewards/chosen": -0.6303333836252932,
+ "rewards/rejected": -2.0422495753013936,
+ "rewards/accuracies": 0.81875,
+ "rewards/margins": 1.4119161891844123,
+ "logps/chosen": -68.51784587502479,
+ "logps/rejected": -89.00889017105102,
+ "epoch": 0.3360470465865221,
+ "step": 420
+ },
+ {
+ "loss": 0.47118396759033204,
+ "grad_norm": 0.03881135582923889,
+ "learning_rate": 6.568000000000001e-06,
+ "entropy": 0.7620953947296948,
+ "num_tokens": 28771581.0,
+ "logits/chosen": 0.8177693994702058,
+ "logits/rejected": 0.9835313458846184,
+ "mean_token_accuracy": 0.6655509194545448,
+ "rewards/chosen": -0.7996256252456078,
+ "rewards/rejected": -2.1756565166643123,
+ "rewards/accuracies": 0.76875,
+ "rewards/margins": 1.3760308918333612,
+ "logps/chosen": -70.287514295578,
+ "logps/rejected": -94.71107230901718,
+ "epoch": 0.34404816674334404,
+ "step": 430
+ },
+ {
+ "loss": 0.41876988410949706,
+ "grad_norm": 0.03742805868387222,
+ "learning_rate": 6.488000000000001e-06,
+ "entropy": 0.7735420475469437,
+ "num_tokens": 29439070.0,
+ "logits/chosen": 0.8423775153844848,
+ "logits/rejected": 0.9741460149609448,
+ "mean_token_accuracy": 0.6611115508247167,
+ "rewards/chosen": -0.6199226884998643,
+ "rewards/rejected": -1.95905202480164,
+ "rewards/accuracies": 0.80875,
+ "rewards/margins": 1.3391293336683885,
+ "logps/chosen": -67.55927532792091,
+ "logps/rejected": -87.32301023721695,
+ "epoch": 0.352049286900166,
+ "step": 440
+ },
+ {
+ "loss": 0.44651355743408205,
+ "grad_norm": 0.027141250669956207,
+ "learning_rate": 6.408000000000001e-06,
+ "entropy": 0.7722219267522451,
+ "num_tokens": 30109902.0,
+ "logits/chosen": 0.8634469613527469,
+ "logits/rejected": 1.0130148540335655,
+ "mean_token_accuracy": 0.6588307336438447,
+ "rewards/chosen": -0.7358856135107271,
+ "rewards/rejected": -2.0964524537087255,
+ "rewards/accuracies": 0.79125,
+ "rewards/margins": 1.360566838402301,
+ "logps/chosen": -72.01057388305664,
+ "logps/rejected": -88.65333864212036,
+ "epoch": 0.360050407056988,
+ "step": 450
+ },
+ {
+ "loss": 0.452637243270874,
+ "grad_norm": 0.02899186685681343,
+ "learning_rate": 6.328000000000001e-06,
+ "entropy": 0.777921670184005,
+ "num_tokens": 30777671.0,
+ "logits/chosen": 0.9423126176557491,
+ "logits/rejected": 1.0900213159289571,
+ "mean_token_accuracy": 0.6588201350346208,
+ "rewards/chosen": -0.6857324316137238,
+ "rewards/rejected": -1.9855998314451426,
+ "rewards/accuracies": 0.795,
+ "rewards/margins": 1.2998674020543695,
+ "logps/chosen": -69.4691036939621,
+ "logps/rejected": -86.50770955562592,
+ "epoch": 0.36805152721380996,
+ "step": 460
+ },
+ {
+ "loss": 0.40210437774658203,
+ "grad_norm": 0.026830365881323814,
+ "learning_rate": 6.248000000000001e-06,
+ "entropy": 0.8067923651146702,
+ "num_tokens": 31447731.0,
+ "logits/chosen": 1.0123578055420643,
+ "logits/rejected": 1.2062729007657655,
+ "mean_token_accuracy": 0.66912337558344,
+ "rewards/chosen": -0.5654079637472751,
+ "rewards/rejected": -2.050150958895101,
+ "rewards/accuracies": 0.84625,
+ "rewards/margins": 1.4847429941548034,
+ "logps/chosen": -68.62741502523423,
+ "logps/rejected": -90.02416857004165,
+ "epoch": 0.3760526473706319,
+ "step": 470
+ },
+ {
+ "loss": 0.4377699375152588,
+ "grad_norm": 0.0313052162528038,
+ "learning_rate": 6.168000000000001e-06,
+ "entropy": 0.8242240695969667,
+ "num_tokens": 32116532.0,
+ "logits/chosen": 1.0703010362241487,
+ "logits/rejected": 1.2458288302922487,
+ "mean_token_accuracy": 0.6617773142643273,
+ "rewards/chosen": -0.5944398860554793,
+ "rewards/rejected": -1.9161300252494402,
+ "rewards/accuracies": 0.8125,
+ "rewards/margins": 1.321690135495737,
+ "logps/chosen": -67.57950053572655,
+ "logps/rejected": -89.14266842842102,
+ "epoch": 0.38405376752745385,
+ "step": 480
+ },
+ {
+ "loss": 0.4258408069610596,
+ "grad_norm": 0.030823541805148125,
+ "learning_rate": 6.088000000000001e-06,
+ "entropy": 0.8232742821751162,
+ "num_tokens": 32785258.0,
+ "logits/chosen": 1.058509937014497,
+ "logits/rejected": 1.2632535002863685,
+ "mean_token_accuracy": 0.6611343866959214,
+ "rewards/chosen": -0.6243098496856692,
+ "rewards/rejected": -2.090642974433722,
+ "rewards/accuracies": 0.8075,
+ "rewards/margins": 1.4663331213686615,
+ "logps/chosen": -67.10051204442978,
+ "logps/rejected": -91.9337764787674,
+ "epoch": 0.3920548876842758,
+ "step": 490
+ },
+ {
+ "loss": 0.4398519039154053,
+ "grad_norm": 0.034121498465538025,
+ "learning_rate": 6.008000000000001e-06,
+ "entropy": 0.8138703954452648,
+ "num_tokens": 33451895.0,
+ "logits/chosen": 1.027927453772456,
+ "logits/rejected": 1.235798934678657,
+ "mean_token_accuracy": 0.6669648469425737,
+ "rewards/chosen": -0.6414771419570752,
+ "rewards/rejected": -2.0651343788561642,
+ "rewards/accuracies": 0.8075,
+ "rewards/margins": 1.42365723485942,
+ "logps/chosen": -63.10917327880859,
+ "logps/rejected": -88.34323552131653,
+ "epoch": 0.40005600784109774,
+ "step": 500
+ },
+ {
+ "eval_loss": 0.4041357934474945,
+ "eval_runtime": 216.6853,
+ "eval_samples_per_second": 4.615,
+ "eval_steps_per_second": 4.615,
+ "eval_entropy": 0.8187362959450111,
+ "eval_num_tokens": 33451895.0,
+ "eval_logits/chosen": 1.0302648436616415,
+ "eval_logits/rejected": 1.2174201609282707,
+ "eval_mean_token_accuracy": 0.6664328058063984,
+ "eval_rewards/chosen": -0.6111570251197264,
+ "eval_rewards/rejected": -2.137786239468027,
+ "eval_rewards/accuracies": 0.824,
+ "eval_rewards/margins": 1.526629216035828,
+ "eval_logps/chosen": -66.57365681552886,
+ "eval_logps/rejected": -89.79568366146087,
+ "epoch": 0.40005600784109774,
+ "step": 500
+ },
+ {
+ "loss": 0.439931583404541,
+ "grad_norm": 0.030598435550928116,
+ "learning_rate": 5.928000000000001e-06,
+ "entropy": 0.7968977816659026,
+ "num_tokens": 34120514.0,
+ "logits/chosen": 1.040446529028589,
+ "logits/rejected": 1.2180045614227497,
+ "mean_token_accuracy": 0.6666285018436611,
+ "rewards/chosen": -0.6471186488174134,
+ "rewards/rejected": -2.0945956809860036,
+ "rewards/accuracies": 0.795,
+ "rewards/margins": 1.4474770293140318,
+ "logps/chosen": -68.99139935493469,
+ "logps/rejected": -89.67447282075882,
+ "epoch": 0.4080571279979197,
+ "step": 510
+ },
+ {
+ "loss": 0.41269855499267577,
+ "grad_norm": 0.034259915351867676,
+ "learning_rate": 5.848000000000001e-06,
+ "entropy": 0.8417493133962853,
+ "num_tokens": 34789388.0,
+ "logits/chosen": 1.0914405103048668,
+ "logits/rejected": 1.2486260757830665,
+ "mean_token_accuracy": 0.6626991921663284,
+ "rewards/chosen": -0.5448232944752089,
+ "rewards/rejected": -1.8674056674454187,
+ "rewards/accuracies": 0.80125,
+ "rewards/margins": 1.322582372322213,
+ "logps/chosen": -67.3422362947464,
+ "logps/rejected": -89.12793979406356,
+ "epoch": 0.4160582481547417,
+ "step": 520
+ },
+ {
+ "loss": 0.4300011157989502,
+ "grad_norm": 0.028924375772476196,
+ "learning_rate": 5.7680000000000005e-06,
+ "entropy": 0.800038969970774,
+ "num_tokens": 35457016.0,
+ "logits/chosen": 1.010426369977227,
+ "logits/rejected": 1.2068138369719763,
+ "mean_token_accuracy": 0.657469975342974,
+ "rewards/chosen": -0.6588539993400627,
+ "rewards/rejected": -2.0767385605734305,
+ "rewards/accuracies": 0.81375,
+ "rewards/margins": 1.4178845623019152,
+ "logps/chosen": -67.54834802865982,
+ "logps/rejected": -88.08037763595581,
+ "epoch": 0.4240593683115636,
+ "step": 530
+ },
+ {
+ "loss": 0.42708277702331543,
+ "grad_norm": 0.03574442118406296,
+ "learning_rate": 5.6880000000000004e-06,
+ "entropy": 0.8132251694751903,
+ "num_tokens": 36128364.0,
+ "logits/chosen": 1.0260850046239467,
+ "logits/rejected": 1.2032431243739152,
+ "mean_token_accuracy": 0.6597580896876752,
+ "rewards/chosen": -0.8153463805263164,
+ "rewards/rejected": -2.388017583406763,
+ "rewards/accuracies": 0.825,
+ "rewards/margins": 1.5726712049450726,
+ "logps/chosen": -71.70862350225448,
+ "logps/rejected": -95.02909649133682,
+ "epoch": 0.4320604884683856,
+ "step": 540
+ },
+ {
+ "loss": 0.44931840896606445,
+ "grad_norm": 0.03517612814903259,
+ "learning_rate": 5.608e-06,
+ "entropy": 0.8036265869461932,
+ "num_tokens": 36796877.0,
+ "logits/chosen": 0.968832698781345,
+ "logits/rejected": 1.2278189388004974,
+ "mean_token_accuracy": 0.6618908178806305,
+ "rewards/chosen": -0.7296826012192469,
+ "rewards/rejected": -2.1867977140343284,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.45711510953377,
+ "logps/chosen": -68.4356790471077,
+ "logps/rejected": -91.23825908184051,
+ "epoch": 0.44006160862520755,
+ "step": 550
+ },
+ {
+ "loss": 0.43575077056884765,
+ "grad_norm": 0.03234167769551277,
+ "learning_rate": 5.528e-06,
+ "entropy": 0.8288327282335376,
+ "num_tokens": 37463909.0,
+ "logits/chosen": 1.0259877874812826,
+ "logits/rejected": 1.2316677721721498,
+ "mean_token_accuracy": 0.6426719821058213,
+ "rewards/chosen": -0.6495068702394201,
+ "rewards/rejected": -2.0214309756543662,
+ "rewards/accuracies": 0.79625,
+ "rewards/margins": 1.3719241059338674,
+ "logps/chosen": -67.9371217560768,
+ "logps/rejected": -89.1365689945221,
+ "epoch": 0.44806272878202946,
+ "step": 560
+ },
+ {
+ "loss": 0.4474793910980225,
+ "grad_norm": 0.03248452767729759,
+ "learning_rate": 5.448e-06,
+ "entropy": 0.822490930550266,
+ "num_tokens": 38132585.0,
+ "logits/chosen": 0.9919691673759055,
+ "logits/rejected": 1.1856925128641767,
+ "mean_token_accuracy": 0.6501407605037093,
+ "rewards/chosen": -0.6451310007286974,
+ "rewards/rejected": -2.070329915027396,
+ "rewards/accuracies": 0.785,
+ "rewards/margins": 1.4251989143295214,
+ "logps/chosen": -68.62252221107482,
+ "logps/rejected": -89.71077305793762,
+ "epoch": 0.45606384893885143,
+ "step": 570
+ },
+ {
+ "loss": 0.4298095703125,
+ "grad_norm": 0.034463562071323395,
+ "learning_rate": 5.368000000000001e-06,
+ "entropy": 0.8107173151255119,
+ "num_tokens": 38800385.0,
+ "logits/chosen": 1.0112625349940034,
+ "logits/rejected": 1.2035294935060166,
+ "mean_token_accuracy": 0.6655292904749512,
+ "rewards/chosen": -0.5617576862563146,
+ "rewards/rejected": -2.0137265094090253,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.4519688209239394,
+ "logps/chosen": -65.35963799476623,
+ "logps/rejected": -88.91398869514465,
+ "epoch": 0.4640649690956734,
+ "step": 580
+ },
+ {
+ "loss": 0.42886834144592284,
+ "grad_norm": 0.0305576641112566,
+ "learning_rate": 5.288000000000001e-06,
+ "entropy": 0.850757884104969,
+ "num_tokens": 39469656.0,
+ "logits/chosen": 1.0339854053135065,
+ "logits/rejected": 1.2066679015493316,
+ "mean_token_accuracy": 0.651202796883881,
+ "rewards/chosen": -0.5649584668217358,
+ "rewards/rejected": -1.9496741295084938,
+ "rewards/accuracies": 0.8075,
+ "rewards/margins": 1.384715657499619,
+ "logps/chosen": -68.9078570830822,
+ "logps/rejected": -88.58427609205246,
+ "epoch": 0.4720660892524953,
+ "step": 590
+ },
+ {
+ "loss": 0.43485217094421386,
+ "grad_norm": 0.03541610389947891,
+ "learning_rate": 5.208000000000001e-06,
+ "entropy": 0.8185462304926477,
+ "num_tokens": 40140015.0,
+ "logits/chosen": 1.020759177433344,
+ "logits/rejected": 1.2234918450532424,
+ "mean_token_accuracy": 0.6691169702727348,
+ "rewards/chosen": -0.5964330812523986,
+ "rewards/rejected": -2.039424882331805,
+ "rewards/accuracies": 0.8,
+ "rewards/margins": 1.4429917992651462,
+ "logps/chosen": -68.23823384642601,
+ "logps/rejected": -92.64348523378372,
+ "epoch": 0.4800672094093173,
+ "step": 600
+ },
+ {
+ "loss": 0.45162158012390136,
+ "grad_norm": 0.033510781824588776,
+ "learning_rate": 5.128000000000001e-06,
+ "entropy": 0.8291073049954139,
+ "num_tokens": 40807645.0,
+ "logits/chosen": 1.0010311819142022,
+ "logits/rejected": 1.1718099957017356,
+ "mean_token_accuracy": 0.6581415209453553,
+ "rewards/chosen": -0.6233529344796443,
+ "rewards/rejected": -2.005291601568242,
+ "rewards/accuracies": 0.77375,
+ "rewards/margins": 1.3819386703660712,
+ "logps/chosen": -67.46263459444046,
+ "logps/rejected": -87.6539410018921,
+ "epoch": 0.48806832956613927,
+ "step": 610
+ },
+ {
+ "loss": 0.4001795768737793,
+ "grad_norm": 0.02592748962342739,
+ "learning_rate": 5.048000000000001e-06,
+ "entropy": 0.7915259440580849,
+ "num_tokens": 41478085.0,
+ "logits/chosen": 0.9424498708706046,
+ "logits/rejected": 1.1908051134340107,
+ "mean_token_accuracy": 0.6566479910630733,
+ "rewards/chosen": -0.7034766381183908,
+ "rewards/rejected": -2.2657433008489534,
+ "rewards/accuracies": 0.82875,
+ "rewards/margins": 1.5622666605748237,
+ "logps/chosen": -70.00140867710114,
+ "logps/rejected": -91.03348273515701,
+ "epoch": 0.49606944972296124,
+ "step": 620
+ },
+ {
+ "eval_loss": 0.39784368872642517,
+ "eval_runtime": 246.52,
+ "eval_samples_per_second": 4.056,
+ "eval_steps_per_second": 4.056,
+ "eval_entropy": 0.8210352072389796,
+ "eval_num_tokens": 41814843.0,
+ "eval_logits/chosen": 0.984598041027552,
+ "eval_logits/rejected": 1.1901518799037578,
+ "eval_mean_token_accuracy": 0.6624880295544863,
+ "eval_rewards/chosen": -0.6323302385269198,
+ "eval_rewards/rejected": -2.2265223917410477,
+ "eval_rewards/accuracies": 0.828,
+ "eval_rewards/margins": 1.594192152224481,
+ "eval_logps/chosen": -66.78538895225525,
+ "eval_logps/rejected": -90.68304518508911,
+ "epoch": 0.5000700098013722,
+ "step": 625
+ },
+ {
+ "loss": 0.39586424827575684,
+ "grad_norm": 0.031716663390398026,
+ "learning_rate": 4.9680000000000005e-06,
+ "entropy": 0.8255119596258738,
+ "num_tokens": 42149874.0,
+ "logits/chosen": 0.9743347225997877,
+ "logits/rejected": 1.148137753184027,
+ "mean_token_accuracy": 0.6619132567942142,
+ "rewards/chosen": -0.6674797656579176,
+ "rewards/rejected": -2.231924351695925,
+ "rewards/accuracies": 0.825,
+ "rewards/margins": 1.5644445876637474,
+ "logps/chosen": -72.87295320034028,
+ "logps/rejected": -93.35041454553604,
+ "epoch": 0.5040705698797832,
+ "step": 630
+ },
+ {
+ "loss": 0.4555098533630371,
+ "grad_norm": 0.03256691247224808,
+ "learning_rate": 4.8880000000000005e-06,
+ "entropy": 0.8626169750606641,
+ "num_tokens": 42819532.0,
+ "logits/chosen": 0.9928484339589345,
+ "logits/rejected": 1.1910316784689614,
+ "mean_token_accuracy": 0.6516608715150505,
+ "rewards/chosen": -0.6784914738946827,
+ "rewards/rejected": -2.058931971938291,
+ "rewards/accuracies": 0.79,
+ "rewards/margins": 1.3804404968768358,
+ "logps/chosen": -70.30101744651795,
+ "logps/rejected": -91.8952933549881,
+ "epoch": 0.5120716900366051,
+ "step": 640
+ },
+ {
+ "loss": 0.39907112121582033,
+ "grad_norm": 0.035284217447042465,
+ "learning_rate": 4.808e-06,
+ "entropy": 0.8281762254203204,
+ "num_tokens": 43486780.0,
+ "logits/chosen": 1.0395317318424935,
+ "logits/rejected": 1.2489178483263352,
+ "mean_token_accuracy": 0.6611321708839387,
+ "rewards/chosen": -0.6159643898750073,
+ "rewards/rejected": -2.1350451210376353,
+ "rewards/accuracies": 0.8275,
+ "rewards/margins": 1.5190807291958481,
+ "logps/chosen": -67.00672106027604,
+ "logps/rejected": -89.23889979362488,
+ "epoch": 0.5200728101934271,
+ "step": 650
+ },
+ {
+ "loss": 0.42336082458496094,
+ "grad_norm": 0.03876621276140213,
+ "learning_rate": 4.728e-06,
+ "entropy": 0.8535072756896261,
+ "num_tokens": 44155749.0,
+ "logits/chosen": 1.0138846105047161,
+ "logits/rejected": 1.2437622753411468,
+ "mean_token_accuracy": 0.6601723604463041,
+ "rewards/chosen": -0.7181494349950662,
+ "rewards/rejected": -2.2160908694111274,
+ "rewards/accuracies": 0.81,
+ "rewards/margins": 1.4979414311470465,
+ "logps/chosen": -69.87262807369233,
+ "logps/rejected": -90.26266839981079,
+ "epoch": 0.5280739303502491,
+ "step": 660
+ },
+ {
+ "loss": 0.4203207492828369,
+ "grad_norm": 0.031068701297044754,
+ "learning_rate": 4.648e-06,
+ "entropy": 0.8250792034156621,
+ "num_tokens": 44825691.0,
+ "logits/chosen": 1.0000705969438701,
+ "logits/rejected": 1.1936529049025115,
+ "mean_token_accuracy": 0.6596701008453966,
+ "rewards/chosen": -0.6805069711917895,
+ "rewards/rejected": -2.1961330182391974,
+ "rewards/accuracies": 0.82,
+ "rewards/margins": 1.5156260441290215,
+ "logps/chosen": -66.9881912457943,
+ "logps/rejected": -92.49107657670974,
+ "epoch": 0.5360750505070709,
+ "step": 670
+ },
+ {
+ "loss": 0.4413760662078857,
+ "grad_norm": 0.037145957350730896,
+ "learning_rate": 4.568e-06,
+ "entropy": 0.8502096946584061,
+ "num_tokens": 45496653.0,
+ "logits/chosen": 0.9376389925959969,
+ "logits/rejected": 1.1631915682891765,
+ "mean_token_accuracy": 0.6566580067947507,
+ "rewards/chosen": -0.7351636211126606,
+ "rewards/rejected": -2.2342318054742645,
+ "rewards/accuracies": 0.79375,
+ "rewards/margins": 1.4990681839408353,
+ "logps/chosen": -70.88615607023239,
+ "logps/rejected": -92.9313056743145,
+ "epoch": 0.5440761706638929,
+ "step": 680
+ },
+ {
+ "loss": 0.4418245792388916,
+ "grad_norm": 0.03285796567797661,
+ "learning_rate": 4.488e-06,
+ "entropy": 0.8708134691650048,
+ "num_tokens": 46165428.0,
+ "logits/chosen": 1.0024080394884036,
+ "logits/rejected": 1.2085236900913934,
+ "mean_token_accuracy": 0.6578917545825242,
+ "rewards/chosen": -0.620168975208071,
+ "rewards/rejected": -2.007820005464309,
+ "rewards/accuracies": 0.785,
+ "rewards/margins": 1.3876510330080054,
+ "logps/chosen": -68.30555811882019,
+ "logps/rejected": -89.2831116116047,
+ "epoch": 0.5520772908207149,
+ "step": 690
+ },
+ {
+ "loss": 0.41495356559753416,
+ "grad_norm": 0.0437500961124897,
+ "learning_rate": 4.408000000000001e-06,
+ "entropy": 0.8640956451097737,
+ "num_tokens": 46835068.0,
+ "logits/chosen": 1.0270284503579012,
+ "logits/rejected": 1.1746015504556322,
+ "mean_token_accuracy": 0.6534906476922333,
+ "rewards/chosen": -0.5982054639050329,
+ "rewards/rejected": -2.088865244037952,
+ "rewards/accuracies": 0.8225,
+ "rewards/margins": 1.4906597804185002,
+ "logps/chosen": -68.62525572896004,
+ "logps/rejected": -89.99094027757644,
+ "epoch": 0.5600784109775369,
+ "step": 700
+ },
+ {
+ "loss": 0.4218575954437256,
+ "grad_norm": 0.031077764928340912,
+ "learning_rate": 4.328000000000001e-06,
+ "entropy": 0.8230296180828008,
+ "num_tokens": 47503448.0,
+ "logits/chosen": 0.9770068476271269,
+ "logits/rejected": 1.1491471264948658,
+ "mean_token_accuracy": 0.6508330240473151,
+ "rewards/chosen": -0.6366723557692421,
+ "rewards/rejected": -2.1254406468151137,
+ "rewards/accuracies": 0.81875,
+ "rewards/margins": 1.4887682919669896,
+ "logps/chosen": -70.09009389519692,
+ "logps/rejected": -88.4571030664444,
+ "epoch": 0.5680795311343588,
+ "step": 710
+ },
+ {
+ "loss": 0.4355219841003418,
+ "grad_norm": 0.03514384478330612,
+ "learning_rate": 4.248000000000001e-06,
+ "entropy": 0.8158625036617742,
+ "num_tokens": 48171986.0,
+ "logits/chosen": 0.9882943464247844,
+ "logits/rejected": 1.158046527064534,
+ "mean_token_accuracy": 0.6569546628184617,
+ "rewards/chosen": -0.6468391339858135,
+ "rewards/rejected": -2.091086347126402,
+ "rewards/accuracies": 0.79125,
+ "rewards/margins": 1.4442472116742282,
+ "logps/chosen": -65.43510097146034,
+ "logps/rejected": -92.05735602140426,
+ "epoch": 0.5760806512911808,
+ "step": 720
+ },
+ {
+ "loss": 0.4080561637878418,
+ "grad_norm": 0.029593531042337418,
+ "learning_rate": 4.168000000000001e-06,
+ "entropy": 0.8207998492859769,
+ "num_tokens": 48840851.0,
+ "logits/chosen": 0.9450742256865056,
+ "logits/rejected": 1.1353487721319035,
+ "mean_token_accuracy": 0.6543917619530112,
+ "rewards/chosen": -0.5942745498075965,
+ "rewards/rejected": -2.0826836202369305,
+ "rewards/accuracies": 0.81375,
+ "rewards/margins": 1.488409067413304,
+ "logps/chosen": -66.98957470655441,
+ "logps/rejected": -88.35660415172578,
+ "epoch": 0.5840817714480028,
+ "step": 730
+ },
+ {
+ "loss": 0.42991070747375487,
+ "grad_norm": 0.03314277157187462,
+ "learning_rate": 4.0880000000000005e-06,
+ "entropy": 0.8131404434971046,
+ "num_tokens": 49510684.0,
+ "logits/chosen": 0.9446147207374227,
+ "logits/rejected": 1.10502900609362,
+ "mean_token_accuracy": 0.6605811486765742,
+ "rewards/chosen": -0.6295872584655444,
+ "rewards/rejected": -2.071510077000421,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.441922816168517,
+ "logps/chosen": -67.57781659841538,
+ "logps/rejected": -90.97830413341522,
+ "epoch": 0.5920828916048246,
+ "step": 740
+ },
+ {
+ "loss": 0.4403865337371826,
+ "grad_norm": 0.03364508971571922,
+ "learning_rate": 4.008e-06,
+ "entropy": 0.8496806487767026,
+ "num_tokens": 50179358.0,
+ "logits/chosen": 0.8881657321865005,
+ "logits/rejected": 1.1181200692564324,
+ "mean_token_accuracy": 0.6447949712444097,
+ "rewards/chosen": -0.659528439897631,
+ "rewards/rejected": -2.1715928015656756,
+ "rewards/accuracies": 0.81375,
+ "rewards/margins": 1.5120643608830868,
+ "logps/chosen": -69.27201319813729,
+ "logps/rejected": -90.62985174059868,
+ "epoch": 0.6000840117616466,
+ "step": 750
+ },
+ {
+ "eval_loss": 0.3905946910381317,
+ "eval_runtime": 280.9968,
+ "eval_samples_per_second": 3.559,
+ "eval_steps_per_second": 3.559,
+ "eval_entropy": 0.8403678885605186,
+ "eval_num_tokens": 50179358.0,
+ "eval_logits/chosen": 0.9554020202510393,
+ "eval_logits/rejected": 1.1666407650875066,
+ "eval_mean_token_accuracy": 0.6619850269705058,
+ "eval_rewards/chosen": -0.6056564345289953,
+ "eval_rewards/rejected": -2.2331584873978283,
+ "eval_rewards/accuracies": 0.833,
+ "eval_rewards/margins": 1.627502056725323,
+ "eval_logps/chosen": -66.51865093421937,
+ "eval_logps/rejected": -90.74940616512299,
+ "epoch": 0.6000840117616466,
+ "step": 750
+ },
+ {
+ "loss": 0.4315373420715332,
+ "grad_norm": 0.05254379287362099,
+ "learning_rate": 3.928e-06,
+ "entropy": 0.8172320466092787,
+ "num_tokens": 50848030.0,
+ "logits/chosen": 0.9638210894226233,
+ "logits/rejected": 1.1683298305109677,
+ "mean_token_accuracy": 0.6649538320116699,
+ "rewards/chosen": -0.700610735580849,
+ "rewards/rejected": -2.2304253363308817,
+ "rewards/accuracies": 0.81375,
+ "rewards/margins": 1.529814600930549,
+ "logps/chosen": -65.90503306984901,
+ "logps/rejected": -91.40787957906723,
+ "epoch": 0.6080851319184686,
+ "step": 760
+ },
+ {
+ "loss": 0.4522406578063965,
+ "grad_norm": 0.03430873528122902,
+ "learning_rate": 3.848e-06,
+ "entropy": 0.8342650856019463,
+ "num_tokens": 51517848.0,
+ "logits/chosen": 0.9867342852132462,
+ "logits/rejected": 1.1760393931884263,
+ "mean_token_accuracy": 0.6546278862282634,
+ "rewards/chosen": -0.6872031458665879,
+ "rewards/rejected": -2.152831002970925,
+ "rewards/accuracies": 0.8025,
+ "rewards/margins": 1.4656278589833527,
+ "logps/chosen": -69.34904262185097,
+ "logps/rejected": -90.72765549182891,
+ "epoch": 0.6160862520752906,
+ "step": 770
+ },
+ {
+ "loss": 0.4273702621459961,
+ "grad_norm": 0.037746552377939224,
+ "learning_rate": 3.7680000000000006e-06,
+ "entropy": 0.8564686650084332,
+ "num_tokens": 52187552.0,
+ "logits/chosen": 0.9985187054787574,
+ "logits/rejected": 1.2113277213347589,
+ "mean_token_accuracy": 0.6595367415063084,
+ "rewards/chosen": -0.6713152281285147,
+ "rewards/rejected": -2.165042465722654,
+ "rewards/accuracies": 0.8025,
+ "rewards/margins": 1.4937272349186241,
+ "logps/chosen": -70.59510239124297,
+ "logps/rejected": -91.22085160732269,
+ "epoch": 0.6240873722321125,
+ "step": 780
+ },
+ {
+ "loss": 0.4381092548370361,
+ "grad_norm": 0.038748253136873245,
+ "learning_rate": 3.6880000000000005e-06,
+ "entropy": 0.8539297018293291,
+ "num_tokens": 52853981.0,
+ "logits/chosen": 0.9752500179059175,
+ "logits/rejected": 1.1755729797655,
+ "mean_token_accuracy": 0.6534031975641846,
+ "rewards/chosen": -0.6332173011422856,
+ "rewards/rejected": -2.053761762228387,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.4205444614309817,
+ "logps/chosen": -65.82179151296616,
+ "logps/rejected": -88.53400151371956,
+ "epoch": 0.6320884923889345,
+ "step": 790
+ },
+ {
+ "loss": 0.44451198577880857,
+ "grad_norm": 0.03472725301980972,
+ "learning_rate": 3.6080000000000004e-06,
+ "entropy": 0.8494848445890238,
+ "num_tokens": 53523227.0,
+ "logits/chosen": 0.9042852588216367,
+ "logits/rejected": 1.0908933438794748,
+ "mean_token_accuracy": 0.6598811968043446,
+ "rewards/chosen": -0.6820450577477459,
+ "rewards/rejected": -2.1446255748937255,
+ "rewards/accuracies": 0.81375,
+ "rewards/margins": 1.4625805140752346,
+ "logps/chosen": -68.17725162267685,
+ "logps/rejected": -93.00321952104568,
+ "epoch": 0.6400896125457564,
+ "step": 800
+ },
+ {
+ "loss": 0.42236781120300293,
+ "grad_norm": 0.04242870211601257,
+ "learning_rate": 3.5280000000000004e-06,
+ "entropy": 0.8503009751974605,
+ "num_tokens": 54192491.0,
+ "logits/chosen": 0.8932810113737385,
+ "logits/rejected": 1.0996539926905944,
+ "mean_token_accuracy": 0.6555284386128187,
+ "rewards/chosen": -0.7142347343725851,
+ "rewards/rejected": -2.2124053180462213,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.498170582773164,
+ "logps/chosen": -70.30286806106568,
+ "logps/rejected": -92.41490193367004,
+ "epoch": 0.6480907327025783,
+ "step": 810
+ },
+ {
+ "loss": 0.42624950408935547,
+ "grad_norm": 0.03398019075393677,
+ "learning_rate": 3.4480000000000003e-06,
+ "entropy": 0.8426565149053932,
+ "num_tokens": 54862343.0,
+ "logits/chosen": 0.8833654460109064,
+ "logits/rejected": 1.15377093984537,
+ "mean_token_accuracy": 0.653633040972054,
+ "rewards/chosen": -0.6783972243926837,
+ "rewards/rejected": -2.323430530915866,
+ "rewards/accuracies": 0.82875,
+ "rewards/margins": 1.64503330163192,
+ "logps/chosen": -70.29749235630035,
+ "logps/rejected": -93.17389137983322,
+ "epoch": 0.6560918528594003,
+ "step": 820
+ },
+ {
+ "loss": 0.4458158493041992,
+ "grad_norm": 0.04128390923142433,
+ "learning_rate": 3.368e-06,
+ "entropy": 0.8945820169046056,
+ "num_tokens": 55534696.0,
+ "logits/chosen": 0.9564383136805112,
+ "logits/rejected": 1.1158353019655596,
+ "mean_token_accuracy": 0.6559794113226235,
+ "rewards/chosen": -0.7165967582234862,
+ "rewards/rejected": -2.2420258207095323,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.5254290606768337,
+ "logps/chosen": -73.92837689638138,
+ "logps/rejected": -95.61369948983193,
+ "epoch": 0.6640929730162223,
+ "step": 830
+ },
+ {
+ "loss": 0.4153374195098877,
+ "grad_norm": 0.03422253951430321,
+ "learning_rate": 3.288e-06,
+ "entropy": 0.8906809307972435,
+ "num_tokens": 56205028.0,
+ "logits/chosen": 0.9781870369567324,
+ "logits/rejected": 1.1856968066960274,
+ "mean_token_accuracy": 0.6539303036406636,
+ "rewards/chosen": -0.6803736175834638,
+ "rewards/rejected": -2.167078277458204,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.48670465889154,
+ "logps/chosen": -70.90936533451081,
+ "logps/rejected": -92.71181347131729,
+ "epoch": 0.6720940931730442,
+ "step": 840
+ },
+ {
+ "loss": 0.43978347778320315,
+ "grad_norm": 0.032033246010541916,
+ "learning_rate": 3.208e-06,
+ "entropy": 0.8340805717546027,
+ "num_tokens": 56872251.0,
+ "logits/chosen": 0.9742372705919218,
+ "logits/rejected": 1.184257694649083,
+ "mean_token_accuracy": 0.6592800302989781,
+ "rewards/chosen": -0.6633233757369453,
+ "rewards/rejected": -2.0870634885225443,
+ "rewards/accuracies": 0.8125,
+ "rewards/margins": 1.4237401100434364,
+ "logps/chosen": -65.23115757584571,
+ "logps/rejected": -88.25671273946762,
+ "epoch": 0.6800952133298662,
+ "step": 850
+ },
+ {
+ "loss": 0.42367067337036135,
+ "grad_norm": 0.029233582317829132,
+ "learning_rate": 3.1280000000000004e-06,
+ "entropy": 0.8729569414258003,
+ "num_tokens": 57540687.0,
+ "logits/chosen": 0.9849000766152969,
+ "logits/rejected": 1.1920839344367296,
+ "mean_token_accuracy": 0.6475752539746463,
+ "rewards/chosen": -0.5576673893872794,
+ "rewards/rejected": -1.9550721618323588,
+ "rewards/accuracies": 0.79375,
+ "rewards/margins": 1.3974047714634799,
+ "logps/chosen": -71.45296947479248,
+ "logps/rejected": -86.27678619146347,
+ "epoch": 0.6880963334866881,
+ "step": 860
+ },
+ {
+ "loss": 0.44771614074707033,
+ "grad_norm": 0.03574584797024727,
+ "learning_rate": 3.0480000000000003e-06,
+ "entropy": 0.8541871241829359,
+ "num_tokens": 58209600.0,
+ "logits/chosen": 0.9407660194085455,
+ "logits/rejected": 1.1319002897367676,
+ "mean_token_accuracy": 0.6590811411850155,
+ "rewards/chosen": -0.6319906624252326,
+ "rewards/rejected": -2.160026735856809,
+ "rewards/accuracies": 0.8125,
+ "rewards/margins": 1.5280360724031925,
+ "logps/chosen": -67.98375816583633,
+ "logps/rejected": -90.60064194440842,
+ "epoch": 0.6960974536435101,
+ "step": 870
+ },
+ {
+ "eval_loss": 0.38891661167144775,
+ "eval_runtime": 312.561,
+ "eval_samples_per_second": 3.199,
+ "eval_steps_per_second": 3.199,
+ "eval_entropy": 0.8718550207260997,
+ "eval_num_tokens": 58542997.0,
+ "eval_logits/chosen": 0.953071309156353,
+ "eval_logits/rejected": 1.171563756401123,
+ "eval_mean_token_accuracy": 0.6624961771219969,
+ "eval_rewards/chosen": -0.5500025833119871,
+ "eval_rewards/rejected": -2.1432462024064733,
+ "eval_rewards/accuracies": 0.832,
+ "eval_rewards/margins": 1.593243618823588,
+ "eval_logps/chosen": -65.96211242198945,
+ "eval_logps/rejected": -89.85028337574005,
+ "epoch": 0.700098013721921,
+ "step": 875
+ },
+ {
+ "loss": 0.4452272891998291,
+ "grad_norm": 0.03952350839972496,
+ "learning_rate": 2.9680000000000002e-06,
+ "entropy": 0.862714468719787,
+ "num_tokens": 58876614.0,
+ "logits/chosen": 0.9177802844600431,
+ "logits/rejected": 1.1281801394280309,
+ "mean_token_accuracy": 0.6562245498690754,
+ "rewards/chosen": -0.6339407643607774,
+ "rewards/rejected": -2.029299714830704,
+ "rewards/accuracies": 0.81125,
+ "rewards/margins": 1.3953589481348172,
+ "logps/chosen": -65.40356833696366,
+ "logps/rejected": -90.87254594087601,
+ "epoch": 0.704098573800332,
+ "step": 880
+ },
+ {
+ "loss": 0.45412406921386717,
+ "grad_norm": 0.03881300240755081,
+ "learning_rate": 2.888e-06,
+ "entropy": 0.8527450338829659,
+ "num_tokens": 59544785.0,
+ "logits/chosen": 0.9487640390288297,
+ "logits/rejected": 1.1518068349457047,
+ "mean_token_accuracy": 0.662967371456325,
+ "rewards/chosen": -0.650298423146014,
+ "rewards/rejected": -2.0994253616167406,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.4491269371297677,
+ "logps/chosen": -66.28779201865196,
+ "logps/rejected": -88.19718941688538,
+ "epoch": 0.712099693957154,
+ "step": 890
+ },
+ {
+ "loss": 0.44872069358825684,
+ "grad_norm": 0.03281624987721443,
+ "learning_rate": 2.808e-06,
+ "entropy": 0.8578792290366255,
+ "num_tokens": 60213449.0,
+ "logits/chosen": 1.0487268798393188,
+ "logits/rejected": 1.2662646021232076,
+ "mean_token_accuracy": 0.6574184788390994,
+ "rewards/chosen": -0.5376099577578134,
+ "rewards/rejected": -1.906648186784296,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.3690382286487148,
+ "logps/chosen": -65.61915182113647,
+ "logps/rejected": -87.26275584936143,
+ "epoch": 0.720100814113976,
+ "step": 900
+ },
+ {
+ "loss": 0.4031352519989014,
+ "grad_norm": 0.030685516074299812,
+ "learning_rate": 2.728e-06,
+ "entropy": 0.8612097218161217,
+ "num_tokens": 60880085.0,
+ "logits/chosen": 1.0143389613035938,
+ "logits/rejected": 1.249641372722458,
+ "mean_token_accuracy": 0.6716222583595663,
+ "rewards/chosen": -0.521080312635895,
+ "rewards/rejected": -1.9784742100659058,
+ "rewards/accuracies": 0.82875,
+ "rewards/margins": 1.4573938966169953,
+ "logps/chosen": -64.77055881500245,
+ "logps/rejected": -85.31282383918762,
+ "epoch": 0.728101934270798,
+ "step": 910
+ },
+ {
+ "loss": 0.43309483528137205,
+ "grad_norm": 0.03589771315455437,
+ "learning_rate": 2.648e-06,
+ "entropy": 0.8547451591311256,
+ "num_tokens": 61547040.0,
+ "logits/chosen": 0.9607552139855732,
+ "logits/rejected": 1.1393559314432558,
+ "mean_token_accuracy": 0.6551316804066301,
+ "rewards/chosen": -0.6062610488014252,
+ "rewards/rejected": -1.9912765481562815,
+ "rewards/accuracies": 0.8025,
+ "rewards/margins": 1.3850154994707555,
+ "logps/chosen": -66.66211392879487,
+ "logps/rejected": -86.9184494805336,
+ "epoch": 0.7361030544276199,
+ "step": 920
+ },
+ {
+ "loss": 0.39836270809173585,
+ "grad_norm": 0.03499443456530571,
+ "learning_rate": 2.568e-06,
+ "entropy": 0.837529921917594,
+ "num_tokens": 62216228.0,
+ "logits/chosen": 0.9273512348782313,
+ "logits/rejected": 1.0906426033962848,
+ "mean_token_accuracy": 0.6469739274866879,
+ "rewards/chosen": -0.6463629278072335,
+ "rewards/rejected": -2.22982905632598,
+ "rewards/accuracies": 0.82875,
+ "rewards/margins": 1.583466123645194,
+ "logps/chosen": -67.06362453579902,
+ "logps/rejected": -93.452269384861,
+ "epoch": 0.7441041745844418,
+ "step": 930
+ },
+ {
+ "loss": 0.42793049812316897,
+ "grad_norm": 0.03505934774875641,
+ "learning_rate": 2.488e-06,
+ "entropy": 0.8360733698453987,
+ "num_tokens": 62884204.0,
+ "logits/chosen": 0.7991276481550503,
+ "logits/rejected": 1.049968788034255,
+ "mean_token_accuracy": 0.6601243341527879,
+ "rewards/chosen": -0.6497158235612005,
+ "rewards/rejected": -2.158851850910287,
+ "rewards/accuracies": 0.80875,
+ "rewards/margins": 1.5091360262827949,
+ "logps/chosen": -65.85698626995087,
+ "logps/rejected": -88.71256803274154,
+ "epoch": 0.7521052947412638,
+ "step": 940
+ },
+ {
+ "loss": 0.45066075325012206,
+ "grad_norm": 0.032886356115341187,
+ "learning_rate": 2.408e-06,
+ "entropy": 0.832636615919182,
+ "num_tokens": 63552133.0,
+ "logits/chosen": 0.8599798307690629,
+ "logits/rejected": 1.0741248958420386,
+ "mean_token_accuracy": 0.649495961368084,
+ "rewards/chosen": -0.7527440831899003,
+ "rewards/rejected": -2.141224974161814,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.3884808889543638,
+ "logps/chosen": -68.56244652748109,
+ "logps/rejected": -88.53944436073303,
+ "epoch": 0.7601064148980857,
+ "step": 950
+ },
+ {
+ "loss": 0.4001929759979248,
+ "grad_norm": 0.0434030182659626,
+ "learning_rate": 2.3280000000000004e-06,
+ "entropy": 0.8262607191543794,
+ "num_tokens": 64222107.0,
+ "logits/chosen": 0.883406699215304,
+ "logits/rejected": 1.1504158259150021,
+ "mean_token_accuracy": 0.66339923880063,
+ "rewards/chosen": -0.6620039639761944,
+ "rewards/rejected": -2.2925733448197025,
+ "rewards/accuracies": 0.83875,
+ "rewards/margins": 1.6305693804193289,
+ "logps/chosen": -68.21887855052948,
+ "logps/rejected": -91.67394641399383,
+ "epoch": 0.7681075350549077,
+ "step": 960
+ },
+ {
+ "loss": 0.43085393905639646,
+ "grad_norm": 0.03178192302584648,
+ "learning_rate": 2.2480000000000003e-06,
+ "entropy": 0.8709142287832219,
+ "num_tokens": 64890263.0,
+ "logits/chosen": 0.8920313960898265,
+ "logits/rejected": 1.128552965912883,
+ "mean_token_accuracy": 0.6419067945331335,
+ "rewards/chosen": -0.6362933055355825,
+ "rewards/rejected": -2.09013077869633,
+ "rewards/accuracies": 0.82125,
+ "rewards/margins": 1.4538374735228716,
+ "logps/chosen": -68.71719403505325,
+ "logps/rejected": -88.28514833807945,
+ "epoch": 0.7761086552117297,
+ "step": 970
+ },
+ {
+ "loss": 0.4230814456939697,
+ "grad_norm": 0.03144161403179169,
+ "learning_rate": 2.1680000000000002e-06,
+ "entropy": 0.86573227096349,
+ "num_tokens": 65560586.0,
+ "logits/chosen": 0.9323236133466635,
+ "logits/rejected": 1.1045293716122209,
+ "mean_token_accuracy": 0.6568231407366693,
+ "rewards/chosen": -0.5575948944242555,
+ "rewards/rejected": -2.0226709698906054,
+ "rewards/accuracies": 0.81,
+ "rewards/margins": 1.4650760741624982,
+ "logps/chosen": -70.52183421492576,
+ "logps/rejected": -89.79245682954789,
+ "epoch": 0.7841097753685516,
+ "step": 980
+ },
+ {
+ "loss": 0.4176182746887207,
+ "grad_norm": 0.034222014248371124,
+ "learning_rate": 2.088e-06,
+ "entropy": 0.8574011596717174,
+ "num_tokens": 66228556.0,
+ "logits/chosen": 0.8027309387040763,
+ "logits/rejected": 1.0642325286382217,
+ "mean_token_accuracy": 0.661268965434283,
+ "rewards/chosen": -0.6723649157090767,
+ "rewards/rejected": -2.184656735348035,
+ "rewards/accuracies": 0.825,
+ "rewards/margins": 1.5122918216045946,
+ "logps/chosen": -67.47515767097474,
+ "logps/rejected": -89.71079513788223,
+ "epoch": 0.7921108955253735,
+ "step": 990
+ },
+ {
+ "loss": 0.42989821434020997,
+ "grad_norm": 0.035680945962667465,
+ "learning_rate": 2.008e-06,
+ "entropy": 0.8599720290006371,
+ "num_tokens": 66897207.0,
+ "logits/chosen": 0.8691081965276974,
+ "logits/rejected": 1.1459109746339848,
+ "mean_token_accuracy": 0.659917052462697,
+ "rewards/chosen": -0.6614104065368883,
+ "rewards/rejected": -2.2487826385660448,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 1.5873722325917334,
+ "logps/chosen": -67.66358781814576,
+ "logps/rejected": -90.84724606990814,
+ "epoch": 0.8001120156821955,
+ "step": 1000
+ },
+ {
+ "eval_loss": 0.38607117533683777,
+ "eval_runtime": 350.7424,
+ "eval_samples_per_second": 2.851,
+ "eval_steps_per_second": 2.851,
+ "eval_entropy": 0.8668703991975635,
+ "eval_num_tokens": 66897207.0,
+ "eval_logits/chosen": 0.8892899907383796,
+ "eval_logits/rejected": 1.1111882686378622,
+ "eval_mean_token_accuracy": 0.659346023902297,
+ "eval_rewards/chosen": -0.582486518596299,
+ "eval_rewards/rejected": -2.2139163566811475,
+ "eval_rewards/accuracies": 0.834,
+ "eval_rewards/margins": 1.631429838422686,
+ "eval_logps/chosen": -66.28695176410675,
+ "eval_logps/rejected": -90.55698481941224,
+ "epoch": 0.8001120156821955,
+ "step": 1000
+ },
+ {
+ "loss": 0.3962708950042725,
+ "grad_norm": 0.030963728204369545,
+ "learning_rate": 1.928e-06,
+ "entropy": 0.8694826575729531,
+ "num_tokens": 67567865.0,
+ "logits/chosen": 0.8826398693945836,
+ "logits/rejected": 1.0751302162055196,
+ "mean_token_accuracy": 0.657812373638153,
+ "rewards/chosen": -0.6352861579412274,
+ "rewards/rejected": -2.261975454816602,
+ "rewards/accuracies": 0.81875,
+ "rewards/margins": 1.6266892972029745,
+ "logps/chosen": -69.32229480504989,
+ "logps/rejected": -95.0387474322319,
+ "epoch": 0.8081131358390174,
+ "step": 1010
+ },
+ {
+ "loss": 0.4258723735809326,
+ "grad_norm": 0.03474907577037811,
+ "learning_rate": 1.8480000000000001e-06,
+ "entropy": 0.8473924996308051,
+ "num_tokens": 68235779.0,
+ "logits/chosen": 0.9070787519392894,
+ "logits/rejected": 1.101770660235957,
+ "mean_token_accuracy": 0.6468252922594547,
+ "rewards/chosen": -0.6778515197880915,
+ "rewards/rejected": -2.2432175894498503,
+ "rewards/accuracies": 0.8025,
+ "rewards/margins": 1.5653660669270903,
+ "logps/chosen": -67.88414312124252,
+ "logps/rejected": -88.61838740348816,
+ "epoch": 0.8161142559958394,
+ "step": 1020
+ },
+ {
+ "loss": 0.40514450073242186,
+ "grad_norm": 0.03416679427027702,
+ "learning_rate": 1.7680000000000003e-06,
+ "entropy": 0.889893756096717,
+ "num_tokens": 68903717.0,
+ "logits/chosen": 0.9059802730828178,
+ "logits/rejected": 1.1393912425461479,
+ "mean_token_accuracy": 0.6406477543432265,
+ "rewards/chosen": -0.609267081760263,
+ "rewards/rejected": -2.1504513141329515,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.54118423323147,
+ "logps/chosen": -69.49155421376229,
+ "logps/rejected": -89.1899503827095,
+ "epoch": 0.8241153761526614,
+ "step": 1030
+ },
+ {
+ "loss": 0.43328490257263186,
+ "grad_norm": 0.053973764181137085,
+ "learning_rate": 1.6880000000000002e-06,
+ "entropy": 0.8649416108580772,
+ "num_tokens": 69573418.0,
+ "logits/chosen": 0.8593249510935347,
+ "logits/rejected": 1.110617254203968,
+ "mean_token_accuracy": 0.6512250990979374,
+ "rewards/chosen": -0.6340019615784694,
+ "rewards/rejected": -2.21387220822362,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.579870247275103,
+ "logps/chosen": -69.59451707363128,
+ "logps/rejected": -91.83544722795486,
+ "epoch": 0.8321164963094834,
+ "step": 1040
+ },
+ {
+ "loss": 0.441865062713623,
+ "grad_norm": 0.033326976001262665,
+ "learning_rate": 1.608e-06,
+ "entropy": 0.8924074355195626,
+ "num_tokens": 70245048.0,
+ "logits/chosen": 0.9512462176944945,
+ "logits/rejected": 1.1714606481888763,
+ "mean_token_accuracy": 0.6525666503142565,
+ "rewards/chosen": -0.6446977265506575,
+ "rewards/rejected": -2.1632383557595314,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.5185406296234578,
+ "logps/chosen": -70.56152170658112,
+ "logps/rejected": -94.69566265106201,
+ "epoch": 0.8401176164663052,
+ "step": 1050
+ },
+ {
+ "loss": 0.418486499786377,
+ "grad_norm": 0.029840189963579178,
+ "learning_rate": 1.528e-06,
+ "entropy": 0.8346750047826208,
+ "num_tokens": 70912031.0,
+ "logits/chosen": 0.899767311968589,
+ "logits/rejected": 1.1073568102098392,
+ "mean_token_accuracy": 0.6593861475214362,
+ "rewards/chosen": -0.55429117706517,
+ "rewards/rejected": -2.0976768376422115,
+ "rewards/accuracies": 0.8175,
+ "rewards/margins": 1.5433856593351811,
+ "logps/chosen": -65.34484733700752,
+ "logps/rejected": -85.84228497505188,
+ "epoch": 0.8481187366231272,
+ "step": 1060
+ },
+ {
+ "loss": 0.4446680545806885,
+ "grad_norm": 0.036603037267923355,
+ "learning_rate": 1.4480000000000002e-06,
+ "entropy": 0.902835673665395,
+ "num_tokens": 71584146.0,
+ "logits/chosen": 0.9127683433896177,
+ "logits/rejected": 1.139923308614523,
+ "mean_token_accuracy": 0.6539772312715649,
+ "rewards/chosen": -0.6741834658858715,
+ "rewards/rejected": -2.1645529880949472,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.490369523759,
+ "logps/chosen": -71.67605731964112,
+ "logps/rejected": -94.79599132776261,
+ "epoch": 0.8561198567799492,
+ "step": 1070
+ },
+ {
+ "loss": 0.41832242012023924,
+ "grad_norm": 0.03776715323328972,
+ "learning_rate": 1.368e-06,
+ "entropy": 0.8873281607031822,
+ "num_tokens": 72253852.0,
+ "logits/chosen": 0.9067677578257508,
+ "logits/rejected": 1.161128675409658,
+ "mean_token_accuracy": 0.6563850439153611,
+ "rewards/chosen": -0.6268765124707716,
+ "rewards/rejected": -2.101546765351668,
+ "rewards/accuracies": 0.81375,
+ "rewards/margins": 1.4746702503063716,
+ "logps/chosen": -70.14818896055222,
+ "logps/rejected": -89.79905828237534,
+ "epoch": 0.8641209769367711,
+ "step": 1080
+ },
+ {
+ "loss": 0.41946825981140134,
+ "grad_norm": 0.03260818123817444,
+ "learning_rate": 1.288e-06,
+ "entropy": 0.8910186088766204,
+ "num_tokens": 72923463.0,
+ "logits/chosen": 0.9292467513931639,
+ "logits/rejected": 1.1392854566021555,
+ "mean_token_accuracy": 0.6597230594418942,
+ "rewards/chosen": -0.6515794447601365,
+ "rewards/rejected": -2.1212883089255774,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 1.4697088625654577,
+ "logps/chosen": -68.16174621343613,
+ "logps/rejected": -91.54474475860596,
+ "epoch": 0.8721220970935931,
+ "step": 1090
+ },
+ {
+ "loss": 0.39578471183776853,
+ "grad_norm": 0.0302139800041914,
+ "learning_rate": 1.2080000000000001e-06,
+ "entropy": 0.8779772936925292,
+ "num_tokens": 73595049.0,
+ "logits/chosen": 0.9384733690310845,
+ "logits/rejected": 1.1711842656368645,
+ "mean_token_accuracy": 0.6657957591675222,
+ "rewards/chosen": -0.6084141440523672,
+ "rewards/rejected": -2.2773913521115903,
+ "rewards/accuracies": 0.8025,
+ "rewards/margins": 1.6689772073482163,
+ "logps/chosen": -68.42859583139419,
+ "logps/rejected": -93.98125689387321,
+ "epoch": 0.8801232172504151,
+ "step": 1100
+ },
+ {
+ "loss": 0.42472043037414553,
+ "grad_norm": 0.03813236951828003,
+ "learning_rate": 1.128e-06,
+ "entropy": 0.8627737898926716,
+ "num_tokens": 74262367.0,
+ "logits/chosen": 0.8667971259571394,
+ "logits/rejected": 1.1435590230927837,
+ "mean_token_accuracy": 0.6563602097705007,
+ "rewards/chosen": -0.6558617835270707,
+ "rewards/rejected": -2.2128921717873893,
+ "rewards/accuracies": 0.80875,
+ "rewards/margins": 1.5570303920283914,
+ "logps/chosen": -68.37327192306519,
+ "logps/rejected": -88.34695291280747,
+ "epoch": 0.8881243374072371,
+ "step": 1110
+ },
+ {
+ "loss": 0.4374277114868164,
+ "grad_norm": 0.050476472824811935,
+ "learning_rate": 1.0480000000000002e-06,
+ "entropy": 0.8711421622475609,
+ "num_tokens": 74930382.0,
+ "logits/chosen": 0.9386477201807173,
+ "logits/rejected": 1.1516541607226247,
+ "mean_token_accuracy": 0.6537962615676224,
+ "rewards/chosen": -0.6622998499354799,
+ "rewards/rejected": -2.1998322988441212,
+ "rewards/accuracies": 0.8025,
+ "rewards/margins": 1.5375324480747805,
+ "logps/chosen": -66.09728898763656,
+ "logps/rejected": -91.47396802663803,
+ "epoch": 0.8961254575640589,
+ "step": 1120
+ },
+ {
+ "eval_loss": 0.38328441977500916,
+ "eval_runtime": 383.2795,
+ "eval_samples_per_second": 2.609,
+ "eval_steps_per_second": 2.609,
+ "eval_entropy": 0.8739720604084432,
+ "eval_num_tokens": 75266854.0,
+ "eval_logits/chosen": 0.8878241154287022,
+ "eval_logits/rejected": 1.1128443841255737,
+ "eval_mean_token_accuracy": 0.661347587838769,
+ "eval_rewards/chosen": -0.6041518464077963,
+ "eval_rewards/rejected": -2.292379699852987,
+ "eval_rewards/accuracies": 0.83,
+ "eval_rewards/margins": 1.6882278505861759,
+ "eval_logps/chosen": -66.5036050491333,
+ "eval_logps/rejected": -91.34161827754974,
+ "epoch": 0.9001260176424699,
+ "step": 1125
+ },
+ {
+ "loss": 0.40437002182006837,
+ "grad_norm": 0.028972849249839783,
+ "learning_rate": 9.68e-07,
+ "entropy": 0.8655532780638896,
+ "num_tokens": 75600528.0,
+ "logits/chosen": 0.8392021783182244,
+ "logits/rejected": 1.0820405521401597,
+ "mean_token_accuracy": 0.6659414252452552,
+ "rewards/chosen": -0.6915992882475621,
+ "rewards/rejected": -2.3597558089741506,
+ "rewards/accuracies": 0.82875,
+ "rewards/margins": 1.6681565197976307,
+ "logps/chosen": -68.61087916851044,
+ "logps/rejected": -93.29826194286346,
+ "epoch": 0.9041265777208809,
+ "step": 1130
+ },
+ {
+ "loss": 0.4108288288116455,
+ "grad_norm": 0.03383871167898178,
+ "learning_rate": 8.880000000000001e-07,
+ "entropy": 0.8478269183557131,
+ "num_tokens": 76268751.0,
+ "logits/chosen": 0.8844847648747343,
+ "logits/rejected": 1.09897104210967,
+ "mean_token_accuracy": 0.6555397282214835,
+ "rewards/chosen": -0.7120777094319055,
+ "rewards/rejected": -2.334874999501044,
+ "rewards/accuracies": 0.83125,
+ "rewards/margins": 1.6227972903219052,
+ "logps/chosen": -68.74041581749916,
+ "logps/rejected": -90.99661464691162,
+ "epoch": 0.9121276978777029,
+ "step": 1140
+ },
+ {
+ "loss": 0.3952176570892334,
+ "grad_norm": 0.03595713898539543,
+ "learning_rate": 8.08e-07,
+ "entropy": 0.8817502701492049,
+ "num_tokens": 76938286.0,
+ "logits/chosen": 0.9065315112356953,
+ "logits/rejected": 1.1068333863860822,
+ "mean_token_accuracy": 0.6470588660426437,
+ "rewards/chosen": -0.6965171393561014,
+ "rewards/rejected": -2.3452763559907908,
+ "rewards/accuracies": 0.8325,
+ "rewards/margins": 1.6487592151854187,
+ "logps/chosen": -68.56175809025764,
+ "logps/rejected": -93.97643731117249,
+ "epoch": 0.9201288180345248,
+ "step": 1150
+ },
+ {
+ "loss": 0.3942981958389282,
+ "grad_norm": 0.037921398878097534,
+ "learning_rate": 7.280000000000001e-07,
+ "entropy": 0.8427206738991663,
+ "num_tokens": 77605061.0,
+ "logits/chosen": 0.8722859004280558,
+ "logits/rejected": 1.1195572515577774,
+ "mean_token_accuracy": 0.6546126752346754,
+ "rewards/chosen": -0.6228112047113246,
+ "rewards/rejected": -2.2704866537463384,
+ "rewards/accuracies": 0.82625,
+ "rewards/margins": 1.6476754496945067,
+ "logps/chosen": -67.8807622051239,
+ "logps/rejected": -88.71758820533752,
+ "epoch": 0.9281299381913468,
+ "step": 1160
+ },
+ {
+ "loss": 0.39456446170806886,
+ "grad_norm": 0.035953767597675323,
+ "learning_rate": 6.48e-07,
+ "entropy": 0.8721329982078169,
+ "num_tokens": 78273137.0,
+ "logits/chosen": 0.9035017024008792,
+ "logits/rejected": 1.1552559042775905,
+ "mean_token_accuracy": 0.6506677641160786,
+ "rewards/chosen": -0.6781392271062942,
+ "rewards/rejected": -2.400411543701921,
+ "rewards/accuracies": 0.82875,
+ "rewards/margins": 1.7222723116065026,
+ "logps/chosen": -68.29391048669815,
+ "logps/rejected": -92.70362775802613,
+ "epoch": 0.9361310583481688,
+ "step": 1170
+ },
+ {
+ "loss": 0.4436354160308838,
+ "grad_norm": 0.034623172134160995,
+ "learning_rate": 5.680000000000001e-07,
+ "entropy": 0.8812280716700479,
+ "num_tokens": 78941920.0,
+ "logits/chosen": 0.876691468200509,
+ "logits/rejected": 1.0672803801925645,
+ "mean_token_accuracy": 0.6573259249515832,
+ "rewards/chosen": -0.6557168781671863,
+ "rewards/rejected": -2.2418328590322925,
+ "rewards/accuracies": 0.805,
+ "rewards/margins": 1.5861159804742784,
+ "logps/chosen": -66.65720802783966,
+ "logps/rejected": -93.701522564888,
+ "epoch": 0.9441321785049906,
+ "step": 1180
+ },
+ {
+ "loss": 0.4059769630432129,
+ "grad_norm": 0.03936178982257843,
+ "learning_rate": 4.88e-07,
+ "entropy": 0.8670852344139712,
+ "num_tokens": 79609692.0,
+ "logits/chosen": 0.8956595470743131,
+ "logits/rejected": 1.0826247228452346,
+ "mean_token_accuracy": 0.660180214960128,
+ "rewards/chosen": -0.6400385223676858,
+ "rewards/rejected": -2.2094940228544875,
+ "rewards/accuracies": 0.82,
+ "rewards/margins": 1.5694554990739562,
+ "logps/chosen": -66.48064380645752,
+ "logps/rejected": -88.73535197257996,
+ "epoch": 0.9521332986618126,
+ "step": 1190
+ },
+ {
+ "loss": 0.40941200256347654,
+ "grad_norm": 0.032248757779598236,
+ "learning_rate": 4.0800000000000005e-07,
+ "entropy": 0.8911747275094968,
+ "num_tokens": 80278040.0,
+ "logits/chosen": 0.891040509998187,
+ "logits/rejected": 1.1391679702752255,
+ "mean_token_accuracy": 0.6495232714340091,
+ "rewards/chosen": -0.645332999396851,
+ "rewards/rejected": -2.2509153579105625,
+ "rewards/accuracies": 0.82125,
+ "rewards/margins": 1.6055823574075476,
+ "logps/chosen": -70.1561289012432,
+ "logps/rejected": -90.579102216959,
+ "epoch": 0.9601344188186346,
+ "step": 1200
+ },
+ {
+ "loss": 0.4071323871612549,
+ "grad_norm": 0.03186144679784775,
+ "learning_rate": 3.280000000000001e-07,
+ "entropy": 0.8457071017683484,
+ "num_tokens": 80944938.0,
+ "logits/chosen": 0.8311079625074299,
+ "logits/rejected": 1.0921373440872542,
+ "mean_token_accuracy": 0.6612742093764246,
+ "rewards/chosen": -0.6387467755692342,
+ "rewards/rejected": -2.2768997981393477,
+ "rewards/accuracies": 0.82625,
+ "rewards/margins": 1.638153019300662,
+ "logps/chosen": -65.99934913992882,
+ "logps/rejected": -87.78630265951156,
+ "epoch": 0.9681355389754566,
+ "step": 1210
+ },
+ {
+ "loss": 0.4300717353820801,
+ "grad_norm": 0.036194413900375366,
+ "learning_rate": 2.48e-07,
+ "entropy": 0.8721460625680629,
+ "num_tokens": 81614945.0,
+ "logits/chosen": 0.8625591955289138,
+ "logits/rejected": 1.0950880850799856,
+ "mean_token_accuracy": 0.6566772576607763,
+ "rewards/chosen": -0.7123425293308902,
+ "rewards/rejected": -2.2779226847983955,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.5655801529623568,
+ "logps/chosen": -70.76329089403153,
+ "logps/rejected": -91.68823468208313,
+ "epoch": 0.9761366591322785,
+ "step": 1220
+ },
+ {
+ "loss": 0.41231770515441896,
+ "grad_norm": 0.041534606367349625,
+ "learning_rate": 1.68e-07,
+ "entropy": 0.8715950054430869,
+ "num_tokens": 82283576.0,
+ "logits/chosen": 0.8665724900597457,
+ "logits/rejected": 1.1105663027294717,
+ "mean_token_accuracy": 0.6669188870489597,
+ "rewards/chosen": -0.6339492637870717,
+ "rewards/rejected": -2.262035603372933,
+ "rewards/accuracies": 0.82125,
+ "rewards/margins": 1.628086341698654,
+ "logps/chosen": -66.2409237909317,
+ "logps/rejected": -92.54593481302261,
+ "epoch": 0.9841377792891005,
+ "step": 1230
+ },
+ {
+ "loss": 0.45244503021240234,
+ "grad_norm": 0.040014103055000305,
+ "learning_rate": 8.800000000000001e-08,
+ "entropy": 0.8759910970041528,
+ "num_tokens": 82951810.0,
+ "logits/chosen": 0.9190710372039126,
+ "logits/rejected": 1.1600554501576563,
+ "mean_token_accuracy": 0.6553594494890421,
+ "rewards/chosen": -0.6744026106092497,
+ "rewards/rejected": -2.2188100884950837,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.5444074777932837,
+ "logps/chosen": -68.02484840035439,
+ "logps/rejected": -91.57242461919785,
+ "epoch": 0.9921388994459225,
+ "step": 1240
+ },
+ {
+ "loss": 0.4069981098175049,
+ "grad_norm": 0.03368917852640152,
+ "learning_rate": 8e-09,
+ "entropy": 0.8927215488708126,
+ "num_tokens": 83608233.0,
+ "logits/chosen": 0.8752754279033991,
+ "logits/rejected": 1.0711386316240197,
+ "mean_token_accuracy": 0.6459781708759813,
+ "rewards/chosen": -0.5922412400234031,
+ "rewards/rejected": -2.1926220250943955,
+ "rewards/accuracies": 0.821882951653944,
+ "rewards/margins": 1.600380785545244,
+ "logps/chosen": -67.47506940516503,
+ "logps/rejected": -92.58554644499723,
+ "epoch": 1.0,
+ "step": 1250
+ },
+ {
+ "eval_loss": 0.3831380307674408,
+ "eval_runtime": 409.2139,
+ "eval_samples_per_second": 2.444,
+ "eval_steps_per_second": 2.444,
+ "eval_entropy": 0.8765238604480401,
+ "eval_num_tokens": 83608233.0,
+ "eval_logits/chosen": 0.8781259246101419,
+ "eval_logits/rejected": 1.107441801738501,
+ "eval_mean_token_accuracy": 0.6608878705352544,
+ "eval_rewards/chosen": -0.6033024535952136,
+ "eval_rewards/rejected": -2.292753183161607,
+ "eval_rewards/accuracies": 0.83,
+ "eval_rewards/margins": 1.689450726993382,
+ "eval_logps/chosen": -66.49511113071442,
+ "eval_logps/rejected": -91.34535308170318,
+ "epoch": 1.0,
+ "step": 1250
+ },
+ {
+ "train_runtime": 37881.4288,
+ "train_samples_per_second": 2.639,
+ "train_steps_per_second": 0.033,
+ "total_flos": 5.0312011545074074e+17,
+ "train_loss": 0.44697874088287354,
+ "epoch": 1.0,
+ "step": 1250
+ }
+ ],
+ "validation_monitor_size": 1000,
+ "validation_monitor_selection": "fixed_seed_random_without_replacement",
+ "validation_monitor_seed": 22,
+ "validation_monitor_indices": [
+ 10,
+ 21,
+ 55,
+ 66,
+ 110,
+ 144,
+ 149,
+ 152,
+ 163,
+ 166,
+ 167,
+ 176,
+ 206,
+ 207,
+ 267,
+ 281,
+ 295,
+ 298,
+ 307,
+ 317,
+ 336,
+ 341,
+ 349,
+ 352,
+ 357,
+ 382,
+ 386,
+ 391,
+ 401,
+ 411,
+ 412,
+ 432,
+ 452,
+ 465,
+ 488,
+ 490,
+ 491,
+ 492,
+ 494,
+ 504,
+ 528,
+ 540,
+ 553,
+ 586,
+ 605,
+ 625,
+ 627,
+ 661,
+ 663,
+ 666,
+ 677,
+ 685,
+ 690,
+ 692,
+ 704,
+ 708,
+ 714,
+ 727,
+ 728,
+ 733,
+ 745,
+ 752,
+ 753,
+ 755,
+ 779,
+ 783,
+ 793,
+ 796,
+ 799,
+ 803,
+ 804,
+ 805,
+ 813,
+ 816,
+ 827,
+ 829,
+ 830,
+ 842,
+ 850,
+ 853,
+ 856,
+ 889,
+ 905,
+ 915,
+ 930,
+ 939,
+ 951,
+ 1002,
+ 1005,
+ 1029,
+ 1049,
+ 1050,
+ 1054,
+ 1067,
+ 1068,
+ 1079,
+ 1091,
+ 1114,
+ 1133,
+ 1140,
+ 1144,
+ 1145,
+ 1186,
+ 1195,
+ 1206,
+ 1223,
+ 1252,
+ 1257,
+ 1259,
+ 1271,
+ 1304,
+ 1305,
+ 1335,
+ 1336,
+ 1343,
+ 1367,
+ 1377,
+ 1403,
+ 1412,
+ 1429,
+ 1443,
+ 1457,
+ 1459,
+ 1460,
+ 1476,
+ 1483,
+ 1486,
+ 1507,
+ 1510,
+ 1519,
+ 1521,
+ 1545,
+ 1570,
+ 1582,
+ 1593,
+ 1595,
+ 1603,
+ 1607,
+ 1614,
+ 1615,
+ 1634,
+ 1639,
+ 1654,
+ 1657,
+ 1673,
+ 1690,
+ 1704,
+ 1746,
+ 1756,
+ 1781,
+ 1783,
+ 1796,
+ 1799,
+ 1809,
+ 1814,
+ 1827,
+ 1829,
+ 1832,
+ 1844,
+ 1847,
+ 1854,
+ 1856,
+ 1858,
+ 1883,
+ 1889,
+ 1925,
+ 1931,
+ 1932,
+ 1934,
+ 1935,
+ 1957,
+ 1962,
+ 1967,
+ 1975,
+ 1982,
+ 1983,
+ 1991,
+ 1998,
+ 2003,
+ 2008,
+ 2017,
+ 2021,
+ 2035,
+ 2040,
+ 2050,
+ 2056,
+ 2077,
+ 2082,
+ 2098,
+ 2100,
+ 2121,
+ 2130,
+ 2134,
+ 2138,
+ 2143,
+ 2166,
+ 2167,
+ 2180,
+ 2181,
+ 2204,
+ 2205,
+ 2212,
+ 2221,
+ 2224,
+ 2230,
+ 2233,
+ 2263,
+ 2269,
+ 2273,
+ 2290,
+ 2291,
+ 2299,
+ 2321,
+ 2323,
+ 2330,
+ 2401,
+ 2417,
+ 2435,
+ 2456,
+ 2488,
+ 2502,
+ 2504,
+ 2532,
+ 2538,
+ 2555,
+ 2558,
+ 2562,
+ 2583,
+ 2585,
+ 2590,
+ 2594,
+ 2596,
+ 2600,
+ 2606,
+ 2607,
+ 2618,
+ 2630,
+ 2647,
+ 2650,
+ 2657,
+ 2679,
+ 2685,
+ 2705,
+ 2712,
+ 2714,
+ 2727,
+ 2740,
+ 2742,
+ 2780,
+ 2784,
+ 2792,
+ 2807,
+ 2808,
+ 2810,
+ 2820,
+ 2831,
+ 2860,
+ 2863,
+ 2866,
+ 2875,
+ 2878,
+ 2898,
+ 2905,
+ 2921,
+ 2922,
+ 2930,
+ 2934,
+ 2944,
+ 2955,
+ 2957,
+ 2959,
+ 2973,
+ 2978,
+ 2998,
+ 3018,
+ 3022,
+ 3028,
+ 3031,
+ 3061,
+ 3085,
+ 3090,
+ 3104,
+ 3105,
+ 3115,
+ 3121,
+ 3122,
+ 3129,
+ 3133,
+ 3135,
+ 3161,
+ 3169,
+ 3194,
+ 3195,
+ 3215,
+ 3225,
+ 3226,
+ 3250,
+ 3265,
+ 3301,
+ 3312,
+ 3329,
+ 3361,
+ 3362,
+ 3376,
+ 3403,
+ 3410,
+ 3419,
+ 3432,
+ 3443,
+ 3452,
+ 3467,
+ 3469,
+ 3475,
+ 3485,
+ 3503,
+ 3515,
+ 3524,
+ 3528,
+ 3544,
+ 3557,
+ 3565,
+ 3637,
+ 3642,
+ 3658,
+ 3659,
+ 3693,
+ 3699,
+ 3722,
+ 3725,
+ 3728,
+ 3731,
+ 3740,
+ 3742,
+ 3762,
+ 3780,
+ 3788,
+ 3794,
+ 3796,
+ 3798,
+ 3817,
+ 3819,
+ 3822,
+ 3839,
+ 3843,
+ 3846,
+ 3851,
+ 3865,
+ 3871,
+ 3884,
+ 3894,
+ 3896,
+ 3907,
+ 3911,
+ 3915,
+ 3919,
+ 3920,
+ 3923,
+ 3933,
+ 3955,
+ 3967,
+ 3972,
+ 3974,
+ 3975,
+ 3997,
+ 4002,
+ 4034,
+ 4063,
+ 4073,
+ 4079,
+ 4082,
+ 4088,
+ 4121,
+ 4145,
+ 4148,
+ 4159,
+ 4161,
+ 4164,
+ 4177,
+ 4188,
+ 4199,
+ 4207,
+ 4213,
+ 4221,
+ 4233,
+ 4241,
+ 4243,
+ 4247,
+ 4264,
+ 4274,
+ 4282,
+ 4286,
+ 4290,
+ 4310,
+ 4313,
+ 4321,
+ 4324,
+ 4327,
+ 4362,
+ 4380,
+ 4407,
+ 4411,
+ 4415,
+ 4417,
+ 4418,
+ 4427,
+ 4431,
+ 4433,
+ 4451,
+ 4466,
+ 4477,
+ 4479,
+ 4493,
+ 4514,
+ 4527,
+ 4539,
+ 4550,
+ 4558,
+ 4568,
+ 4579,
+ 4583,
+ 4584,
+ 4586,
+ 4587,
+ 4590,
+ 4599,
+ 4602,
+ 4610,
+ 4626,
+ 4627,
+ 4630,
+ 4655,
+ 4656,
+ 4657,
+ 4661,
+ 4685,
+ 4714,
+ 4715,
+ 4731,
+ 4752,
+ 4769,
+ 4791,
+ 4805,
+ 4818,
+ 4829,
+ 4839,
+ 4843,
+ 4871,
+ 4879,
+ 4885,
+ 4888,
+ 4895,
+ 4900,
+ 4923,
+ 4966,
+ 4968,
+ 4972,
+ 4989,
+ 4994,
+ 4998,
+ 5001,
+ 5019,
+ 5026,
+ 5032,
+ 5034,
+ 5046,
+ 5055,
+ 5085,
+ 5086,
+ 5088,
+ 5089,
+ 5090,
+ 5095,
+ 5119,
+ 5121,
+ 5133,
+ 5154,
+ 5169,
+ 5178,
+ 5222,
+ 5223,
+ 5232,
+ 5233,
+ 5240,
+ 5256,
+ 5259,
+ 5264,
+ 5276,
+ 5279,
+ 5335,
+ 5337,
+ 5345,
+ 5348,
+ 5373,
+ 5378,
+ 5422,
+ 5442,
+ 5443,
+ 5445,
+ 5477,
+ 5485,
+ 5495,
+ 5497,
+ 5504,
+ 5526,
+ 5542,
+ 5564,
+ 5570,
+ 5579,
+ 5608,
+ 5610,
+ 5624,
+ 5630,
+ 5638,
+ 5651,
+ 5663,
+ 5670,
+ 5675,
+ 5691,
+ 5700,
+ 5706,
+ 5715,
+ 5720,
+ 5721,
+ 5732,
+ 5738,
+ 5741,
+ 5769,
+ 5771,
+ 5795,
+ 5796,
+ 5800,
+ 5809,
+ 5810,
+ 5815,
+ 5819,
+ 5827,
+ 5848,
+ 5849,
+ 5852,
+ 5859,
+ 5880,
+ 5907,
+ 5912,
+ 5919,
+ 5931,
+ 5932,
+ 5941,
+ 5948,
+ 5950,
+ 5953,
+ 5981,
+ 6000,
+ 6003,
+ 6010,
+ 6018,
+ 6075,
+ 6103,
+ 6106,
+ 6109,
+ 6114,
+ 6123,
+ 6131,
+ 6138,
+ 6139,
+ 6164,
+ 6173,
+ 6180,
+ 6185,
+ 6189,
+ 6190,
+ 6221,
+ 6223,
+ 6237,
+ 6255,
+ 6262,
+ 6265,
+ 6293,
+ 6305,
+ 6331,
+ 6336,
+ 6355,
+ 6366,
+ 6371,
+ 6396,
+ 6399,
+ 6402,
+ 6408,
+ 6432,
+ 6433,
+ 6441,
+ 6456,
+ 6465,
+ 6486,
+ 6489,
+ 6507,
+ 6508,
+ 6522,
+ 6528,
+ 6537,
+ 6551,
+ 6564,
+ 6590,
+ 6598,
+ 6599,
+ 6611,
+ 6613,
+ 6615,
+ 6621,
+ 6634,
+ 6647,
+ 6649,
+ 6654,
+ 6676,
+ 6690,
+ 6708,
+ 6729,
+ 6744,
+ 6749,
+ 6756,
+ 6761,
+ 6763,
+ 6773,
+ 6788,
+ 6790,
+ 6796,
+ 6797,
+ 6811,
+ 6850,
+ 6869,
+ 6871,
+ 6882,
+ 6895,
+ 6906,
+ 6911,
+ 6914,
+ 6952,
+ 6966,
+ 6985,
+ 7001,
+ 7021,
+ 7031,
+ 7038,
+ 7041,
+ 7045,
+ 7052,
+ 7057,
+ 7073,
+ 7076,
+ 7102,
+ 7107,
+ 7109,
+ 7117,
+ 7122,
+ 7125,
+ 7207,
+ 7212,
+ 7215,
+ 7232,
+ 7246,
+ 7250,
+ 7258,
+ 7263,
+ 7267,
+ 7270,
+ 7274,
+ 7280,
+ 7286,
+ 7293,
+ 7298,
+ 7306,
+ 7316,
+ 7325,
+ 7326,
+ 7356,
+ 7367,
+ 7385,
+ 7392,
+ 7405,
+ 7416,
+ 7421,
+ 7426,
+ 7452,
+ 7519,
+ 7534,
+ 7542,
+ 7546,
+ 7601,
+ 7604,
+ 7606,
+ 7609,
+ 7649,
+ 7653,
+ 7682,
+ 7699,
+ 7738,
+ 7750,
+ 7798,
+ 7800,
+ 7801,
+ 7805,
+ 7811,
+ 7832,
+ 7837,
+ 7849,
+ 7856,
+ 7876,
+ 7877,
+ 7887,
+ 7905,
+ 7919,
+ 7920,
+ 7922,
+ 7923,
+ 7926,
+ 7966,
+ 7970,
+ 7973,
+ 7974,
+ 7976,
+ 7986,
+ 8027,
+ 8028,
+ 8068,
+ 8074,
+ 8091,
+ 8120,
+ 8122,
+ 8125,
+ 8152,
+ 8153,
+ 8164,
+ 8167,
+ 8169,
+ 8171,
+ 8177,
+ 8189,
+ 8192,
+ 8196,
+ 8212,
+ 8217,
+ 8231,
+ 8242,
+ 8244,
+ 8250,
+ 8256,
+ 8257,
+ 8265,
+ 8270,
+ 8274,
+ 8283,
+ 8290,
+ 8294,
+ 8302,
+ 8307,
+ 8318,
+ 8326,
+ 8338,
+ 8350,
+ 8353,
+ 8357,
+ 8371,
+ 8377,
+ 8380,
+ 8387,
+ 8388,
+ 8396,
+ 8402,
+ 8419,
+ 8423,
+ 8428,
+ 8435,
+ 8439,
+ 8442,
+ 8444,
+ 8453,
+ 8475,
+ 8481,
+ 8495,
+ 8498,
+ 8523,
+ 8531,
+ 8562,
+ 8568,
+ 8584,
+ 8588,
+ 8589,
+ 8592,
+ 8597,
+ 8608,
+ 8611,
+ 8624,
+ 8636,
+ 8637,
+ 8654,
+ 8657,
+ 8661,
+ 8680,
+ 8683,
+ 8687,
+ 8693,
+ 8695,
+ 8697,
+ 8704,
+ 8713,
+ 8714,
+ 8755,
+ 8758,
+ 8761,
+ 8772,
+ 8781,
+ 8800,
+ 8808,
+ 8812,
+ 8815,
+ 8822,
+ 8825,
+ 8830,
+ 8837,
+ 8863,
+ 8867,
+ 8894,
+ 8898,
+ 8924,
+ 8937,
+ 8955,
+ 8964,
+ 8978,
+ 8995,
+ 9007,
+ 9018,
+ 9034,
+ 9043,
+ 9074,
+ 9075,
+ 9099,
+ 9103,
+ 9112,
+ 9115,
+ 9127,
+ 9137,
+ 9142,
+ 9163,
+ 9174,
+ 9179,
+ 9196,
+ 9208,
+ 9213,
+ 9222,
+ 9228,
+ 9230,
+ 9274,
+ 9278,
+ 9293,
+ 9328,
+ 9338,
+ 9342,
+ 9346,
+ 9368,
+ 9385,
+ 9403,
+ 9420,
+ 9429,
+ 9432,
+ 9453,
+ 9455,
+ 9463,
+ 9502,
+ 9511,
+ 9517,
+ 9545,
+ 9548,
+ 9550,
+ 9558,
+ 9566,
+ 9574,
+ 9577,
+ 9578,
+ 9589,
+ 9606,
+ 9618,
+ 9659,
+ 9672,
+ 9673,
+ 9706,
+ 9717,
+ 9728,
+ 9732,
+ 9758,
+ 9764,
+ 9773,
+ 9778,
+ 9779,
+ 9798,
+ 9801,
+ 9803,
+ 9804,
+ 9845,
+ 9849,
+ 9862,
+ 9876,
+ 9890,
+ 9911,
+ 9959,
+ 9966,
+ 9967,
+ 9968,
+ 9974,
+ 9983,
+ 10007,
+ 10032,
+ 10042,
+ 10044,
+ 10061,
+ 10062,
+ 10080,
+ 10085,
+ 10095,
+ 10098,
+ 10132,
+ 10144,
+ 10175,
+ 10176,
+ 10197,
+ 10220,
+ 10230,
+ 10233,
+ 10242,
+ 10260,
+ 10265,
+ 10305,
+ 10313,
+ 10314,
+ 10332,
+ 10338,
+ 10341,
+ 10342,
+ 10347,
+ 10353,
+ 10354,
+ 10360,
+ 10381,
+ 10390,
+ 10396,
+ 10402,
+ 10404,
+ 10418,
+ 10432,
+ 10445,
+ 10447,
+ 10450,
+ 10454,
+ 10489,
+ 10523,
+ 10539,
+ 10541,
+ 10550,
+ 10561,
+ 10562,
+ 10573,
+ 10577,
+ 10583,
+ 10589,
+ 10596,
+ 10601,
+ 10605,
+ 10633,
+ 10634,
+ 10663,
+ 10671,
+ 10672,
+ 10685,
+ 10710,
+ 10730,
+ 10733,
+ 10740,
+ 10744,
+ 10774,
+ 10786,
+ 10829,
+ 10833,
+ 10838,
+ 10855,
+ 10879,
+ 10907,
+ 10909,
+ 10911,
+ 10928,
+ 10936,
+ 10956,
+ 10989,
+ 11022,
+ 11030,
+ 11031,
+ 11041,
+ 11055,
+ 11058,
+ 11075,
+ 11077,
+ 11085,
+ 11096
+ ],
+ "early_stopping_patience": 3
+}
diff --git a/phase1/qwen-main-dpo-v3/README.md b/phase1/qwen-main-dpo-v3/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..da378ab29d01c6df5db9f17bb0858ce8a4a0c022
--- /dev/null
+++ b/phase1/qwen-main-dpo-v3/README.md
@@ -0,0 +1,81 @@
+---
+library_name: peft
+model_name: phase1-qwen-main-dpo-v3
+tags:
+- base_model:adapter:Qwen/Qwen2.5-1.5B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+license: apache-2.0
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+pipeline_tag: text-generation
+---
+
+# Phase 1 Qwen main DPO adapter
+
+This is the Phase 1 main LoRA-DPO adapter fine-tuned from
+[Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct)
+with 99,986 preference pairs. It has been trained using
+[TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo_id = "geyuxu/york-milestone-project-self-traing-loop-model"
+subfolder = "phase1/qwen-main-dpo-v3"
+base_id = "Qwen/Qwen2.5-1.5B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(repo_id, subfolder=subfolder)
+base_model = AutoModelForCausalLM.from_pretrained(
+ base_id, torch_dtype="auto", device_map="auto"
+)
+model = PeftModel.from_pretrained(base_model, repo_id, subfolder=subfolder)
+```
+
+## Training procedure
+
+
+
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 0.29.0
+- Transformers: 5.3.0
+- Pytorch: 2.10.0
+- Datasets: 4.6.1
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+ title = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+ author = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+ year = 2023,
+ booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+ url = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+ editor = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+ title = {{TRL: Transformers Reinforcement Learning}},
+ author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+ license = {Apache-2.0},
+ url = {https://github.com/huggingface/trl},
+ year = {2020}
+}
+```
diff --git a/phase1/qwen-main-dpo-v3/adapter_config.json b/phase1/qwen-main-dpo-v3/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..af2c37433449bd4e20212b848583833bdaf67495
--- /dev/null
+++ b/phase1/qwen-main-dpo-v3/adapter_config.json
@@ -0,0 +1,43 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.05,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 16,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "q_proj",
+ "v_proj",
+ "o_proj",
+ "k_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
diff --git a/phase1/qwen-main-dpo-v3/adapter_model.safetensors b/phase1/qwen-main-dpo-v3/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b52689a8199fe204047852071d0a06e9b9b0a328
--- /dev/null
+++ b/phase1/qwen-main-dpo-v3/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc60e5fe47be9bf4f64d734d40d89310a4826d8156d0ab8221b466f21c5efd58
+size 17462432
diff --git a/phase1/qwen-main-dpo-v3/chat_template.jinja b/phase1/qwen-main-dpo-v3/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e
--- /dev/null
+++ b/phase1/qwen-main-dpo-v3/chat_template.jinja
@@ -0,0 +1,54 @@
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- messages[0]['content'] }}
+ {%- else %}
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+ {%- endif %}
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0]['role'] == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+ {%- else %}
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {{- '<|im_start|>' + message.role }}
+ {%- if message.content %}
+ {{- '\n' + message.content }}
+ {%- endif %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- '}\n' }}
+ {%- endfor %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- message.content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}
diff --git a/phase1/qwen-main-dpo-v3/tokenizer.json b/phase1/qwen-main-dpo-v3/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..d73428d91463b495bc017fb6a2fb3a656646482b
--- /dev/null
+++ b/phase1/qwen-main-dpo-v3/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5
+size 11422166
diff --git a/phase1/qwen-main-dpo-v3/tokenizer_config.json b/phase1/qwen-main-dpo-v3/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9fd0fb48e73786f54fb04e98892f5f5a0ebeebdb
--- /dev/null
+++ b/phase1/qwen-main-dpo-v3/tokenizer_config.json
@@ -0,0 +1,29 @@
+{
+ "add_prefix_space": false,
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|im_start|>",
+ "<|im_end|>",
+ "<|object_ref_start|>",
+ "<|object_ref_end|>",
+ "<|box_start|>",
+ "<|box_end|>",
+ "<|quad_start|>",
+ "<|quad_end|>",
+ "<|vision_start|>",
+ "<|vision_end|>",
+ "<|vision_pad|>",
+ "<|image_pad|>",
+ "<|video_pad|>"
+ ],
+ "is_local": true,
+ "model_max_length": 131072,
+ "pad_token": "<|endoftext|>",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null
+}
diff --git a/phase1/qwen-main-dpo-v3/training_args.bin b/phase1/qwen-main-dpo-v3/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..b081cff1f7eda31f03912fcff25d6834200864e0
--- /dev/null
+++ b/phase1/qwen-main-dpo-v3/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1578389f7f0fe332b393d7d42285d2a1a804b2efc9a174e73d1805a5ca67e41b
+size 5841
diff --git a/phase1/qwen-main-dpo-v3/training_recipe.json b/phase1/qwen-main-dpo-v3/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..630e13c7dfb5f2ee4029860a455f88e95625413c
--- /dev/null
+++ b/phase1/qwen-main-dpo-v3/training_recipe.json
@@ -0,0 +1,20 @@
+{
+ "format_version": 1,
+ "experiment": "main_dpo",
+ "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+ "train_membership_sha256": "e105dbe9f9e652f095541415523cdcda51cc9e8d12d3ba4ca03e29e40296324f",
+ "validation_membership_sha256": "a46c1c0d0a6c3c444a3ac940322e42a2f1be5ec8e5f5f729a002e7484742551b",
+ "num_train_pairs": 99986,
+ "num_validation_pairs": 11110,
+ "learning_rate": 1e-05,
+ "beta": 0.1,
+ "num_epochs": 1.0,
+ "seed": 22,
+ "max_length": 512,
+ "max_prompt_length": 384,
+ "gradient_accumulation_steps": 80,
+ "save_steps": 125,
+ "save_total_limit": 12,
+ "preference_pairs_format_version": 2,
+ "dpo_prompt_mode": "rag_prompt_shared"
+}
diff --git a/phase1/qwen-main-dpo-v3/training_summary.json b/phase1/qwen-main-dpo-v3/training_summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..67835b2227501dde15ad8b132857fed94a61b348
--- /dev/null
+++ b/phase1/qwen-main-dpo-v3/training_summary.json
@@ -0,0 +1,3493 @@
+{
+ "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+ "resolved_model": "Qwen/Qwen2.5-1.5B-Instruct",
+ "num_train_records": 99986,
+ "num_validation_records_available": 11110,
+ "training_config": {
+ "lora_rank": 16,
+ "lora_alpha": 32,
+ "learning_rate": 1e-05,
+ "num_epochs": 1.0,
+ "per_device_batch_size": 1,
+ "per_device_eval_batch_size": 1,
+ "gradient_accumulation_steps": 80,
+ "beta": 0.1,
+ "max_length": 512,
+ "max_prompt_length": 384,
+ "seed": 22,
+ "save_steps": 125,
+ "save_total_limit": 12,
+ "resume_from_checkpoint": null,
+ "precision_dtype": "torch.bfloat16",
+ "bf16": true,
+ "fp16": false,
+ "tokenizer_add_bos_token": false
+ },
+ "global_step": 1250,
+ "best_metric": 0.3715994358062744,
+ "best_model_checkpoint": "outputs/dpo_phase1_qwen_all_unique_repair_v3/lora/checkpoint-1250",
+ "train_metrics": {
+ "train_runtime": 54275.4778,
+ "train_samples_per_second": 1.842,
+ "train_steps_per_second": 0.023,
+ "total_flos": 6.794900050947748e+17,
+ "train_loss": 0.4424690731048584
+ },
+ "log_history": [
+ {
+ "loss": 0.6901532173156738,
+ "grad_norm": 0.012621426023542881,
+ "learning_rate": 9.928e-06,
+ "entropy": 0.8093121654563583,
+ "num_tokens": 670963.0,
+ "logits/chosen": -0.9584369752729957,
+ "logits/rejected": -0.9518610903521089,
+ "mean_token_accuracy": 0.7313421124406159,
+ "rewards/chosen": 0.00045646784230086724,
+ "rewards/rejected": -0.005912241121634452,
+ "rewards/accuracies": 0.52625,
+ "rewards/margins": 0.006368708992449683,
+ "logps/chosen": -53.02946231007576,
+ "logps/rejected": -52.86994596719742,
+ "epoch": 0.008001120156821955,
+ "step": 10
+ },
+ {
+ "loss": 0.674683666229248,
+ "grad_norm": 0.012483606114983559,
+ "learning_rate": 9.848000000000001e-06,
+ "entropy": 0.7939381037396379,
+ "num_tokens": 1341997.0,
+ "logits/chosen": -0.9796888803192282,
+ "logits/rejected": -0.9018265656875972,
+ "mean_token_accuracy": 0.7365013980120421,
+ "rewards/chosen": -0.01624998665111889,
+ "rewards/rejected": -0.05539245237303021,
+ "rewards/accuracies": 0.70875,
+ "rewards/margins": 0.03914246566069778,
+ "logps/chosen": -50.256172008514405,
+ "logps/rejected": -54.854298782348636,
+ "epoch": 0.01600224031364391,
+ "step": 20
+ },
+ {
+ "loss": 0.6438151359558105,
+ "grad_norm": 0.012461220845580101,
+ "learning_rate": 9.768e-06,
+ "entropy": 0.7936106454930268,
+ "num_tokens": 2012140.0,
+ "logits/chosen": -0.9887686294803811,
+ "logits/rejected": -0.8958026605139604,
+ "mean_token_accuracy": 0.7354880513995886,
+ "rewards/chosen": -0.06927679138132589,
+ "rewards/rejected": -0.1785658121076267,
+ "rewards/accuracies": 0.77875,
+ "rewards/margins": 0.10928902073879726,
+ "logps/chosen": -49.68967272758484,
+ "logps/rejected": -54.42679405450821,
+ "epoch": 0.024003360470465866,
+ "step": 30
+ },
+ {
+ "loss": 0.6171675205230713,
+ "grad_norm": 0.012906152755022049,
+ "learning_rate": 9.688000000000001e-06,
+ "entropy": 0.8316492229385767,
+ "num_tokens": 2685435.0,
+ "logits/chosen": -0.9345810579875565,
+ "logits/rejected": -0.8415264475290734,
+ "mean_token_accuracy": 0.7263862137310206,
+ "rewards/chosen": -0.17557695602539752,
+ "rewards/rejected": -0.35647496176256027,
+ "rewards/accuracies": 0.76375,
+ "rewards/margins": 0.18089800578542053,
+ "logps/chosen": -55.840644490718844,
+ "logps/rejected": -58.12323925256729,
+ "epoch": 0.03200448062728782,
+ "step": 40
+ },
+ {
+ "loss": 0.5752066612243653,
+ "grad_norm": 0.012006422504782677,
+ "learning_rate": 9.608e-06,
+ "entropy": 0.7856921868515201,
+ "num_tokens": 3355193.0,
+ "logits/chosen": -0.9370843916521988,
+ "logits/rejected": -0.8159164053898426,
+ "mean_token_accuracy": 0.723189832996577,
+ "rewards/chosen": -0.2943884494334634,
+ "rewards/rejected": -0.6045056528400164,
+ "rewards/accuracies": 0.785,
+ "rewards/margins": 0.31011720293667167,
+ "logps/chosen": -52.23467300653458,
+ "logps/rejected": -58.18705793142319,
+ "epoch": 0.04000560078410977,
+ "step": 50
+ },
+ {
+ "loss": 0.5501768589019775,
+ "grad_norm": 0.013944911770522594,
+ "learning_rate": 9.528000000000001e-06,
+ "entropy": 0.8087824353948235,
+ "num_tokens": 4027753.0,
+ "logits/chosen": -1.0292918722485356,
+ "logits/rejected": -0.855586984143943,
+ "mean_token_accuracy": 0.7112987640220672,
+ "rewards/chosen": -0.5289613087896577,
+ "rewards/rejected": -0.9568682179228198,
+ "rewards/accuracies": 0.76375,
+ "rewards/margins": 0.4279069083847571,
+ "logps/chosen": -56.717787961959836,
+ "logps/rejected": -66.04619632005692,
+ "epoch": 0.04800672094093173,
+ "step": 60
+ },
+ {
+ "loss": 0.5261342525482178,
+ "grad_norm": 0.012163989245891571,
+ "learning_rate": 9.448e-06,
+ "entropy": 0.7831143385136966,
+ "num_tokens": 4699664.0,
+ "logits/chosen": -1.101007950411393,
+ "logits/rejected": -0.9435466428316124,
+ "mean_token_accuracy": 0.7068579371087254,
+ "rewards/chosen": -0.7223313848793623,
+ "rewards/rejected": -1.2714019845274742,
+ "rewards/accuracies": 0.77875,
+ "rewards/margins": 0.5490705984598026,
+ "logps/chosen": -61.18258552312851,
+ "logps/rejected": -65.89279877901077,
+ "epoch": 0.05600784109775368,
+ "step": 70
+ },
+ {
+ "loss": 0.5274584293365479,
+ "grad_norm": 0.015226946212351322,
+ "learning_rate": 9.368e-06,
+ "entropy": 0.7488077865628293,
+ "num_tokens": 5372157.0,
+ "logits/chosen": -1.2634839435133882,
+ "logits/rejected": -1.082128236350263,
+ "mean_token_accuracy": 0.7112827113177628,
+ "rewards/chosen": -0.9723736402860231,
+ "rewards/rejected": -1.626036137866322,
+ "rewards/accuracies": 0.745,
+ "rewards/margins": 0.6536624957341701,
+ "logps/chosen": -62.329527735710144,
+ "logps/rejected": -69.63982899188996,
+ "epoch": 0.06400896125457564,
+ "step": 80
+ },
+ {
+ "loss": 0.5142660140991211,
+ "grad_norm": 0.013633078895509243,
+ "learning_rate": 9.288e-06,
+ "entropy": 0.7427371655139723,
+ "num_tokens": 6040573.0,
+ "logits/chosen": -1.311929577255716,
+ "logits/rejected": -1.125822709345973,
+ "mean_token_accuracy": 0.7006962174363435,
+ "rewards/chosen": -1.0937171235348797,
+ "rewards/rejected": -1.8326766119452076,
+ "rewards/accuracies": 0.74625,
+ "rewards/margins": 0.7389594866335392,
+ "logps/chosen": -60.460937385559085,
+ "logps/rejected": -69.68028839588165,
+ "epoch": 0.0720100814113976,
+ "step": 90
+ },
+ {
+ "loss": 0.48309645652770994,
+ "grad_norm": 0.013873466290533543,
+ "learning_rate": 9.208e-06,
+ "entropy": 0.7353724740154576,
+ "num_tokens": 6713961.0,
+ "logits/chosen": -1.3981686101953354,
+ "logits/rejected": -1.1492534001655728,
+ "mean_token_accuracy": 0.7068778586760164,
+ "rewards/chosen": -1.275778706202982,
+ "rewards/rejected": -2.205598971605068,
+ "rewards/accuracies": 0.77875,
+ "rewards/margins": 0.9298202639631927,
+ "logps/chosen": -63.14240853786468,
+ "logps/rejected": -78.55453793764114,
+ "epoch": 0.08001120156821954,
+ "step": 100
+ },
+ {
+ "loss": 0.47826247215270995,
+ "grad_norm": 0.01541591715067625,
+ "learning_rate": 9.128e-06,
+ "entropy": 0.7661040782314376,
+ "num_tokens": 7387335.0,
+ "logits/chosen": -1.3620669095703972,
+ "logits/rejected": -1.1219973012975202,
+ "mean_token_accuracy": 0.6907096220739186,
+ "rewards/chosen": -1.3754910858121001,
+ "rewards/rejected": -2.3353658849513157,
+ "rewards/accuracies": 0.78375,
+ "rewards/margins": 0.9598747962340712,
+ "logps/chosen": -67.88601873874664,
+ "logps/rejected": -78.68544527292252,
+ "epoch": 0.0880123217250415,
+ "step": 110
+ },
+ {
+ "loss": 0.5143081188201905,
+ "grad_norm": 0.017378821969032288,
+ "learning_rate": 9.048e-06,
+ "entropy": 0.7460291436497937,
+ "num_tokens": 8057646.0,
+ "logits/chosen": -1.4079163351569763,
+ "logits/rejected": -1.1285183809325192,
+ "mean_token_accuracy": 0.700520682586357,
+ "rewards/chosen": -1.3658788571986589,
+ "rewards/rejected": -2.2463605564041065,
+ "rewards/accuracies": 0.75125,
+ "rewards/margins": 0.8804816985502839,
+ "logps/chosen": -65.51872705698014,
+ "logps/rejected": -74.3196389889717,
+ "epoch": 0.09601344188186346,
+ "step": 120
+ },
+ {
+ "eval_loss": 0.4647688567638397,
+ "eval_runtime": 195.6609,
+ "eval_samples_per_second": 5.111,
+ "eval_steps_per_second": 5.111,
+ "eval_entropy": 0.7556589982062578,
+ "eval_num_tokens": 8395356.0,
+ "eval_logits/chosen": -1.2958382112131164,
+ "eval_logits/rejected": -1.0266528869050695,
+ "eval_mean_token_accuracy": 0.6990812446027994,
+ "eval_rewards/chosen": -1.2314954024991966,
+ "eval_rewards/rejected": -2.241709055064479,
+ "eval_rewards/accuracies": 0.791,
+ "eval_rewards/margins": 1.0102136510759592,
+ "eval_logps/chosen": -62.685266920089724,
+ "eval_logps/rejected": -75.3382277507782,
+ "epoch": 0.10001400196027443,
+ "step": 125
+ },
+ {
+ "loss": 0.4783907890319824,
+ "grad_norm": 0.01220870204269886,
+ "learning_rate": 8.968000000000001e-06,
+ "entropy": 0.7789282223425107,
+ "num_tokens": 8731040.0,
+ "logits/chosen": -1.2888471366805734,
+ "logits/rejected": -1.0516192191884435,
+ "mean_token_accuracy": 0.691539853401482,
+ "rewards/chosen": -1.2831247101316694,
+ "rewards/rejected": -2.242197908805683,
+ "rewards/accuracies": 0.78,
+ "rewards/margins": 0.9590731968544424,
+ "logps/chosen": -68.42604200839996,
+ "logps/rejected": -76.86017044305801,
+ "epoch": 0.10401456203868542,
+ "step": 130
+ },
+ {
+ "loss": 0.4924801826477051,
+ "grad_norm": 0.012252936139702797,
+ "learning_rate": 8.888e-06,
+ "entropy": 0.7674391892907443,
+ "num_tokens": 9401640.0,
+ "logits/chosen": -1.2537443916085618,
+ "logits/rejected": -0.9872851940228372,
+ "mean_token_accuracy": 0.7068716604355723,
+ "rewards/chosen": -1.2096528004054563,
+ "rewards/rejected": -2.1564956095546948,
+ "rewards/accuracies": 0.77875,
+ "rewards/margins": 0.946842809554655,
+ "logps/chosen": -61.44815753698349,
+ "logps/rejected": -75.21420559167862,
+ "epoch": 0.11201568219550737,
+ "step": 140
+ },
+ {
+ "loss": 0.4433901786804199,
+ "grad_norm": 0.019120773300528526,
+ "learning_rate": 8.808000000000001e-06,
+ "entropy": 0.7849294722545892,
+ "num_tokens": 10074158.0,
+ "logits/chosen": -1.2328529211448207,
+ "logits/rejected": -0.9243871139015541,
+ "mean_token_accuracy": 0.6956806232780218,
+ "rewards/chosen": -1.2763914220364496,
+ "rewards/rejected": -2.3580219121300616,
+ "rewards/accuracies": 0.8,
+ "rewards/margins": 1.081630490552634,
+ "logps/chosen": -64.5562591767311,
+ "logps/rejected": -80.11041006326676,
+ "epoch": 0.12001680235232932,
+ "step": 150
+ },
+ {
+ "loss": 0.4346295356750488,
+ "grad_norm": 0.013042928650975227,
+ "learning_rate": 8.728e-06,
+ "entropy": 0.7355541310226544,
+ "num_tokens": 10743004.0,
+ "logits/chosen": -1.4005078205253927,
+ "logits/rejected": -1.0567861199616662,
+ "mean_token_accuracy": 0.7047591439820826,
+ "rewards/chosen": -1.2316719387439299,
+ "rewards/rejected": -2.3673254120955245,
+ "rewards/accuracies": 0.81,
+ "rewards/margins": 1.1356534744799136,
+ "logps/chosen": -59.515862419605256,
+ "logps/rejected": -76.50743375062943,
+ "epoch": 0.12801792250915128,
+ "step": 160
+ },
+ {
+ "loss": 0.5103495597839356,
+ "grad_norm": 0.015699906274676323,
+ "learning_rate": 8.648000000000001e-06,
+ "entropy": 0.740672427273239,
+ "num_tokens": 11410696.0,
+ "logits/chosen": -1.404414859203772,
+ "logits/rejected": -1.1005658892552497,
+ "mean_token_accuracy": 0.7076463782973588,
+ "rewards/chosen": -1.2404756933083991,
+ "rewards/rejected": -2.2894487548015605,
+ "rewards/accuracies": 0.77875,
+ "rewards/margins": 1.0489730596914888,
+ "logps/chosen": -60.35841114163399,
+ "logps/rejected": -74.72909695386886,
+ "epoch": 0.13601904266597323,
+ "step": 170
+ },
+ {
+ "loss": 0.46210732460021975,
+ "grad_norm": 0.013843734748661518,
+ "learning_rate": 8.568e-06,
+ "entropy": 0.7232949527283199,
+ "num_tokens": 12080027.0,
+ "logits/chosen": -1.5137382104169737,
+ "logits/rejected": -1.1591507879383176,
+ "mean_token_accuracy": 0.7147883488237858,
+ "rewards/chosen": -1.1584112935088342,
+ "rewards/rejected": -2.2342659830115736,
+ "rewards/accuracies": 0.80125,
+ "rewards/margins": 1.075854687690735,
+ "logps/chosen": -59.70499998569488,
+ "logps/rejected": -75.35983695030212,
+ "epoch": 0.1440201628227952,
+ "step": 180
+ },
+ {
+ "loss": 0.48125648498535156,
+ "grad_norm": 0.0137063292786479,
+ "learning_rate": 8.488e-06,
+ "entropy": 0.7297163017120328,
+ "num_tokens": 12751327.0,
+ "logits/chosen": -1.444744739736003,
+ "logits/rejected": -1.1977971036132493,
+ "mean_token_accuracy": 0.7079235365241766,
+ "rewards/chosen": -1.1817194805917097,
+ "rewards/rejected": -2.165882707394485,
+ "rewards/accuracies": 0.77625,
+ "rewards/margins": 0.9841632262058556,
+ "logps/chosen": -62.84384714603424,
+ "logps/rejected": -74.69614839792251,
+ "epoch": 0.15202128297961714,
+ "step": 190
+ },
+ {
+ "loss": 0.4559768199920654,
+ "grad_norm": 0.017784949392080307,
+ "learning_rate": 8.408e-06,
+ "entropy": 0.7738335855977493,
+ "num_tokens": 13423121.0,
+ "logits/chosen": -1.3036348294549531,
+ "logits/rejected": -1.091831282302383,
+ "mean_token_accuracy": 0.704848392214626,
+ "rewards/chosen": -1.0437373626657063,
+ "rewards/rejected": -2.0530137880844994,
+ "rewards/accuracies": 0.8125,
+ "rewards/margins": 1.0092764250934123,
+ "logps/chosen": -64.31396059989929,
+ "logps/rejected": -72.96644197225571,
+ "epoch": 0.1600224031364391,
+ "step": 200
+ },
+ {
+ "loss": 0.44863576889038087,
+ "grad_norm": 0.014046410098671913,
+ "learning_rate": 8.328e-06,
+ "entropy": 0.782024933876237,
+ "num_tokens": 14095754.0,
+ "logits/chosen": -1.3947293172547142,
+ "logits/rejected": -1.0955241061061023,
+ "mean_token_accuracy": 0.7042262899316847,
+ "rewards/chosen": -1.0590654362243368,
+ "rewards/rejected": -2.181202197938692,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 1.1221367592178284,
+ "logps/chosen": -63.88484351158142,
+ "logps/rejected": -77.38630795001984,
+ "epoch": 0.16802352329326106,
+ "step": 210
+ },
+ {
+ "loss": 0.44264936447143555,
+ "grad_norm": 0.012177016586065292,
+ "learning_rate": 8.248e-06,
+ "entropy": 0.7621519005317532,
+ "num_tokens": 14766898.0,
+ "logits/chosen": -1.4800165515522479,
+ "logits/rejected": -1.2002405014776387,
+ "mean_token_accuracy": 0.705511124804616,
+ "rewards/chosen": -1.0531316056221112,
+ "rewards/rejected": -2.159446875175927,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 1.1063152687205002,
+ "logps/chosen": -62.64944897651672,
+ "logps/rejected": -76.60918622970581,
+ "epoch": 0.176024643450083,
+ "step": 220
+ },
+ {
+ "loss": 0.4280844688415527,
+ "grad_norm": 0.016551608219742775,
+ "learning_rate": 8.168e-06,
+ "entropy": 0.7501697012913064,
+ "num_tokens": 15435708.0,
+ "logits/chosen": -1.4906831771255906,
+ "logits/rejected": -1.2600194493747174,
+ "mean_token_accuracy": 0.7034410179778934,
+ "rewards/chosen": -1.0560207770811394,
+ "rewards/rejected": -2.279763128710911,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.2237423502095044,
+ "logps/chosen": -62.1078541469574,
+ "logps/rejected": -74.43184190511704,
+ "epoch": 0.18402576360690498,
+ "step": 230
+ },
+ {
+ "loss": 0.4562046051025391,
+ "grad_norm": 0.018661431968212128,
+ "learning_rate": 8.088e-06,
+ "entropy": 0.7657817163813161,
+ "num_tokens": 16105021.0,
+ "logits/chosen": -1.4795107714667275,
+ "logits/rejected": -1.2074811268885899,
+ "mean_token_accuracy": 0.6954782837070524,
+ "rewards/chosen": -1.187200614688045,
+ "rewards/rejected": -2.371643460559426,
+ "rewards/accuracies": 0.79625,
+ "rewards/margins": 1.1844428442558275,
+ "logps/chosen": -65.60020695924759,
+ "logps/rejected": -75.24097845077515,
+ "epoch": 0.19202688376372692,
+ "step": 240
+ },
+ {
+ "loss": 0.43615498542785647,
+ "grad_norm": 0.017426082864403725,
+ "learning_rate": 8.008e-06,
+ "entropy": 0.7539033056423068,
+ "num_tokens": 16775473.0,
+ "logits/chosen": -1.5293930977344743,
+ "logits/rejected": -1.2039755376829944,
+ "mean_token_accuracy": 0.705405270345509,
+ "rewards/chosen": -1.153575651863182,
+ "rewards/rejected": -2.4536339438945287,
+ "rewards/accuracies": 0.8,
+ "rewards/margins": 1.300058292048052,
+ "logps/chosen": -61.453006939888,
+ "logps/rejected": -78.41303983211517,
+ "epoch": 0.20002800392054887,
+ "step": 250
+ },
+ {
+ "eval_loss": 0.41755250096321106,
+ "eval_runtime": 228.8761,
+ "eval_samples_per_second": 4.369,
+ "eval_steps_per_second": 4.369,
+ "eval_entropy": 0.7515761259810534,
+ "eval_num_tokens": 16775473.0,
+ "eval_logits/chosen": -1.5818283479057658,
+ "eval_logits/rejected": -1.2564869641723404,
+ "eval_mean_token_accuracy": 0.7046490426361561,
+ "eval_rewards/chosen": -1.1224118193469477,
+ "eval_rewards/rejected": -2.479533482246101,
+ "eval_rewards/accuracies": 0.817,
+ "eval_rewards/margins": 1.3571216629631817,
+ "eval_logps/chosen": -61.59443111038208,
+ "eval_logps/rejected": -77.71647195625305,
+ "epoch": 0.20002800392054887,
+ "step": 250
+ },
+ {
+ "loss": 0.45821270942687986,
+ "grad_norm": 0.021657424047589302,
+ "learning_rate": 7.928e-06,
+ "entropy": 0.7126325222587911,
+ "num_tokens": 17443745.0,
+ "logits/chosen": -1.688095090193879,
+ "logits/rejected": -1.3269386596926678,
+ "mean_token_accuracy": 0.7153355416469276,
+ "rewards/chosen": -1.0613955771701875,
+ "rewards/rejected": -2.3263194395392204,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.2649238619487733,
+ "logps/chosen": -58.97456979036331,
+ "logps/rejected": -75.75996887207032,
+ "epoch": 0.20802912407737084,
+ "step": 260
+ },
+ {
+ "loss": 0.45466012954711915,
+ "grad_norm": 0.01559050939977169,
+ "learning_rate": 7.848000000000002e-06,
+ "entropy": 0.7546081965982012,
+ "num_tokens": 18114630.0,
+ "logits/chosen": -1.6243488424681385,
+ "logits/rejected": -1.2826702078202163,
+ "mean_token_accuracy": 0.7035466120112688,
+ "rewards/chosen": -0.9812952489028248,
+ "rewards/rejected": -2.2022523112816272,
+ "rewards/accuracies": 0.79625,
+ "rewards/margins": 1.2209570601163433,
+ "logps/chosen": -59.83277177095413,
+ "logps/rejected": -76.06529054880143,
+ "epoch": 0.2160302442341928,
+ "step": 270
+ },
+ {
+ "loss": 0.45614118576049806,
+ "grad_norm": 0.019957436248660088,
+ "learning_rate": 7.768e-06,
+ "entropy": 0.7647551147075137,
+ "num_tokens": 18785905.0,
+ "logits/chosen": -1.5628276684430993,
+ "logits/rejected": -1.2313030623237793,
+ "mean_token_accuracy": 0.7110288896784186,
+ "rewards/chosen": -0.9674690414072393,
+ "rewards/rejected": -2.1841950027487473,
+ "rewards/accuracies": 0.7975,
+ "rewards/margins": 1.2167259603133425,
+ "logps/chosen": -61.66511924505234,
+ "logps/rejected": -76.08382419586182,
+ "epoch": 0.22403136439101473,
+ "step": 280
+ },
+ {
+ "loss": 0.4528087615966797,
+ "grad_norm": 0.016109870746731758,
+ "learning_rate": 7.688000000000002e-06,
+ "entropy": 0.7966391498795565,
+ "num_tokens": 19454926.0,
+ "logits/chosen": -1.5498234098285903,
+ "logits/rejected": -1.1901957881893803,
+ "mean_token_accuracy": 0.7049976014718413,
+ "rewards/chosen": -0.9073732627101344,
+ "rewards/rejected": -2.0697916117409476,
+ "rewards/accuracies": 0.805,
+ "rewards/margins": 1.1624183485191315,
+ "logps/chosen": -59.853264043331144,
+ "logps/rejected": -74.87920271635056,
+ "epoch": 0.2320324845478367,
+ "step": 290
+ },
+ {
+ "loss": 0.47883009910583496,
+ "grad_norm": 0.016857173293828964,
+ "learning_rate": 7.608000000000001e-06,
+ "entropy": 0.8085904457367724,
+ "num_tokens": 20126111.0,
+ "logits/chosen": -1.458701673585645,
+ "logits/rejected": -1.1795248022037903,
+ "mean_token_accuracy": 0.7017660610564053,
+ "rewards/chosen": -0.909867911524816,
+ "rewards/rejected": -1.9736081816648947,
+ "rewards/accuracies": 0.79125,
+ "rewards/margins": 1.06374026496429,
+ "logps/chosen": -63.4786088347435,
+ "logps/rejected": -73.5995323753357,
+ "epoch": 0.24003360470465865,
+ "step": 300
+ },
+ {
+ "loss": 0.44984683990478513,
+ "grad_norm": 0.014970983378589153,
+ "learning_rate": 7.528000000000001e-06,
+ "entropy": 0.7699917011187063,
+ "num_tokens": 20797038.0,
+ "logits/chosen": -1.446817589316724,
+ "logits/rejected": -1.1859943999402387,
+ "mean_token_accuracy": 0.7064874805696308,
+ "rewards/chosen": -0.9031680591187978,
+ "rewards/rejected": -2.074450860506622,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.1712828036351128,
+ "logps/chosen": -59.92485630989075,
+ "logps/rejected": -73.00901054382324,
+ "epoch": 0.24803472486148062,
+ "step": 310
+ },
+ {
+ "loss": 0.4490029335021973,
+ "grad_norm": 0.022525979205965996,
+ "learning_rate": 7.4480000000000005e-06,
+ "entropy": 0.7794862373394426,
+ "num_tokens": 21469537.0,
+ "logits/chosen": -1.4607136962944067,
+ "logits/rejected": -1.2216475012442696,
+ "mean_token_accuracy": 0.7070466516911984,
+ "rewards/chosen": -0.9359806089089397,
+ "rewards/rejected": -2.1456712255581807,
+ "rewards/accuracies": 0.80125,
+ "rewards/margins": 1.2096906119165942,
+ "logps/chosen": -63.570755321979526,
+ "logps/rejected": -74.33635073184968,
+ "epoch": 0.25603584501830257,
+ "step": 320
+ },
+ {
+ "loss": 0.42673392295837403,
+ "grad_norm": 0.01500563696026802,
+ "learning_rate": 7.3680000000000004e-06,
+ "entropy": 0.7719001133486745,
+ "num_tokens": 22139392.0,
+ "logits/chosen": -1.5307678604859354,
+ "logits/rejected": -1.1878700116418397,
+ "mean_token_accuracy": 0.7252580110915006,
+ "rewards/chosen": -0.9217133387317881,
+ "rewards/rejected": -2.2019231244851833,
+ "rewards/accuracies": 0.81625,
+ "rewards/margins": 1.280209785089828,
+ "logps/chosen": -57.07231194496155,
+ "logps/rejected": -75.99781235456467,
+ "epoch": 0.26403696517512454,
+ "step": 330
+ },
+ {
+ "loss": 0.43677473068237305,
+ "grad_norm": 0.016953062266111374,
+ "learning_rate": 7.288e-06,
+ "entropy": 0.7901253036441631,
+ "num_tokens": 22810806.0,
+ "logits/chosen": -1.5143094582014855,
+ "logits/rejected": -1.1739754756461656,
+ "mean_token_accuracy": 0.7107993514090777,
+ "rewards/chosen": -1.0534828628422475,
+ "rewards/rejected": -2.3567850939719936,
+ "rewards/accuracies": 0.79,
+ "rewards/margins": 1.3033022294938563,
+ "logps/chosen": -62.018573169708255,
+ "logps/rejected": -77.68898005723953,
+ "epoch": 0.27203808533194646,
+ "step": 340
+ },
+ {
+ "loss": 0.4813384056091309,
+ "grad_norm": 0.020505042746663094,
+ "learning_rate": 7.208e-06,
+ "entropy": 0.7920829233201221,
+ "num_tokens": 23482313.0,
+ "logits/chosen": -1.5573064750111225,
+ "logits/rejected": -1.2691127637031634,
+ "mean_token_accuracy": 0.7065368478372693,
+ "rewards/chosen": -1.0512184386729495,
+ "rewards/rejected": -2.255847032779711,
+ "rewards/accuracies": 0.77375,
+ "rewards/margins": 1.2046285913512111,
+ "logps/chosen": -63.08674698352814,
+ "logps/rejected": -78.29900344133377,
+ "epoch": 0.28003920548876843,
+ "step": 350
+ },
+ {
+ "loss": 0.4246501922607422,
+ "grad_norm": 0.014883480034768581,
+ "learning_rate": 7.128e-06,
+ "entropy": 0.7865306049736682,
+ "num_tokens": 24153305.0,
+ "logits/chosen": -1.537738084026084,
+ "logits/rejected": -1.1983942055842802,
+ "mean_token_accuracy": 0.7134439899772406,
+ "rewards/chosen": -0.9302535154198995,
+ "rewards/rejected": -2.318044547114114,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.387791032139212,
+ "logps/chosen": -61.84508213996887,
+ "logps/rejected": -76.50247138738632,
+ "epoch": 0.2880403256455904,
+ "step": 360
+ },
+ {
+ "loss": 0.45168180465698243,
+ "grad_norm": 0.01625877432525158,
+ "learning_rate": 7.048e-06,
+ "entropy": 0.7856893282332749,
+ "num_tokens": 24824508.0,
+ "logits/chosen": -1.49558786923791,
+ "logits/rejected": -1.2264479343669743,
+ "mean_token_accuracy": 0.7111923243664205,
+ "rewards/chosen": -0.859120489389345,
+ "rewards/rejected": -2.0333363087201723,
+ "rewards/accuracies": 0.8,
+ "rewards/margins": 1.1742158197984098,
+ "logps/chosen": -60.18622805953026,
+ "logps/rejected": -72.6990950036049,
+ "epoch": 0.2960414458024123,
+ "step": 370
+ },
+ {
+ "eval_loss": 0.40108245611190796,
+ "eval_runtime": 261.8573,
+ "eval_samples_per_second": 3.819,
+ "eval_steps_per_second": 3.819,
+ "eval_entropy": 0.7854411639701575,
+ "eval_num_tokens": 25157973.0,
+ "eval_logits/chosen": -1.5470742603207561,
+ "eval_logits/rejected": -1.224529391292129,
+ "eval_mean_token_accuracy": 0.7116532544195652,
+ "eval_rewards/chosen": -0.744420228220697,
+ "eval_rewards/rejected": -2.09283788038115,
+ "eval_rewards/accuracies": 0.84,
+ "eval_rewards/margins": 1.3484176517128945,
+ "eval_logps/chosen": -57.814515258789065,
+ "eval_logps/rejected": -73.84951600837708,
+ "epoch": 0.3000420058808233,
+ "step": 375
+ },
+ {
+ "loss": 0.43377981185913084,
+ "grad_norm": 0.017510831356048584,
+ "learning_rate": 6.968e-06,
+ "entropy": 0.7978322333609685,
+ "num_tokens": 25494641.0,
+ "logits/chosen": -1.4614303553748496,
+ "logits/rejected": -1.1762243386882418,
+ "mean_token_accuracy": 0.709608536399901,
+ "rewards/chosen": -0.8106335669964756,
+ "rewards/rejected": -2.098603892197134,
+ "rewards/accuracies": 0.81375,
+ "rewards/margins": 1.2879703220631926,
+ "logps/chosen": -59.16475499391556,
+ "logps/rejected": -73.48036357402802,
+ "epoch": 0.3040425659592343,
+ "step": 380
+ },
+ {
+ "loss": 0.4555358409881592,
+ "grad_norm": 0.019583892077207565,
+ "learning_rate": 6.888e-06,
+ "entropy": 0.8092104942744481,
+ "num_tokens": 26166528.0,
+ "logits/chosen": -1.5254947324121024,
+ "logits/rejected": -1.2240848010049827,
+ "mean_token_accuracy": 0.7108938498422503,
+ "rewards/chosen": -0.8702591723093428,
+ "rewards/rejected": -2.1351046158939244,
+ "rewards/accuracies": 0.7875,
+ "rewards/margins": 1.2648454446252435,
+ "logps/chosen": -61.42998598575592,
+ "logps/rejected": -76.14873928785325,
+ "epoch": 0.31204368611605626,
+ "step": 390
+ },
+ {
+ "loss": 0.46687164306640627,
+ "grad_norm": 0.017899347469210625,
+ "learning_rate": 6.808e-06,
+ "entropy": 0.8109981114600668,
+ "num_tokens": 26837910.0,
+ "logits/chosen": -1.5435783477572906,
+ "logits/rejected": -1.2533959900020157,
+ "mean_token_accuracy": 0.7053979247622192,
+ "rewards/chosen": -0.8435966002533678,
+ "rewards/rejected": -1.9857485976428142,
+ "rewards/accuracies": 0.79125,
+ "rewards/margins": 1.142151997378096,
+ "logps/chosen": -59.731822919845584,
+ "logps/rejected": -73.6962216091156,
+ "epoch": 0.3200448062728782,
+ "step": 400
+ },
+ {
+ "loss": 0.4498548984527588,
+ "grad_norm": 0.017566895112395287,
+ "learning_rate": 6.728e-06,
+ "entropy": 0.7847930058425118,
+ "num_tokens": 27508253.0,
+ "logits/chosen": -1.5778073680801266,
+ "logits/rejected": -1.2930986244729112,
+ "mean_token_accuracy": 0.7119706268794834,
+ "rewards/chosen": -0.8386218428566281,
+ "rewards/rejected": -2.119775672905962,
+ "rewards/accuracies": 0.7975,
+ "rewards/margins": 1.2811538278777153,
+ "logps/chosen": -59.29162739753723,
+ "logps/rejected": -73.53354480028152,
+ "epoch": 0.32804592642970015,
+ "step": 410
+ },
+ {
+ "loss": 0.40497851371765137,
+ "grad_norm": 0.01540332194417715,
+ "learning_rate": 6.648e-06,
+ "entropy": 0.7516823384127929,
+ "num_tokens": 28178623.0,
+ "logits/chosen": -1.6650058681382822,
+ "logits/rejected": -1.3997253712734676,
+ "mean_token_accuracy": 0.7158960890956223,
+ "rewards/chosen": -0.8034676910203415,
+ "rewards/rejected": -2.2201583882281555,
+ "rewards/accuracies": 0.81125,
+ "rewards/margins": 1.4166906926268712,
+ "logps/chosen": -59.174430742263795,
+ "logps/rejected": -74.63985994577408,
+ "epoch": 0.3360470465865221,
+ "step": 420
+ },
+ {
+ "loss": 0.4662345886230469,
+ "grad_norm": 0.018290609121322632,
+ "learning_rate": 6.568000000000001e-06,
+ "entropy": 0.7800224199614604,
+ "num_tokens": 28851605.0,
+ "logits/chosen": -1.6067276746321397,
+ "logits/rejected": -1.345846387955557,
+ "mean_token_accuracy": 0.7173346922919154,
+ "rewards/chosen": -0.9026323649001938,
+ "rewards/rejected": -2.25237619676278,
+ "rewards/accuracies": 0.79125,
+ "rewards/margins": 1.349743832880631,
+ "logps/chosen": -60.28967917919159,
+ "logps/rejected": -78.79194824457169,
+ "epoch": 0.34404816674334404,
+ "step": 430
+ },
+ {
+ "loss": 0.4252736568450928,
+ "grad_norm": 0.01900332234799862,
+ "learning_rate": 6.488000000000001e-06,
+ "entropy": 0.7889413698905264,
+ "num_tokens": 29521003.0,
+ "logits/chosen": -1.5594821986624388,
+ "logits/rejected": -1.284990779442694,
+ "mean_token_accuracy": 0.7114941450580955,
+ "rewards/chosen": -0.6923378604184837,
+ "rewards/rejected": -1.9594628562072467,
+ "rewards/accuracies": 0.8125,
+ "rewards/margins": 1.2671249943878502,
+ "logps/chosen": -57.641191160678865,
+ "logps/rejected": -71.39436632752418,
+ "epoch": 0.352049286900166,
+ "step": 440
+ },
+ {
+ "loss": 0.44404311180114747,
+ "grad_norm": 0.014757134020328522,
+ "learning_rate": 6.408000000000001e-06,
+ "entropy": 0.8029435851906601,
+ "num_tokens": 30193628.0,
+ "logits/chosen": -1.5528246220337487,
+ "logits/rejected": -1.306353942278804,
+ "mean_token_accuracy": 0.7148870236054062,
+ "rewards/chosen": -0.7460464937914366,
+ "rewards/rejected": -2.0762319400499107,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 1.3301854438055307,
+ "logps/chosen": -60.94721186161041,
+ "logps/rejected": -73.36143920898438,
+ "epoch": 0.360050407056988,
+ "step": 450
+ },
+ {
+ "loss": 0.4490717887878418,
+ "grad_norm": 0.015265013091266155,
+ "learning_rate": 6.328000000000001e-06,
+ "entropy": 0.7935610801939038,
+ "num_tokens": 30863088.0,
+ "logits/chosen": -1.4974854166216225,
+ "logits/rejected": -1.2316522249854684,
+ "mean_token_accuracy": 0.7104171360097825,
+ "rewards/chosen": -0.7627023357424195,
+ "rewards/rejected": -2.0078364689345474,
+ "rewards/accuracies": 0.7925,
+ "rewards/margins": 1.2451341325975955,
+ "logps/chosen": -59.97197327375412,
+ "logps/rejected": -72.11604419708252,
+ "epoch": 0.36805152721380996,
+ "step": 460
+ },
+ {
+ "loss": 0.41155333518981935,
+ "grad_norm": 0.013331553898751736,
+ "learning_rate": 6.248000000000001e-06,
+ "entropy": 0.8216456134367036,
+ "num_tokens": 31534941.0,
+ "logits/chosen": -1.3811194428474023,
+ "logits/rejected": -1.1138888568445255,
+ "mean_token_accuracy": 0.7219325851276517,
+ "rewards/chosen": -0.6609289994760001,
+ "rewards/rejected": -2.0405853765274515,
+ "rewards/accuracies": 0.8225,
+ "rewards/margins": 1.3796563766803593,
+ "logps/chosen": -58.38505304098129,
+ "logps/rejected": -74.04946865081787,
+ "epoch": 0.3760526473706319,
+ "step": 470
+ },
+ {
+ "loss": 0.4466897964477539,
+ "grad_norm": 0.01753743551671505,
+ "learning_rate": 6.168000000000001e-06,
+ "entropy": 0.8334384193710139,
+ "num_tokens": 32205656.0,
+ "logits/chosen": -1.3827546708003007,
+ "logits/rejected": -1.0339429527817656,
+ "mean_token_accuracy": 0.7131594355031848,
+ "rewards/chosen": -0.7439785542327445,
+ "rewards/rejected": -1.960863492001299,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.216884934026748,
+ "logps/chosen": -58.21757227420807,
+ "logps/rejected": -73.37372440934182,
+ "epoch": 0.38405376752745385,
+ "step": 480
+ },
+ {
+ "loss": 0.40377373695373536,
+ "grad_norm": 0.01816868782043457,
+ "learning_rate": 6.088000000000001e-06,
+ "entropy": 0.8024617000628496,
+ "num_tokens": 32876163.0,
+ "logits/chosen": -1.4179740237506409,
+ "logits/rejected": -1.0827905872881898,
+ "mean_token_accuracy": 0.7154640745930374,
+ "rewards/chosen": -0.7735933150124038,
+ "rewards/rejected": -2.22035426512899,
+ "rewards/accuracies": 0.81375,
+ "rewards/margins": 1.4467609539348631,
+ "logps/chosen": -57.90766100645065,
+ "logps/rejected": -77.59653000831604,
+ "epoch": 0.3920548876842758,
+ "step": 490
+ },
+ {
+ "loss": 0.41811537742614746,
+ "grad_norm": 0.019719718024134636,
+ "learning_rate": 6.008000000000001e-06,
+ "entropy": 0.7722080520819873,
+ "num_tokens": 33544655.0,
+ "logits/chosen": -1.5976364903353601,
+ "logits/rejected": -1.254849461857967,
+ "mean_token_accuracy": 0.7171562075056136,
+ "rewards/chosen": -0.8839642912911404,
+ "rewards/rejected": -2.397825531034032,
+ "rewards/accuracies": 0.825,
+ "rewards/margins": 1.5138612392637878,
+ "logps/chosen": -55.453014096021654,
+ "logps/rejected": -76.40599090814591,
+ "epoch": 0.40005600784109774,
+ "step": 500
+ },
+ {
+ "eval_loss": 0.38985610008239746,
+ "eval_runtime": 295.4363,
+ "eval_samples_per_second": 3.385,
+ "eval_steps_per_second": 3.385,
+ "eval_entropy": 0.7853469266706379,
+ "eval_num_tokens": 33544655.0,
+ "eval_logits/chosen": -1.5529572877113846,
+ "eval_logits/rejected": -1.2052691634940973,
+ "eval_mean_token_accuracy": 0.7127959969043731,
+ "eval_rewards/chosen": -0.8540078571253689,
+ "eval_rewards/rejected": -2.4217219229266047,
+ "eval_rewards/accuracies": 0.839,
+ "eval_rewards/margins": 1.567714064385742,
+ "eval_logps/chosen": -58.910391542434695,
+ "eval_logps/rejected": -77.13835635185242,
+ "epoch": 0.40005600784109774,
+ "step": 500
+ },
+ {
+ "loss": 0.4106621265411377,
+ "grad_norm": 0.01843755505979061,
+ "learning_rate": 5.928000000000001e-06,
+ "entropy": 0.7777362689268194,
+ "num_tokens": 34215067.0,
+ "logits/chosen": -1.5562227061384784,
+ "logits/rejected": -1.2522495552006923,
+ "mean_token_accuracy": 0.7179272715188563,
+ "rewards/chosen": -0.8717791981572373,
+ "rewards/rejected": -2.4079101788741535,
+ "rewards/accuracies": 0.81375,
+ "rewards/margins": 1.5361309775570406,
+ "logps/chosen": -60.166661179065706,
+ "logps/rejected": -77.11534287214279,
+ "epoch": 0.4080571279979197,
+ "step": 510
+ },
+ {
+ "loss": 0.395180869102478,
+ "grad_norm": 0.01829616166651249,
+ "learning_rate": 5.848000000000001e-06,
+ "entropy": 0.8065995912099606,
+ "num_tokens": 34885788.0,
+ "logits/chosen": -1.5098659115660116,
+ "logits/rejected": -1.157861895612393,
+ "mean_token_accuracy": 0.720119754821062,
+ "rewards/chosen": -0.7933310749317752,
+ "rewards/rejected": -2.2485245560633484,
+ "rewards/accuracies": 0.825,
+ "rewards/margins": 1.455193478623405,
+ "logps/chosen": -58.70744555234909,
+ "logps/rejected": -76.83435313940048,
+ "epoch": 0.4160582481547417,
+ "step": 520
+ },
+ {
+ "loss": 0.44591546058654785,
+ "grad_norm": 0.01715761423110962,
+ "learning_rate": 5.7680000000000005e-06,
+ "entropy": 0.773575337145885,
+ "num_tokens": 35555395.0,
+ "logits/chosen": -1.5592401225110262,
+ "logits/rejected": -1.2181071316955396,
+ "mean_token_accuracy": 0.7086451490409672,
+ "rewards/chosen": -0.8721873250235512,
+ "rewards/rejected": -2.2501098511656163,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.377922523468733,
+ "logps/chosen": -58.998940489292146,
+ "logps/rejected": -75.27887318134307,
+ "epoch": 0.4240593683115636,
+ "step": 530
+ },
+ {
+ "loss": 0.4328285217285156,
+ "grad_norm": 0.02316458150744438,
+ "learning_rate": 5.6880000000000004e-06,
+ "entropy": 0.7934144282658235,
+ "num_tokens": 36228638.0,
+ "logits/chosen": -1.4998075336206127,
+ "logits/rejected": -1.2020413938133552,
+ "mean_token_accuracy": 0.7126592384837568,
+ "rewards/chosen": -0.8903547990875086,
+ "rewards/rejected": -2.427313156935852,
+ "rewards/accuracies": 0.8025,
+ "rewards/margins": 1.5369583551678807,
+ "logps/chosen": -60.6658417057991,
+ "logps/rejected": -78.67574594974518,
+ "epoch": 0.4320604884683856,
+ "step": 540
+ },
+ {
+ "loss": 0.4486415863037109,
+ "grad_norm": 0.022637424990534782,
+ "learning_rate": 5.608e-06,
+ "entropy": 0.7808621303540713,
+ "num_tokens": 36899036.0,
+ "logits/chosen": -1.5604184278130042,
+ "logits/rejected": -1.2256144527197506,
+ "mean_token_accuracy": 0.7166568437963724,
+ "rewards/chosen": -0.8569655304190382,
+ "rewards/rejected": -2.2920745618618095,
+ "rewards/accuracies": 0.80625,
+ "rewards/margins": 1.4351090330723673,
+ "logps/chosen": -58.39388742446899,
+ "logps/rejected": -76.57233754873276,
+ "epoch": 0.44006160862520755,
+ "step": 550
+ },
+ {
+ "loss": 0.43186440467834475,
+ "grad_norm": 0.015806999057531357,
+ "learning_rate": 5.528e-06,
+ "entropy": 0.829354452828411,
+ "num_tokens": 37567966.0,
+ "logits/chosen": -1.3967952813692375,
+ "logits/rejected": -1.1090892220325619,
+ "mean_token_accuracy": 0.6998139720410108,
+ "rewards/chosen": -0.7105526073317014,
+ "rewards/rejected": -2.0654750111402245,
+ "rewards/accuracies": 0.81125,
+ "rewards/margins": 1.3549224025500006,
+ "logps/chosen": -57.93950361251831,
+ "logps/rejected": -72.99591649293899,
+ "epoch": 0.44806272878202946,
+ "step": 560
+ },
+ {
+ "loss": 0.43549156188964844,
+ "grad_norm": 0.021929794922471046,
+ "learning_rate": 5.448e-06,
+ "entropy": 0.8052812117640861,
+ "num_tokens": 38238610.0,
+ "logits/chosen": -1.4406510095404184,
+ "logits/rejected": -1.1625784526819891,
+ "mean_token_accuracy": 0.7065126617625356,
+ "rewards/chosen": -0.7944385814492125,
+ "rewards/rejected": -2.1914290168089794,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.396990433158353,
+ "logps/chosen": -59.14217608451843,
+ "logps/rejected": -75.0877578663826,
+ "epoch": 0.45606384893885143,
+ "step": 570
+ },
+ {
+ "loss": 0.4129189968109131,
+ "grad_norm": 0.01712222397327423,
+ "learning_rate": 5.368000000000001e-06,
+ "entropy": 0.8094228478927107,
+ "num_tokens": 38908212.0,
+ "logits/chosen": -1.407300765264523,
+ "logits/rejected": -1.0415596670678384,
+ "mean_token_accuracy": 0.7172896678000689,
+ "rewards/chosen": -0.7117314671058557,
+ "rewards/rejected": -2.1860597877670807,
+ "rewards/accuracies": 0.8175,
+ "rewards/margins": 1.4743283203756437,
+ "logps/chosen": -56.23722167730332,
+ "logps/rejected": -74.95896697044373,
+ "epoch": 0.4640649690956734,
+ "step": 580
+ },
+ {
+ "loss": 0.43604726791381837,
+ "grad_norm": 0.015477149747312069,
+ "learning_rate": 5.288000000000001e-06,
+ "entropy": 0.8548492413954227,
+ "num_tokens": 39579214.0,
+ "logits/chosen": -1.2922011359963301,
+ "logits/rejected": -0.9956562484834224,
+ "mean_token_accuracy": 0.7064971200935543,
+ "rewards/chosen": -0.7065712768954109,
+ "rewards/rejected": -2.107755633329507,
+ "rewards/accuracies": 0.82125,
+ "rewards/margins": 1.4011843568086624,
+ "logps/chosen": -59.846063117980954,
+ "logps/rejected": -74.58847686052323,
+ "epoch": 0.4720660892524953,
+ "step": 590
+ },
+ {
+ "loss": 0.4239354610443115,
+ "grad_norm": 0.02124476432800293,
+ "learning_rate": 5.208000000000001e-06,
+ "entropy": 0.824605501004844,
+ "num_tokens": 40251415.0,
+ "logits/chosen": -1.3701909734490212,
+ "logits/rejected": -0.9900921016783352,
+ "mean_token_accuracy": 0.7211226490046829,
+ "rewards/chosen": -0.7936599334669882,
+ "rewards/rejected": -2.3192991207691374,
+ "rewards/accuracies": 0.81625,
+ "rewards/margins": 1.5256391859147698,
+ "logps/chosen": -59.08933132648468,
+ "logps/rejected": -79.14344486474991,
+ "epoch": 0.4800672094093173,
+ "step": 600
+ },
+ {
+ "loss": 0.4354452133178711,
+ "grad_norm": 0.023743441328406334,
+ "learning_rate": 5.128000000000001e-06,
+ "entropy": 0.825452755644219,
+ "num_tokens": 40920910.0,
+ "logits/chosen": -1.390550910390904,
+ "logits/rejected": -1.0735959651479647,
+ "mean_token_accuracy": 0.713898301422596,
+ "rewards/chosen": -0.789569493080653,
+ "rewards/rejected": -2.1930771789352,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.403507683072239,
+ "logps/chosen": -58.420460934638974,
+ "logps/rejected": -74.17758195877076,
+ "epoch": 0.48806832956613927,
+ "step": 610
+ },
+ {
+ "loss": 0.4045247077941895,
+ "grad_norm": 0.01405557431280613,
+ "learning_rate": 5.048000000000001e-06,
+ "entropy": 0.8011420608652406,
+ "num_tokens": 41593142.0,
+ "logits/chosen": -1.3975533947414562,
+ "logits/rejected": -1.0831290848973116,
+ "mean_token_accuracy": 0.7130240154080093,
+ "rewards/chosen": -0.8486350445944845,
+ "rewards/rejected": -2.4122694664489246,
+ "rewards/accuracies": 0.83,
+ "rewards/margins": 1.5636344207916408,
+ "logps/chosen": -60.129820008277896,
+ "logps/rejected": -76.31982692241668,
+ "epoch": 0.49606944972296124,
+ "step": 620
+ },
+ {
+ "eval_loss": 0.38206014037132263,
+ "eval_runtime": 326.7329,
+ "eval_samples_per_second": 3.061,
+ "eval_steps_per_second": 3.061,
+ "eval_entropy": 0.8252472431890201,
+ "eval_num_tokens": 41930819.0,
+ "eval_logits/chosen": -1.36153454710921,
+ "eval_logits/rejected": -1.0058315979395118,
+ "eval_mean_token_accuracy": 0.7101194297969341,
+ "eval_rewards/chosen": -0.7108273756076232,
+ "eval_rewards/rejected": -2.305952288910863,
+ "eval_rewards/accuracies": 0.846,
+ "eval_rewards/margins": 1.5951249125674367,
+ "eval_logps/chosen": -57.47858674812317,
+ "eval_logps/rejected": -75.98066004753113,
+ "epoch": 0.5000700098013722,
+ "step": 625
+ },
+ {
+ "loss": 0.3930924654006958,
+ "grad_norm": 0.01451192144304514,
+ "learning_rate": 4.9680000000000005e-06,
+ "entropy": 0.8378929265070474,
+ "num_tokens": 42266772.0,
+ "logits/chosen": -1.315587886950433,
+ "logits/rejected": -1.0099089304924136,
+ "mean_token_accuracy": 0.7140151102561504,
+ "rewards/chosen": -0.7489332604239461,
+ "rewards/rejected": -2.328624072318198,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.5796908099856228,
+ "logps/chosen": -62.18757668733597,
+ "logps/rejected": -77.6511843419075,
+ "epoch": 0.5040705698797832,
+ "step": 630
+ },
+ {
+ "loss": 0.44039368629455566,
+ "grad_norm": 0.017432237043976784,
+ "learning_rate": 4.8880000000000005e-06,
+ "entropy": 0.8770715677752742,
+ "num_tokens": 42938338.0,
+ "logits/chosen": -1.3235264756797693,
+ "logits/rejected": -0.9964174353869055,
+ "mean_token_accuracy": 0.7056646738946438,
+ "rewards/chosen": -0.6992467068618862,
+ "rewards/rejected": -2.0929239434680857,
+ "rewards/accuracies": 0.8025,
+ "rewards/margins": 1.3936772356089204,
+ "logps/chosen": -59.22969470500946,
+ "logps/rejected": -76.437605779171,
+ "epoch": 0.5120716900366051,
+ "step": 640
+ },
+ {
+ "loss": 0.4005030632019043,
+ "grad_norm": 0.016337474808096886,
+ "learning_rate": 4.808e-06,
+ "entropy": 0.8228744911667308,
+ "num_tokens": 43607368.0,
+ "logits/chosen": -1.365069605441217,
+ "logits/rejected": -1.040179422743084,
+ "mean_token_accuracy": 0.7150637634471059,
+ "rewards/chosen": -0.6480746366505628,
+ "rewards/rejected": -2.150714794436935,
+ "rewards/accuracies": 0.835,
+ "rewards/margins": 1.5026401576166972,
+ "logps/chosen": -56.05587344884872,
+ "logps/rejected": -73.34227110147476,
+ "epoch": 0.5200728101934271,
+ "step": 650
+ },
+ {
+ "loss": 0.4082051753997803,
+ "grad_norm": 0.018142661079764366,
+ "learning_rate": 4.728e-06,
+ "entropy": 0.8530499871546635,
+ "num_tokens": 44278185.0,
+ "logits/chosen": -1.3381476572038138,
+ "logits/rejected": -0.9844825739754265,
+ "mean_token_accuracy": 0.711031482918188,
+ "rewards/chosen": -0.7167994258418912,
+ "rewards/rejected": -2.222132046652259,
+ "rewards/accuracies": 0.8325,
+ "rewards/margins": 1.5053326183510944,
+ "logps/chosen": -58.75081548690796,
+ "logps/rejected": -74.44041372299195,
+ "epoch": 0.5280739303502491,
+ "step": 660
+ },
+ {
+ "loss": 0.4182604789733887,
+ "grad_norm": 0.020815948024392128,
+ "learning_rate": 4.648e-06,
+ "entropy": 0.8168351130496012,
+ "num_tokens": 44950052.0,
+ "logits/chosen": -1.4143566707732802,
+ "logits/rejected": -1.0639793929044592,
+ "mean_token_accuracy": 0.7124683810770511,
+ "rewards/chosen": -0.7617488951507266,
+ "rewards/rejected": -2.276013866282883,
+ "rewards/accuracies": 0.81375,
+ "rewards/margins": 1.5142649730667472,
+ "logps/chosen": -56.72195715665817,
+ "logps/rejected": -77.04312695026398,
+ "epoch": 0.5360750505070709,
+ "step": 670
+ },
+ {
+ "loss": 0.4252179145812988,
+ "grad_norm": 0.020110271871089935,
+ "learning_rate": 4.568e-06,
+ "entropy": 0.8308680512331194,
+ "num_tokens": 45623022.0,
+ "logits/chosen": -1.3950534629812166,
+ "logits/rejected": -1.0998214083286344,
+ "mean_token_accuracy": 0.7056502989120781,
+ "rewards/chosen": -0.8144988723962023,
+ "rewards/rejected": -2.389123064740561,
+ "rewards/accuracies": 0.80875,
+ "rewards/margins": 1.5746241921419277,
+ "logps/chosen": -60.53876167535782,
+ "logps/rejected": -78.16164077043533,
+ "epoch": 0.5440761706638929,
+ "step": 680
+ },
+ {
+ "loss": 0.45481224060058595,
+ "grad_norm": 0.01787034422159195,
+ "learning_rate": 4.488e-06,
+ "entropy": 0.8564485525735654,
+ "num_tokens": 46293504.0,
+ "logits/chosen": -1.3725143150009782,
+ "logits/rejected": -1.0610108838467847,
+ "mean_token_accuracy": 0.7121961736120284,
+ "rewards/chosen": -0.7007349021799746,
+ "rewards/rejected": -2.1405522361784826,
+ "rewards/accuracies": 0.80875,
+ "rewards/margins": 1.4398173329560087,
+ "logps/chosen": -58.63401502370834,
+ "logps/rejected": -75.10802860021592,
+ "epoch": 0.5520772908207149,
+ "step": 690
+ },
+ {
+ "loss": 0.4242884635925293,
+ "grad_norm": 0.019236121326684952,
+ "learning_rate": 4.408000000000001e-06,
+ "entropy": 0.8544818184821633,
+ "num_tokens": 46964981.0,
+ "logits/chosen": -1.349378895644786,
+ "logits/rejected": -1.0686512966747685,
+ "mean_token_accuracy": 0.7098362519778312,
+ "rewards/chosen": -0.6842107876761293,
+ "rewards/rejected": -2.1552941927440408,
+ "rewards/accuracies": 0.82125,
+ "rewards/margins": 1.4710834045434604,
+ "logps/chosen": -58.09884097576141,
+ "logps/rejected": -74.92867855072022,
+ "epoch": 0.5600784109775369,
+ "step": 700
+ },
+ {
+ "loss": 0.4102015018463135,
+ "grad_norm": 0.02260979637503624,
+ "learning_rate": 4.328000000000001e-06,
+ "entropy": 0.8242619492893573,
+ "num_tokens": 47635031.0,
+ "logits/chosen": -1.4417073731696255,
+ "logits/rejected": -1.15610772613027,
+ "mean_token_accuracy": 0.711150385774672,
+ "rewards/chosen": -0.6375206779901055,
+ "rewards/rejected": -2.1217567729702567,
+ "rewards/accuracies": 0.81625,
+ "rewards/margins": 1.484236096283421,
+ "logps/chosen": -58.721065254211425,
+ "logps/rejected": -73.21105932235717,
+ "epoch": 0.5680795311343588,
+ "step": 710
+ },
+ {
+ "loss": 0.4387828826904297,
+ "grad_norm": 0.02488075941801071,
+ "learning_rate": 4.248000000000001e-06,
+ "entropy": 0.7898647125309799,
+ "num_tokens": 48305521.0,
+ "logits/chosen": -1.535454146525396,
+ "logits/rejected": -1.1988366438783475,
+ "mean_token_accuracy": 0.7144390594959259,
+ "rewards/chosen": -0.6357523593801306,
+ "rewards/rejected": -2.0578468732352486,
+ "rewards/accuracies": 0.8,
+ "rewards/margins": 1.4220945093501358,
+ "logps/chosen": -54.41331891536713,
+ "logps/rejected": -75.98147065877914,
+ "epoch": 0.5760806512911808,
+ "step": 720
+ },
+ {
+ "loss": 0.39289658069610595,
+ "grad_norm": 0.015159406699240208,
+ "learning_rate": 4.168000000000001e-06,
+ "entropy": 0.8010593402700033,
+ "num_tokens": 48976282.0,
+ "logits/chosen": -1.55113046097786,
+ "logits/rejected": -1.2506834203141584,
+ "mean_token_accuracy": 0.7138257564418018,
+ "rewards/chosen": -0.5929771073686425,
+ "rewards/rejected": -2.1302011678872077,
+ "rewards/accuracies": 0.83375,
+ "rewards/margins": 1.5372240616590716,
+ "logps/chosen": -56.13849130153656,
+ "logps/rejected": -73.13589999198913,
+ "epoch": 0.5840817714480028,
+ "step": 730
+ },
+ {
+ "loss": 0.41160149574279786,
+ "grad_norm": 0.01860692724585533,
+ "learning_rate": 4.0880000000000005e-06,
+ "entropy": 0.7805629429890542,
+ "num_tokens": 49648195.0,
+ "logits/chosen": -1.6207133992642588,
+ "logits/rejected": -1.2987192623027932,
+ "mean_token_accuracy": 0.7162860633060336,
+ "rewards/chosen": -0.7419188594727893,
+ "rewards/rejected": -2.275629919102648,
+ "rewards/accuracies": 0.825,
+ "rewards/margins": 1.5337110587162897,
+ "logps/chosen": -58.221080236434936,
+ "logps/rejected": -76.93056843519211,
+ "epoch": 0.5920828916048246,
+ "step": 740
+ },
+ {
+ "loss": 0.4270487785339355,
+ "grad_norm": 0.016421068459749222,
+ "learning_rate": 4.008e-06,
+ "entropy": 0.8073938890940917,
+ "num_tokens": 50318801.0,
+ "logits/chosen": -1.5882363637098387,
+ "logits/rejected": -1.247667502316742,
+ "mean_token_accuracy": 0.7069075590185822,
+ "rewards/chosen": -0.8003950071084546,
+ "rewards/rejected": -2.452392807676806,
+ "rewards/accuracies": 0.825,
+ "rewards/margins": 1.651997799584642,
+ "logps/chosen": -59.83541061878204,
+ "logps/rejected": -78.04929527282715,
+ "epoch": 0.6000840117616466,
+ "step": 750
+ },
+ {
+ "eval_loss": 0.3747462034225464,
+ "eval_runtime": 359.9421,
+ "eval_samples_per_second": 2.778,
+ "eval_steps_per_second": 2.778,
+ "eval_entropy": 0.7970051126877079,
+ "eval_num_tokens": 50318801.0,
+ "eval_logits/chosen": -1.6053054796594703,
+ "eval_logits/rejected": -1.247482501886518,
+ "eval_mean_token_accuracy": 0.7094336547851563,
+ "eval_rewards/chosen": -0.7686092760775937,
+ "eval_rewards/rejected": -2.5153538503656163,
+ "eval_rewards/accuracies": 0.841,
+ "eval_rewards/margins": 1.7467445721179247,
+ "eval_logps/chosen": -58.05640573692322,
+ "eval_logps/rejected": -78.07467563819885,
+ "epoch": 0.6000840117616466,
+ "step": 750
+ },
+ {
+ "loss": 0.4169970989227295,
+ "grad_norm": 0.03688720986247063,
+ "learning_rate": 3.928e-06,
+ "entropy": 0.7846462618000806,
+ "num_tokens": 50989250.0,
+ "logits/chosen": -1.6120100895242033,
+ "logits/rejected": -1.2415178312508848,
+ "mean_token_accuracy": 0.7159298903495074,
+ "rewards/chosen": -0.839654815679096,
+ "rewards/rejected": -2.502948221999686,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.6632934046909214,
+ "logps/chosen": -56.75322003364563,
+ "logps/rejected": -78.04070549488068,
+ "epoch": 0.6080851319184686,
+ "step": 760
+ },
+ {
+ "loss": 0.4282489776611328,
+ "grad_norm": 0.017638148739933968,
+ "learning_rate": 3.848e-06,
+ "entropy": 0.798651073614019,
+ "num_tokens": 51660938.0,
+ "logits/chosen": -1.5328797896282882,
+ "logits/rejected": -1.2321443010934947,
+ "mean_token_accuracy": 0.7048248733580113,
+ "rewards/chosen": -0.8160745739022969,
+ "rewards/rejected": -2.404342279869743,
+ "rewards/accuracies": 0.82625,
+ "rewards/margins": 1.5882677041646094,
+ "logps/chosen": -59.79865051269531,
+ "logps/rejected": -77.78480454921723,
+ "epoch": 0.6160862520752906,
+ "step": 770
+ },
+ {
+ "loss": 0.4250539779663086,
+ "grad_norm": 0.01926429010927677,
+ "learning_rate": 3.7680000000000006e-06,
+ "entropy": 0.8206474994809833,
+ "num_tokens": 52332511.0,
+ "logits/chosen": -1.5045491055990166,
+ "logits/rejected": -1.1643194357904556,
+ "mean_token_accuracy": 0.7142482680641115,
+ "rewards/chosen": -0.7649500152390101,
+ "rewards/rejected": -2.319859855013201,
+ "rewards/accuracies": 0.8125,
+ "rewards/margins": 1.5549098403338575,
+ "logps/chosen": -60.40905921697617,
+ "logps/rejected": -77.19861435890198,
+ "epoch": 0.6240873722321125,
+ "step": 780
+ },
+ {
+ "loss": 0.42780022621154784,
+ "grad_norm": 0.01716187596321106,
+ "learning_rate": 3.6880000000000005e-06,
+ "entropy": 0.7971815115597565,
+ "num_tokens": 53000638.0,
+ "logits/chosen": -1.585354376363884,
+ "logits/rejected": -1.2273074579325267,
+ "mean_token_accuracy": 0.707803654409945,
+ "rewards/chosen": -0.7402846401413262,
+ "rewards/rejected": -2.2537980271608102,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.5135133841168136,
+ "logps/chosen": -56.16835569143295,
+ "logps/rejected": -75.9030461883545,
+ "epoch": 0.6320884923889345,
+ "step": 790
+ },
+ {
+ "loss": 0.439426851272583,
+ "grad_norm": 0.021782318130135536,
+ "learning_rate": 3.6080000000000004e-06,
+ "entropy": 0.8156255233831325,
+ "num_tokens": 53671873.0,
+ "logits/chosen": -1.5731947810052915,
+ "logits/rejected": -1.2208077437038316,
+ "mean_token_accuracy": 0.7152832678332924,
+ "rewards/chosen": -0.7687084915324521,
+ "rewards/rejected": -2.2547003319143553,
+ "rewards/accuracies": 0.81125,
+ "rewards/margins": 1.4859918395150453,
+ "logps/chosen": -58.14893871307373,
+ "logps/rejected": -78.18987871646881,
+ "epoch": 0.6400896125457564,
+ "step": 800
+ },
+ {
+ "loss": 0.420841646194458,
+ "grad_norm": 0.025692911818623543,
+ "learning_rate": 3.5280000000000004e-06,
+ "entropy": 0.821183467732335,
+ "num_tokens": 54343089.0,
+ "logits/chosen": -1.5207735061927403,
+ "logits/rejected": -1.2057331729540248,
+ "mean_token_accuracy": 0.7104452795907855,
+ "rewards/chosen": -0.780645934283275,
+ "rewards/rejected": -2.3124329587374812,
+ "rewards/accuracies": 0.8075,
+ "rewards/margins": 1.531787025486119,
+ "logps/chosen": -60.014517648220064,
+ "logps/rejected": -78.12048140048981,
+ "epoch": 0.6480907327025783,
+ "step": 810
+ },
+ {
+ "loss": 0.4105512619018555,
+ "grad_norm": 0.01884465478360653,
+ "learning_rate": 3.4480000000000003e-06,
+ "entropy": 0.8063031704328023,
+ "num_tokens": 55014978.0,
+ "logits/chosen": -1.547767386796032,
+ "logits/rejected": -1.1820112378623413,
+ "mean_token_accuracy": 0.7089173524640501,
+ "rewards/chosen": -0.7608894053113181,
+ "rewards/rejected": -2.441992430248065,
+ "rewards/accuracies": 0.8325,
+ "rewards/margins": 1.6811030247248708,
+ "logps/chosen": -60.38896199703217,
+ "logps/rejected": -78.01072539567947,
+ "epoch": 0.6560918528594003,
+ "step": 820
+ },
+ {
+ "loss": 0.4294763088226318,
+ "grad_norm": 0.019730493426322937,
+ "learning_rate": 3.368e-06,
+ "entropy": 0.8684769855998457,
+ "num_tokens": 55689151.0,
+ "logits/chosen": -1.3910758939427397,
+ "logits/rejected": -1.1155818673408202,
+ "mean_token_accuracy": 0.7053723630867899,
+ "rewards/chosen": -0.7968793028977234,
+ "rewards/rejected": -2.375062556483317,
+ "rewards/accuracies": 0.825,
+ "rewards/margins": 1.5781832543481142,
+ "logps/chosen": -63.42195160150528,
+ "logps/rejected": -80.02929930448532,
+ "epoch": 0.6640929730162223,
+ "step": 830
+ },
+ {
+ "loss": 0.4067112445831299,
+ "grad_norm": 0.015371326357126236,
+ "learning_rate": 3.288e-06,
+ "entropy": 0.8669515595107805,
+ "num_tokens": 56361294.0,
+ "logits/chosen": -1.4120339153963328,
+ "logits/rejected": -1.0943624791595064,
+ "mean_token_accuracy": 0.7072863762266934,
+ "rewards/chosen": -0.7408881269737049,
+ "rewards/rejected": -2.295605608313781,
+ "rewards/accuracies": 0.805,
+ "rewards/margins": 1.5547174850385637,
+ "logps/chosen": -60.02739023685455,
+ "logps/rejected": -77.22145056962967,
+ "epoch": 0.6720940931730442,
+ "step": 840
+ },
+ {
+ "loss": 0.4195101261138916,
+ "grad_norm": 0.017242170870304108,
+ "learning_rate": 3.208e-06,
+ "entropy": 0.8097485016728752,
+ "num_tokens": 57030353.0,
+ "logits/chosen": -1.5199316608931759,
+ "logits/rejected": -1.1795340725304932,
+ "mean_token_accuracy": 0.7123510077036918,
+ "rewards/chosen": -0.7133299406401784,
+ "rewards/rejected": -2.192705774551141,
+ "rewards/accuracies": 0.82625,
+ "rewards/margins": 1.4793758322671056,
+ "logps/chosen": -55.20628922462463,
+ "logps/rejected": -73.9845145535469,
+ "epoch": 0.6800952133298662,
+ "step": 850
+ },
+ {
+ "loss": 0.4102588176727295,
+ "grad_norm": 0.01566699519753456,
+ "learning_rate": 3.1280000000000004e-06,
+ "entropy": 0.8263511486476637,
+ "num_tokens": 57700689.0,
+ "logits/chosen": -1.4269044114958789,
+ "logits/rejected": -1.1771139106072332,
+ "mean_token_accuracy": 0.7052110943943262,
+ "rewards/chosen": -0.6530551976517165,
+ "rewards/rejected": -2.1147736522718334,
+ "rewards/accuracies": 0.8125,
+ "rewards/margins": 1.461718455082737,
+ "logps/chosen": -61.08844540596008,
+ "logps/rejected": -72.37894055843353,
+ "epoch": 0.6880963334866881,
+ "step": 860
+ },
+ {
+ "loss": 0.42830562591552734,
+ "grad_norm": 0.02185574173927307,
+ "learning_rate": 3.0480000000000003e-06,
+ "entropy": 0.8228394509857754,
+ "num_tokens": 58371582.0,
+ "logits/chosen": -1.4636678357654866,
+ "logits/rejected": -1.117573851293101,
+ "mean_token_accuracy": 0.7181230850890279,
+ "rewards/chosen": -0.7092919610657191,
+ "rewards/rejected": -2.242413058922393,
+ "rewards/accuracies": 0.82625,
+ "rewards/margins": 1.5331210961006583,
+ "logps/chosen": -57.20385055541992,
+ "logps/rejected": -76.02769055843353,
+ "epoch": 0.6960974536435101,
+ "step": 870
+ },
+ {
+ "eval_loss": 0.3759123682975769,
+ "eval_runtime": 399.2571,
+ "eval_samples_per_second": 2.505,
+ "eval_steps_per_second": 2.505,
+ "eval_entropy": 0.8344861168425995,
+ "eval_num_tokens": 58705924.0,
+ "eval_logits/chosen": -1.4575140190597924,
+ "eval_logits/rejected": -1.1039781761695373,
+ "eval_mean_token_accuracy": 0.711262987613678,
+ "eval_rewards/chosen": -0.6297580211221648,
+ "eval_rewards/rejected": -2.276288676444092,
+ "eval_rewards/accuracies": 0.842,
+ "eval_rewards/margins": 1.6465306548774243,
+ "eval_logps/chosen": -56.667893226623534,
+ "eval_logps/rejected": -75.6840239276886,
+ "epoch": 0.700098013721921,
+ "step": 875
+ },
+ {
+ "loss": 0.4234786510467529,
+ "grad_norm": 0.02065058797597885,
+ "learning_rate": 2.9680000000000002e-06,
+ "entropy": 0.8277974076462852,
+ "num_tokens": 59040465.0,
+ "logits/chosen": -1.5135564880748953,
+ "logits/rejected": -1.1432497618107653,
+ "mean_token_accuracy": 0.7117055612802505,
+ "rewards/chosen": -0.696259319268429,
+ "rewards/rejected": -2.2431890765414573,
+ "rewards/accuracies": 0.8075,
+ "rewards/margins": 1.5469297548476606,
+ "logps/chosen": -55.49664348840714,
+ "logps/rejected": -77.08198328495025,
+ "epoch": 0.704098573800332,
+ "step": 880
+ },
+ {
+ "loss": 0.4353479862213135,
+ "grad_norm": 0.02186797559261322,
+ "learning_rate": 2.888e-06,
+ "entropy": 0.8144791065354365,
+ "num_tokens": 59710660.0,
+ "logits/chosen": -1.4794269513186382,
+ "logits/rejected": -1.1591747705292565,
+ "mean_token_accuracy": 0.7170553574152291,
+ "rewards/chosen": -0.7273707000938157,
+ "rewards/rejected": -2.210361190639669,
+ "rewards/accuracies": 0.80125,
+ "rewards/margins": 1.4829904899653048,
+ "logps/chosen": -56.91487885832787,
+ "logps/rejected": -74.57020094871521,
+ "epoch": 0.712099693957154,
+ "step": 890
+ },
+ {
+ "loss": 0.43715639114379884,
+ "grad_norm": 0.020525585860013962,
+ "learning_rate": 2.808e-06,
+ "entropy": 0.8120913207791455,
+ "num_tokens": 60381052.0,
+ "logits/chosen": -1.4459924833017133,
+ "logits/rejected": -1.1256531892985777,
+ "mean_token_accuracy": 0.7126112458109856,
+ "rewards/chosen": -0.6970637990878459,
+ "rewards/rejected": -2.1881442813354077,
+ "rewards/accuracies": 0.8025,
+ "rewards/margins": 1.4910804780339821,
+ "logps/chosen": -56.196629449129105,
+ "logps/rejected": -74.68293124079705,
+ "epoch": 0.720100814113976,
+ "step": 900
+ },
+ {
+ "loss": 0.4009076118469238,
+ "grad_norm": 0.0200913455337286,
+ "learning_rate": 2.728e-06,
+ "entropy": 0.820758284192998,
+ "num_tokens": 61049450.0,
+ "logits/chosen": -1.468269785823377,
+ "logits/rejected": -1.1344321171992913,
+ "mean_token_accuracy": 0.7174195778183639,
+ "rewards/chosen": -0.709326482572651,
+ "rewards/rejected": -2.28662730479904,
+ "rewards/accuracies": 0.81875,
+ "rewards/margins": 1.5773008212819695,
+ "logps/chosen": -56.747891561985014,
+ "logps/rejected": -73.50010628700257,
+ "epoch": 0.728101934270798,
+ "step": 910
+ },
+ {
+ "loss": 0.4346621513366699,
+ "grad_norm": 0.01706886664032936,
+ "learning_rate": 2.648e-06,
+ "entropy": 0.8205121580697596,
+ "num_tokens": 61718430.0,
+ "logits/chosen": -1.4375778086851516,
+ "logits/rejected": -1.161296226506176,
+ "mean_token_accuracy": 0.7136667492613197,
+ "rewards/chosen": -0.7132218919423212,
+ "rewards/rejected": -2.13055392677139,
+ "rewards/accuracies": 0.80125,
+ "rewards/margins": 1.4173320323741063,
+ "logps/chosen": -57.20179939746857,
+ "logps/rejected": -72.76975549459458,
+ "epoch": 0.7361030544276199,
+ "step": 920
+ },
+ {
+ "loss": 0.3956768035888672,
+ "grad_norm": 0.01797674037516117,
+ "learning_rate": 2.568e-06,
+ "entropy": 0.8012085157257388,
+ "num_tokens": 62389582.0,
+ "logits/chosen": -1.532120015247838,
+ "logits/rejected": -1.208119469186731,
+ "mean_token_accuracy": 0.7055639943294227,
+ "rewards/chosen": -0.7599997415716644,
+ "rewards/rejected": -2.4362060464013484,
+ "rewards/accuracies": 0.825,
+ "rewards/margins": 1.6762063066801056,
+ "logps/chosen": -57.179485994577405,
+ "logps/rejected": -79.18268291473389,
+ "epoch": 0.7441041745844418,
+ "step": 930
+ },
+ {
+ "loss": 0.4311192035675049,
+ "grad_norm": 0.017888743430376053,
+ "learning_rate": 2.488e-06,
+ "entropy": 0.7985861063818447,
+ "num_tokens": 63059623.0,
+ "logits/chosen": -1.558568974228593,
+ "logits/rejected": -1.2086813305598347,
+ "mean_token_accuracy": 0.7144212306663394,
+ "rewards/chosen": -0.7640767650106136,
+ "rewards/rejected": -2.2844264058230874,
+ "rewards/accuracies": 0.8075,
+ "rewards/margins": 1.5203496412816457,
+ "logps/chosen": -56.44099218845368,
+ "logps/rejected": -74.95756293773651,
+ "epoch": 0.7521052947412638,
+ "step": 940
+ },
+ {
+ "loss": 0.44293651580810545,
+ "grad_norm": 0.024403788149356842,
+ "learning_rate": 2.408e-06,
+ "entropy": 0.8035592026286759,
+ "num_tokens": 63729323.0,
+ "logits/chosen": -1.5457893383254793,
+ "logits/rejected": -1.196683114693986,
+ "mean_token_accuracy": 0.7051368881203234,
+ "rewards/chosen": -0.8423608328169212,
+ "rewards/rejected": -2.2869279081953935,
+ "rewards/accuracies": 0.8125,
+ "rewards/margins": 1.4445670791901648,
+ "logps/chosen": -58.55195621013641,
+ "logps/rejected": -75.49890763282775,
+ "epoch": 0.7601064148980857,
+ "step": 950
+ },
+ {
+ "loss": 0.4008366107940674,
+ "grad_norm": 0.01861320622265339,
+ "learning_rate": 2.3280000000000004e-06,
+ "entropy": 0.7981485390500166,
+ "num_tokens": 64401251.0,
+ "logits/chosen": -1.5139035842305524,
+ "logits/rejected": -1.1744702669983618,
+ "mean_token_accuracy": 0.7120618709363044,
+ "rewards/chosen": -0.7763427560425771,
+ "rewards/rejected": -2.4696535951615077,
+ "rewards/accuracies": 0.84,
+ "rewards/margins": 1.6933108403813093,
+ "logps/chosen": -58.61193150997162,
+ "logps/rejected": -78.02690306186676,
+ "epoch": 0.7681075350549077,
+ "step": 960
+ },
+ {
+ "loss": 0.4489262104034424,
+ "grad_norm": 0.015494938008487225,
+ "learning_rate": 2.2480000000000003e-06,
+ "entropy": 0.8330775782934506,
+ "num_tokens": 65071349.0,
+ "logits/chosen": -1.4378532668123365,
+ "logits/rejected": -1.1536408081373142,
+ "mean_token_accuracy": 0.7020319653488696,
+ "rewards/chosen": -0.7739227567915805,
+ "rewards/rejected": -2.280776680170675,
+ "rewards/accuracies": 0.81375,
+ "rewards/margins": 1.5068539200443773,
+ "logps/chosen": -59.13198090314865,
+ "logps/rejected": -74.87351196527482,
+ "epoch": 0.7761086552117297,
+ "step": 970
+ },
+ {
+ "loss": 0.42146644592285154,
+ "grad_norm": 0.01722661592066288,
+ "learning_rate": 2.1680000000000002e-06,
+ "entropy": 0.83052067089302,
+ "num_tokens": 65743620.0,
+ "logits/chosen": -1.4442045437350117,
+ "logits/rejected": -1.1513308155926538,
+ "mean_token_accuracy": 0.7084798959642649,
+ "rewards/chosen": -0.7037231323859305,
+ "rewards/rejected": -2.252449466900289,
+ "rewards/accuracies": 0.82125,
+ "rewards/margins": 1.5487263337243349,
+ "logps/chosen": -60.383059858083726,
+ "logps/rejected": -76.19223057866097,
+ "epoch": 0.7841097753685516,
+ "step": 980
+ },
+ {
+ "loss": 0.4122499942779541,
+ "grad_norm": 0.019420692697167397,
+ "learning_rate": 2.088e-06,
+ "entropy": 0.8109630790716619,
+ "num_tokens": 66413395.0,
+ "logits/chosen": -1.541120669321979,
+ "logits/rejected": -1.2107965185782945,
+ "mean_token_accuracy": 0.7163080858811736,
+ "rewards/chosen": -0.7509561662202031,
+ "rewards/rejected": -2.339945316381636,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.5889891442121007,
+ "logps/chosen": -57.66542109251022,
+ "logps/rejected": -76.55372718334198,
+ "epoch": 0.7921108955253735,
+ "step": 990
+ },
+ {
+ "loss": 0.4178761005401611,
+ "grad_norm": 0.020704196766018867,
+ "learning_rate": 2.008e-06,
+ "entropy": 0.816696729575342,
+ "num_tokens": 67084018.0,
+ "logits/chosen": -1.4885645002637296,
+ "logits/rejected": -1.1673584722357746,
+ "mean_token_accuracy": 0.7150707822386175,
+ "rewards/chosen": -0.741308625468664,
+ "rewards/rejected": -2.406399813084354,
+ "rewards/accuracies": 0.825,
+ "rewards/margins": 1.6650911844847724,
+ "logps/chosen": -57.74228939056397,
+ "logps/rejected": -77.05882190465927,
+ "epoch": 0.8001120156821955,
+ "step": 1000
+ },
+ {
+ "eval_loss": 0.37232494354248047,
+ "eval_runtime": 431.9338,
+ "eval_samples_per_second": 2.315,
+ "eval_steps_per_second": 2.315,
+ "eval_entropy": 0.816042770717293,
+ "eval_num_tokens": 67084018.0,
+ "eval_logits/chosen": -1.533623475146162,
+ "eval_logits/rejected": -1.1799501516011692,
+ "eval_mean_token_accuracy": 0.7080534865856171,
+ "eval_rewards/chosen": -0.6996203318120678,
+ "eval_rewards/rejected": -2.440506931256736,
+ "eval_rewards/accuracies": 0.842,
+ "eval_rewards/margins": 1.7408866018354894,
+ "eval_logps/chosen": -57.36651630592346,
+ "eval_logps/rejected": -77.32620645523072,
+ "epoch": 0.8001120156821955,
+ "step": 1000
+ },
+ {
+ "loss": 0.37767949104309084,
+ "grad_norm": 0.02488139644265175,
+ "learning_rate": 1.928e-06,
+ "entropy": 0.834057361006926,
+ "num_tokens": 67756375.0,
+ "logits/chosen": -1.4910327476329437,
+ "logits/rejected": -1.1754918204218958,
+ "mean_token_accuracy": 0.7165563557669521,
+ "rewards/chosen": -0.74927385541072,
+ "rewards/rejected": -2.466023100640159,
+ "rewards/accuracies": 0.83875,
+ "rewards/margins": 1.716749248035485,
+ "logps/chosen": -59.18668337464332,
+ "logps/rejected": -79.89627618789673,
+ "epoch": 0.8081131358390174,
+ "step": 1010
+ },
+ {
+ "loss": 0.4235543251037598,
+ "grad_norm": 0.02607184834778309,
+ "learning_rate": 1.8480000000000001e-06,
+ "entropy": 0.7991531542147277,
+ "num_tokens": 68426131.0,
+ "logits/chosen": -1.53151378613551,
+ "logits/rejected": -1.2543500149118787,
+ "mean_token_accuracy": 0.7038963638059795,
+ "rewards/chosen": -0.7901757431415899,
+ "rewards/rejected": -2.433681526587461,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.6435057808365672,
+ "logps/chosen": -58.34346651315689,
+ "logps/rejected": -76.04588947534562,
+ "epoch": 0.8161142559958394,
+ "step": 1020
+ },
+ {
+ "loss": 0.40994958877563475,
+ "grad_norm": 0.02169707603752613,
+ "learning_rate": 1.7680000000000003e-06,
+ "entropy": 0.8521039391926024,
+ "num_tokens": 69096137.0,
+ "logits/chosen": -1.415748264102772,
+ "logits/rejected": -1.1016163433476274,
+ "mean_token_accuracy": 0.6977022993937134,
+ "rewards/chosen": -0.7218073151416684,
+ "rewards/rejected": -2.3258816314843718,
+ "rewards/accuracies": 0.8225,
+ "rewards/margins": 1.6040743189398199,
+ "logps/chosen": -59.84083811759949,
+ "logps/rejected": -75.67940702915192,
+ "epoch": 0.8241153761526614,
+ "step": 1030
+ },
+ {
+ "loss": 0.4198129177093506,
+ "grad_norm": 0.026502352207899094,
+ "learning_rate": 1.6880000000000002e-06,
+ "entropy": 0.8326882326963824,
+ "num_tokens": 69767741.0,
+ "logits/chosen": -1.4916627174188057,
+ "logits/rejected": -1.1632308065666426,
+ "mean_token_accuracy": 0.7085826633311808,
+ "rewards/chosen": -0.7927563856748747,
+ "rewards/rejected": -2.4268036293244224,
+ "rewards/accuracies": 0.815,
+ "rewards/margins": 1.6340472471993417,
+ "logps/chosen": -60.4176405954361,
+ "logps/rejected": -78.03718240499497,
+ "epoch": 0.8321164963094834,
+ "step": 1040
+ },
+ {
+ "loss": 0.4403986930847168,
+ "grad_norm": 0.020527468994259834,
+ "learning_rate": 1.608e-06,
+ "entropy": 0.8563040402246407,
+ "num_tokens": 70441281.0,
+ "logits/chosen": -1.372570142753147,
+ "logits/rejected": -1.0752142516375143,
+ "mean_token_accuracy": 0.7006468511372804,
+ "rewards/chosen": -0.8503002758337243,
+ "rewards/rejected": -2.4302191760434653,
+ "rewards/accuracies": 0.79875,
+ "rewards/margins": 1.5799189011985435,
+ "logps/chosen": -61.7722567152977,
+ "logps/rejected": -80.11385835170746,
+ "epoch": 0.8401176164663052,
+ "step": 1050
+ },
+ {
+ "loss": 0.43283534049987793,
+ "grad_norm": 0.015235912054777145,
+ "learning_rate": 1.528e-06,
+ "entropy": 0.7852728144929279,
+ "num_tokens": 71110390.0,
+ "logits/chosen": -1.4950714916636278,
+ "logits/rejected": -1.1911925690553107,
+ "mean_token_accuracy": 0.7106820711679757,
+ "rewards/chosen": -0.8049419036525068,
+ "rewards/rejected": -2.4248900110144316,
+ "rewards/accuracies": 0.7975,
+ "rewards/margins": 1.6199481027945877,
+ "logps/chosen": -57.78098263502121,
+ "logps/rejected": -74.53830781698227,
+ "epoch": 0.8481187366231272,
+ "step": 1060
+ },
+ {
+ "loss": 0.43035149574279785,
+ "grad_norm": 0.019019614905118942,
+ "learning_rate": 1.4480000000000002e-06,
+ "entropy": 0.8613008195675502,
+ "num_tokens": 71784543.0,
+ "logits/chosen": -1.3445209097630133,
+ "logits/rejected": -1.0426056675988673,
+ "mean_token_accuracy": 0.7076550057157874,
+ "rewards/chosen": -0.8018456945472281,
+ "rewards/rejected": -2.346686116975907,
+ "rewards/accuracies": 0.8075,
+ "rewards/margins": 1.5448404226545245,
+ "logps/chosen": -61.45916315793991,
+ "logps/rejected": -79.62759203910828,
+ "epoch": 0.8561198567799492,
+ "step": 1070
+ },
+ {
+ "loss": 0.41099891662597654,
+ "grad_norm": 0.016506865620613098,
+ "learning_rate": 1.368e-06,
+ "entropy": 0.8535030201455811,
+ "num_tokens": 72456138.0,
+ "logits/chosen": -1.3566193304510508,
+ "logits/rejected": -1.0452022279083852,
+ "mean_token_accuracy": 0.7095106438919901,
+ "rewards/chosen": -0.7196678565689945,
+ "rewards/rejected": -2.2712431223198655,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.5515752672031522,
+ "logps/chosen": -59.758289806842804,
+ "logps/rejected": -75.89903884649277,
+ "epoch": 0.8641209769367711,
+ "step": 1080
+ },
+ {
+ "loss": 0.40389509201049806,
+ "grad_norm": 0.017898013815283775,
+ "learning_rate": 1.288e-06,
+ "entropy": 0.8495773486199323,
+ "num_tokens": 73127658.0,
+ "logits/chosen": -1.3875953147732851,
+ "logits/rejected": -1.071999496568926,
+ "mean_token_accuracy": 0.7125035657919944,
+ "rewards/chosen": -0.6880159555127102,
+ "rewards/rejected": -2.2900046212004965,
+ "rewards/accuracies": 0.8275,
+ "rewards/margins": 1.6019886623835191,
+ "logps/chosen": -57.14114458799362,
+ "logps/rejected": -76.71981400489807,
+ "epoch": 0.8721220970935931,
+ "step": 1090
+ },
+ {
+ "loss": 0.41137285232543946,
+ "grad_norm": 0.02073514461517334,
+ "learning_rate": 1.2080000000000001e-06,
+ "entropy": 0.8456397303473204,
+ "num_tokens": 73801143.0,
+ "logits/chosen": -1.3381752214246538,
+ "logits/rejected": -1.0179612502984496,
+ "mean_token_accuracy": 0.7174369205720723,
+ "rewards/chosen": -0.7226426570682087,
+ "rewards/rejected": -2.3872625685081585,
+ "rewards/accuracies": 0.805,
+ "rewards/margins": 1.6646199140977114,
+ "logps/chosen": -58.33309064865112,
+ "logps/rejected": -78.428830742836,
+ "epoch": 0.8801232172504151,
+ "step": 1100
+ },
+ {
+ "loss": 0.3985079050064087,
+ "grad_norm": 0.02392728626728058,
+ "learning_rate": 1.128e-06,
+ "entropy": 0.8126404450691189,
+ "num_tokens": 74470337.0,
+ "logits/chosen": -1.4301342412115163,
+ "logits/rejected": -1.0898322773092897,
+ "mean_token_accuracy": 0.7102809575386345,
+ "rewards/chosen": -0.72720633818215,
+ "rewards/rejected": -2.3956915115077573,
+ "rewards/accuracies": 0.81875,
+ "rewards/margins": 1.6684851680183783,
+ "logps/chosen": -58.585094909667966,
+ "logps/rejected": -75.48162739038467,
+ "epoch": 0.8881243374072371,
+ "step": 1110
+ },
+ {
+ "loss": 0.4233971118927002,
+ "grad_norm": 0.030219033360481262,
+ "learning_rate": 1.0480000000000002e-06,
+ "entropy": 0.8403116905188653,
+ "num_tokens": 75140132.0,
+ "logits/chosen": -1.4380602392142443,
+ "logits/rejected": -1.0473352436693573,
+ "mean_token_accuracy": 0.7092700024694204,
+ "rewards/chosen": -0.7306968126030552,
+ "rewards/rejected": -2.3540891151165124,
+ "rewards/accuracies": 0.81875,
+ "rewards/margins": 1.6233923038374634,
+ "logps/chosen": -56.10472262263298,
+ "logps/rejected": -78.65561183929444,
+ "epoch": 0.8961254575640589,
+ "step": 1120
+ },
+ {
+ "eval_loss": 0.37198907136917114,
+ "eval_runtime": 461.3493,
+ "eval_samples_per_second": 2.168,
+ "eval_steps_per_second": 2.168,
+ "eval_entropy": 0.8254555847709999,
+ "eval_num_tokens": 75477459.0,
+ "eval_logits/chosen": -1.4423002773926261,
+ "eval_logits/rejected": -1.08621697552761,
+ "eval_mean_token_accuracy": 0.7080234110057354,
+ "eval_rewards/chosen": -0.688676969906548,
+ "eval_rewards/rejected": -2.4224348444156347,
+ "eval_rewards/accuracies": 0.848,
+ "eval_rewards/margins": 1.7337578756753356,
+ "eval_logps/chosen": -57.25708269691467,
+ "eval_logps/rejected": -77.14548556137085,
+ "epoch": 0.9001260176424699,
+ "step": 1125
+ },
+ {
+ "loss": 0.3935527324676514,
+ "grad_norm": 0.022325722500681877,
+ "learning_rate": 9.68e-07,
+ "entropy": 0.8382952943560668,
+ "num_tokens": 75812014.0,
+ "logits/chosen": -1.418837708765449,
+ "logits/rejected": -1.0779699632670043,
+ "mean_token_accuracy": 0.7202487983740866,
+ "rewards/chosen": -0.7877505103487056,
+ "rewards/rejected": -2.4676480230064772,
+ "rewards/accuracies": 0.8275,
+ "rewards/margins": 1.679897514770273,
+ "logps/chosen": -58.169000940322874,
+ "logps/rejected": -78.61564015626908,
+ "epoch": 0.9041265777208809,
+ "step": 1130
+ },
+ {
+ "loss": 0.3964625597000122,
+ "grad_norm": 0.019324010238051414,
+ "learning_rate": 8.880000000000001e-07,
+ "entropy": 0.8130056881319615,
+ "num_tokens": 76482016.0,
+ "logits/chosen": -1.419070527008814,
+ "logits/rejected": -1.137954775488449,
+ "mean_token_accuracy": 0.7059694546088576,
+ "rewards/chosen": -0.7633834109694726,
+ "rewards/rejected": -2.414881606222825,
+ "rewards/accuracies": 0.82875,
+ "rewards/margins": 1.651498195346794,
+ "logps/chosen": -58.853646819591525,
+ "logps/rejected": -76.24682815074921,
+ "epoch": 0.9121276978777029,
+ "step": 1140
+ },
+ {
+ "loss": 0.3915949583053589,
+ "grad_norm": 0.01624801754951477,
+ "learning_rate": 8.08e-07,
+ "entropy": 0.8315947725270234,
+ "num_tokens": 77153403.0,
+ "logits/chosen": -1.4183466503678914,
+ "logits/rejected": -1.0919806119809055,
+ "mean_token_accuracy": 0.7022176405414939,
+ "rewards/chosen": -0.7714760913598729,
+ "rewards/rejected": -2.467881916766055,
+ "rewards/accuracies": 0.84,
+ "rewards/margins": 1.6964058211818338,
+ "logps/chosen": -58.059779484272006,
+ "logps/rejected": -79.21186244726181,
+ "epoch": 0.9201288180345248,
+ "step": 1150
+ },
+ {
+ "loss": 0.38826522827148435,
+ "grad_norm": 0.026730364188551903,
+ "learning_rate": 7.280000000000001e-07,
+ "entropy": 0.792092831293121,
+ "num_tokens": 77821905.0,
+ "logits/chosen": -1.565466490680308,
+ "logits/rejected": -1.178379751278885,
+ "mean_token_accuracy": 0.7027372723817825,
+ "rewards/chosen": -0.7491817296582304,
+ "rewards/rejected": -2.426239545307035,
+ "rewards/accuracies": 0.83125,
+ "rewards/margins": 1.6770578184351326,
+ "logps/chosen": -58.95333029031754,
+ "logps/rejected": -75.34961170434951,
+ "epoch": 0.9281299381913468,
+ "step": 1160
+ },
+ {
+ "loss": 0.40007843971252444,
+ "grad_norm": 0.018569432199001312,
+ "learning_rate": 6.48e-07,
+ "entropy": 0.8224156517500524,
+ "num_tokens": 78491775.0,
+ "logits/chosen": -1.5012509494746065,
+ "logits/rejected": -1.1467693293032915,
+ "mean_token_accuracy": 0.7090437597036362,
+ "rewards/chosen": -0.7536068202524256,
+ "rewards/rejected": -2.5228245135350154,
+ "rewards/accuracies": 0.81875,
+ "rewards/margins": 1.7692176943086089,
+ "logps/chosen": -58.17639876842499,
+ "logps/rejected": -78.52316574573517,
+ "epoch": 0.9361310583481688,
+ "step": 1170
+ },
+ {
+ "loss": 0.4408255100250244,
+ "grad_norm": 0.024299506098031998,
+ "learning_rate": 5.680000000000001e-07,
+ "entropy": 0.830776419856702,
+ "num_tokens": 79162540.0,
+ "logits/chosen": -1.4830534321700366,
+ "logits/rejected": -1.1111014440667628,
+ "mean_token_accuracy": 0.7090770015493035,
+ "rewards/chosen": -0.7984052636207344,
+ "rewards/rejected": -2.4370353724921006,
+ "rewards/accuracies": 0.795,
+ "rewards/margins": 1.6386301069799811,
+ "logps/chosen": -57.70949717998505,
+ "logps/rejected": -79.33320981264114,
+ "epoch": 0.9441321785049906,
+ "step": 1180
+ },
+ {
+ "loss": 0.416896915435791,
+ "grad_norm": 0.019658422097563744,
+ "learning_rate": 4.88e-07,
+ "entropy": 0.8127233781164978,
+ "num_tokens": 79832047.0,
+ "logits/chosen": -1.4123649851333377,
+ "logits/rejected": -1.1466187299790185,
+ "mean_token_accuracy": 0.7103831494599581,
+ "rewards/chosen": -0.7157201138033997,
+ "rewards/rejected": -2.3061228878897966,
+ "rewards/accuracies": 0.8125,
+ "rewards/margins": 1.590402772258967,
+ "logps/chosen": -56.48203515172005,
+ "logps/rejected": -75.60109715938569,
+ "epoch": 0.9521332986618126,
+ "step": 1190
+ },
+ {
+ "loss": 0.4019536018371582,
+ "grad_norm": 0.017900435253977776,
+ "learning_rate": 4.0800000000000005e-07,
+ "entropy": 0.843690790355904,
+ "num_tokens": 80502194.0,
+ "logits/chosen": -1.3902989435653317,
+ "logits/rejected": -1.0785033512547093,
+ "mean_token_accuracy": 0.7014491311274469,
+ "rewards/chosen": -0.7470678079669597,
+ "rewards/rejected": -2.3619672801207345,
+ "rewards/accuracies": 0.81125,
+ "rewards/margins": 1.614899470116943,
+ "logps/chosen": -60.23810036301613,
+ "logps/rejected": -76.19441313266753,
+ "epoch": 0.9601344188186346,
+ "step": 1200
+ },
+ {
+ "loss": 0.405252742767334,
+ "grad_norm": 0.026110496371984482,
+ "learning_rate": 3.280000000000001e-07,
+ "entropy": 0.7792508405505214,
+ "num_tokens": 81171107.0,
+ "logits/chosen": -1.5459746228809121,
+ "logits/rejected": -1.1853865636356122,
+ "mean_token_accuracy": 0.7177962634526193,
+ "rewards/chosen": -0.7623023401526734,
+ "rewards/rejected": -2.425694184426102,
+ "rewards/accuracies": 0.81875,
+ "rewards/margins": 1.6633918406302108,
+ "logps/chosen": -56.42661242365837,
+ "logps/rejected": -75.50851482152939,
+ "epoch": 0.9681355389754566,
+ "step": 1210
+ },
+ {
+ "loss": 0.4123998165130615,
+ "grad_norm": 0.018806418403983116,
+ "learning_rate": 2.48e-07,
+ "entropy": 0.831228096887935,
+ "num_tokens": 81842883.0,
+ "logits/chosen": -1.4490407879391156,
+ "logits/rejected": -1.1519401234903923,
+ "mean_token_accuracy": 0.7097262334264814,
+ "rewards/chosen": -0.7701989058920299,
+ "rewards/rejected": -2.4217049359795055,
+ "rewards/accuracies": 0.805,
+ "rewards/margins": 1.65150602793321,
+ "logps/chosen": -60.28224401473999,
+ "logps/rejected": -77.09071898698807,
+ "epoch": 0.9761366591322785,
+ "step": 1220
+ },
+ {
+ "loss": 0.40515909194946287,
+ "grad_norm": 0.027354199439287186,
+ "learning_rate": 1.68e-07,
+ "entropy": 0.8178756858388079,
+ "num_tokens": 82513579.0,
+ "logits/chosen": -1.4675544175368733,
+ "logits/rejected": -1.1366583786224183,
+ "mean_token_accuracy": 0.7170148695632815,
+ "rewards/chosen": -0.7370101418271952,
+ "rewards/rejected": -2.4574845099955565,
+ "rewards/accuracies": 0.8425,
+ "rewards/margins": 1.7204743684129788,
+ "logps/chosen": -56.6686701130867,
+ "logps/rejected": -78.64371829748154,
+ "epoch": 0.9841377792891005,
+ "step": 1230
+ },
+ {
+ "loss": 0.47594127655029295,
+ "grad_norm": 0.02245207503437996,
+ "learning_rate": 8.800000000000001e-08,
+ "entropy": 0.8239703281945548,
+ "num_tokens": 83183854.0,
+ "logits/chosen": -1.4181352852579936,
+ "logits/rejected": -1.088310710386729,
+ "mean_token_accuracy": 0.7088763321656734,
+ "rewards/chosen": -0.802495680890861,
+ "rewards/rejected": -2.3516381712537258,
+ "rewards/accuracies": 0.80375,
+ "rewards/margins": 1.5491424936638214,
+ "logps/chosen": -58.71147965669632,
+ "logps/rejected": -76.95230464458466,
+ "epoch": 0.9921388994459225,
+ "step": 1240
+ },
+ {
+ "loss": 0.4114171028137207,
+ "grad_norm": 0.02189941145479679,
+ "learning_rate": 8e-09,
+ "entropy": 0.8552226083004805,
+ "num_tokens": 83841856.0,
+ "logits/chosen": -1.4202063989012872,
+ "logits/rejected": -1.060263823141246,
+ "mean_token_accuracy": 0.7003038972805777,
+ "rewards/chosen": -0.6637024932304197,
+ "rewards/rejected": -2.3128979423820364,
+ "rewards/accuracies": 0.821882951653944,
+ "rewards/margins": 1.6491954480902156,
+ "logps/chosen": -57.31481355565195,
+ "logps/rejected": -77.89109719982584,
+ "epoch": 1.0,
+ "step": 1250
+ },
+ {
+ "eval_loss": 0.3715994358062744,
+ "eval_runtime": 456.0719,
+ "eval_samples_per_second": 2.193,
+ "eval_steps_per_second": 2.193,
+ "eval_entropy": 0.8287503328006715,
+ "eval_num_tokens": 83841856.0,
+ "eval_logits/chosen": -1.4355893608780679,
+ "eval_logits/rejected": -1.0769000715683428,
+ "eval_mean_token_accuracy": 0.7080028675496578,
+ "eval_rewards/chosen": -0.6952986993879313,
+ "eval_rewards/rejected": -2.446402875666041,
+ "eval_rewards/accuracies": 0.843,
+ "eval_rewards/margins": 1.7511041724067182,
+ "eval_logps/chosen": -57.32329998397827,
+ "eval_logps/rejected": -77.3851658782959,
+ "epoch": 1.0,
+ "step": 1250
+ },
+ {
+ "train_runtime": 54275.4778,
+ "train_samples_per_second": 1.842,
+ "train_steps_per_second": 0.023,
+ "total_flos": 6.794900050947748e+17,
+ "train_loss": 0.4424690731048584,
+ "epoch": 1.0,
+ "step": 1250
+ }
+ ],
+ "validation_monitor_size": 1000,
+ "validation_monitor_selection": "fixed_seed_random_without_replacement",
+ "validation_monitor_seed": 22,
+ "validation_monitor_indices": [
+ 10,
+ 21,
+ 55,
+ 66,
+ 110,
+ 144,
+ 149,
+ 152,
+ 163,
+ 166,
+ 167,
+ 176,
+ 206,
+ 207,
+ 267,
+ 281,
+ 295,
+ 298,
+ 307,
+ 317,
+ 336,
+ 341,
+ 349,
+ 352,
+ 357,
+ 382,
+ 386,
+ 391,
+ 401,
+ 411,
+ 412,
+ 432,
+ 452,
+ 465,
+ 488,
+ 490,
+ 491,
+ 492,
+ 494,
+ 504,
+ 528,
+ 540,
+ 553,
+ 586,
+ 605,
+ 625,
+ 627,
+ 661,
+ 663,
+ 666,
+ 677,
+ 685,
+ 690,
+ 692,
+ 704,
+ 708,
+ 714,
+ 727,
+ 728,
+ 733,
+ 745,
+ 752,
+ 753,
+ 755,
+ 779,
+ 783,
+ 793,
+ 796,
+ 799,
+ 803,
+ 804,
+ 805,
+ 813,
+ 816,
+ 827,
+ 829,
+ 830,
+ 842,
+ 850,
+ 853,
+ 856,
+ 889,
+ 905,
+ 915,
+ 930,
+ 939,
+ 951,
+ 1002,
+ 1005,
+ 1029,
+ 1049,
+ 1050,
+ 1054,
+ 1067,
+ 1068,
+ 1079,
+ 1091,
+ 1114,
+ 1133,
+ 1140,
+ 1144,
+ 1145,
+ 1186,
+ 1195,
+ 1206,
+ 1223,
+ 1252,
+ 1257,
+ 1259,
+ 1271,
+ 1304,
+ 1305,
+ 1335,
+ 1336,
+ 1343,
+ 1367,
+ 1377,
+ 1403,
+ 1412,
+ 1429,
+ 1443,
+ 1457,
+ 1459,
+ 1460,
+ 1476,
+ 1483,
+ 1486,
+ 1507,
+ 1510,
+ 1519,
+ 1521,
+ 1545,
+ 1570,
+ 1582,
+ 1593,
+ 1595,
+ 1603,
+ 1607,
+ 1614,
+ 1615,
+ 1634,
+ 1639,
+ 1654,
+ 1657,
+ 1673,
+ 1690,
+ 1704,
+ 1746,
+ 1756,
+ 1781,
+ 1783,
+ 1796,
+ 1799,
+ 1809,
+ 1814,
+ 1827,
+ 1829,
+ 1832,
+ 1844,
+ 1847,
+ 1854,
+ 1856,
+ 1858,
+ 1883,
+ 1889,
+ 1925,
+ 1931,
+ 1932,
+ 1934,
+ 1935,
+ 1957,
+ 1962,
+ 1967,
+ 1975,
+ 1982,
+ 1983,
+ 1991,
+ 1998,
+ 2003,
+ 2008,
+ 2017,
+ 2021,
+ 2035,
+ 2040,
+ 2050,
+ 2056,
+ 2077,
+ 2082,
+ 2098,
+ 2100,
+ 2121,
+ 2130,
+ 2134,
+ 2138,
+ 2143,
+ 2166,
+ 2167,
+ 2180,
+ 2181,
+ 2204,
+ 2205,
+ 2212,
+ 2221,
+ 2224,
+ 2230,
+ 2233,
+ 2263,
+ 2269,
+ 2273,
+ 2290,
+ 2291,
+ 2299,
+ 2321,
+ 2323,
+ 2330,
+ 2401,
+ 2417,
+ 2435,
+ 2456,
+ 2488,
+ 2502,
+ 2504,
+ 2532,
+ 2538,
+ 2555,
+ 2558,
+ 2562,
+ 2583,
+ 2585,
+ 2590,
+ 2594,
+ 2596,
+ 2600,
+ 2606,
+ 2607,
+ 2618,
+ 2630,
+ 2647,
+ 2650,
+ 2657,
+ 2679,
+ 2685,
+ 2705,
+ 2712,
+ 2714,
+ 2727,
+ 2740,
+ 2742,
+ 2780,
+ 2784,
+ 2792,
+ 2807,
+ 2808,
+ 2810,
+ 2820,
+ 2831,
+ 2860,
+ 2863,
+ 2866,
+ 2875,
+ 2878,
+ 2898,
+ 2905,
+ 2921,
+ 2922,
+ 2930,
+ 2934,
+ 2944,
+ 2955,
+ 2957,
+ 2959,
+ 2973,
+ 2978,
+ 2998,
+ 3018,
+ 3022,
+ 3028,
+ 3031,
+ 3061,
+ 3085,
+ 3090,
+ 3104,
+ 3105,
+ 3115,
+ 3121,
+ 3122,
+ 3129,
+ 3133,
+ 3135,
+ 3161,
+ 3169,
+ 3194,
+ 3195,
+ 3215,
+ 3225,
+ 3226,
+ 3250,
+ 3265,
+ 3301,
+ 3312,
+ 3329,
+ 3361,
+ 3362,
+ 3376,
+ 3403,
+ 3410,
+ 3419,
+ 3432,
+ 3443,
+ 3452,
+ 3467,
+ 3469,
+ 3475,
+ 3485,
+ 3503,
+ 3515,
+ 3524,
+ 3528,
+ 3544,
+ 3557,
+ 3565,
+ 3637,
+ 3642,
+ 3658,
+ 3659,
+ 3693,
+ 3699,
+ 3722,
+ 3725,
+ 3728,
+ 3731,
+ 3740,
+ 3742,
+ 3762,
+ 3780,
+ 3788,
+ 3794,
+ 3796,
+ 3798,
+ 3817,
+ 3819,
+ 3822,
+ 3839,
+ 3843,
+ 3846,
+ 3851,
+ 3865,
+ 3871,
+ 3884,
+ 3894,
+ 3896,
+ 3907,
+ 3911,
+ 3915,
+ 3919,
+ 3920,
+ 3923,
+ 3933,
+ 3955,
+ 3967,
+ 3972,
+ 3974,
+ 3975,
+ 3997,
+ 4002,
+ 4034,
+ 4063,
+ 4073,
+ 4079,
+ 4082,
+ 4088,
+ 4121,
+ 4145,
+ 4148,
+ 4159,
+ 4161,
+ 4164,
+ 4177,
+ 4188,
+ 4199,
+ 4207,
+ 4213,
+ 4221,
+ 4233,
+ 4241,
+ 4243,
+ 4247,
+ 4264,
+ 4274,
+ 4282,
+ 4286,
+ 4290,
+ 4310,
+ 4313,
+ 4321,
+ 4324,
+ 4327,
+ 4362,
+ 4380,
+ 4407,
+ 4411,
+ 4415,
+ 4417,
+ 4418,
+ 4427,
+ 4431,
+ 4433,
+ 4451,
+ 4466,
+ 4477,
+ 4479,
+ 4493,
+ 4514,
+ 4527,
+ 4539,
+ 4550,
+ 4558,
+ 4568,
+ 4579,
+ 4583,
+ 4584,
+ 4586,
+ 4587,
+ 4590,
+ 4599,
+ 4602,
+ 4610,
+ 4626,
+ 4627,
+ 4630,
+ 4655,
+ 4656,
+ 4657,
+ 4661,
+ 4685,
+ 4714,
+ 4715,
+ 4731,
+ 4752,
+ 4769,
+ 4791,
+ 4805,
+ 4818,
+ 4829,
+ 4839,
+ 4843,
+ 4871,
+ 4879,
+ 4885,
+ 4888,
+ 4895,
+ 4900,
+ 4923,
+ 4966,
+ 4968,
+ 4972,
+ 4989,
+ 4994,
+ 4998,
+ 5001,
+ 5019,
+ 5026,
+ 5032,
+ 5034,
+ 5046,
+ 5055,
+ 5085,
+ 5086,
+ 5088,
+ 5089,
+ 5090,
+ 5095,
+ 5119,
+ 5121,
+ 5133,
+ 5154,
+ 5169,
+ 5178,
+ 5222,
+ 5223,
+ 5232,
+ 5233,
+ 5240,
+ 5256,
+ 5259,
+ 5264,
+ 5276,
+ 5279,
+ 5335,
+ 5337,
+ 5345,
+ 5348,
+ 5373,
+ 5378,
+ 5422,
+ 5442,
+ 5443,
+ 5445,
+ 5477,
+ 5485,
+ 5495,
+ 5497,
+ 5504,
+ 5526,
+ 5542,
+ 5564,
+ 5570,
+ 5579,
+ 5608,
+ 5610,
+ 5624,
+ 5630,
+ 5638,
+ 5651,
+ 5663,
+ 5670,
+ 5675,
+ 5691,
+ 5700,
+ 5706,
+ 5715,
+ 5720,
+ 5721,
+ 5732,
+ 5738,
+ 5741,
+ 5769,
+ 5771,
+ 5795,
+ 5796,
+ 5800,
+ 5809,
+ 5810,
+ 5815,
+ 5819,
+ 5827,
+ 5848,
+ 5849,
+ 5852,
+ 5859,
+ 5880,
+ 5907,
+ 5912,
+ 5919,
+ 5931,
+ 5932,
+ 5941,
+ 5948,
+ 5950,
+ 5953,
+ 5981,
+ 6000,
+ 6003,
+ 6010,
+ 6018,
+ 6075,
+ 6103,
+ 6106,
+ 6109,
+ 6114,
+ 6123,
+ 6131,
+ 6138,
+ 6139,
+ 6164,
+ 6173,
+ 6180,
+ 6185,
+ 6189,
+ 6190,
+ 6221,
+ 6223,
+ 6237,
+ 6255,
+ 6262,
+ 6265,
+ 6293,
+ 6305,
+ 6331,
+ 6336,
+ 6355,
+ 6366,
+ 6371,
+ 6396,
+ 6399,
+ 6402,
+ 6408,
+ 6432,
+ 6433,
+ 6441,
+ 6456,
+ 6465,
+ 6486,
+ 6489,
+ 6507,
+ 6508,
+ 6522,
+ 6528,
+ 6537,
+ 6551,
+ 6564,
+ 6590,
+ 6598,
+ 6599,
+ 6611,
+ 6613,
+ 6615,
+ 6621,
+ 6634,
+ 6647,
+ 6649,
+ 6654,
+ 6676,
+ 6690,
+ 6708,
+ 6729,
+ 6744,
+ 6749,
+ 6756,
+ 6761,
+ 6763,
+ 6773,
+ 6788,
+ 6790,
+ 6796,
+ 6797,
+ 6811,
+ 6850,
+ 6869,
+ 6871,
+ 6882,
+ 6895,
+ 6906,
+ 6911,
+ 6914,
+ 6952,
+ 6966,
+ 6985,
+ 7001,
+ 7021,
+ 7031,
+ 7038,
+ 7041,
+ 7045,
+ 7052,
+ 7057,
+ 7073,
+ 7076,
+ 7102,
+ 7107,
+ 7109,
+ 7117,
+ 7122,
+ 7125,
+ 7207,
+ 7212,
+ 7215,
+ 7232,
+ 7246,
+ 7250,
+ 7258,
+ 7263,
+ 7267,
+ 7270,
+ 7274,
+ 7280,
+ 7286,
+ 7293,
+ 7298,
+ 7306,
+ 7316,
+ 7325,
+ 7326,
+ 7356,
+ 7367,
+ 7385,
+ 7392,
+ 7405,
+ 7416,
+ 7421,
+ 7426,
+ 7452,
+ 7519,
+ 7534,
+ 7542,
+ 7546,
+ 7601,
+ 7604,
+ 7606,
+ 7609,
+ 7649,
+ 7653,
+ 7682,
+ 7699,
+ 7738,
+ 7750,
+ 7798,
+ 7800,
+ 7801,
+ 7805,
+ 7811,
+ 7832,
+ 7837,
+ 7849,
+ 7856,
+ 7876,
+ 7877,
+ 7887,
+ 7905,
+ 7919,
+ 7920,
+ 7922,
+ 7923,
+ 7926,
+ 7966,
+ 7970,
+ 7973,
+ 7974,
+ 7976,
+ 7986,
+ 8027,
+ 8028,
+ 8068,
+ 8074,
+ 8091,
+ 8120,
+ 8122,
+ 8125,
+ 8152,
+ 8153,
+ 8164,
+ 8167,
+ 8169,
+ 8171,
+ 8177,
+ 8189,
+ 8192,
+ 8196,
+ 8212,
+ 8217,
+ 8231,
+ 8242,
+ 8244,
+ 8250,
+ 8256,
+ 8257,
+ 8265,
+ 8270,
+ 8274,
+ 8283,
+ 8290,
+ 8294,
+ 8302,
+ 8307,
+ 8318,
+ 8326,
+ 8338,
+ 8350,
+ 8353,
+ 8357,
+ 8371,
+ 8377,
+ 8380,
+ 8387,
+ 8388,
+ 8396,
+ 8402,
+ 8419,
+ 8423,
+ 8428,
+ 8435,
+ 8439,
+ 8442,
+ 8444,
+ 8453,
+ 8475,
+ 8481,
+ 8495,
+ 8498,
+ 8523,
+ 8531,
+ 8562,
+ 8568,
+ 8584,
+ 8588,
+ 8589,
+ 8592,
+ 8597,
+ 8608,
+ 8611,
+ 8624,
+ 8636,
+ 8637,
+ 8654,
+ 8657,
+ 8661,
+ 8680,
+ 8683,
+ 8687,
+ 8693,
+ 8695,
+ 8697,
+ 8704,
+ 8713,
+ 8714,
+ 8755,
+ 8758,
+ 8761,
+ 8772,
+ 8781,
+ 8800,
+ 8808,
+ 8812,
+ 8815,
+ 8822,
+ 8825,
+ 8830,
+ 8837,
+ 8863,
+ 8867,
+ 8894,
+ 8898,
+ 8924,
+ 8937,
+ 8955,
+ 8964,
+ 8978,
+ 8995,
+ 9007,
+ 9018,
+ 9034,
+ 9043,
+ 9074,
+ 9075,
+ 9099,
+ 9103,
+ 9112,
+ 9115,
+ 9127,
+ 9137,
+ 9142,
+ 9163,
+ 9174,
+ 9179,
+ 9196,
+ 9208,
+ 9213,
+ 9222,
+ 9228,
+ 9230,
+ 9274,
+ 9278,
+ 9293,
+ 9328,
+ 9338,
+ 9342,
+ 9346,
+ 9368,
+ 9385,
+ 9403,
+ 9420,
+ 9429,
+ 9432,
+ 9453,
+ 9455,
+ 9463,
+ 9502,
+ 9511,
+ 9517,
+ 9545,
+ 9548,
+ 9550,
+ 9558,
+ 9566,
+ 9574,
+ 9577,
+ 9578,
+ 9589,
+ 9606,
+ 9618,
+ 9659,
+ 9672,
+ 9673,
+ 9706,
+ 9717,
+ 9728,
+ 9732,
+ 9758,
+ 9764,
+ 9773,
+ 9778,
+ 9779,
+ 9798,
+ 9801,
+ 9803,
+ 9804,
+ 9845,
+ 9849,
+ 9862,
+ 9876,
+ 9890,
+ 9911,
+ 9959,
+ 9966,
+ 9967,
+ 9968,
+ 9974,
+ 9983,
+ 10007,
+ 10032,
+ 10042,
+ 10044,
+ 10061,
+ 10062,
+ 10080,
+ 10085,
+ 10095,
+ 10098,
+ 10132,
+ 10144,
+ 10175,
+ 10176,
+ 10197,
+ 10220,
+ 10230,
+ 10233,
+ 10242,
+ 10260,
+ 10265,
+ 10305,
+ 10313,
+ 10314,
+ 10332,
+ 10338,
+ 10341,
+ 10342,
+ 10347,
+ 10353,
+ 10354,
+ 10360,
+ 10381,
+ 10390,
+ 10396,
+ 10402,
+ 10404,
+ 10418,
+ 10432,
+ 10445,
+ 10447,
+ 10450,
+ 10454,
+ 10489,
+ 10523,
+ 10539,
+ 10541,
+ 10550,
+ 10561,
+ 10562,
+ 10573,
+ 10577,
+ 10583,
+ 10589,
+ 10596,
+ 10601,
+ 10605,
+ 10633,
+ 10634,
+ 10663,
+ 10671,
+ 10672,
+ 10685,
+ 10710,
+ 10730,
+ 10733,
+ 10740,
+ 10744,
+ 10774,
+ 10786,
+ 10829,
+ 10833,
+ 10838,
+ 10855,
+ 10879,
+ 10907,
+ 10909,
+ 10911,
+ 10928,
+ 10936,
+ 10956,
+ 10989,
+ 11022,
+ 11030,
+ 11031,
+ 11041,
+ 11055,
+ 11058,
+ 11075,
+ 11077,
+ 11085,
+ 11096
+ ],
+ "early_stopping_patience": 3
+}