Publish Phase 1 LoRA adapters

.gitattributes

@@ -23,6 +23,7 @@
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
+**/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
 *.tar filter=lfs diff=lfs merge=lfs -text

MANIFEST.sha256

@@ -0,0 +1,72 @@
+e7046ded3c2209e9a672bafd5d88ef404de69d10cc8a31cb0f3f29814ae80d8e  phase1/ablations/empty_context/README.md
+eb964f79f8e6f7ea2891c7d62fd4e4e0080de87c6998c03e0007524e57d7f34f  phase1/ablations/empty_context/adapter_config.json
+79a128d64b87a92a3b4450386e35821dc796e494dca46b4d8c421b8580dbfd63  phase1/ablations/empty_context/adapter_model.safetensors
+cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f  phase1/ablations/empty_context/chat_template.jinja
+6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5  phase1/ablations/empty_context/tokenizer.json
+d9bf1d78a80bad5eb65b83c2608910e3be00a9d3250c8153a5a393ad1257e1d6  phase1/ablations/empty_context/tokenizer_config.json
+4980eeedcdde01c95d37224943c3c64dcbeed7f0894c9e83bd99f12534a473fd  phase1/ablations/empty_context/training_args.bin
+38881ae95ca349d9434dc02bf513b6fc157c642b6dd0f1671dc387eb59214767  phase1/ablations/empty_context/training_recipe.json
+f7af5bbec6236061230e3955c84120341cea767726734f7d0e0240cdd83996c0  phase1/ablations/empty_context/training_summary.json
+510e129e1772049396993a6dee55abaa81f63b5764d03fadbb519ef6fafd076f  phase1/ablations/helps_only/README.md
+eb964f79f8e6f7ea2891c7d62fd4e4e0080de87c6998c03e0007524e57d7f34f  phase1/ablations/helps_only/adapter_config.json
+a143ad6c49a7a1b1d98e81af88412c91b367a394fa48a89517d6057fddf91748  phase1/ablations/helps_only/adapter_model.safetensors
+cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f  phase1/ablations/helps_only/chat_template.jinja
+6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5  phase1/ablations/helps_only/tokenizer.json
+d9bf1d78a80bad5eb65b83c2608910e3be00a9d3250c8153a5a393ad1257e1d6  phase1/ablations/helps_only/tokenizer_config.json
+101d92ecb945e3232d561989c7a979ff91d184b4cccbe34a200a56060e3c5ce6  phase1/ablations/helps_only/training_args.bin
+99d8a8dd262b1543c87ce9adbffa30a7017e0a8eda928fa2d15e51213b3d563a  phase1/ablations/helps_only/training_recipe.json
+b28011dd1e9881bf6cd76d49411c2eb1d600ffd9b7fddb02e4a225fe6cbf10ba  phase1/ablations/helps_only/training_summary.json
+6e949508412323e8d11526b703bec49eb0cf3b1f7d82d0cc1be01f2ed7f6e685  phase1/ablations/matched_clean/README.md
+eb964f79f8e6f7ea2891c7d62fd4e4e0080de87c6998c03e0007524e57d7f34f  phase1/ablations/matched_clean/adapter_config.json
+724b1e847a25563f7fe04d27dfa05df9213b97348490c4cf5b8f1c0cbdb90b68  phase1/ablations/matched_clean/adapter_model.safetensors
+cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f  phase1/ablations/matched_clean/chat_template.jinja
+6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5  phase1/ablations/matched_clean/tokenizer.json
+d9bf1d78a80bad5eb65b83c2608910e3be00a9d3250c8153a5a393ad1257e1d6  phase1/ablations/matched_clean/tokenizer_config.json
+cfafc3f2a531cdc31948a992577839650613ac2f09616d9263508e3e97da8555  phase1/ablations/matched_clean/training_args.bin
+beb473536db2f62b9573ce70a0627393a709176bc1e3e19c140cec99f1db6112  phase1/ablations/matched_clean/training_recipe.json
+bc5409a34a85f904abaac080ebb4be57ced1c64ee94e94587717876a6565a884  phase1/ablations/matched_clean/training_summary.json
+077a73a8683dea6ef8f3b30879101c55f2c7c3061ac00342e272f788ae48779f  phase1/ablations/no_context/README.md
+eb964f79f8e6f7ea2891c7d62fd4e4e0080de87c6998c03e0007524e57d7f34f  phase1/ablations/no_context/adapter_config.json
+6f3a8ff3c19fd3e0ff728feab4e0a73a33c71608510f37d9dd27e53b74f97523  phase1/ablations/no_context/adapter_model.safetensors
+cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f  phase1/ablations/no_context/chat_template.jinja
+6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5  phase1/ablations/no_context/tokenizer.json
+d9bf1d78a80bad5eb65b83c2608910e3be00a9d3250c8153a5a393ad1257e1d6  phase1/ablations/no_context/tokenizer_config.json
+dd47c7f20edcbf52ada253a96ebd7ddcb8045bdd2a75f52fdd0f51ad7bc0183e  phase1/ablations/no_context/training_args.bin
+1c7e4ab6940afc79f279336e739811171ce4a85c1198fb2a8baf4fd4cc6160b7  phase1/ablations/no_context/training_recipe.json
+91d8ec7a27b0ffe87b52ac7370ceffd78c910bacf142b7a9e942274ce26c8d0b  phase1/ablations/no_context/training_summary.json
+81302140e113697d25464f222778a32a3a55dc576752d834b3b120e690833e5e  phase1/ablations/random_labels/README.md
+eb964f79f8e6f7ea2891c7d62fd4e4e0080de87c6998c03e0007524e57d7f34f  phase1/ablations/random_labels/adapter_config.json
+7bebded9e10f63603bfd9d6a135dda87c8bcac25b4a43b365e2b1ce882bf4c93  phase1/ablations/random_labels/adapter_model.safetensors
+cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f  phase1/ablations/random_labels/chat_template.jinja
+6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5  phase1/ablations/random_labels/tokenizer.json
+d9bf1d78a80bad5eb65b83c2608910e3be00a9d3250c8153a5a393ad1257e1d6  phase1/ablations/random_labels/tokenizer_config.json
+ad84a00370d0c977fcd38a1fce033b130494f3fffd4a3c0cbc44a9dd43c0bdbf  phase1/ablations/random_labels/training_args.bin
+986dd7375539ac325bec654e90e7d077b4bf7738a471d83d7aa47c4d9f1898fb  phase1/ablations/random_labels/training_recipe.json
+4fcf1c2cc9c584de1ac096a5e7cb48babbd74b25d74c9973e654976f42d0be74  phase1/ablations/random_labels/training_summary.json
+6399ea44e270928871d5a50c1bc2c06eeb83d7ca10d5deaaeb78e4d41e859933  phase1/ablations/shuffled_documents/README.md
+eb964f79f8e6f7ea2891c7d62fd4e4e0080de87c6998c03e0007524e57d7f34f  phase1/ablations/shuffled_documents/adapter_config.json
+c9d8b830ce2a3b583eb54eab7470a7b3acb9c163dee812de0313c454462de064  phase1/ablations/shuffled_documents/adapter_model.safetensors
+cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f  phase1/ablations/shuffled_documents/chat_template.jinja
+6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5  phase1/ablations/shuffled_documents/tokenizer.json
+d9bf1d78a80bad5eb65b83c2608910e3be00a9d3250c8153a5a393ad1257e1d6  phase1/ablations/shuffled_documents/tokenizer_config.json
+eaedd299a54d4c58a5c10cd9e379b80d44bbfd7922ccd578d5731f3e7adeb7cb  phase1/ablations/shuffled_documents/training_args.bin
+d37a5d995c8264d0abffeff9171478803b7b22412780c4e8226bf0b857f96235  phase1/ablations/shuffled_documents/training_recipe.json
+2e8d728d08c0bcc0212162c3cdd9295f5c982d92f65316f19f78c055a8c5589c  phase1/ablations/shuffled_documents/training_summary.json
+4d1f59c2d09544c2e0397996d8032202fff701109f5e7320c48a72c7fa005b70  phase1/llama-dpo-v3/README.md
+18239fcf982739e03a128d680ac81ec7f05e0f058c032bed56a51376d7da731e  phase1/llama-dpo-v3/adapter_config.json
+2633d2be48f838d5c4404786ca9baa38ff2b0ed66d2fa5d7d11eb74fe63c178d  phase1/llama-dpo-v3/adapter_model.safetensors
+5816fce10444e03c2e9ee1ef8a4a1ea61ae7e69e438613f3b17b69d0426223a4  phase1/llama-dpo-v3/chat_template.jinja
+b20d148821d1e209454ef0d54a1d2bb1ff12cd365a0ff5391d96010e897b9644  phase1/llama-dpo-v3/tokenizer.json
+85795daec85614d3918187e7bd6d3548a0de064dc66e365db30bdea9d7b71dc3  phase1/llama-dpo-v3/tokenizer_config.json
+fced63a90802dbed1aa84b5b0863e967f730e08882f30d6a2cec2f76954c2675  phase1/llama-dpo-v3/training_args.bin
+a8d5a947873a04a1bbbc7972465db555eee1cf238d06c7f30f3d698056ee3d4e  phase1/llama-dpo-v3/training_recipe.json
+55c87632a89ed152627178af35294d6c5c11096f8af740c682141925ad0aa122  phase1/llama-dpo-v3/training_summary.json
+512f4df0376b902685fe95ad685ea8b06c1112bec0cdf3ae7c7b7db7eb462664  phase1/qwen-main-dpo-v3/README.md
+f59cbec9982518824aa800f9fbc56b764fc73098422cb4e81f2e70d6099d26a2  phase1/qwen-main-dpo-v3/adapter_config.json
+cc60e5fe47be9bf4f64d734d40d89310a4826d8156d0ab8221b466f21c5efd58  phase1/qwen-main-dpo-v3/adapter_model.safetensors
+cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f  phase1/qwen-main-dpo-v3/chat_template.jinja
+6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5  phase1/qwen-main-dpo-v3/tokenizer.json
+d9bf1d78a80bad5eb65b83c2608910e3be00a9d3250c8153a5a393ad1257e1d6  phase1/qwen-main-dpo-v3/tokenizer_config.json
+1578389f7f0fe332b393d7d42285d2a1a804b2efc9a174e73d1805a5ca67e41b  phase1/qwen-main-dpo-v3/training_args.bin
+3e8d64f2b8fcf2d9259644c9b919fc9597a450d3ec89eae738a467fded895a88  phase1/qwen-main-dpo-v3/training_recipe.json
+5263a105fa846c7cf3ddbb03ede58f4956c25f0e7a8ae38f9f8974930d16518b  phase1/qwen-main-dpo-v3/training_summary.json

README.md

@@ -0,0 +1,88 @@
+---
+library_name: peft
+tags:
+- dpo
+- lora
+- peft
+- self-training
+- transformers
+- trl
+pipeline_tag: text-generation
+---
+
+# York Milestone Project — Phase 1 adapters
+
+Hugging Face 仓库：
+[`geyuxu/york-milestone-project-self-traing-loop-model`](https://huggingface.co/geyuxu/york-milestone-project-self-traing-loop-model)
+
+本仓库保存一期 v3 的最终 PEFT LoRA-DPO adapter，用于复现实验和后续阶段复用。
+它不包含基础模型权重、完整训练 checkpoint、合成数据、评测输出或原始 API 响应。
+
+## 内容
+
+| 路径 | 基础模型 | 用途 | 训练 / 验证偏好对 |
+| --- | --- | --- | ---: |
+| `phase1/qwen-main-dpo-v3/` | `Qwen/Qwen2.5-1.5B-Instruct` | 一期主实验 | 99,986 / 11,110 |
+| `phase1/llama-dpo-v3/` | `meta-llama/Llama-3.2-1B-Instruct` | 跨模型迁移实验 | 99,986 / 11,110 |
+| `phase1/ablations/empty_context/` | Qwen2.5-1.5B-Instruct | 诊断消融 | 77,124 / 8,563 |
+| `phase1/ablations/helps_only/` | Qwen2.5-1.5B-Instruct | 诊断消融 | 77,124 / 8,563 |
+| `phase1/ablations/matched_clean/` | Qwen2.5-1.5B-Instruct | 诊断消融 | 77,124 / 8,563 |
+| `phase1/ablations/no_context/` | Qwen2.5-1.5B-Instruct | 诊断消融 | 77,124 / 8,563 |
+| `phase1/ablations/random_labels/` | Qwen2.5-1.5B-Instruct | 诊断消融 | 77,124 / 8,563 |
+| `phase1/ablations/shuffled_documents/` | Qwen2.5-1.5B-Instruct | 诊断消融 | 77,124 / 8,563 |
+
+每个目录包含：
+
+- `adapter_model.safetensors` 和 `adapter_config.json`；
+- 与导出时一致的 tokenizer 和 chat template；
+- `training_recipe.json`：冻结的数据成员哈希及核心训练参数；
+- `training_summary.json`：训练步数、日志和最终摘要；
+- `training_args.bin`：Transformers 训练参数快照。
+
+## 加载 adapter
+
+以下示例加载 Qwen 主实验。其他 adapter 只需替换 `subfolder` 和对应的
+`base_id`：
+
+```python
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo_id = "geyuxu/york-milestone-project-self-traing-loop-model"
+subfolder = "phase1/qwen-main-dpo-v3"
+base_id = "Qwen/Qwen2.5-1.5B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(repo_id, subfolder=subfolder)
+base_model = AutoModelForCausalLM.from_pretrained(
+    base_id,
+    torch_dtype="auto",
+    device_map="auto",
+)
+model = PeftModel.from_pretrained(
+    base_model,
+    repo_id,
+    subfolder=subfolder,
+)
+```
+
+Llama adapter 需要先在 Hugging Face 接受
+`meta-llama/Llama-3.2-1B-Instruct` 的访问条款，并使用有权限的账户登录。
+
+## 完整性验证
+
+从仓库根目录执行：
+
+```bash
+sha256sum -c MANIFEST.sha256
+```
+
+`MANIFEST.sha256` 覆盖 `phase1/` 下发布的全部文件。Safetensors 通过 Git LFS
+管理；基础模型权重不会被提交到本仓库。
+
+## 许可证与限制
+
+- Qwen adapter 依赖 Apache-2.0 许可的 Qwen2.5 基础模型；
+- Llama adapter 的使用受 Llama 3.2 Community License 及其访问条款约束；
+- 训练数据的来源、许可和限制以配套数据仓库的 dataset card 为准。
+
+这些 adapter 是研究实验产物，未针对生产安全、事实准确性或具体行业用途做保证。

phase1/ablations/empty_context/README.md

@@ -0,0 +1,80 @@
+---
+library_name: peft
+model_name: phase1-qwen-empty-context-ablation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-1.5B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+license: apache-2.0
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+pipeline_tag: text-generation
+---
+
+# Phase 1 Qwen `empty_context` diagnostic ablation
+
+This is the Phase 1 `empty_context` diagnostic LoRA-DPO adapter fine-tuned from
+[Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct).
+The frozen training recipe and membership hashes are included in this directory.
+
+## Quick start
+
+```python
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo_id = "geyuxu/york-milestone-project-self-traing-loop-model"
+subfolder = "phase1/ablations/empty_context"
+base_id = "Qwen/Qwen2.5-1.5B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(repo_id, subfolder=subfolder)
+base_model = AutoModelForCausalLM.from_pretrained(
+    base_id, torch_dtype="auto", device_map="auto"
+)
+model = PeftModel.from_pretrained(base_model, repo_id, subfolder=subfolder)
+```
+
+## Training procedure
+
+
+
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 0.29.0
+- Transformers: 5.3.0
+- Pytorch: 2.10.0
+- Datasets: 4.6.1
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+    title        = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+    author       = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+    year         = 2023,
+    booktitle    = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+    url          = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+    editor       = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

phase1/ablations/empty_context/adapter_config.json

@@ -0,0 +1,43 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "k_proj",
+    "o_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

phase1/ablations/empty_context/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79a128d64b87a92a3b4450386e35821dc796e494dca46b4d8c421b8580dbfd63
+size 17462432

phase1/ablations/empty_context/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

phase1/ablations/empty_context/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5
+size 11422166

phase1/ablations/empty_context/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

phase1/ablations/empty_context/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4980eeedcdde01c95d37224943c3c64dcbeed7f0894c9e83bd99f12534a473fd
+size 5841

phase1/ablations/empty_context/training_recipe.json

@@ -0,0 +1,22 @@
+{
+  "format_version": 1,
+  "experiment": "confusion_ablation",
+  "ablation": "empty_context",
+  "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+  "train_membership_sha256": "b497fa765223a9810ad784f31e1598edfa992a162146d6f94d9cdf4adcedd87d",
+  "validation_membership_sha256": "2f609467438cd6a01fd84b9d80c0fcd84c44767e3b2ed98f4f6a83d68a62c181",
+  "num_train_pairs": 77124,
+  "num_validation_pairs": 8563,
+  "transformation_seed": 22,
+  "validation_prompt_mode": "clean_rag",
+  "learning_rate": 1e-05,
+  "beta": 0.1,
+  "num_epochs": 1.0,
+  "seed": 22,
+  "max_length": 512,
+  "max_prompt_length": 384,
+  "gradient_accumulation_steps": 80,
+  "save_steps": 125,
+  "save_total_limit": 12,
+  "preference_pairs_format_version": 2
+}

phase1/ablations/empty_context/training_summary.json

@@ -0,0 +1,2925 @@
+{
+  "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+  "resolved_model": "Qwen/Qwen2.5-1.5B-Instruct",
+  "num_train_records": 77124,
+  "num_validation_records_available": 8563,
+  "training_config": {
+    "lora_rank": 16,
+    "lora_alpha": 32,
+    "learning_rate": 1e-05,
+    "num_epochs": 1.0,
+    "per_device_batch_size": 1,
+    "per_device_eval_batch_size": 1,
+    "gradient_accumulation_steps": 80,
+    "beta": 0.1,
+    "max_length": 512,
+    "max_prompt_length": 384,
+    "seed": 22,
+    "save_steps": 125,
+    "save_total_limit": 12,
+    "resume_from_checkpoint": null,
+    "precision_dtype": "torch.bfloat16",
+    "bf16": true,
+    "fp16": false,
+    "tokenizer_add_bos_token": false
+  },
+  "global_step": 965,
+  "best_metric": 0.5264487266540527,
+  "best_model_checkpoint": "outputs/confusion_ablation_phase1_repair_v3/empty_context/lora/checkpoint-875",
+  "train_metrics": {
+    "train_runtime": 22398.3946,
+    "train_samples_per_second": 3.443,
+    "train_steps_per_second": 0.043,
+    "total_flos": 1.3308435135621734e+17,
+    "train_loss": 0.48160764293967134
+  },
+  "log_history": [
+    {
+      "loss": 0.6885997772216796,
+      "grad_norm": 0.012969385832548141,
+      "learning_rate": 9.906735751295338e-06,
+      "entropy": 1.0798783786967396,
+      "num_tokens": 156534.0,
+      "logits/chosen": -0.7346464429003529,
+      "logits/rejected": -0.7633830132317224,
+      "mean_token_accuracy": 0.689994627982378,
+      "rewards/chosen": 0.004463997096299863,
+      "rewards/rejected": -0.005229698287648716,
+      "rewards/accuracies": 0.5475,
+      "rewards/margins": 0.009693695359455887,
+      "logps/chosen": -53.64696934700012,
+      "logps/rejected": -45.47284424066544,
+      "epoch": 0.010372905969607386,
+      "step": 10
+    },
+    {
+      "loss": 0.6514244079589844,
+      "grad_norm": 0.01852896623313427,
+      "learning_rate": 9.803108808290156e-06,
+      "entropy": 1.1224612561706453,
+      "num_tokens": 313607.0,
+      "logits/chosen": -0.6341513052007935,
+      "logits/rejected": -0.6562745898278561,
+      "mean_token_accuracy": 0.6891892428323626,
+      "rewards/chosen": 0.03406709198377939,
+      "rewards/rejected": -0.05670502469384701,
+      "rewards/accuracies": 0.7825,
+      "rewards/margins": 0.09077211681949848,
+      "logps/chosen": -55.87430178999901,
+      "logps/rejected": -44.03523090600967,
+      "epoch": 0.02074581193921477,
+      "step": 20
+    },
+    {
+      "loss": 0.6161895275115967,
+      "grad_norm": 0.009809216484427452,
+      "learning_rate": 9.699481865284975e-06,
+      "entropy": 1.2153802848607302,
+      "num_tokens": 471990.0,
+      "logits/chosen": -0.5183433123187651,
+      "logits/rejected": -0.48201045809287074,
+      "mean_token_accuracy": 0.6826715680211782,
+      "rewards/chosen": 0.03547020315836562,
+      "rewards/rejected": -0.15197590536190547,
+      "rewards/accuracies": 0.76625,
+      "rewards/margins": 0.18744610846129944,
+      "logps/chosen": -55.871621309518815,
+      "logps/rejected": -47.164069904088976,
+      "epoch": 0.031118717908822157,
+      "step": 30
+    },
+    {
+      "loss": 0.5837482929229736,
+      "grad_norm": 0.009694655425846577,
+      "learning_rate": 9.595854922279793e-06,
+      "entropy": 1.2649927862174808,
+      "num_tokens": 630457.0,
+      "logits/chosen": -0.41558199378068605,
+      "logits/rejected": -0.36016731520911593,
+      "mean_token_accuracy": 0.6745159946288913,
+      "rewards/chosen": 0.0030272094297106376,
+      "rewards/rejected": -0.28438122774517977,
+      "rewards/accuracies": 0.78,
+      "rewards/margins": 0.2874084369814955,
+      "logps/chosen": -56.05272542119026,
+      "logps/rejected": -47.54765802979469,
+      "epoch": 0.04149162387842954,
+      "step": 40
+    },
+    {
+      "loss": 0.5699157238006591,
+      "grad_norm": 0.00790126621723175,
+      "learning_rate": 9.492227979274612e-06,
+      "entropy": 1.3643021282646806,
+      "num_tokens": 788279.0,
+      "logits/chosen": -0.2938073242511318,
+      "logits/rejected": -0.2156574113712712,
+      "mean_token_accuracy": 0.6769538759440183,
+      "rewards/chosen": -0.057574519852860245,
+      "rewards/rejected": -0.4283463936719636,
+      "rewards/accuracies": 0.77,
+      "rewards/margins": 0.37077187476679685,
+      "logps/chosen": -57.178628134727475,
+      "logps/rejected": -48.340812134742734,
+      "epoch": 0.051864529848036925,
+      "step": 50
+    },
+    {
+      "loss": 0.5565647125244141,
+      "grad_norm": 0.007377359550446272,
+      "learning_rate": 9.388601036269432e-06,
+      "entropy": 1.4374694915488362,
+      "num_tokens": 944837.0,
+      "logits/chosen": -0.24100234533637963,
+      "logits/rejected": -0.15933266229535895,
+      "mean_token_accuracy": 0.6763953970745206,
+      "rewards/chosen": -0.11264537643779476,
+      "rewards/rejected": -0.5672522658561502,
+      "rewards/accuracies": 0.77125,
+      "rewards/margins": 0.4546068909333553,
+      "logps/chosen": -56.07041595697403,
+      "logps/rejected": -49.03205164551735,
+      "epoch": 0.062237435817644314,
+      "step": 60
+    },
+    {
+      "loss": 0.5234037876129151,
+      "grad_norm": 0.008119487203657627,
+      "learning_rate": 9.284974093264249e-06,
+      "entropy": 1.4613975173421203,
+      "num_tokens": 1102438.0,
+      "logits/chosen": -0.2570220845928096,
+      "logits/rejected": -0.08345125228558167,
+      "mean_token_accuracy": 0.6671836027130484,
+      "rewards/chosen": -0.20022683466155286,
+      "rewards/rejected": -0.7679086389223813,
+      "rewards/accuracies": 0.7875,
+      "rewards/margins": 0.5676818046835251,
+      "logps/chosen": -58.67170790791511,
+      "logps/rejected": -51.08670456886291,
+      "epoch": 0.0726103417872517,
+      "step": 70
+    },
+    {
+      "loss": 0.5317386150360107,
+      "grad_norm": 0.008479549549520016,
+      "learning_rate": 9.181347150259067e-06,
+      "entropy": 1.5662438894622028,
+      "num_tokens": 1259718.0,
+      "logits/chosen": -0.19741540588436432,
+      "logits/rejected": -0.05688106650755805,
+      "mean_token_accuracy": 0.6640376079455018,
+      "rewards/chosen": -0.30314797887514944,
+      "rewards/rejected": -0.9170537282113219,
+      "rewards/accuracies": 0.775,
+      "rewards/margins": 0.6139057491905987,
+      "logps/chosen": -58.94739377617836,
+      "logps/rejected": -54.774017692804335,
+      "epoch": 0.08298324775685909,
+      "step": 80
+    },
+    {
+      "loss": 0.524826192855835,
+      "grad_norm": 0.008542119525372982,
+      "learning_rate": 9.077720207253888e-06,
+      "entropy": 1.5676167696528136,
+      "num_tokens": 1416640.0,
+      "logits/chosen": -0.19087733562866938,
+      "logits/rejected": -0.06961324446158064,
+      "mean_token_accuracy": 0.6602621851768344,
+      "rewards/chosen": -0.32906408929731695,
+      "rewards/rejected": -1.0247976127301808,
+      "rewards/accuracies": 0.78,
+      "rewards/margins": 0.6957335224747658,
+      "logps/chosen": -59.972526935338976,
+      "logps/rejected": -54.91353523254394,
+      "epoch": 0.09335615372646647,
+      "step": 90
+    },
+    {
+      "loss": 0.5366811752319336,
+      "grad_norm": 0.01028946042060852,
+      "learning_rate": 8.974093264248706e-06,
+      "entropy": 1.6082658682204782,
+      "num_tokens": 1578582.0,
+      "logits/chosen": -0.13524073540751783,
+      "logits/rejected": -0.03532241128049209,
+      "mean_token_accuracy": 0.6632538431324064,
+      "rewards/chosen": -0.45744188758188103,
+      "rewards/rejected": -1.1608894589025294,
+      "rewards/accuracies": 0.75625,
+      "rewards/margins": 0.7034475702582859,
+      "logps/chosen": -65.19879947781563,
+      "logps/rejected": -61.238030140399935,
+      "epoch": 0.10372905969607385,
+      "step": 100
+    },
+    {
+      "loss": 0.5021177291870117,
+      "grad_norm": 0.009549791924655437,
+      "learning_rate": 8.870466321243523e-06,
+      "entropy": 1.580866142194718,
+      "num_tokens": 1736902.0,
+      "logits/chosen": -0.16430359268125203,
+      "logits/rejected": -0.02982257553996645,
+      "mean_token_accuracy": 0.6615490332618356,
+      "rewards/chosen": -0.4121622362524067,
+      "rewards/rejected": -1.1860957287461498,
+      "rewards/accuracies": 0.80375,
+      "rewards/margins": 0.7739334937836975,
+      "logps/chosen": -60.39180130243302,
+      "logps/rejected": -55.96378324866295,
+      "epoch": 0.11410196566568125,
+      "step": 110
+    },
+    {
+      "loss": 0.5247333526611329,
+      "grad_norm": 0.007295758463442326,
+      "learning_rate": 8.766839378238343e-06,
+      "entropy": 1.5716826527751981,
+      "num_tokens": 1894607.0,
+      "logits/chosen": -0.21342730308044772,
+      "logits/rejected": -0.08283230979493357,
+      "mean_token_accuracy": 0.668023902028799,
+      "rewards/chosen": -0.4445982250826205,
+      "rewards/rejected": -1.1845778720156523,
+      "rewards/accuracies": 0.76875,
+      "rewards/margins": 0.7399796470795991,
+      "logps/chosen": -61.616042573452,
+      "logps/rejected": -53.51006484746933,
+      "epoch": 0.12447487163528863,
+      "step": 120
+    },
+    {
+      "eval_loss": 0.5866048336029053,
+      "eval_runtime": 186.3701,
+      "eval_samples_per_second": 5.366,
+      "eval_steps_per_second": 5.366,
+      "eval_entropy": 1.026585803925991,
+      "eval_num_tokens": 1973585.0,
+      "eval_logits/chosen": -0.27179382896787163,
+      "eval_logits/rejected": -0.1471519449829466,
+      "eval_mean_token_accuracy": 0.7038057380318642,
+      "eval_rewards/chosen": -0.4332459507464664,
+      "eval_rewards/rejected": -0.8382879723370134,
+      "eval_rewards/accuracies": 0.673,
+      "eval_rewards/margins": 0.40504202180169524,
+      "eval_logps/chosen": -54.505446267127994,
+      "eval_logps/rejected": -61.022751308441165,
+      "epoch": 0.1296613246200923,
+      "step": 125
+    },
+    {
+      "loss": 0.5197542190551758,
+      "grad_norm": 0.007421356160193682,
+      "learning_rate": 8.663212435233162e-06,
+      "entropy": 1.6153548054210842,
+      "num_tokens": 2053549.0,
+      "logits/chosen": -0.16180727962310634,
+      "logits/rejected": -0.02674481546655947,
+      "mean_token_accuracy": 0.6631045359373092,
+      "rewards/chosen": -0.44111566214839515,
+      "rewards/rejected": -1.2121200907422462,
+      "rewards/accuracies": 0.7875,
+      "rewards/margins": 0.7710044272383675,
+      "logps/chosen": -60.93198619604111,
+      "logps/rejected": -57.747130882740024,
+      "epoch": 0.13484777760489602,
+      "step": 130
+    },
+    {
+      "loss": 0.5147777080535889,
+      "grad_norm": 0.009197528474032879,
+      "learning_rate": 8.55958549222798e-06,
+      "entropy": 1.5458942975103855,
+      "num_tokens": 2209979.0,
+      "logits/chosen": -0.2903924655131609,
+      "logits/rejected": -0.10041574268765137,
+      "mean_token_accuracy": 0.6736215281113982,
+      "rewards/chosen": -0.420290390338123,
+      "rewards/rejected": -1.1833371208855532,
+      "rewards/accuracies": 0.78875,
+      "rewards/margins": 0.7630467313993722,
+      "logps/chosen": -55.64869555354118,
+      "logps/rejected": -56.70225902199745,
+      "epoch": 0.1452206835745034,
+      "step": 140
+    },
+    {
+      "loss": 0.48405823707580564,
+      "grad_norm": 0.009587124921381474,
+      "learning_rate": 8.455958549222799e-06,
+      "entropy": 1.5990170714166014,
+      "num_tokens": 2369571.0,
+      "logits/chosen": -0.24899731487552476,
+      "logits/rejected": -0.0959463068576649,
+      "mean_token_accuracy": 0.6553366997931153,
+      "rewards/chosen": -0.3976931418222512,
+      "rewards/rejected": -1.2890572238096503,
+      "rewards/accuracies": 0.8025,
+      "rewards/margins": 0.8913640801142901,
+      "logps/chosen": -60.26506376743317,
+      "logps/rejected": -59.52328753948212,
+      "epoch": 0.1555935895441108,
+      "step": 150
+    },
+    {
+      "loss": 0.5164113998413086,
+      "grad_norm": 0.009895331226289272,
+      "learning_rate": 8.352331606217617e-06,
+      "entropy": 1.6323789210617543,
+      "num_tokens": 2528769.0,
+      "logits/chosen": -0.2223647205235664,
+      "logits/rejected": -0.09436181620109207,
+      "mean_token_accuracy": 0.6549394633993506,
+      "rewards/chosen": -0.5111256973049604,
+      "rewards/rejected": -1.2863853847890276,
+      "rewards/accuracies": 0.77875,
+      "rewards/margins": 0.7752596874302253,
+      "logps/chosen": -64.33949285626412,
+      "logps/rejected": -59.65175414085388,
+      "epoch": 0.16596649551371817,
+      "step": 160
+    },
+    {
+      "loss": 0.5000368118286133,
+      "grad_norm": 0.008498546667397022,
+      "learning_rate": 8.248704663212436e-06,
+      "entropy": 1.6361813547555357,
+      "num_tokens": 2688485.0,
+      "logits/chosen": -0.16995661589474917,
+      "logits/rejected": -0.058628515103994625,
+      "mean_token_accuracy": 0.6581607956252992,
+      "rewards/chosen": -0.4723435838901787,
+      "rewards/rejected": -1.3314759727602359,
+      "rewards/accuracies": 0.79,
+      "rewards/margins": 0.8591323888232,
+      "logps/chosen": -63.09673968315124,
+      "logps/rejected": -58.408678278923034,
+      "epoch": 0.17633940148332555,
+      "step": 170
+    },
+    {
+      "loss": 0.48834381103515623,
+      "grad_norm": 0.0076238978654146194,
+      "learning_rate": 8.145077720207254e-06,
+      "entropy": 1.6382397083751856,
+      "num_tokens": 2845521.0,
+      "logits/chosen": -0.15718930734615982,
+      "logits/rejected": -0.05206926899191467,
+      "mean_token_accuracy": 0.6570399883762001,
+      "rewards/chosen": -0.44634571393646183,
+      "rewards/rejected": -1.3426366519206203,
+      "rewards/accuracies": 0.80125,
+      "rewards/margins": 0.8962909340648912,
+      "logps/chosen": -60.25443920731544,
+      "logps/rejected": -57.17802233457565,
+      "epoch": 0.18671230745293294,
+      "step": 180
+    },
+    {
+      "loss": 0.47557435035705564,
+      "grad_norm": 0.008195850066840649,
+      "learning_rate": 8.041450777202073e-06,
+      "entropy": 1.664605896770954,
+      "num_tokens": 3002074.0,
+      "logits/chosen": -0.18363723049103992,
+      "logits/rejected": -0.02745318001992498,
+      "mean_token_accuracy": 0.6470572968758642,
+      "rewards/chosen": -0.4189180153416373,
+      "rewards/rejected": -1.3144572579705618,
+      "rewards/accuracies": 0.80625,
+      "rewards/margins": 0.895539241922088,
+      "logps/chosen": -62.842660636901854,
+      "logps/rejected": -55.66383082151413,
+      "epoch": 0.19708521342254032,
+      "step": 190
+    },
+    {
+      "loss": 0.534032154083252,
+      "grad_norm": 0.009718852117657661,
+      "learning_rate": 7.937823834196891e-06,
+      "entropy": 1.624278563093394,
+      "num_tokens": 3158718.0,
+      "logits/chosen": -0.1727993139137808,
+      "logits/rejected": -0.03551834474392872,
+      "mean_token_accuracy": 0.659155095871538,
+      "rewards/chosen": -0.4744971431684098,
+      "rewards/rejected": -1.2906594037846661,
+      "rewards/accuracies": 0.7525,
+      "rewards/margins": 0.8161622601863928,
+      "logps/chosen": -59.0803443980217,
+      "logps/rejected": -56.69621436238289,
+      "epoch": 0.2074581193921477,
+      "step": 200
+    },
+    {
+      "loss": 0.49621829986572263,
+      "grad_norm": 0.006965890992432833,
+      "learning_rate": 7.834196891191712e-06,
+      "entropy": 1.6065182481892406,
+      "num_tokens": 3312685.0,
+      "logits/chosen": -0.27075287980432916,
+      "logits/rejected": -0.0948328996504896,
+      "mean_token_accuracy": 0.655901060551405,
+      "rewards/chosen": -0.37993197983159915,
+      "rewards/rejected": -1.2266587249445728,
+      "rewards/accuracies": 0.7925,
+      "rewards/margins": 0.8467267453856766,
+      "logps/chosen": -56.4505591404438,
+      "logps/rejected": -53.79004474401474,
+      "epoch": 0.2178310253617551,
+      "step": 210
+    },
+    {
+      "loss": 0.48090167045593263,
+      "grad_norm": 0.009547212161123753,
+      "learning_rate": 7.730569948186528e-06,
+      "entropy": 1.6623543649539352,
+      "num_tokens": 3472099.0,
+      "logits/chosen": -0.13968818712728476,
+      "logits/rejected": 0.0031978122884744786,
+      "mean_token_accuracy": 0.6451264442130923,
+      "rewards/chosen": -0.482780237980769,
+      "rewards/rejected": -1.413478313506348,
+      "rewards/accuracies": 0.795,
+      "rewards/margins": 0.930698074856773,
+      "logps/chosen": -60.84456259608269,
+      "logps/rejected": -58.714372800588606,
+      "epoch": 0.2282039313313625,
+      "step": 220
+    },
+    {
+      "loss": 0.4559622764587402,
+      "grad_norm": 0.007603341713547707,
+      "learning_rate": 7.626943005181348e-06,
+      "entropy": 1.6722249686904251,
+      "num_tokens": 3632168.0,
+      "logits/chosen": -0.10667086636674211,
+      "logits/rejected": -0.022223443209434823,
+      "mean_token_accuracy": 0.6569089805148542,
+      "rewards/chosen": -0.4016924969325191,
+      "rewards/rejected": -1.4414992470349535,
+      "rewards/accuracies": 0.8125,
+      "rewards/margins": 1.0398067526565864,
+      "logps/chosen": -65.72550685882568,
+      "logps/rejected": -59.37691457271576,
+      "epoch": 0.23857683730096987,
+      "step": 230
+    },
+    {
+      "loss": 0.4861691951751709,
+      "grad_norm": 0.0118019487708807,
+      "learning_rate": 7.523316062176167e-06,
+      "entropy": 1.663213079739362,
+      "num_tokens": 3790720.0,
+      "logits/chosen": -0.084852375856096,
+      "logits/rejected": 0.020669056782071628,
+      "mean_token_accuracy": 0.6473133432120085,
+      "rewards/chosen": -0.48122090814737023,
+      "rewards/rejected": -1.4429112731330678,
+      "rewards/accuracies": 0.7925,
+      "rewards/margins": 0.9616903657466174,
+      "logps/chosen": -63.50809263586998,
+      "logps/rejected": -59.43233494758606,
+      "epoch": 0.24894974327057726,
+      "step": 240
+    },
+    {
+      "loss": 0.4908440589904785,
+      "grad_norm": 0.009277455508708954,
+      "learning_rate": 7.419689119170985e-06,
+      "entropy": 1.6366083236783744,
+      "num_tokens": 3949401.0,
+      "logits/chosen": -0.1602433276424981,
+      "logits/rejected": 0.01983137979948086,
+      "mean_token_accuracy": 0.6576688695885241,
+      "rewards/chosen": -0.4815865662365832,
+      "rewards/rejected": -1.3985355464673193,
+      "rewards/accuracies": 0.78375,
+      "rewards/margins": 0.9169489816110581,
+      "logps/chosen": -62.95811867237091,
+      "logps/rejected": -59.38419282078743,
+      "epoch": 0.2593226492401846,
+      "step": 250
+    },
+    {
+      "eval_loss": 0.5700510144233704,
+      "eval_runtime": 215.5827,
+      "eval_samples_per_second": 4.639,
+      "eval_steps_per_second": 4.639,
+      "eval_entropy": 1.0522404039800166,
+      "eval_num_tokens": 3949401.0,
+      "eval_logits/chosen": -0.2592554757373893,
+      "eval_logits/rejected": -0.12886324215802,
+      "eval_mean_token_accuracy": 0.6999065904319286,
+      "eval_rewards/chosen": -0.5199006232493557,
+      "eval_rewards/rejected": -1.0152434906987473,
+      "eval_rewards/accuracies": 0.689,
+      "eval_rewards/margins": 0.4953428685963154,
+      "eval_logps/chosen": -55.371993000030514,
+      "eval_logps/rejected": -62.79230645370483,
+      "epoch": 0.2593226492401846,
+      "step": 250
+    },
+    {
+      "loss": 0.5332072734832763,
+      "grad_norm": 0.01310813333839178,
+      "learning_rate": 7.3160621761658035e-06,
+      "entropy": 1.6763470254838466,
+      "num_tokens": 4109572.0,
+      "logits/chosen": -0.11223109996092892,
+      "logits/rejected": 0.00783857225474676,
+      "mean_token_accuracy": 0.6527093886770308,
+      "rewards/chosen": -0.5290841975083458,
+      "rewards/rejected": -1.4143161330505973,
+      "rewards/accuracies": 0.76875,
+      "rewards/margins": 0.8852319360524415,
+      "logps/chosen": -65.61116208314895,
+      "logps/rejected": -60.50024994492531,
+      "epoch": 0.26969555520979205,
+      "step": 260
+    },
+    {
+      "loss": 0.46380929946899413,
+      "grad_norm": 0.008016095496714115,
+      "learning_rate": 7.212435233160623e-06,
+      "entropy": 1.6493263538647442,
+      "num_tokens": 4264194.0,
+      "logits/chosen": -0.15574262186655452,
+      "logits/rejected": -0.00402961174777503,
+      "mean_token_accuracy": 0.6535617489926517,
+      "rewards/chosen": -0.39940022186754504,
+      "rewards/rejected": -1.3337873129447688,
+      "rewards/accuracies": 0.80625,
+      "rewards/margins": 0.9343870897591114,
+      "logps/chosen": -58.40460694432259,
+      "logps/rejected": -56.3271455013752,
+      "epoch": 0.28006846117939943,
+      "step": 270
+    },
+    {
+      "loss": 0.46946086883544924,
+      "grad_norm": 0.010439381934702396,
+      "learning_rate": 7.108808290155441e-06,
+      "entropy": 1.6875111198704689,
+      "num_tokens": 4422991.0,
+      "logits/chosen": -0.15225440851382163,
+      "logits/rejected": -0.012922279572286165,
+      "mean_token_accuracy": 0.6428867661487311,
+      "rewards/chosen": -0.42678490347389014,
+      "rewards/rejected": -1.3960492441387031,
+      "rewards/accuracies": 0.815,
+      "rewards/margins": 0.9692643392784521,
+      "logps/chosen": -63.95781509041786,
+      "logps/rejected": -57.44645619392395,
+      "epoch": 0.2904413671490068,
+      "step": 280
+    },
+    {
+      "loss": 0.4867039680480957,
+      "grad_norm": 0.008255927823483944,
+      "learning_rate": 7.005181347150259e-06,
+      "entropy": 1.6560454944707452,
+      "num_tokens": 4582080.0,
+      "logits/chosen": -0.12475022342462339,
+      "logits/rejected": -0.004146134714328987,
+      "mean_token_accuracy": 0.6578386729583144,
+      "rewards/chosen": -0.4762678133821464,
+      "rewards/rejected": -1.3799970698577817,
+      "rewards/accuracies": 0.78875,
+      "rewards/margins": 0.9037292560562491,
+      "logps/chosen": -62.563043652772905,
+      "logps/rejected": -58.98709165453911,
+      "epoch": 0.3008142731186142,
+      "step": 290
+    },
+    {
+      "loss": 0.46946206092834475,
+      "grad_norm": 0.008176690898835659,
+      "learning_rate": 6.9015544041450784e-06,
+      "entropy": 1.636359941866249,
+      "num_tokens": 4740383.0,
+      "logits/chosen": -0.14494295129202733,
+      "logits/rejected": -0.012990473919638954,
+      "mean_token_accuracy": 0.6567367980629206,
+      "rewards/chosen": -0.5025469059059469,
+      "rewards/rejected": -1.4605811326974072,
+      "rewards/accuracies": 0.80125,
+      "rewards/margins": 0.9580342254508287,
+      "logps/chosen": -61.66068306684494,
+      "logps/rejected": -59.60216732621193,
+      "epoch": 0.3111871790882216,
+      "step": 300
+    },
+    {
+      "loss": 0.4926762580871582,
+      "grad_norm": 0.008404894731938839,
+      "learning_rate": 6.797927461139897e-06,
+      "entropy": 1.6317085930332542,
+      "num_tokens": 4901454.0,
+      "logits/chosen": -0.10420462296034713,
+      "logits/rejected": -0.0032643015884895763,
+      "mean_token_accuracy": 0.6517730862647295,
+      "rewards/chosen": -0.4740483855601633,
+      "rewards/rejected": -1.467887537285569,
+      "rewards/accuracies": 0.78625,
+      "rewards/margins": 0.9938391536381096,
+      "logps/chosen": -62.19328094601631,
+      "logps/rejected": -62.36802620172501,
+      "epoch": 0.32156008505782896,
+      "step": 310
+    },
+    {
+      "loss": 0.4758056640625,
+      "grad_norm": 0.01086408831179142,
+      "learning_rate": 6.6943005181347155e-06,
+      "entropy": 1.6321182049531489,
+      "num_tokens": 5058889.0,
+      "logits/chosen": -0.1815479056351674,
+      "logits/rejected": -0.07081081249938541,
+      "mean_token_accuracy": 0.6414232835173607,
+      "rewards/chosen": -0.5062945262032008,
+      "rewards/rejected": -1.4949601662519854,
+      "rewards/accuracies": 0.81,
+      "rewards/margins": 0.9886656388547271,
+      "logps/chosen": -63.52185977697373,
+      "logps/rejected": -59.69021632432938,
+      "epoch": 0.33193299102743634,
+      "step": 320
+    },
+    {
+      "loss": 0.4784576892852783,
+      "grad_norm": 0.010599741712212563,
+      "learning_rate": 6.590673575129535e-06,
+      "entropy": 1.6721400913968683,
+      "num_tokens": 5216549.0,
+      "logits/chosen": -0.1229586102306177,
+      "logits/rejected": -0.033501552710855156,
+      "mean_token_accuracy": 0.639937344957143,
+      "rewards/chosen": -0.5294519958324235,
+      "rewards/rejected": -1.616222132177645,
+      "rewards/accuracies": 0.78,
+      "rewards/margins": 1.086770132854581,
+      "logps/chosen": -63.12015713572502,
+      "logps/rejected": -62.12025535821915,
+      "epoch": 0.3423058969970437,
+      "step": 330
+    },
+    {
+      "loss": 0.4864365100860596,
+      "grad_norm": 0.010677792131900787,
+      "learning_rate": 6.487046632124353e-06,
+      "entropy": 1.5796756833419203,
+      "num_tokens": 5374161.0,
+      "logits/chosen": -0.17744578636097688,
+      "logits/rejected": -0.08292268708566802,
+      "mean_token_accuracy": 0.6436819550767541,
+      "rewards/chosen": -0.5906799642526311,
+      "rewards/rejected": -1.6022152152087075,
+      "rewards/accuracies": 0.78375,
+      "rewards/margins": 1.0115352519135923,
+      "logps/chosen": -62.16209160447121,
+      "logps/rejected": -60.97210006713867,
+      "epoch": 0.3526788029666511,
+      "step": 340
+    },
+    {
+      "loss": 0.5056374549865723,
+      "grad_norm": 0.013692691922187805,
+      "learning_rate": 6.383419689119171e-06,
+      "entropy": 1.5922023140452801,
+      "num_tokens": 5533185.0,
+      "logits/chosen": -0.11285970908028921,
+      "logits/rejected": -0.018185535407032168,
+      "mean_token_accuracy": 0.6443661257252097,
+      "rewards/chosen": -0.580222937125145,
+      "rewards/rejected": -1.5960858852404636,
+      "rewards/accuracies": 0.7725,
+      "rewards/margins": 1.0158629460725934,
+      "logps/chosen": -64.89270892024041,
+      "logps/rejected": -62.29878480195999,
+      "epoch": 0.3630517089362585,
+      "step": 350
+    },
+    {
+      "loss": 0.48989334106445315,
+      "grad_norm": 0.012253508903086185,
+      "learning_rate": 6.2797927461139905e-06,
+      "entropy": 1.5395906928181649,
+      "num_tokens": 5691159.0,
+      "logits/chosen": -0.14245793212958643,
+      "logits/rejected": -0.015230929526604408,
+      "mean_token_accuracy": 0.6611733642220498,
+      "rewards/chosen": -0.5437271582013921,
+      "rewards/rejected": -1.5548808443109738,
+      "rewards/accuracies": 0.7975,
+      "rewards/margins": 1.011153682768345,
+      "logps/chosen": -60.83840570926666,
+      "logps/rejected": -59.6216096830368,
+      "epoch": 0.37342461490586587,
+      "step": 360
+    },
+    {
+      "loss": 0.48537001609802244,
+      "grad_norm": 0.00956793874502182,
+      "learning_rate": 6.176165803108809e-06,
+      "entropy": 1.6442958949133755,
+      "num_tokens": 5850088.0,
+      "logits/chosen": -0.058320485277038855,
+      "logits/rejected": 0.06270940886949956,
+      "mean_token_accuracy": 0.6433443369530142,
+      "rewards/chosen": -0.5535192088356417,
+      "rewards/rejected": -1.5760743238392751,
+      "rewards/accuracies": 0.7975,
+      "rewards/margins": 1.0225551136396824,
+      "logps/chosen": -66.1269180560112,
+      "logps/rejected": -60.73978069186211,
+      "epoch": 0.38379752087547325,
+      "step": 370
+    },
+    {
+      "eval_loss": 0.5541799664497375,
+      "eval_runtime": 245.9984,
+      "eval_samples_per_second": 4.065,
+      "eval_steps_per_second": 4.065,
+      "eval_entropy": 1.0474999970644712,
+      "eval_num_tokens": 5928735.0,
+      "eval_logits/chosen": -0.23271217653526577,
+      "eval_logits/rejected": -0.10208672570441625,
+      "eval_mean_token_accuracy": 0.6993389547616243,
+      "eval_rewards/chosen": -0.571883928276773,
+      "eval_rewards/rejected": -1.1227601580685005,
+      "eval_rewards/accuracies": 0.707,
+      "eval_rewards/margins": 0.5508762280121445,
+      "eval_logps/chosen": -55.89182602119446,
+      "eval_logps/rejected": -63.867473125457764,
+      "epoch": 0.38898397386027694,
+      "step": 375
+    },
+    {
+      "loss": 0.447767972946167,
+      "grad_norm": 0.010045935399830341,
+      "learning_rate": 6.0725388601036275e-06,
+      "entropy": 1.5958718929998577,
+      "num_tokens": 6007948.0,
+      "logits/chosen": -0.010203895356886221,
+      "logits/rejected": 0.0953480252296025,
+      "mean_token_accuracy": 0.6510641277581454,
+      "rewards/chosen": -0.46026305966945075,
+      "rewards/rejected": -1.566630380146671,
+      "rewards/accuracies": 0.825,
+      "rewards/margins": 1.1063673190772534,
+      "logps/chosen": -61.75375435590744,
+      "logps/rejected": -58.965001332759854,
+      "epoch": 0.39417042684508063,
+      "step": 380
+    },
+    {
+      "loss": 0.4722391128540039,
+      "grad_norm": 0.010498611256480217,
+      "learning_rate": 5.968911917098445e-06,
+      "entropy": 1.6217002650536596,
+      "num_tokens": 6167000.0,
+      "logits/chosen": -0.010942938609718112,
+      "logits/rejected": 0.13150225704237622,
+      "mean_token_accuracy": 0.6487077697180211,
+      "rewards/chosen": -0.4893380562632228,
+      "rewards/rejected": -1.546268537521828,
+      "rewards/accuracies": 0.79875,
+      "rewards/margins": 1.056930480999872,
+      "logps/chosen": -63.818650953769684,
+      "logps/rejected": -61.81397884130478,
+      "epoch": 0.404543332814688,
+      "step": 390
+    },
+    {
+      "loss": 0.4782541275024414,
+      "grad_norm": 0.009464209899306297,
+      "learning_rate": 5.865284974093265e-06,
+      "entropy": 1.6495763343665748,
+      "num_tokens": 6325205.0,
+      "logits/chosen": -0.011458384051150232,
+      "logits/rejected": 0.09236410504123445,
+      "mean_token_accuracy": 0.6448470102529973,
+      "rewards/chosen": -0.51347439962934,
+      "rewards/rejected": -1.5123917219531722,
+      "rewards/accuracies": 0.79125,
+      "rewards/margins": 0.9989173209294677,
+      "logps/chosen": -63.590197974443434,
+      "logps/rejected": -61.28543629407883,
+      "epoch": 0.4149162387842954,
+      "step": 400
+    },
+    {
+      "loss": 0.4769914627075195,
+      "grad_norm": 0.011201824992895126,
+      "learning_rate": 5.761658031088083e-06,
+      "entropy": 1.6099769476987422,
+      "num_tokens": 6483981.0,
+      "logits/chosen": -0.008573553675108867,
+      "logits/rejected": 0.10661449974217808,
+      "mean_token_accuracy": 0.6476613377220929,
+      "rewards/chosen": -0.5158215398409811,
+      "rewards/rejected": -1.5142960718111136,
+      "rewards/accuracies": 0.795,
+      "rewards/margins": 0.9984745322261006,
+      "logps/chosen": -62.939886302948,
+      "logps/rejected": -60.21168664813042,
+      "epoch": 0.4252891447539028,
+      "step": 410
+    },
+    {
+      "loss": 0.4447061061859131,
+      "grad_norm": 0.01015845313668251,
+      "learning_rate": 5.658031088082902e-06,
+      "entropy": 1.6073184362612665,
+      "num_tokens": 6642561.0,
+      "logits/chosen": 0.007230915169619996,
+      "logits/rejected": 0.10328242233161539,
+      "mean_token_accuracy": 0.6508273026905954,
+      "rewards/chosen": -0.4930744762084214,
+      "rewards/rejected": -1.6134738429238495,
+      "rewards/accuracies": 0.82,
+      "rewards/margins": 1.1203993632458151,
+      "logps/chosen": -65.08479910731316,
+      "logps/rejected": -60.65289543390274,
+      "epoch": 0.4356620507235102,
+      "step": 420
+    },
+    {
+      "loss": 0.44136753082275393,
+      "grad_norm": 0.008634388446807861,
+      "learning_rate": 5.554404145077721e-06,
+      "entropy": 1.62525453383103,
+      "num_tokens": 6802172.0,
+      "logits/chosen": 0.09677644223113024,
+      "logits/rejected": 0.19686068222909975,
+      "mean_token_accuracy": 0.6414820049889386,
+      "rewards/chosen": -0.530476398501778,
+      "rewards/rejected": -1.7060055632909643,
+      "rewards/accuracies": 0.815,
+      "rewards/margins": 1.1755291634099558,
+      "logps/chosen": -63.63275898098946,
+      "logps/rejected": -61.548430292606355,
+      "epoch": 0.4460349566931176,
+      "step": 430
+    },
+    {
+      "loss": 0.44923830032348633,
+      "grad_norm": 0.011443796567618847,
+      "learning_rate": 5.4507772020725395e-06,
+      "entropy": 1.6008457892294974,
+      "num_tokens": 6960698.0,
+      "logits/chosen": 0.03864486058133284,
+      "logits/rejected": 0.16183756452396064,
+      "mean_token_accuracy": 0.6556359087582677,
+      "rewards/chosen": -0.5446211060311179,
+      "rewards/rejected": -1.6678383790072986,
+      "rewards/accuracies": 0.8275,
+      "rewards/margins": 1.1232172738574446,
+      "logps/chosen": -61.80046648740768,
+      "logps/rejected": -60.58844954848289,
+      "epoch": 0.456407862662725,
+      "step": 440
+    },
+    {
+      "loss": 0.4775514602661133,
+      "grad_norm": 0.015888305380940437,
+      "learning_rate": 5.347150259067357e-06,
+      "entropy": 1.6431135883461685,
+      "num_tokens": 7120292.0,
+      "logits/chosen": 0.058064731861634356,
+      "logits/rejected": 0.2062258352022907,
+      "mean_token_accuracy": 0.6481080191396177,
+      "rewards/chosen": -0.5665322134363828,
+      "rewards/rejected": -1.696948098014691,
+      "rewards/accuracies": 0.79375,
+      "rewards/margins": 1.1304158817767165,
+      "logps/chosen": -62.31974795341492,
+      "logps/rejected": -62.94689394712448,
+      "epoch": 0.46678076863233237,
+      "step": 450
+    },
+    {
+      "loss": 0.45407419204711913,
+      "grad_norm": 0.011245348490774632,
+      "learning_rate": 5.243523316062177e-06,
+      "entropy": 1.6142372595332564,
+      "num_tokens": 7277503.0,
+      "logits/chosen": 0.027211253980977537,
+      "logits/rejected": 0.14270901371261632,
+      "mean_token_accuracy": 0.6483659755066037,
+      "rewards/chosen": -0.5206575704316492,
+      "rewards/rejected": -1.627284916813951,
+      "rewards/accuracies": 0.82875,
+      "rewards/margins": 1.1066273492295295,
+      "logps/chosen": -62.51640429496765,
+      "logps/rejected": -59.92736347913742,
+      "epoch": 0.47715367460193975,
+      "step": 460
+    },
+    {
+      "loss": 0.4705537796020508,
+      "grad_norm": 0.017906704917550087,
+      "learning_rate": 5.139896373056995e-06,
+      "entropy": 1.6305865264125168,
+      "num_tokens": 7433576.0,
+      "logits/chosen": -0.01868226836908457,
+      "logits/rejected": 0.10559852450806628,
+      "mean_token_accuracy": 0.6382040186412632,
+      "rewards/chosen": -0.5739967311238433,
+      "rewards/rejected": -1.6218135237134994,
+      "rewards/accuracies": 0.78125,
+      "rewards/margins": 1.0478167911106722,
+      "logps/chosen": -61.798225450515744,
+      "logps/rejected": -59.17569625377655,
+      "epoch": 0.48752658057154713,
+      "step": 470
+    },
+    {
+      "loss": 0.44884982109069826,
+      "grad_norm": 0.009567538276314735,
+      "learning_rate": 5.036269430051814e-06,
+      "entropy": 1.6556390350870789,
+      "num_tokens": 7593962.0,
+      "logits/chosen": 0.02396342091178786,
+      "logits/rejected": 0.15151965562177133,
+      "mean_token_accuracy": 0.6423015125840902,
+      "rewards/chosen": -0.6105934929932119,
+      "rewards/rejected": -1.7286542325050687,
+      "rewards/accuracies": 0.79875,
+      "rewards/margins": 1.1180607356689871,
+      "logps/chosen": -65.96984986424447,
+      "logps/rejected": -63.567516083717344,
+      "epoch": 0.4978994865411545,
+      "step": 480
+    },
+    {
+      "loss": 0.46419124603271483,
+      "grad_norm": 0.010610009543597698,
+      "learning_rate": 4.932642487046633e-06,
+      "entropy": 1.63694656169042,
+      "num_tokens": 7753841.0,
+      "logits/chosen": 0.05542139131963341,
+      "logits/rejected": 0.1531588473091942,
+      "mean_token_accuracy": 0.6532461035437882,
+      "rewards/chosen": -0.6174891357401794,
+      "rewards/rejected": -1.7478586566343437,
+      "rewards/accuracies": 0.8125,
+      "rewards/margins": 1.130369521221146,
+      "logps/chosen": -63.278802993297575,
+      "logps/rejected": -63.9350430393219,
+      "epoch": 0.5082723925107618,
+      "step": 490
+    },
+    {
+      "loss": 0.4718832015991211,
+      "grad_norm": 0.01074894331395626,
+      "learning_rate": 4.829015544041451e-06,
+      "entropy": 1.6344914321228863,
+      "num_tokens": 7911511.0,
+      "logits/chosen": 0.039370719731205275,
+      "logits/rejected": 0.16598465687796668,
+      "mean_token_accuracy": 0.6492240923829377,
+      "rewards/chosen": -0.5642123090758104,
+      "rewards/rejected": -1.6763065649938653,
+      "rewards/accuracies": 0.78625,
+      "rewards/margins": 1.1120942557882518,
+      "logps/chosen": -61.73503879547119,
+      "logps/rejected": -62.51426592111588,
+      "epoch": 0.5186452984803692,
+      "step": 500
+    },
+    {
+      "eval_loss": 0.5400257706642151,
+      "eval_runtime": 278.2919,
+      "eval_samples_per_second": 3.593,
+      "eval_steps_per_second": 3.593,
+      "eval_entropy": 1.0458819408267737,
+      "eval_num_tokens": 7911511.0,
+      "eval_logits/chosen": -0.1978596412920271,
+      "eval_logits/rejected": -0.06171821412221096,
+      "eval_mean_token_accuracy": 0.6972633041292429,
+      "eval_rewards/chosen": -0.6398490693312487,
+      "eval_rewards/rejected": -1.2539186353033873,
+      "eval_rewards/accuracies": 0.727,
+      "eval_rewards/margins": 0.6140695666372776,
+      "eval_logps/chosen": -56.57147743225098,
+      "eval_logps/rejected": -65.17905788230895,
+      "epoch": 0.5186452984803692,
+      "step": 500
+    },
+    {
+      "loss": 0.4755256175994873,
+      "grad_norm": 0.011379857547581196,
+      "learning_rate": 4.72538860103627e-06,
+      "entropy": 1.6379237968847156,
+      "num_tokens": 8070081.0,
+      "logits/chosen": 0.08480375218267698,
+      "logits/rejected": 0.18248930696878024,
+      "mean_token_accuracy": 0.6486436153575778,
+      "rewards/chosen": -0.6162076763511868,
+      "rewards/rejected": -1.6715471999935108,
+      "rewards/accuracies": 0.80625,
+      "rewards/margins": 1.0553395241731778,
+      "logps/chosen": -64.42356903076171,
+      "logps/rejected": -59.52313063383102,
+      "epoch": 0.5290182044499767,
+      "step": 510
+    },
+    {
+      "loss": 0.4912434101104736,
+      "grad_norm": 0.009543896652758121,
+      "learning_rate": 4.621761658031089e-06,
+      "entropy": 1.62886246021837,
+      "num_tokens": 8226032.0,
+      "logits/chosen": 0.04373571989101422,
+      "logits/rejected": 0.1632708972191895,
+      "mean_token_accuracy": 0.6450024632364512,
+      "rewards/chosen": -0.5994134752946411,
+      "rewards/rejected": -1.5973126511012379,
+      "rewards/accuracies": 0.79,
+      "rewards/margins": 0.9978991763386875,
+      "logps/chosen": -61.06594479799271,
+      "logps/rejected": -59.51982655286789,
+      "epoch": 0.5393911104195841,
+      "step": 520
+    },
+    {
+      "loss": 0.4588636875152588,
+      "grad_norm": 0.010315664112567902,
+      "learning_rate": 4.518134715025907e-06,
+      "entropy": 1.6835303687490524,
+      "num_tokens": 8384377.0,
+      "logits/chosen": 0.06022681958823317,
+      "logits/rejected": 0.182036032657229,
+      "mean_token_accuracy": 0.63875184844248,
+      "rewards/chosen": -0.5799533515534131,
+      "rewards/rejected": -1.7233658448676579,
+      "rewards/accuracies": 0.80875,
+      "rewards/margins": 1.143412495199591,
+      "logps/chosen": -64.0477586555481,
+      "logps/rejected": -62.54838394999504,
+      "epoch": 0.5497640163891915,
+      "step": 530
+    },
+    {
+      "loss": 0.4572054386138916,
+      "grad_norm": 0.01254233904182911,
+      "learning_rate": 4.414507772020726e-06,
+      "entropy": 1.6456316580064594,
+      "num_tokens": 8543279.0,
+      "logits/chosen": 0.06499475886537846,
+      "logits/rejected": 0.19926284731697655,
+      "mean_token_accuracy": 0.6476211673207581,
+      "rewards/chosen": -0.6441993828896193,
+      "rewards/rejected": -1.7510575192118996,
+      "rewards/accuracies": 0.83,
+      "rewards/margins": 1.106858134865761,
+      "logps/chosen": -62.7225538957119,
+      "logps/rejected": -62.64689356803894,
+      "epoch": 0.5601369223587989,
+      "step": 540
+    },
+    {
+      "loss": 0.46482481956481936,
+      "grad_norm": 0.033692456781864166,
+      "learning_rate": 4.310880829015544e-06,
+      "entropy": 1.694882277622819,
+      "num_tokens": 8703115.0,
+      "logits/chosen": 0.15109038640147246,
+      "logits/rejected": 0.2297669311708141,
+      "mean_token_accuracy": 0.6399877128005028,
+      "rewards/chosen": -0.661507485556067,
+      "rewards/rejected": -1.8317748800176197,
+      "rewards/accuracies": 0.79875,
+      "rewards/margins": 1.170267395619303,
+      "logps/chosen": -66.47220434308052,
+      "logps/rejected": -66.16992118120193,
+      "epoch": 0.5705098283284062,
+      "step": 550
+    },
+    {
+      "loss": 0.46529335975646974,
+      "grad_norm": 0.012274126522243023,
+      "learning_rate": 4.207253886010363e-06,
+      "entropy": 1.6411442287266254,
+      "num_tokens": 8862439.0,
+      "logits/chosen": 0.07749967645058822,
+      "logits/rejected": 0.17964104042378107,
+      "mean_token_accuracy": 0.6407143748924136,
+      "rewards/chosen": -0.6461584532729012,
+      "rewards/rejected": -1.7939777009031967,
+      "rewards/accuracies": 0.8,
+      "rewards/margins": 1.1478192471479998,
+      "logps/chosen": -64.53955813646317,
+      "logps/rejected": -62.56636171579361,
+      "epoch": 0.5808827342980136,
+      "step": 560
+    },
+    {
+      "loss": 0.4430724620819092,
+      "grad_norm": 0.014654034748673439,
+      "learning_rate": 4.103626943005182e-06,
+      "entropy": 1.641110150022432,
+      "num_tokens": 9019532.0,
+      "logits/chosen": 0.1006488560296452,
+      "logits/rejected": 0.21211194322628596,
+      "mean_token_accuracy": 0.6409156301990151,
+      "rewards/chosen": -0.6455532481150612,
+      "rewards/rejected": -1.7706630855146797,
+      "rewards/accuracies": 0.835,
+      "rewards/margins": 1.1251098392903804,
+      "logps/chosen": -63.84049278497696,
+      "logps/rejected": -60.347349853515624,
+      "epoch": 0.591255640267621,
+      "step": 570
+    },
+    {
+      "loss": 0.46980915069580076,
+      "grad_norm": 0.015018550679087639,
+      "learning_rate": 4.000000000000001e-06,
+      "entropy": 1.6142543618567289,
+      "num_tokens": 9176151.0,
+      "logits/chosen": 0.008260599334212472,
+      "logits/rejected": 0.13603789421792953,
+      "mean_token_accuracy": 0.6421346751507372,
+      "rewards/chosen": -0.6516643855780512,
+      "rewards/rejected": -1.7954043288715185,
+      "rewards/accuracies": 0.805,
+      "rewards/margins": 1.14373994121328,
+      "logps/chosen": -61.326625975370405,
+      "logps/rejected": -61.17676810503006,
+      "epoch": 0.6016285462372284,
+      "step": 580
+    },
+    {
+      "loss": 0.5050342559814454,
+      "grad_norm": 0.018550831824541092,
+      "learning_rate": 3.896373056994819e-06,
+      "entropy": 1.6450188556872307,
+      "num_tokens": 9335435.0,
+      "logits/chosen": 0.03664537931858989,
+      "logits/rejected": 0.1512234776259551,
+      "mean_token_accuracy": 0.6435695023182779,
+      "rewards/chosen": -0.7144207920093322,
+      "rewards/rejected": -1.727617413054686,
+      "rewards/accuracies": 0.77375,
+      "rewards/margins": 1.013196619511582,
+      "logps/chosen": -65.46060695648194,
+      "logps/rejected": -63.5906271135807,
+      "epoch": 0.6120014522068358,
+      "step": 590
+    },
+    {
+      "loss": 0.4836301326751709,
+      "grad_norm": 0.016474206000566483,
+      "learning_rate": 3.7927461139896377e-06,
+      "entropy": 1.6528891836851836,
+      "num_tokens": 9491363.0,
+      "logits/chosen": 0.08386782463631755,
+      "logits/rejected": 0.20943303131643703,
+      "mean_token_accuracy": 0.6487930232100189,
+      "rewards/chosen": -0.6163708032899012,
+      "rewards/rejected": -1.6733668212150223,
+      "rewards/accuracies": 0.78375,
+      "rewards/margins": 1.0569960164930672,
+      "logps/chosen": -60.280444753170016,
+      "logps/rejected": -61.18290127515793,
+      "epoch": 0.6223743581764432,
+      "step": 600
+    },
+    {
+      "loss": 0.4617151260375977,
+      "grad_norm": 0.010820780880749226,
+      "learning_rate": 3.6891191709844567e-06,
+      "entropy": 1.6939242084044963,
+      "num_tokens": 9648643.0,
+      "logits/chosen": 0.09977307296605359,
+      "logits/rejected": 0.20379420233565199,
+      "mean_token_accuracy": 0.6439293037727475,
+      "rewards/chosen": -0.6212675093274447,
+      "rewards/rejected": -1.7044480502128136,
+      "rewards/accuracies": 0.82,
+      "rewards/margins": 1.0831805411481765,
+      "logps/chosen": -60.97681099057198,
+      "logps/rejected": -62.301487598419186,
+      "epoch": 0.6327472641460505,
+      "step": 610
+    },
+    {
+      "loss": 0.45804176330566404,
+      "grad_norm": 0.012919292785227299,
+      "learning_rate": 3.5854922279792748e-06,
+      "entropy": 1.619003531029448,
+      "num_tokens": 9806029.0,
+      "logits/chosen": 0.0483610800015232,
+      "logits/rejected": 0.1797610236274009,
+      "mean_token_accuracy": 0.6475096028484404,
+      "rewards/chosen": -0.5888219272701827,
+      "rewards/rejected": -1.706802941111964,
+      "rewards/accuracies": 0.8075,
+      "rewards/margins": 1.1179810122167693,
+      "logps/chosen": -59.50200087547302,
+      "logps/rejected": -61.049637752771375,
+      "epoch": 0.6431201701156579,
+      "step": 620
+    },
+    {
+      "eval_loss": 0.5324422717094421,
+      "eval_runtime": 288.7421,
+      "eval_samples_per_second": 3.463,
+      "eval_steps_per_second": 3.463,
+      "eval_entropy": 1.0448875862136484,
+      "eval_num_tokens": 9884893.0,
+      "eval_logits/chosen": -0.19975289350595532,
+      "eval_logits/rejected": -0.060755203565891945,
+      "eval_mean_token_accuracy": 0.6964090894907713,
+      "eval_rewards/chosen": -0.6912122842722456,
+      "eval_rewards/rejected": -1.345464039585553,
+      "eval_rewards/accuracies": 0.736,
+      "eval_rewards/margins": 0.6542517534568906,
+      "eval_logps/chosen": -57.08510956954956,
+      "eval_logps/rejected": -66.09451188468933,
+      "epoch": 0.6483066231004616,
+      "step": 625
+    },
+    {
+      "loss": 0.45544919967651365,
+      "grad_norm": 0.010571232065558434,
+      "learning_rate": 3.4818652849740937e-06,
+      "entropy": 1.6706199841760099,
+      "num_tokens": 9965103.0,
+      "logits/chosen": 0.1180553175480971,
+      "logits/rejected": 0.21143161932841895,
+      "mean_token_accuracy": 0.6404067935794592,
+      "rewards/chosen": -0.6081898983999418,
+      "rewards/rejected": -1.7734553063547356,
+      "rewards/accuracies": 0.81,
+      "rewards/margins": 1.1652654066774994,
+      "logps/chosen": -62.56314527273178,
+      "logps/rejected": -63.61036643981934,
+      "epoch": 0.6534930760852653,
+      "step": 630
+    },
+    {
+      "loss": 0.44363651275634763,
+      "grad_norm": 0.011373443529009819,
+      "learning_rate": 3.3782383419689123e-06,
+      "entropy": 1.661266395803541,
+      "num_tokens": 10122828.0,
+      "logits/chosen": 0.13310990911343398,
+      "logits/rejected": 0.2430608346820568,
+      "mean_token_accuracy": 0.6414054480008781,
+      "rewards/chosen": -0.645503295796052,
+      "rewards/rejected": -1.7806003371396217,
+      "rewards/accuracies": 0.81,
+      "rewards/margins": 1.1350970419961959,
+      "logps/chosen": -61.5342090690136,
+      "logps/rejected": -63.98482904434204,
+      "epoch": 0.6638659820548727,
+      "step": 640
+    },
+    {
+      "loss": 0.4796154499053955,
+      "grad_norm": 0.012532204389572144,
+      "learning_rate": 3.274611398963731e-06,
+      "entropy": 1.6701466926187276,
+      "num_tokens": 10279859.0,
+      "logits/chosen": 0.07329429847103937,
+      "logits/rejected": 0.1878599203293891,
+      "mean_token_accuracy": 0.6414687449485064,
+      "rewards/chosen": -0.661306283445374,
+      "rewards/rejected": -1.7610867334110663,
+      "rewards/accuracies": 0.80375,
+      "rewards/margins": 1.099780449680984,
+      "logps/chosen": -62.595525816679,
+      "logps/rejected": -61.356811988353726,
+      "epoch": 0.6742388880244801,
+      "step": 650
+    },
+    {
+      "loss": 0.4748543739318848,
+      "grad_norm": 0.01547182071954012,
+      "learning_rate": 3.1709844559585493e-06,
+      "entropy": 1.6658864275645464,
+      "num_tokens": 10436075.0,
+      "logits/chosen": 0.10011936781303017,
+      "logits/rejected": 0.2041365355242361,
+      "mean_token_accuracy": 0.6389276959933341,
+      "rewards/chosen": -0.6364252381109691,
+      "rewards/rejected": -1.735760998390615,
+      "rewards/accuracies": 0.80625,
+      "rewards/margins": 1.099335762821138,
+      "logps/chosen": -61.83362093210221,
+      "logps/rejected": -61.003912779092786,
+      "epoch": 0.6846117939940874,
+      "step": 660
+    },
+    {
+      "loss": 0.4296245098114014,
+      "grad_norm": 0.011170756071805954,
+      "learning_rate": 3.0673575129533683e-06,
+      "entropy": 1.6508124286774546,
+      "num_tokens": 10594936.0,
+      "logits/chosen": 0.10534840408886,
+      "logits/rejected": 0.20408601168915652,
+      "mean_token_accuracy": 0.6367748867347837,
+      "rewards/chosen": -0.6433876353609412,
+      "rewards/rejected": -1.8510402401175816,
+      "rewards/accuracies": 0.8175,
+      "rewards/margins": 1.2076526060514152,
+      "logps/chosen": -64.62543644189834,
+      "logps/rejected": -62.8101417517662,
+      "epoch": 0.6949846999636948,
+      "step": 670
+    },
+    {
+      "loss": 0.4698948383331299,
+      "grad_norm": 0.019366616383194923,
+      "learning_rate": 2.963730569948187e-06,
+      "entropy": 1.6687841359246522,
+      "num_tokens": 10754603.0,
+      "logits/chosen": 0.13530835426877952,
+      "logits/rejected": 0.25161180215337653,
+      "mean_token_accuracy": 0.6364573692902923,
+      "rewards/chosen": -0.6632168094060035,
+      "rewards/rejected": -1.8288732153433376,
+      "rewards/accuracies": 0.79875,
+      "rewards/margins": 1.165656405221671,
+      "logps/chosen": -65.38578713893891,
+      "logps/rejected": -64.20987098813058,
+      "epoch": 0.7053576059333022,
+      "step": 680
+    },
+    {
+      "loss": 0.5016210079193115,
+      "grad_norm": 0.022646328434348106,
+      "learning_rate": 2.8601036269430053e-06,
+      "entropy": 1.711660223044455,
+      "num_tokens": 10913690.0,
+      "logits/chosen": 0.16971737857777713,
+      "logits/rejected": 0.2760859463634132,
+      "mean_token_accuracy": 0.6390431644208729,
+      "rewards/chosen": -0.6508824050683597,
+      "rewards/rejected": -1.6932820503300172,
+      "rewards/accuracies": 0.79375,
+      "rewards/margins": 1.0423996420949697,
+      "logps/chosen": -66.28588567495346,
+      "logps/rejected": -62.060355219841,
+      "epoch": 0.7157305119029096,
+      "step": 690
+    },
+    {
+      "loss": 0.43436241149902344,
+      "grad_norm": 0.009509176947176456,
+      "learning_rate": 2.7564766839378243e-06,
+      "entropy": 1.6765110883302987,
+      "num_tokens": 11071134.0,
+      "logits/chosen": 0.12124103042581971,
+      "logits/rejected": 0.2600804952933655,
+      "mean_token_accuracy": 0.6427443787083029,
+      "rewards/chosen": -0.5906095821626696,
+      "rewards/rejected": -1.720989261372597,
+      "rewards/accuracies": 0.835,
+      "rewards/margins": 1.130379677421879,
+      "logps/chosen": -63.95436416387558,
+      "logps/rejected": -59.14868986845016,
+      "epoch": 0.726103417872517,
+      "step": 700
+    },
+    {
+      "loss": 0.4556713581085205,
+      "grad_norm": 0.01146204024553299,
+      "learning_rate": 2.6528497409326424e-06,
+      "entropy": 1.6694943796936423,
+      "num_tokens": 11230405.0,
+      "logits/chosen": 0.07533675440714994,
+      "logits/rejected": 0.19347860103992787,
+      "mean_token_accuracy": 0.6436189501732588,
+      "rewards/chosen": -0.5904611392628294,
+      "rewards/rejected": -1.7462449892124277,
+      "rewards/accuracies": 0.8025,
+      "rewards/margins": 1.155783847218845,
+      "logps/chosen": -66.84839742302894,
+      "logps/rejected": -61.99807296037674,
+      "epoch": 0.7364763238421244,
+      "step": 710
+    },
+    {
+      "loss": 0.4577175140380859,
+      "grad_norm": 0.012849503196775913,
+      "learning_rate": 2.5492227979274614e-06,
+      "entropy": 1.6657723085489125,
+      "num_tokens": 11386296.0,
+      "logits/chosen": 0.05922319803504693,
+      "logits/rejected": 0.17808647771296027,
+      "mean_token_accuracy": 0.6431331102643162,
+      "rewards/chosen": -0.5677624635772373,
+      "rewards/rejected": -1.6767527183049242,
+      "rewards/accuracies": 0.8225,
+      "rewards/margins": 1.1089902543462813,
+      "logps/chosen": -61.68554986834526,
+      "logps/rejected": -58.83755177259445,
+      "epoch": 0.7468492298117317,
+      "step": 720
+    },
+    {
+      "loss": 0.4547234535217285,
+      "grad_norm": 0.014395375736057758,
+      "learning_rate": 2.44559585492228e-06,
+      "entropy": 1.6862239858694374,
+      "num_tokens": 11546081.0,
+      "logits/chosen": 0.1350684718563505,
+      "logits/rejected": 0.2427451158459563,
+      "mean_token_accuracy": 0.64147017583251,
+      "rewards/chosen": -0.5967396693397313,
+      "rewards/rejected": -1.765362592941492,
+      "rewards/accuracies": 0.80375,
+      "rewards/margins": 1.1686229225434364,
+      "logps/chosen": -63.325855791568756,
+      "logps/rejected": -65.51477008223533,
+      "epoch": 0.7572221357813391,
+      "step": 730
+    },
+    {
+      "loss": 0.4426294803619385,
+      "grad_norm": 0.012110771611332893,
+      "learning_rate": 2.3419689119170984e-06,
+      "entropy": 1.707785174669698,
+      "num_tokens": 11703833.0,
+      "logits/chosen": 0.0927098814711561,
+      "logits/rejected": 0.23603759404490104,
+      "mean_token_accuracy": 0.6414807749167085,
+      "rewards/chosen": -0.5864860777647118,
+      "rewards/rejected": -1.7438211480446626,
+      "rewards/accuracies": 0.83125,
+      "rewards/margins": 1.1573350698873401,
+      "logps/chosen": -63.334676113128666,
+      "logps/rejected": -63.40963178634644,
+      "epoch": 0.7675950417509465,
+      "step": 740
+    },
+    {
+      "loss": 0.46600141525268557,
+      "grad_norm": 0.013532786630094051,
+      "learning_rate": 2.2383419689119174e-06,
+      "entropy": 1.6768190996162593,
+      "num_tokens": 11865175.0,
+      "logits/chosen": 0.1273617797525129,
+      "logits/rejected": 0.22872866333573835,
+      "mean_token_accuracy": 0.642769878860563,
+      "rewards/chosen": -0.6710381170455002,
+      "rewards/rejected": -1.8427071001776494,
+      "rewards/accuracies": 0.805,
+      "rewards/margins": 1.1716689813882113,
+      "logps/chosen": -66.42985866189002,
+      "logps/rejected": -66.3072749710083,
+      "epoch": 0.7779679477205539,
+      "step": 750
+    },
+    {
+      "eval_loss": 0.5291131734848022,
+      "eval_runtime": 316.7312,
+      "eval_samples_per_second": 3.157,
+      "eval_steps_per_second": 3.157,
+      "eval_entropy": 1.0425387567952276,
+      "eval_num_tokens": 11865175.0,
+      "eval_logits/chosen": -0.19368413704948595,
+      "eval_logits/rejected": -0.05676454023564469,
+      "eval_mean_token_accuracy": 0.6966218997985124,
+      "eval_rewards/chosen": -0.6922309857774526,
+      "eval_rewards/rejected": -1.3544460057471879,
+      "eval_rewards/accuracies": 0.747,
+      "eval_rewards/margins": 0.6622150189429522,
+      "eval_logps/chosen": -57.09529658699036,
+      "eval_logps/rejected": -66.18433157348633,
+      "epoch": 0.7779679477205539,
+      "step": 750
+    },
+    {
+      "loss": 0.43166389465332033,
+      "grad_norm": 0.012500453740358353,
+      "learning_rate": 2.134715025906736e-06,
+      "entropy": 1.6828834046982228,
+      "num_tokens": 12024093.0,
+      "logits/chosen": 0.09826941010263189,
+      "logits/rejected": 0.215735024556398,
+      "mean_token_accuracy": 0.6342808033898473,
+      "rewards/chosen": -0.6309049091633642,
+      "rewards/rejected": -1.7712371364238242,
+      "rewards/accuracies": 0.815,
+      "rewards/margins": 1.1403322232328355,
+      "logps/chosen": -64.024556722641,
+      "logps/rejected": -62.57202324867249,
+      "epoch": 0.7883408536901613,
+      "step": 760
+    },
+    {
+      "loss": 0.4471421718597412,
+      "grad_norm": 0.014410309493541718,
+      "learning_rate": 2.0310880829015544e-06,
+      "entropy": 1.6857702764961868,
+      "num_tokens": 12185744.0,
+      "logits/chosen": 0.14331582088471329,
+      "logits/rejected": 0.2682991323037694,
+      "mean_token_accuracy": 0.6441437931358814,
+      "rewards/chosen": -0.6643577039731394,
+      "rewards/rejected": -1.8807749817200239,
+      "rewards/accuracies": 0.81125,
+      "rewards/margins": 1.216417273581028,
+      "logps/chosen": -67.24417539000511,
+      "logps/rejected": -66.79223707199097,
+      "epoch": 0.7987137596597687,
+      "step": 770
+    },
+    {
+      "loss": 0.4785769939422607,
+      "grad_norm": 0.010570192709565163,
+      "learning_rate": 1.9274611398963734e-06,
+      "entropy": 1.6633455219957978,
+      "num_tokens": 12343401.0,
+      "logits/chosen": 0.07438493724116783,
+      "logits/rejected": 0.23364041511381434,
+      "mean_token_accuracy": 0.639512750543654,
+      "rewards/chosen": -0.6736341681498743,
+      "rewards/rejected": -1.796194753185846,
+      "rewards/accuracies": 0.79,
+      "rewards/margins": 1.1225605825753882,
+      "logps/chosen": -62.37282539129257,
+      "logps/rejected": -63.56037598967552,
+      "epoch": 0.809086665629376,
+      "step": 780
+    },
+    {
+      "loss": 0.45286383628845217,
+      "grad_norm": 0.011962756514549255,
+      "learning_rate": 1.823834196891192e-06,
+      "entropy": 1.6700926853902638,
+      "num_tokens": 12502157.0,
+      "logits/chosen": 0.1286325604504252,
+      "logits/rejected": 0.2370263101476654,
+      "mean_token_accuracy": 0.6403315839730204,
+      "rewards/chosen": -0.6551112483125325,
+      "rewards/rejected": -1.8369905388413463,
+      "rewards/accuracies": 0.81625,
+      "rewards/margins": 1.1818792873900383,
+      "logps/chosen": -63.591456497907636,
+      "logps/rejected": -63.02011847734451,
+      "epoch": 0.8194595715989834,
+      "step": 790
+    },
+    {
+      "loss": 0.4665197849273682,
+      "grad_norm": 0.015845410525798798,
+      "learning_rate": 1.7202072538860104e-06,
+      "entropy": 1.7264064208138734,
+      "num_tokens": 12659952.0,
+      "logits/chosen": 0.12747972368676874,
+      "logits/rejected": 0.2234084493421016,
+      "mean_token_accuracy": 0.6323709175549448,
+      "rewards/chosen": -0.670276442007671,
+      "rewards/rejected": -1.800412250665977,
+      "rewards/accuracies": 0.81375,
+      "rewards/margins": 1.1301358060259372,
+      "logps/chosen": -65.14966633319855,
+      "logps/rejected": -62.890252923965456,
+      "epoch": 0.8298324775685908,
+      "step": 800
+    },
+    {
+      "loss": 0.46672878265380857,
+      "grad_norm": 0.011275322176516056,
+      "learning_rate": 1.6165803108808292e-06,
+      "entropy": 1.688749819751829,
+      "num_tokens": 12818219.0,
+      "logits/chosen": 0.08963352847012644,
+      "logits/rejected": 0.1937745526364166,
+      "mean_token_accuracy": 0.6342837125249207,
+      "rewards/chosen": -0.6473926620587008,
+      "rewards/rejected": -1.7900561995629687,
+      "rewards/accuracies": 0.7925,
+      "rewards/margins": 1.142663535233587,
+      "logps/chosen": -63.99472352027893,
+      "logps/rejected": -63.061451687812806,
+      "epoch": 0.8402053835381982,
+      "step": 810
+    },
+    {
+      "loss": 0.4595985412597656,
+      "grad_norm": 0.009922225028276443,
+      "learning_rate": 1.5129533678756477e-06,
+      "entropy": 1.6839555408246816,
+      "num_tokens": 12975834.0,
+      "logits/chosen": 0.14187639603561716,
+      "logits/rejected": 0.2228106795045077,
+      "mean_token_accuracy": 0.6386101646535098,
+      "rewards/chosen": -0.6652093456028524,
+      "rewards/rejected": -1.8065621317276963,
+      "rewards/accuracies": 0.8,
+      "rewards/margins": 1.1413527865428477,
+      "logps/chosen": -65.14974474191666,
+      "logps/rejected": -61.74604295730591,
+      "epoch": 0.8505782895078056,
+      "step": 820
+    },
+    {
+      "loss": 0.4635880470275879,
+      "grad_norm": 0.015552631579339504,
+      "learning_rate": 1.4093264248704663e-06,
+      "entropy": 1.684992496855557,
+      "num_tokens": 13133432.0,
+      "logits/chosen": 0.09258615948240319,
+      "logits/rejected": 0.23631826346334864,
+      "mean_token_accuracy": 0.6466719186119735,
+      "rewards/chosen": -0.6487690168057452,
+      "rewards/rejected": -1.7847515585264773,
+      "rewards/accuracies": 0.81125,
+      "rewards/margins": 1.1359825418051333,
+      "logps/chosen": -61.15254833102226,
+      "logps/rejected": -62.26296893358231,
+      "epoch": 0.860951195477413,
+      "step": 830
+    },
+    {
+      "loss": 0.44598889350891113,
+      "grad_norm": 0.015216046944260597,
+      "learning_rate": 1.3056994818652852e-06,
+      "entropy": 1.709428556431085,
+      "num_tokens": 13291850.0,
+      "logits/chosen": 0.15097671106129862,
+      "logits/rejected": 0.25655990051725586,
+      "mean_token_accuracy": 0.6395855396427215,
+      "rewards/chosen": -0.5992260871930921,
+      "rewards/rejected": -1.8031220447564555,
+      "rewards/accuracies": 0.81125,
+      "rewards/margins": 1.2038959555141628,
+      "logps/chosen": -65.10634205937386,
+      "logps/rejected": -63.04532386302948,
+      "epoch": 0.8713241014470204,
+      "step": 840
+    },
+    {
+      "loss": 0.46810121536254884,
+      "grad_norm": 0.009856506250798702,
+      "learning_rate": 1.2020725388601037e-06,
+      "entropy": 1.70108500065282,
+      "num_tokens": 13450070.0,
+      "logits/chosen": 0.15703350726422605,
+      "logits/rejected": 0.2533505205489093,
+      "mean_token_accuracy": 0.643168338034302,
+      "rewards/chosen": -0.62836980179698,
+      "rewards/rejected": -1.7627987958928224,
+      "rewards/accuracies": 0.80125,
+      "rewards/margins": 1.1344289934798144,
+      "logps/chosen": -64.12525810718536,
+      "logps/rejected": -62.792922837734224,
+      "epoch": 0.8816970074166278,
+      "step": 850
+    },
+    {
+      "loss": 0.44599761962890627,
+      "grad_norm": 0.012146789580583572,
+      "learning_rate": 1.0984455958549225e-06,
+      "entropy": 1.70398797435686,
+      "num_tokens": 13610138.0,
+      "logits/chosen": 0.17104202048628422,
+      "logits/rejected": 0.262595645620993,
+      "mean_token_accuracy": 0.642651722766459,
+      "rewards/chosen": -0.6636459323119197,
+      "rewards/rejected": -1.85319078174929,
+      "rewards/accuracies": 0.8125,
+      "rewards/margins": 1.1895448501838837,
+      "logps/chosen": -67.19200572252274,
+      "logps/rejected": -64.02687373161316,
+      "epoch": 0.8920699133862352,
+      "step": 860
+    },
+    {
+      "loss": 0.4463825702667236,
+      "grad_norm": 0.013926991261541843,
+      "learning_rate": 9.94818652849741e-07,
+      "entropy": 1.6875281669571995,
+      "num_tokens": 13769613.0,
+      "logits/chosen": 0.1617350362536588,
+      "logits/rejected": 0.25974264117481755,
+      "mean_token_accuracy": 0.6424587191455067,
+      "rewards/chosen": -0.6411357878561103,
+      "rewards/rejected": -1.7520794819842558,
+      "rewards/accuracies": 0.8275,
+      "rewards/margins": 1.1109436923218892,
+      "logps/chosen": -66.13908497691155,
+      "logps/rejected": -62.40538552284241,
+      "epoch": 0.9024428193558426,
+      "step": 870
+    },
+    {
+      "eval_loss": 0.5264487266540527,
+      "eval_runtime": 346.8502,
+      "eval_samples_per_second": 2.883,
+      "eval_steps_per_second": 2.883,
+      "eval_entropy": 1.0440044164210558,
+      "eval_num_tokens": 13848578.0,
+      "eval_logits/chosen": -0.18929922542314412,
+      "eval_logits/rejected": -0.05010199907599402,
+      "eval_mean_token_accuracy": 0.6949034848362208,
+      "eval_rewards/chosen": -0.7382756498559029,
+      "eval_rewards/rejected": -1.426999929712445,
+      "eval_rewards/accuracies": 0.743,
+      "eval_rewards/margins": 0.6887242792025209,
+      "eval_logps/chosen": -57.555743215560916,
+      "eval_logps/rejected": -66.9098708076477,
+      "epoch": 0.9076292723406463,
+      "step": 875
+    },
+    {
+      "loss": 0.43689889907836915,
+      "grad_norm": 0.010173802264034748,
+      "learning_rate": 8.911917098445596e-07,
+      "entropy": 1.6774525323137641,
+      "num_tokens": 13928284.0,
+      "logits/chosen": 0.15889503262817226,
+      "logits/rejected": 0.27439704581314006,
+      "mean_token_accuracy": 0.6466628183331341,
+      "rewards/chosen": -0.6104124534751736,
+      "rewards/rejected": -1.7897700341788003,
+      "rewards/accuracies": 0.81625,
+      "rewards/margins": 1.1793575775902718,
+      "logps/chosen": -62.02614854454994,
+      "logps/rejected": -64.84975938558578,
+      "epoch": 0.91281572532545,
+      "step": 880
+    },
+    {
+      "loss": 0.48273696899414065,
+      "grad_norm": 0.017326118424534798,
+      "learning_rate": 7.875647668393784e-07,
+      "entropy": 1.6614145183190703,
+      "num_tokens": 14086112.0,
+      "logits/chosen": 0.07985394351840137,
+      "logits/rejected": 0.20892557638470582,
+      "mean_token_accuracy": 0.6420493371598422,
+      "rewards/chosen": -0.7163908848223582,
+      "rewards/rejected": -1.8062069240648997,
+      "rewards/accuracies": 0.79875,
+      "rewards/margins": 1.0898160382080824,
+      "logps/chosen": -63.230327110290524,
+      "logps/rejected": -64.38345015048981,
+      "epoch": 0.9231886312950573,
+      "step": 890
+    },
+    {
+      "loss": 0.43041396141052246,
+      "grad_norm": 0.017108725383877754,
+      "learning_rate": 6.839378238341969e-07,
+      "entropy": 1.6680658238008619,
+      "num_tokens": 14245558.0,
+      "logits/chosen": 0.14648505909070886,
+      "logits/rejected": 0.26673792896430265,
+      "mean_token_accuracy": 0.6451807205565274,
+      "rewards/chosen": -0.6325454073216497,
+      "rewards/rejected": -1.9007032794272527,
+      "rewards/accuracies": 0.82125,
+      "rewards/margins": 1.2681578750582412,
+      "logps/chosen": -62.82921484231949,
+      "logps/rejected": -66.73553317070008,
+      "epoch": 0.9335615372646647,
+      "step": 900
+    },
+    {
+      "loss": 0.4457756519317627,
+      "grad_norm": 0.010929541662335396,
+      "learning_rate": 5.803108808290156e-07,
+      "entropy": 1.7130034917313606,
+      "num_tokens": 14404540.0,
+      "logits/chosen": 0.12710906584716752,
+      "logits/rejected": 0.2675685432938819,
+      "mean_token_accuracy": 0.6400158200226724,
+      "rewards/chosen": -0.6709339278266998,
+      "rewards/rejected": -1.8822100719693118,
+      "rewards/accuracies": 0.825,
+      "rewards/margins": 1.2112761446926743,
+      "logps/chosen": -64.97942503213882,
+      "logps/rejected": -64.39793292522431,
+      "epoch": 0.9439344432342721,
+      "step": 910
+    },
+    {
+      "loss": 0.45868711471557616,
+      "grad_norm": 0.012205246835947037,
+      "learning_rate": 4.7668393782383424e-07,
+      "entropy": 1.6709410886280238,
+      "num_tokens": 14560645.0,
+      "logits/chosen": 0.10928400241500846,
+      "logits/rejected": 0.2426133711223395,
+      "mean_token_accuracy": 0.6370881532691419,
+      "rewards/chosen": -0.6589926348004519,
+      "rewards/rejected": -1.8069635758572258,
+      "rewards/accuracies": 0.81375,
+      "rewards/margins": 1.14797093979083,
+      "logps/chosen": -61.491244453191754,
+      "logps/rejected": -61.11656880378723,
+      "epoch": 0.9543073492038795,
+      "step": 920
+    },
+    {
+      "loss": 0.4510962963104248,
+      "grad_norm": 0.016680743545293808,
+      "learning_rate": 3.730569948186528e-07,
+      "entropy": 1.7099607919715345,
+      "num_tokens": 14720454.0,
+      "logits/chosen": 0.11883571728281689,
+      "logits/rejected": 0.20641075253811259,
+      "mean_token_accuracy": 0.6349475438706577,
+      "rewards/chosen": -0.6969622136335238,
+      "rewards/rejected": -1.859341476371046,
+      "rewards/accuracies": 0.81,
+      "rewards/margins": 1.162379261488095,
+      "logps/chosen": -65.8184747338295,
+      "logps/rejected": -65.03840645551682,
+      "epoch": 0.9646802551734869,
+      "step": 930
+    },
+    {
+      "loss": 0.44940953254699706,
+      "grad_norm": 0.013420112431049347,
+      "learning_rate": 2.694300518134715e-07,
+      "entropy": 1.6909510315582157,
+      "num_tokens": 14879971.0,
+      "logits/chosen": 0.18477739616368072,
+      "logits/rejected": 0.2645026210979778,
+      "mean_token_accuracy": 0.6382351936399937,
+      "rewards/chosen": -0.6513557667902206,
+      "rewards/rejected": -1.7696480820770375,
+      "rewards/accuracies": 0.815,
+      "rewards/margins": 1.1182923125289381,
+      "logps/chosen": -65.54289688110352,
+      "logps/rejected": -62.99382516860962,
+      "epoch": 0.9750531611430943,
+      "step": 940
+    },
+    {
+      "loss": 0.47352304458618166,
+      "grad_norm": 0.013681693933904171,
+      "learning_rate": 1.6580310880829015e-07,
+      "entropy": 1.661503377771005,
+      "num_tokens": 15038091.0,
+      "logits/chosen": 0.11480745743066594,
+      "logits/rejected": 0.22102661224680678,
+      "mean_token_accuracy": 0.6446126988902688,
+      "rewards/chosen": -0.7158770331276173,
+      "rewards/rejected": -1.8211175910846213,
+      "rewards/accuracies": 0.78875,
+      "rewards/margins": 1.1052405576594173,
+      "logps/chosen": -63.61789976716042,
+      "logps/rejected": -62.64833914756775,
+      "epoch": 0.9854260671127016,
+      "step": 950
+    },
+    {
+      "loss": 0.46666803359985354,
+      "grad_norm": 0.014694225043058395,
+      "learning_rate": 6.217616580310881e-08,
+      "entropy": 1.6944513383321465,
+      "num_tokens": 15198513.0,
+      "logits/chosen": 0.1733613867125754,
+      "logits/rejected": 0.27222349129564544,
+      "mean_token_accuracy": 0.6361824345402419,
+      "rewards/chosen": -0.7022936144633786,
+      "rewards/rejected": -1.8512264592042629,
+      "rewards/accuracies": 0.7925,
+      "rewards/margins": 1.148932844074443,
+      "logps/chosen": -66.14034503817558,
+      "logps/rejected": -63.98270012617111,
+      "epoch": 0.995798973082309,
+      "step": 960
+    },
+    {
+      "train_runtime": 22398.3946,
+      "train_samples_per_second": 3.443,
+      "train_steps_per_second": 0.043,
+      "total_flos": 1.3308435135621734e+17,
+      "train_loss": 0.48160764293967134,
+      "entropy": 1.7446463192686623,
+      "num_tokens": 15263900.0,
+      "logits/chosen": 0.2563561171205477,
+      "logits/rejected": 0.3359260186024246,
+      "mean_token_accuracy": 0.6413786758059337,
+      "rewards/chosen": -0.7268365635874359,
+      "rewards/rejected": -1.856159317700974,
+      "rewards/accuracies": 0.8209876543209876,
+      "rewards/margins": 1.1293227543801436,
+      "logps/chosen": -67.83627563052707,
+      "logps/rejected": -68.1628461767126,
+      "epoch": 1.0,
+      "step": 965
+    }
+  ],
+  "validation_monitor_size": 1000,
+  "validation_monitor_selection": "fixed_seed_random_without_replacement",
+  "validation_monitor_seed": 22,
+  "validation_monitor_indices": [
+    2,
+    10,
+    14,
+    21,
+    55,
+    63,
+    66,
+    69,
+    107,
+    110,
+    142,
+    144,
+    149,
+    150,
+    151,
+    152,
+    160,
+    163,
+    166,
+    167,
+    176,
+    181,
+    206,
+    207,
+    259,
+    267,
+    281,
+    295,
+    298,
+    306,
+    307,
+    317,
+    321,
+    331,
+    336,
+    341,
+    346,
+    349,
+    351,
+    352,
+    357,
+    358,
+    369,
+    370,
+    382,
+    386,
+    391,
+    401,
+    411,
+    412,
+    432,
+    437,
+    452,
+    462,
+    465,
+    488,
+    490,
+    491,
+    492,
+    494,
+    503,
+    504,
+    508,
+    528,
+    538,
+    540,
+    551,
+    553,
+    567,
+    586,
+    605,
+    606,
+    625,
+    627,
+    661,
+    663,
+    666,
+    677,
+    685,
+    690,
+    692,
+    704,
+    708,
+    714,
+    715,
+    727,
+    728,
+    733,
+    745,
+    752,
+    753,
+    755,
+    764,
+    773,
+    779,
+    783,
+    793,
+    796,
+    799,
+    803,
+    804,
+    805,
+    813,
+    816,
+    818,
+    823,
+    827,
+    829,
+    830,
+    837,
+    842,
+    845,
+    850,
+    853,
+    856,
+    889,
+    890,
+    905,
+    915,
+    924,
+    930,
+    939,
+    951,
+    988,
+    1002,
+    1005,
+    1023,
+    1029,
+    1038,
+    1049,
+    1050,
+    1054,
+    1056,
+    1067,
+    1068,
+    1079,
+    1088,
+    1091,
+    1103,
+    1114,
+    1118,
+    1133,
+    1140,
+    1144,
+    1145,
+    1155,
+    1186,
+    1195,
+    1206,
+    1223,
+    1251,
+    1252,
+    1257,
+    1259,
+    1270,
+    1271,
+    1295,
+    1303,
+    1304,
+    1305,
+    1329,
+    1335,
+    1336,
+    1343,
+    1367,
+    1373,
+    1377,
+    1403,
+    1412,
+    1429,
+    1443,
+    1457,
+    1459,
+    1460,
+    1476,
+    1483,
+    1486,
+    1494,
+    1507,
+    1510,
+    1519,
+    1521,
+    1522,
+    1545,
+    1551,
+    1570,
+    1582,
+    1593,
+    1595,
+    1603,
+    1607,
+    1614,
+    1615,
+    1625,
+    1634,
+    1639,
+    1648,
+    1654,
+    1657,
+    1662,
+    1667,
+    1673,
+    1690,
+    1704,
+    1746,
+    1756,
+    1781,
+    1783,
+    1796,
+    1799,
+    1809,
+    1814,
+    1827,
+    1829,
+    1832,
+    1844,
+    1847,
+    1854,
+    1856,
+    1857,
+    1858,
+    1874,
+    1883,
+    1889,
+    1924,
+    1925,
+    1931,
+    1932,
+    1934,
+    1935,
+    1938,
+    1950,
+    1957,
+    1962,
+    1967,
+    1975,
+    1982,
+    1983,
+    1991,
+    1998,
+    2003,
+    2008,
+    2017,
+    2021,
+    2026,
+    2035,
+    2040,
+    2050,
+    2056,
+    2077,
+    2079,
+    2082,
+    2098,
+    2100,
+    2108,
+    2121,
+    2130,
+    2133,
+    2134,
+    2138,
+    2143,
+    2166,
+    2167,
+    2180,
+    2181,
+    2185,
+    2204,
+    2205,
+    2212,
+    2221,
+    2224,
+    2226,
+    2230,
+    2233,
+    2256,
+    2261,
+    2263,
+    2269,
+    2273,
+    2290,
+    2291,
+    2299,
+    2301,
+    2321,
+    2323,
+    2330,
+    2384,
+    2401,
+    2417,
+    2420,
+    2421,
+    2424,
+    2430,
+    2435,
+    2456,
+    2488,
+    2502,
+    2504,
+    2519,
+    2527,
+    2532,
+    2538,
+    2542,
+    2553,
+    2555,
+    2558,
+    2559,
+    2562,
+    2578,
+    2583,
+    2585,
+    2590,
+    2592,
+    2594,
+    2596,
+    2600,
+    2606,
+    2607,
+    2618,
+    2630,
+    2637,
+    2647,
+    2650,
+    2657,
+    2679,
+    2685,
+    2705,
+    2706,
+    2712,
+    2713,
+    2714,
+    2727,
+    2740,
+    2742,
+    2755,
+    2780,
+    2784,
+    2792,
+    2807,
+    2808,
+    2810,
+    2820,
+    2831,
+    2839,
+    2860,
+    2862,
+    2863,
+    2866,
+    2875,
+    2878,
+    2898,
+    2905,
+    2921,
+    2922,
+    2926,
+    2930,
+    2934,
+    2944,
+    2955,
+    2957,
+    2959,
+    2973,
+    2978,
+    2998,
+    3018,
+    3022,
+    3028,
+    3031,
+    3061,
+    3085,
+    3090,
+    3104,
+    3105,
+    3111,
+    3115,
+    3121,
+    3122,
+    3129,
+    3133,
+    3135,
+    3161,
+    3162,
+    3169,
+    3173,
+    3194,
+    3195,
+    3215,
+    3225,
+    3226,
+    3241,
+    3247,
+    3250,
+    3253,
+    3265,
+    3268,
+    3293,
+    3301,
+    3312,
+    3329,
+    3352,
+    3361,
+    3362,
+    3376,
+    3396,
+    3401,
+    3403,
+    3410,
+    3419,
+    3432,
+    3443,
+    3452,
+    3467,
+    3469,
+    3475,
+    3485,
+    3503,
+    3514,
+    3515,
+    3524,
+    3528,
+    3538,
+    3544,
+    3557,
+    3560,
+    3565,
+    3600,
+    3637,
+    3642,
+    3658,
+    3659,
+    3692,
+    3693,
+    3699,
+    3722,
+    3725,
+    3728,
+    3731,
+    3740,
+    3742,
+    3762,
+    3766,
+    3780,
+    3788,
+    3794,
+    3796,
+    3798,
+    3805,
+    3817,
+    3819,
+    3822,
+    3837,
+    3839,
+    3843,
+    3846,
+    3851,
+    3854,
+    3865,
+    3870,
+    3871,
+    3884,
+    3894,
+    3895,
+    3896,
+    3907,
+    3911,
+    3915,
+    3919,
+    3920,
+    3923,
+    3933,
+    3955,
+    3967,
+    3972,
+    3974,
+    3975,
+    3984,
+    3985,
+    3997,
+    4002,
+    4010,
+    4034,
+    4044,
+    4063,
+    4073,
+    4079,
+    4082,
+    4088,
+    4121,
+    4145,
+    4148,
+    4159,
+    4161,
+    4164,
+    4177,
+    4188,
+    4199,
+    4203,
+    4207,
+    4208,
+    4213,
+    4221,
+    4225,
+    4233,
+    4241,
+    4243,
+    4247,
+    4264,
+    4274,
+    4282,
+    4286,
+    4290,
+    4308,
+    4310,
+    4313,
+    4321,
+    4324,
+    4327,
+    4349,
+    4362,
+    4370,
+    4374,
+    4380,
+    4407,
+    4409,
+    4411,
+    4415,
+    4417,
+    4418,
+    4427,
+    4431,
+    4433,
+    4451,
+    4466,
+    4477,
+    4478,
+    4479,
+    4493,
+    4494,
+    4514,
+    4527,
+    4539,
+    4550,
+    4558,
+    4568,
+    4579,
+    4583,
+    4584,
+    4586,
+    4587,
+    4590,
+    4599,
+    4602,
+    4610,
+    4626,
+    4627,
+    4630,
+    4641,
+    4647,
+    4655,
+    4656,
+    4657,
+    4661,
+    4685,
+    4714,
+    4715,
+    4731,
+    4749,
+    4752,
+    4769,
+    4777,
+    4782,
+    4791,
+    4805,
+    4818,
+    4829,
+    4833,
+    4837,
+    4839,
+    4843,
+    4871,
+    4875,
+    4879,
+    4880,
+    4885,
+    4888,
+    4895,
+    4900,
+    4923,
+    4966,
+    4968,
+    4972,
+    4989,
+    4994,
+    4998,
+    5001,
+    5013,
+    5019,
+    5026,
+    5032,
+    5034,
+    5046,
+    5055,
+    5085,
+    5086,
+    5088,
+    5089,
+    5090,
+    5095,
+    5108,
+    5110,
+    5119,
+    5120,
+    5121,
+    5133,
+    5148,
+    5154,
+    5160,
+    5169,
+    5178,
+    5222,
+    5223,
+    5232,
+    5233,
+    5240,
+    5256,
+    5259,
+    5264,
+    5271,
+    5276,
+    5279,
+    5294,
+    5300,
+    5303,
+    5321,
+    5335,
+    5337,
+    5345,
+    5348,
+    5365,
+    5373,
+    5378,
+    5384,
+    5422,
+    5442,
+    5443,
+    5445,
+    5477,
+    5485,
+    5495,
+    5497,
+    5504,
+    5526,
+    5533,
+    5542,
+    5564,
+    5570,
+    5579,
+    5587,
+    5592,
+    5608,
+    5610,
+    5624,
+    5630,
+    5638,
+    5651,
+    5663,
+    5670,
+    5675,
+    5691,
+    5700,
+    5706,
+    5707,
+    5715,
+    5720,
+    5721,
+    5722,
+    5732,
+    5738,
+    5741,
+    5769,
+    5771,
+    5775,
+    5795,
+    5796,
+    5800,
+    5809,
+    5810,
+    5815,
+    5819,
+    5827,
+    5848,
+    5849,
+    5852,
+    5859,
+    5880,
+    5884,
+    5907,
+    5909,
+    5912,
+    5919,
+    5931,
+    5932,
+    5934,
+    5941,
+    5948,
+    5950,
+    5952,
+    5953,
+    5969,
+    5981,
+    6000,
+    6003,
+    6010,
+    6018,
+    6041,
+    6065,
+    6075,
+    6090,
+    6103,
+    6106,
+    6109,
+    6114,
+    6123,
+    6131,
+    6136,
+    6138,
+    6139,
+    6164,
+    6165,
+    6171,
+    6173,
+    6180,
+    6185,
+    6189,
+    6190,
+    6221,
+    6223,
+    6237,
+    6255,
+    6262,
+    6265,
+    6293,
+    6296,
+    6305,
+    6318,
+    6331,
+    6336,
+    6340,
+    6355,
+    6366,
+    6371,
+    6396,
+    6399,
+    6402,
+    6408,
+    6432,
+    6433,
+    6441,
+    6456,
+    6465,
+    6478,
+    6486,
+    6489,
+    6507,
+    6508,
+    6520,
+    6522,
+    6528,
+    6537,
+    6551,
+    6564,
+    6589,
+    6590,
+    6598,
+    6599,
+    6611,
+    6613,
+    6615,
+    6621,
+    6634,
+    6647,
+    6649,
+    6654,
+    6676,
+    6680,
+    6690,
+    6708,
+    6729,
+    6736,
+    6744,
+    6749,
+    6756,
+    6761,
+    6763,
+    6773,
+    6788,
+    6790,
+    6796,
+    6797,
+    6811,
+    6824,
+    6850,
+    6869,
+    6871,
+    6882,
+    6892,
+    6895,
+    6906,
+    6911,
+    6912,
+    6914,
+    6927,
+    6952,
+    6966,
+    6985,
+    7001,
+    7021,
+    7031,
+    7035,
+    7038,
+    7041,
+    7045,
+    7052,
+    7057,
+    7058,
+    7072,
+    7073,
+    7076,
+    7086,
+    7102,
+    7107,
+    7109,
+    7117,
+    7122,
+    7125,
+    7166,
+    7182,
+    7199,
+    7207,
+    7212,
+    7215,
+    7232,
+    7243,
+    7246,
+    7250,
+    7253,
+    7258,
+    7263,
+    7267,
+    7270,
+    7274,
+    7280,
+    7286,
+    7293,
+    7298,
+    7302,
+    7306,
+    7316,
+    7325,
+    7326,
+    7334,
+    7356,
+    7357,
+    7363,
+    7364,
+    7367,
+    7385,
+    7392,
+    7399,
+    7405,
+    7416,
+    7420,
+    7421,
+    7426,
+    7452,
+    7476,
+    7481,
+    7519,
+    7534,
+    7542,
+    7546,
+    7573,
+    7585,
+    7601,
+    7604,
+    7606,
+    7609,
+    7629,
+    7649,
+    7653,
+    7667,
+    7682,
+    7699,
+    7738,
+    7750,
+    7786,
+    7798,
+    7800,
+    7801,
+    7805,
+    7806,
+    7811,
+    7813,
+    7832,
+    7837,
+    7849,
+    7856,
+    7876,
+    7877,
+    7887,
+    7905,
+    7916,
+    7919,
+    7920,
+    7922,
+    7923,
+    7926,
+    7944,
+    7961,
+    7966,
+    7970,
+    7973,
+    7974,
+    7976,
+    7986,
+    7999,
+    8027,
+    8028,
+    8034,
+    8047,
+    8068,
+    8074,
+    8077,
+    8091,
+    8120,
+    8122,
+    8125,
+    8152,
+    8153,
+    8164,
+    8167,
+    8169,
+    8171,
+    8177,
+    8184,
+    8189,
+    8192,
+    8196,
+    8202,
+    8212,
+    8217,
+    8231,
+    8242,
+    8244,
+    8250,
+    8256,
+    8257,
+    8265,
+    8270,
+    8274,
+    8283,
+    8290,
+    8294,
+    8301,
+    8302,
+    8307,
+    8318,
+    8326,
+    8338,
+    8349,
+    8350,
+    8353,
+    8357,
+    8371,
+    8374,
+    8377,
+    8380,
+    8387,
+    8388,
+    8396,
+    8402,
+    8419,
+    8423,
+    8428,
+    8435,
+    8439,
+    8442,
+    8444,
+    8453,
+    8461,
+    8475,
+    8481,
+    8485,
+    8493,
+    8495,
+    8498,
+    8523,
+    8530,
+    8531,
+    8562
+  ],
+  "early_stopping_patience": 3
+}

phase1/ablations/helps_only/README.md

@@ -0,0 +1,80 @@
+---
+library_name: peft
+model_name: phase1-qwen-helps-only-ablation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-1.5B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+license: apache-2.0
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+pipeline_tag: text-generation
+---
+
+# Phase 1 Qwen `helps_only` diagnostic ablation
+
+This is the Phase 1 `helps_only` diagnostic LoRA-DPO adapter fine-tuned from
+[Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct).
+The frozen training recipe and membership hashes are included in this directory.
+
+## Quick start
+
+```python
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo_id = "geyuxu/york-milestone-project-self-traing-loop-model"
+subfolder = "phase1/ablations/helps_only"
+base_id = "Qwen/Qwen2.5-1.5B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(repo_id, subfolder=subfolder)
+base_model = AutoModelForCausalLM.from_pretrained(
+    base_id, torch_dtype="auto", device_map="auto"
+)
+model = PeftModel.from_pretrained(base_model, repo_id, subfolder=subfolder)
+```
+
+## Training procedure
+
+
+
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 0.29.0
+- Transformers: 5.3.0
+- Pytorch: 2.10.0
+- Datasets: 4.6.1
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+    title        = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+    author       = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+    year         = 2023,
+    booktitle    = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+    url          = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+    editor       = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

phase1/ablations/helps_only/adapter_config.json

@@ -0,0 +1,43 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "k_proj",
+    "o_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

phase1/ablations/helps_only/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a143ad6c49a7a1b1d98e81af88412c91b367a394fa48a89517d6057fddf91748
+size 17462432

phase1/ablations/helps_only/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

phase1/ablations/helps_only/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5
+size 11422166

phase1/ablations/helps_only/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

phase1/ablations/helps_only/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:101d92ecb945e3232d561989c7a979ff91d184b4cccbe34a200a56060e3c5ce6
+size 5841

phase1/ablations/helps_only/training_recipe.json

@@ -0,0 +1,22 @@
+{
+  "format_version": 1,
+  "experiment": "confusion_ablation",
+  "ablation": "helps_only",
+  "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+  "train_membership_sha256": "5febb0b71375db4da6a27fdaa6d3226681631022eab8e57a816d93519c4854ab",
+  "validation_membership_sha256": "2f609467438cd6a01fd84b9d80c0fcd84c44767e3b2ed98f4f6a83d68a62c181",
+  "num_train_pairs": 77124,
+  "num_validation_pairs": 8563,
+  "transformation_seed": 22,
+  "validation_prompt_mode": "clean_rag",
+  "learning_rate": 1e-05,
+  "beta": 0.1,
+  "num_epochs": 1.0,
+  "seed": 22,
+  "max_length": 512,
+  "max_prompt_length": 384,
+  "gradient_accumulation_steps": 80,
+  "save_steps": 125,
+  "save_total_limit": 12,
+  "preference_pairs_format_version": 2
+}

phase1/ablations/helps_only/training_summary.json

@@ -0,0 +1,2029 @@
+{
+  "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+  "resolved_model": "Qwen/Qwen2.5-1.5B-Instruct",
+  "num_train_records": 77124,
+  "num_validation_records_available": 8563,
+  "training_config": {
+    "lora_rank": 16,
+    "lora_alpha": 32,
+    "learning_rate": 1e-05,
+    "num_epochs": 1.0,
+    "per_device_batch_size": 1,
+    "per_device_eval_batch_size": 1,
+    "gradient_accumulation_steps": 80,
+    "beta": 0.1,
+    "max_length": 512,
+    "max_prompt_length": 384,
+    "seed": 22,
+    "save_steps": 125,
+    "save_total_limit": 12,
+    "resume_from_checkpoint": null,
+    "precision_dtype": "torch.bfloat16",
+    "bf16": true,
+    "fp16": false,
+    "tokenizer_add_bos_token": false
+  },
+  "global_step": 500,
+  "best_metric": 0.5006195306777954,
+  "best_model_checkpoint": "outputs/confusion_ablation_phase1_repair_v3/helps_only/lora/checkpoint-125",
+  "train_metrics": {
+    "train_runtime": 16864.5913,
+    "train_samples_per_second": 4.573,
+    "train_steps_per_second": 0.057,
+    "total_flos": 2.7163725934432666e+17,
+    "train_loss": 0.23282903826236725
+  },
+  "log_history": [
+    {
+      "loss": 0.6858583450317383,
+      "grad_norm": 0.016611717641353607,
+      "learning_rate": 9.906735751295338e-06,
+      "entropy": 0.794021682399325,
+      "num_tokens": 670561.0,
+      "logits/chosen": -0.9187091639020697,
+      "logits/rejected": -0.9434830672895622,
+      "mean_token_accuracy": 0.736259245146066,
+      "rewards/chosen": 0.0011638513133243578,
+      "rewards/rejected": -0.013860612378184669,
+      "rewards/accuracies": 0.5925,
+      "rewards/margins": 0.015024463631707476,
+      "logps/chosen": -54.79746845722198,
+      "logps/rejected": -49.96550277948379,
+      "epoch": 0.010372905969607386,
+      "step": 10
+    },
+    {
+      "loss": 0.6576845169067382,
+      "grad_norm": 0.016666823998093605,
+      "learning_rate": 9.803108808290156e-06,
+      "entropy": 0.7863238480500877,
+      "num_tokens": 1340958.0,
+      "logits/chosen": -0.9552881883230906,
+      "logits/rejected": -0.9189709572546179,
+      "mean_token_accuracy": 0.7423679579421878,
+      "rewards/chosen": -0.013600413688568552,
+      "rewards/rejected": -0.08799497530219376,
+      "rewards/accuracies": 0.85875,
+      "rewards/margins": 0.07439456147782039,
+      "logps/chosen": -55.26659148097038,
+      "logps/rejected": -49.557674720287324,
+      "epoch": 0.02074581193921477,
+      "step": 20
+    },
+    {
+      "loss": 0.6105106353759766,
+      "grad_norm": 0.015847327187657356,
+      "learning_rate": 9.699481865284975e-06,
+      "entropy": 0.846787225897424,
+      "num_tokens": 2013796.0,
+      "logits/chosen": -0.8573241884593181,
+      "logits/rejected": -0.8453699552830984,
+      "mean_token_accuracy": 0.727089679390192,
+      "rewards/chosen": -0.06516332936625986,
+      "rewards/rejected": -0.24710432796498935,
+      "rewards/accuracies": 0.8575,
+      "rewards/margins": 0.18194099859509152,
+      "logps/chosen": -59.86930260896683,
+      "logps/rejected": -53.61334676504135,
+      "epoch": 0.031118717908822157,
+      "step": 30
+    },
+    {
+      "loss": 0.560779619216919,
+      "grad_norm": 0.01469502318650484,
+      "learning_rate": 9.595854922279793e-06,
+      "entropy": 0.7988823162578046,
+      "num_tokens": 2684543.0,
+      "logits/chosen": -0.9062229187635736,
+      "logits/rejected": -0.8287031519573034,
+      "mean_token_accuracy": 0.726212237663567,
+      "rewards/chosen": -0.15750733469684747,
+      "rewards/rejected": -0.47408700099374984,
+      "rewards/accuracies": 0.8625,
+      "rewards/margins": 0.3165796657791361,
+      "logps/chosen": -57.612799718379975,
+      "logps/rejected": -53.58559124469757,
+      "epoch": 0.04149162387842954,
+      "step": 40
+    },
+    {
+      "loss": 0.5105207920074463,
+      "grad_norm": 0.014577334746718407,
+      "learning_rate": 9.492227979274612e-06,
+      "entropy": 0.7784375699353404,
+      "num_tokens": 3356031.0,
+      "logits/chosen": -0.913959642386141,
+      "logits/rejected": -0.8075734227458465,
+      "mean_token_accuracy": 0.7303825634159148,
+      "rewards/chosen": -0.32432046058405833,
+      "rewards/rejected": -0.8036991727564601,
+      "rewards/accuracies": 0.8625,
+      "rewards/margins": 0.4793787114601582,
+      "logps/chosen": -60.06104364156723,
+      "logps/rejected": -56.74762117385864,
+      "epoch": 0.051864529848036925,
+      "step": 50
+    },
+    {
+      "loss": 0.4511364459991455,
+      "grad_norm": 0.01305594015866518,
+      "learning_rate": 9.388601036269432e-06,
+      "entropy": 0.7942980694322614,
+      "num_tokens": 4026577.0,
+      "logits/chosen": -0.9856768630883611,
+      "logits/rejected": -0.8317867858234134,
+      "mean_token_accuracy": 0.7197631917707622,
+      "rewards/chosen": -0.5297682328737574,
+      "rewards/rejected": -1.217431449344149,
+      "rewards/accuracies": 0.88625,
+      "rewards/margins": 0.6876632150541991,
+      "logps/chosen": -60.84162630081177,
+      "logps/rejected": -61.641977578401566,
+      "epoch": 0.062237435817644314,
+      "step": 60
+    },
+    {
+      "loss": 0.422989559173584,
+      "grad_norm": 0.01258748210966587,
+      "learning_rate": 9.284974093264249e-06,
+      "entropy": 0.7553169909078861,
+      "num_tokens": 4696820.0,
+      "logits/chosen": -1.1534164984013175,
+      "logits/rejected": -0.9453616152186384,
+      "mean_token_accuracy": 0.7059021528251469,
+      "rewards/chosen": -0.8479185369245533,
+      "rewards/rejected": -1.733439953867346,
+      "rewards/accuracies": 0.85125,
+      "rewards/margins": 0.8855214151740074,
+      "logps/chosen": -64.60678691148757,
+      "logps/rejected": -65.6156524682045,
+      "epoch": 0.0726103417872517,
+      "step": 70
+    },
+    {
+      "loss": 0.37308950424194337,
+      "grad_norm": 0.013154606334865093,
+      "learning_rate": 9.181347150259067e-06,
+      "entropy": 0.7961554935819004,
+      "num_tokens": 5368225.0,
+      "logits/chosen": -1.1909632763159101,
+      "logits/rejected": -0.9626243884256602,
+      "mean_token_accuracy": 0.7057545288372785,
+      "rewards/chosen": -1.0665127522181137,
+      "rewards/rejected": -2.248069805242121,
+      "rewards/accuracies": 0.8875,
+      "rewards/margins": 1.1815570489689708,
+      "logps/chosen": -68.12463062763214,
+      "logps/rejected": -72.84404395580292,
+      "epoch": 0.08298324775685909,
+      "step": 80
+    },
+    {
+      "loss": 0.36935536861419677,
+      "grad_norm": 0.015311975963413715,
+      "learning_rate": 9.077720207253888e-06,
+      "entropy": 0.7894015382009093,
+      "num_tokens": 6040068.0,
+      "logits/chosen": -1.3107291135601724,
+      "logits/rejected": -1.0484886964785651,
+      "mean_token_accuracy": 0.7019536628946662,
+      "rewards/chosen": -1.3442938781030533,
+      "rewards/rejected": -2.594835725099547,
+      "rewards/accuracies": 0.875,
+      "rewards/margins": 1.2505418445914984,
+      "logps/chosen": -72.38709788918496,
+      "logps/rejected": -75.64441309213639,
+      "epoch": 0.09335615372646647,
+      "step": 90
+    },
+    {
+      "loss": 0.32566983699798585,
+      "grad_norm": 0.013686278834939003,
+      "learning_rate": 8.974093264248706e-06,
+      "entropy": 0.7407993375349906,
+      "num_tokens": 6710899.0,
+      "logits/chosen": -1.491155676700035,
+      "logits/rejected": -1.1507826984501721,
+      "mean_token_accuracy": 0.7010396635532379,
+      "rewards/chosen": -1.472484585774364,
+      "rewards/rejected": -3.0050534189864995,
+      "rewards/accuracies": 0.87375,
+      "rewards/margins": 1.5325688306987286,
+      "logps/chosen": -71.50509068489075,
+      "logps/rejected": -79.74594541549682,
+      "epoch": 0.10372905969607385,
+      "step": 100
+    },
+    {
+      "loss": 0.3260276556015015,
+      "grad_norm": 0.015970611944794655,
+      "learning_rate": 8.870466321243523e-06,
+      "entropy": 0.755189473812934,
+      "num_tokens": 7384308.0,
+      "logits/chosen": -1.4866158399686538,
+      "logits/rejected": -1.2556732947348208,
+      "mean_token_accuracy": 0.6936355204880238,
+      "rewards/chosen": -1.6455462214258296,
+      "rewards/rejected": -3.314337354162708,
+      "rewards/accuracies": 0.875,
+      "rewards/margins": 1.6687911279685794,
+      "logps/chosen": -75.27282932758331,
+      "logps/rejected": -83.60757318258285,
+      "epoch": 0.11410196566568125,
+      "step": 110
+    },
+    {
+      "loss": 0.29123826026916505,
+      "grad_norm": 0.011552904732525349,
+      "learning_rate": 8.766839378238343e-06,
+      "entropy": 0.7474201882537455,
+      "num_tokens": 8056453.0,
+      "logits/chosen": -1.6142062577777776,
+      "logits/rejected": -1.264399125865953,
+      "mean_token_accuracy": 0.7005275747552514,
+      "rewards/chosen": -1.5997523294587155,
+      "rewards/rejected": -3.4206406278908252,
+      "rewards/accuracies": 0.89875,
+      "rewards/margins": 1.8208882979303598,
+      "logps/chosen": -72.69924384593963,
+      "logps/rejected": -84.9409653043747,
+      "epoch": 0.12447487163528863,
+      "step": 120
+    },
+    {
+      "eval_loss": 0.5006195306777954,
+      "eval_runtime": 185.028,
+      "eval_samples_per_second": 5.405,
+      "eval_steps_per_second": 5.405,
+      "eval_entropy": 0.7257204860020429,
+      "eval_num_tokens": 8393345.0,
+      "eval_logits/chosen": -1.6364654488183503,
+      "eval_logits/rejected": -1.3224665456826323,
+      "eval_mean_token_accuracy": 0.6896886819005013,
+      "eval_rewards/chosen": -1.8563280573266092,
+      "eval_rewards/rejected": -3.186325779806066,
+      "eval_rewards/accuracies": 0.791,
+      "eval_rewards/margins": 1.329997718065977,
+      "eval_logps/chosen": -68.73626706314087,
+      "eval_logps/rejected": -84.50312891960144,
+      "epoch": 0.1296613246200923,
+      "step": 125
+    },
+    {
+      "loss": 0.2636831045150757,
+      "grad_norm": 0.013076480478048325,
+      "learning_rate": 8.663212435233162e-06,
+      "entropy": 0.7181003772222903,
+      "num_tokens": 8728634.0,
+      "logits/chosen": -1.671642872950834,
+      "logits/rejected": -1.343902642880392,
+      "mean_token_accuracy": 0.705937882065773,
+      "rewards/chosen": -1.5789886895833478,
+      "rewards/rejected": -3.5362877251021563,
+      "rewards/accuracies": 0.90625,
+      "rewards/margins": 1.9572990308701992,
+      "logps/chosen": -72.90494430541992,
+      "logps/rejected": -84.75396874666214,
+      "epoch": 0.13484777760489602,
+      "step": 130
+    },
+    {
+      "loss": 0.2740552186965942,
+      "grad_norm": 0.021369587630033493,
+      "learning_rate": 8.55958549222798e-06,
+      "entropy": 0.7297478528990178,
+      "num_tokens": 9401159.0,
+      "logits/chosen": -1.7250535840926815,
+      "logits/rejected": -1.360942676206459,
+      "mean_token_accuracy": 0.7058176286891102,
+      "rewards/chosen": -1.711061445976993,
+      "rewards/rejected": -3.760642220210284,
+      "rewards/accuracies": 0.89875,
+      "rewards/margins": 2.0495807684585454,
+      "logps/chosen": -75.34392234086991,
+      "logps/rejected": -86.37021635055542,
+      "epoch": 0.1452206835745034,
+      "step": 140
+    },
+    {
+      "loss": 0.22246203422546387,
+      "grad_norm": 0.011216296814382076,
+      "learning_rate": 8.455958549222799e-06,
+      "entropy": 0.7156421507499181,
+      "num_tokens": 10070639.0,
+      "logits/chosen": -1.8058072450366334,
+      "logits/rejected": -1.453452492851942,
+      "mean_token_accuracy": 0.6968013681657612,
+      "rewards/chosen": -1.7734466221980256,
+      "rewards/rejected": -3.9852739233896135,
+      "rewards/accuracies": 0.935,
+      "rewards/margins": 2.2118273005262017,
+      "logps/chosen": -71.75837573051453,
+      "logps/rejected": -88.40371996164322,
+      "epoch": 0.1555935895441108,
+      "step": 150
+    },
+    {
+      "loss": 0.22982723712921144,
+      "grad_norm": 0.012776419520378113,
+      "learning_rate": 8.352331606217617e-06,
+      "entropy": 0.6933300006289209,
+      "num_tokens": 10739089.0,
+      "logits/chosen": -1.9515460256771149,
+      "logits/rejected": -1.5484639940247698,
+      "mean_token_accuracy": 0.7057104521989822,
+      "rewards/chosen": -1.9016535378903792,
+      "rewards/rejected": -4.284137947391718,
+      "rewards/accuracies": 0.92625,
+      "rewards/margins": 2.3824844037741424,
+      "logps/chosen": -71.04089045763016,
+      "logps/rejected": -91.5341923880577,
+      "epoch": 0.16596649551371817,
+      "step": 160
+    },
+    {
+      "loss": 0.23764348030090332,
+      "grad_norm": 0.014385406859219074,
+      "learning_rate": 8.248704663212436e-06,
+      "entropy": 0.7189149663675926,
+      "num_tokens": 11409834.0,
+      "logits/chosen": -1.891959825395243,
+      "logits/rejected": -1.548625273180476,
+      "mean_token_accuracy": 0.6981369163282216,
+      "rewards/chosen": -1.8744462743058101,
+      "rewards/rejected": -4.231724939988926,
+      "rewards/accuracies": 0.90875,
+      "rewards/margins": 2.3572786602377893,
+      "logps/chosen": -74.21431750059128,
+      "logps/rejected": -91.71514132022858,
+      "epoch": 0.17633940148332555,
+      "step": 170
+    },
+    {
+      "loss": 0.20213358402252196,
+      "grad_norm": 0.01709628663957119,
+      "learning_rate": 8.145077720207254e-06,
+      "entropy": 0.7331272099440684,
+      "num_tokens": 12080449.0,
+      "logits/chosen": -1.8974689940297265,
+      "logits/rejected": -1.5145953081096013,
+      "mean_token_accuracy": 0.6983240643888712,
+      "rewards/chosen": -1.801844048615967,
+      "rewards/rejected": -4.443616501316428,
+      "rewards/accuracies": 0.92875,
+      "rewards/margins": 2.6417724495008588,
+      "logps/chosen": -74.37266374826432,
+      "logps/rejected": -93.39013815402984,
+      "epoch": 0.18671230745293294,
+      "step": 180
+    },
+    {
+      "loss": 0.22085981369018554,
+      "grad_norm": 0.017351916059851646,
+      "learning_rate": 8.041450777202073e-06,
+      "entropy": 0.7038006630554446,
+      "num_tokens": 12750629.0,
+      "logits/chosen": -2.036204467140802,
+      "logits/rejected": -1.5767073775724174,
+      "mean_token_accuracy": 0.7034011324681342,
+      "rewards/chosen": -1.9227157408563653,
+      "rewards/rejected": -4.568975618286058,
+      "rewards/accuracies": 0.9175,
+      "rewards/margins": 2.6462598730251194,
+      "logps/chosen": -71.0266849398613,
+      "logps/rejected": -95.72787975311279,
+      "epoch": 0.19708521342254032,
+      "step": 190
+    },
+    {
+      "loss": 0.21095876693725585,
+      "grad_norm": 0.011923568323254585,
+      "learning_rate": 7.937823834196891e-06,
+      "entropy": 0.7554844116524327,
+      "num_tokens": 13423352.0,
+      "logits/chosen": -1.86974235439135,
+      "logits/rejected": -1.501746707999648,
+      "mean_token_accuracy": 0.6907317889854312,
+      "rewards/chosen": -1.9836193190991616,
+      "rewards/rejected": -4.729873430356383,
+      "rewards/accuracies": 0.92,
+      "rewards/margins": 2.7462541125901043,
+      "logps/chosen": -76.15088132619857,
+      "logps/rejected": -98.10042838811874,
+      "epoch": 0.2074581193921477,
+      "step": 200
+    },
+    {
+      "loss": 0.20679113864898682,
+      "grad_norm": 0.016088780015707016,
+      "learning_rate": 7.834196891191712e-06,
+      "entropy": 0.7792452401274932,
+      "num_tokens": 14095948.0,
+      "logits/chosen": -1.8705466560278834,
+      "logits/rejected": -1.461863531116033,
+      "mean_token_accuracy": 0.6862335817702114,
+      "rewards/chosen": -1.8493201506307742,
+      "rewards/rejected": -4.571774728436139,
+      "rewards/accuracies": 0.9275,
+      "rewards/margins": 2.7224545743037014,
+      "logps/chosen": -74.57652376651764,
+      "logps/rejected": -96.64638544797897,
+      "epoch": 0.2178310253617551,
+      "step": 210
+    },
+    {
+      "loss": 0.1841462254524231,
+      "grad_norm": 0.018083656206727028,
+      "learning_rate": 7.730569948186528e-06,
+      "entropy": 0.7562577223840344,
+      "num_tokens": 14765355.0,
+      "logits/chosen": -1.9466678828790673,
+      "logits/rejected": -1.556764617717985,
+      "mean_token_accuracy": 0.6736517621856183,
+      "rewards/chosen": -1.9498732457583536,
+      "rewards/rejected": -4.814249985329806,
+      "rewards/accuracies": 0.94375,
+      "rewards/margins": 2.8643767323717473,
+      "logps/chosen": -74.6550524687767,
+      "logps/rejected": -96.928872423172,
+      "epoch": 0.2282039313313625,
+      "step": 220
+    },
+    {
+      "loss": 0.18351367712020875,
+      "grad_norm": 0.02809528261423111,
+      "learning_rate": 7.626943005181348e-06,
+      "entropy": 0.7285784264818358,
+      "num_tokens": 15435816.0,
+      "logits/chosen": -2.0795930087592787,
+      "logits/rejected": -1.636840952128652,
+      "mean_token_accuracy": 0.6832175821252168,
+      "rewards/chosen": -2.231121272182354,
+      "rewards/rejected": -5.6053673730045555,
+      "rewards/accuracies": 0.935,
+      "rewards/margins": 3.374246101528406,
+      "logps/chosen": -77.43112219810486,
+      "logps/rejected": -106.47521061182022,
+      "epoch": 0.23857683730096987,
+      "step": 230
+    },
+    {
+      "loss": 0.19073803424835206,
+      "grad_norm": 0.01874386891722679,
+      "learning_rate": 7.523316062176167e-06,
+      "entropy": 0.7100621155637782,
+      "num_tokens": 16103129.0,
+      "logits/chosen": -2.111952753872817,
+      "logits/rejected": -1.7125841187317576,
+      "mean_token_accuracy": 0.6768746449425816,
+      "rewards/chosen": -2.3892388375883455,
+      "rewards/rejected": -5.755352979376912,
+      "rewards/accuracies": 0.935,
+      "rewards/margins": 3.366114140301943,
+      "logps/chosen": -76.3891685450077,
+      "logps/rejected": -104.74123107671738,
+      "epoch": 0.24894974327057726,
+      "step": 240
+    },
+    {
+      "loss": 0.15556421279907226,
+      "grad_norm": 0.01747226156294346,
+      "learning_rate": 7.419689119170985e-06,
+      "entropy": 0.733566192787257,
+      "num_tokens": 16774426.0,
+      "logits/chosen": -1.9979488002914059,
+      "logits/rejected": -1.5990273070071686,
+      "mean_token_accuracy": 0.6710057172738015,
+      "rewards/chosen": -2.2422293889999856,
+      "rewards/rejected": -5.71058324098587,
+      "rewards/accuracies": 0.95125,
+      "rewards/margins": 3.4683538476377724,
+      "logps/chosen": -77.04140710353852,
+      "logps/rejected": -106.12069250583649,
+      "epoch": 0.2593226492401846,
+      "step": 250
+    },
+    {
+      "eval_loss": 0.6056406497955322,
+      "eval_runtime": 214.9765,
+      "eval_samples_per_second": 4.652,
+      "eval_steps_per_second": 4.652,
+      "eval_entropy": 0.7675776538113133,
+      "eval_num_tokens": 16774426.0,
+      "eval_logits/chosen": -1.9016790324834714,
+      "eval_logits/rejected": -1.5678728323483735,
+      "eval_mean_token_accuracy": 0.638426756888628,
+      "eval_rewards/chosen": -2.858136704711709,
+      "eval_rewards/rejected": -5.103179955605418,
+      "eval_rewards/accuracies": 0.779,
+      "eval_rewards/margins": 2.245043248832226,
+      "eval_logps/chosen": -78.75435346412658,
+      "eval_logps/rejected": -103.67167038726807,
+      "epoch": 0.2593226492401846,
+      "step": 250
+    },
+    {
+      "loss": 0.15468125343322753,
+      "grad_norm": 0.023065408691763878,
+      "learning_rate": 7.3160621761658035e-06,
+      "entropy": 0.7432888356089825,
+      "num_tokens": 17443716.0,
+      "logits/chosen": -2.0815265223306243,
+      "logits/rejected": -1.6500056931946074,
+      "mean_token_accuracy": 0.665775734577328,
+      "rewards/chosen": -2.3973246609413765,
+      "rewards/rejected": -5.997232149764895,
+      "rewards/accuracies": 0.95,
+      "rewards/margins": 3.599907489940524,
+      "logps/chosen": -77.37100106954574,
+      "logps/rejected": -108.72272551059723,
+      "epoch": 0.26969555520979205,
+      "step": 260
+    },
+    {
+      "loss": 0.17242192029953002,
+      "grad_norm": 0.018100790679454803,
+      "learning_rate": 7.212435233160623e-06,
+      "entropy": 0.7815824018983403,
+      "num_tokens": 18115456.0,
+      "logits/chosen": -1.8929626691100117,
+      "logits/rejected": -1.568634429940542,
+      "mean_token_accuracy": 0.6482958744466305,
+      "rewards/chosen": -2.8240217579288767,
+      "rewards/rejected": -6.417929985076189,
+      "rewards/accuracies": 0.93875,
+      "rewards/margins": 3.593908224105835,
+      "logps/chosen": -86.06786829471588,
+      "logps/rejected": -114.08773995637894,
+      "epoch": 0.28006846117939943,
+      "step": 270
+    },
+    {
+      "loss": 0.13711202144622803,
+      "grad_norm": 0.013445369899272919,
+      "learning_rate": 7.108808290155441e-06,
+      "entropy": 0.7558632073167246,
+      "num_tokens": 18787182.0,
+      "logits/chosen": -1.978579450117466,
+      "logits/rejected": -1.6631533533075546,
+      "mean_token_accuracy": 0.6569508682191372,
+      "rewards/chosen": -2.7565071246563457,
+      "rewards/rejected": -6.632811545878649,
+      "rewards/accuracies": 0.9575,
+      "rewards/margins": 3.8763044214993716,
+      "logps/chosen": -84.94609524965286,
+      "logps/rejected": -115.75064901828766,
+      "epoch": 0.2904413671490068,
+      "step": 280
+    },
+    {
+      "loss": 0.14471136331558226,
+      "grad_norm": 0.03034352697432041,
+      "learning_rate": 7.005181347150259e-06,
+      "entropy": 0.7719684389990289,
+      "num_tokens": 19458826.0,
+      "logits/chosen": -1.9281082628928359,
+      "logits/rejected": -1.5931914351338883,
+      "mean_token_accuracy": 0.6512335392460227,
+      "rewards/chosen": -3.03531946905714,
+      "rewards/rejected": -7.0661579599231485,
+      "rewards/accuracies": 0.94875,
+      "rewards/margins": 4.030838481113315,
+      "logps/chosen": -85.57453865528106,
+      "logps/rejected": -120.38242918252945,
+      "epoch": 0.3008142731186142,
+      "step": 290
+    },
+    {
+      "loss": 0.14207189083099364,
+      "grad_norm": 0.01976640149950981,
+      "learning_rate": 6.9015544041450784e-06,
+      "entropy": 0.775840242926497,
+      "num_tokens": 20129807.0,
+      "logits/chosen": -1.9291908069467354,
+      "logits/rejected": -1.529736851381803,
+      "mean_token_accuracy": 0.647318363133818,
+      "rewards/chosen": -3.2047973467002158,
+      "rewards/rejected": -7.190337324440479,
+      "rewards/accuracies": 0.94375,
+      "rewards/margins": 3.985539976321161,
+      "logps/chosen": -88.11508201122284,
+      "logps/rejected": -121.52326109170913,
+      "epoch": 0.3111871790882216,
+      "step": 300
+    },
+    {
+      "loss": 0.13882720470428467,
+      "grad_norm": 0.019125595688819885,
+      "learning_rate": 6.797927461139897e-06,
+      "entropy": 0.7558516392938327,
+      "num_tokens": 20801777.0,
+      "logits/chosen": -1.9366754550881657,
+      "logits/rejected": -1.5732930849415423,
+      "mean_token_accuracy": 0.6629200987983495,
+      "rewards/chosen": -3.214626377870736,
+      "rewards/rejected": -7.299282197505236,
+      "rewards/accuracies": 0.9475,
+      "rewards/margins": 4.084655814617872,
+      "logps/chosen": -88.43090007543564,
+      "logps/rejected": -121.64797079086304,
+      "epoch": 0.32156008505782896,
+      "step": 310
+    },
+    {
+      "loss": 0.15239068269729614,
+      "grad_norm": 0.026703625917434692,
+      "learning_rate": 6.6943005181347155e-06,
+      "entropy": 0.759519019573927,
+      "num_tokens": 21473384.0,
+      "logits/chosen": -1.9384710945696861,
+      "logits/rejected": -1.6229014066134837,
+      "mean_token_accuracy": 0.6460824762284756,
+      "rewards/chosen": -3.5289745971094817,
+      "rewards/rejected": -7.814981915429234,
+      "rewards/accuracies": 0.9425,
+      "rewards/margins": 4.286007315814495,
+      "logps/chosen": -95.04349534749984,
+      "logps/rejected": -127.0398188662529,
+      "epoch": 0.33193299102743634,
+      "step": 320
+    },
+    {
+      "loss": 0.13009344339370726,
+      "grad_norm": 0.018585270270705223,
+      "learning_rate": 6.590673575129535e-06,
+      "entropy": 0.7342186298122396,
+      "num_tokens": 22142477.0,
+      "logits/chosen": -2.0363266451123923,
+      "logits/rejected": -1.67308820389407,
+      "mean_token_accuracy": 0.6532345769926906,
+      "rewards/chosen": -3.3065084332806873,
+      "rewards/rejected": -7.662135322336107,
+      "rewards/accuracies": 0.95625,
+      "rewards/margins": 4.355626890957356,
+      "logps/chosen": -86.15242535829545,
+      "logps/rejected": -125.1262379026413,
+      "epoch": 0.3423058969970437,
+      "step": 330
+    },
+    {
+      "loss": 0.1294613003730774,
+      "grad_norm": 0.024174772202968597,
+      "learning_rate": 6.487046632124353e-06,
+      "entropy": 0.778732093811268,
+      "num_tokens": 22814406.0,
+      "logits/chosen": -1.8744846753614508,
+      "logits/rejected": -1.557779354695435,
+      "mean_token_accuracy": 0.6483545247651636,
+      "rewards/chosen": -3.3343913850979243,
+      "rewards/rejected": -7.493394564837217,
+      "rewards/accuracies": 0.95625,
+      "rewards/margins": 4.1590031705796715,
+      "logps/chosen": -91.52018271446228,
+      "logps/rejected": -125.49517280578613,
+      "epoch": 0.3526788029666511,
+      "step": 340
+    },
+    {
+      "loss": 0.12839078903198242,
+      "grad_norm": 0.021780716255307198,
+      "learning_rate": 6.383419689119171e-06,
+      "entropy": 0.7516101838089526,
+      "num_tokens": 23486707.0,
+      "logits/chosen": -1.9700945184752572,
+      "logits/rejected": -1.6397432545695403,
+      "mean_token_accuracy": 0.6522356096096337,
+      "rewards/chosen": -3.294995879915514,
+      "rewards/rejected": -7.7126641423255204,
+      "rewards/accuracies": 0.9575,
+      "rewards/margins": 4.4176682551950215,
+      "logps/chosen": -90.83600348949432,
+      "logps/rejected": -126.77610003948212,
+      "epoch": 0.3630517089362585,
+      "step": 350
+    },
+    {
+      "loss": 0.1488456130027771,
+      "grad_norm": 0.020714374259114265,
+      "learning_rate": 6.2797927461139905e-06,
+      "entropy": 0.7544214495958295,
+      "num_tokens": 24156469.0,
+      "logits/chosen": -1.92354159972201,
+      "logits/rejected": -1.5932112088234438,
+      "mean_token_accuracy": 0.6473389553651213,
+      "rewards/chosen": -3.2932736888423095,
+      "rewards/rejected": -7.400682945102453,
+      "rewards/accuracies": 0.945,
+      "rewards/margins": 4.107409240156412,
+      "logps/chosen": -88.0268507552147,
+      "logps/rejected": -122.07945943832398,
+      "epoch": 0.37342461490586587,
+      "step": 360
+    },
+    {
+      "loss": 0.12627313137054444,
+      "grad_norm": 0.01646515727043152,
+      "learning_rate": 6.176165803108809e-06,
+      "entropy": 0.8060684028314427,
+      "num_tokens": 24831471.0,
+      "logits/chosen": -1.7544659981832322,
+      "logits/rejected": -1.463259412146846,
+      "mean_token_accuracy": 0.6474123366177083,
+      "rewards/chosen": -3.231792522999458,
+      "rewards/rejected": -7.640311172902584,
+      "rewards/accuracies": 0.95875,
+      "rewards/margins": 4.408518632799387,
+      "logps/chosen": -94.10357024669648,
+      "logps/rejected": -128.06225858926774,
+      "epoch": 0.38379752087547325,
+      "step": 370
+    },
+    {
+      "eval_loss": 0.7162045836448669,
+      "eval_runtime": 245.5758,
+      "eval_samples_per_second": 4.072,
+      "eval_steps_per_second": 4.072,
+      "eval_entropy": 0.7526722595356404,
+      "eval_num_tokens": 25165755.0,
+      "eval_logits/chosen": -1.90159736974585,
+      "eval_logits/rejected": -1.5791501459591346,
+      "eval_mean_token_accuracy": 0.6217855967730284,
+      "eval_rewards/chosen": -4.221462745887227,
+      "eval_rewards/rejected": -7.149939286857843,
+      "eval_rewards/accuracies": 0.783,
+      "eval_rewards/margins": 2.9284765325784683,
+      "eval_logps/chosen": -92.38761359214783,
+      "eval_logps/rejected": -124.13926333999633,
+      "epoch": 0.38898397386027694,
+      "step": 375
+    },
+    {
+      "loss": 0.12565295696258544,
+      "grad_norm": 0.023654216900467873,
+      "learning_rate": 6.0725388601036275e-06,
+      "entropy": 0.7168624554201961,
+      "num_tokens": 25500679.0,
+      "logits/chosen": -2.056052749523911,
+      "logits/rejected": -1.643372895075378,
+      "mean_token_accuracy": 0.6568564863596111,
+      "rewards/chosen": -3.4278560562586065,
+      "rewards/rejected": -7.977554326504469,
+      "rewards/accuracies": 0.95,
+      "rewards/margins": 4.549698264449835,
+      "logps/chosen": -86.61842272043228,
+      "logps/rejected": -128.30167908906935,
+      "epoch": 0.39417042684508063,
+      "step": 380
+    },
+    {
+      "loss": 0.11409251689910889,
+      "grad_norm": 0.024783629924058914,
+      "learning_rate": 5.968911917098445e-06,
+      "entropy": 0.7501401096500921,
+      "num_tokens": 26171757.0,
+      "logits/chosen": -1.9333745257284571,
+      "logits/rejected": -1.567348165840012,
+      "mean_token_accuracy": 0.6470369586162269,
+      "rewards/chosen": -3.536213577263552,
+      "rewards/rejected": -8.32627578780055,
+      "rewards/accuracies": 0.96,
+      "rewards/margins": 4.790062201619148,
+      "logps/chosen": -91.70740163564682,
+      "logps/rejected": -133.4269912481308,
+      "epoch": 0.404543332814688,
+      "step": 390
+    },
+    {
+      "loss": 0.10429913997650146,
+      "grad_norm": 0.01578158512711525,
+      "learning_rate": 5.865284974093265e-06,
+      "entropy": 0.7956635004619602,
+      "num_tokens": 26844715.0,
+      "logits/chosen": -1.8049366638144833,
+      "logits/rejected": -1.5104901431761746,
+      "mean_token_accuracy": 0.6389342303480953,
+      "rewards/chosen": -3.4948488865431866,
+      "rewards/rejected": -8.024183837622404,
+      "rewards/accuracies": 0.96375,
+      "rewards/margins": 4.529334946647286,
+      "logps/chosen": -94.648166680336,
+      "logps/rejected": -130.52285103082656,
+      "epoch": 0.4149162387842954,
+      "step": 400
+    },
+    {
+      "loss": 0.13800268173217772,
+      "grad_norm": 0.019957033917307854,
+      "learning_rate": 5.761658031088083e-06,
+      "entropy": 0.7793120911484585,
+      "num_tokens": 27516725.0,
+      "logits/chosen": -1.8549602335857083,
+      "logits/rejected": -1.5267175160987139,
+      "mean_token_accuracy": 0.6423193990625441,
+      "rewards/chosen": -3.492969937432499,
+      "rewards/rejected": -8.126144999023527,
+      "rewards/accuracies": 0.96125,
+      "rewards/margins": 4.633175050020218,
+      "logps/chosen": -93.58816482782363,
+      "logps/rejected": -131.41816306352615,
+      "epoch": 0.4252891447539028,
+      "step": 410
+    },
+    {
+      "loss": 0.11805713176727295,
+      "grad_norm": 0.0187994334846735,
+      "learning_rate": 5.658031088082902e-06,
+      "entropy": 0.7545757652306929,
+      "num_tokens": 28188135.0,
+      "logits/chosen": -1.873550110920591,
+      "logits/rejected": -1.5043320525754367,
+      "mean_token_accuracy": 0.6394615587592125,
+      "rewards/chosen": -3.3145995734280405,
+      "rewards/rejected": -7.947769758999348,
+      "rewards/accuracies": 0.9575,
+      "rewards/margins": 4.633170171380043,
+      "logps/chosen": -87.63653787136077,
+      "logps/rejected": -129.56220105171204,
+      "epoch": 0.4356620507235102,
+      "step": 420
+    },
+    {
+      "loss": 0.12740592956542968,
+      "grad_norm": 0.01738053746521473,
+      "learning_rate": 5.554404145077721e-06,
+      "entropy": 0.75883277257788,
+      "num_tokens": 28858804.0,
+      "logits/chosen": -1.9298376085920284,
+      "logits/rejected": -1.560056586818597,
+      "mean_token_accuracy": 0.6475845011882484,
+      "rewards/chosen": -3.515412328981329,
+      "rewards/rejected": -8.071005233377218,
+      "rewards/accuracies": 0.96,
+      "rewards/margins": 4.555592897236347,
+      "logps/chosen": -92.735348944664,
+      "logps/rejected": -129.75473901987075,
+      "epoch": 0.4460349566931176,
+      "step": 430
+    },
+    {
+      "loss": 0.14025732278823852,
+      "grad_norm": 0.03447360917925835,
+      "learning_rate": 5.4507772020725395e-06,
+      "entropy": 0.7607411430426874,
+      "num_tokens": 29529999.0,
+      "logits/chosen": -1.8672980078366863,
+      "logits/rejected": -1.5365591751796719,
+      "mean_token_accuracy": 0.6453877515532076,
+      "rewards/chosen": -3.5431255162111484,
+      "rewards/rejected": -8.013637196272612,
+      "rewards/accuracies": 0.94625,
+      "rewards/margins": 4.4705116748809814,
+      "logps/chosen": -93.00428878068924,
+      "logps/rejected": -129.0281007218361,
+      "epoch": 0.456407862662725,
+      "step": 440
+    },
+    {
+      "loss": 0.09569562077522278,
+      "grad_norm": 0.019115449860692024,
+      "learning_rate": 5.347150259067357e-06,
+      "entropy": 0.787961185129825,
+      "num_tokens": 30203505.0,
+      "logits/chosen": -1.9089318864847349,
+      "logits/rejected": -1.4739854222647117,
+      "mean_token_accuracy": 0.654497133269906,
+      "rewards/chosen": -3.3072420401062117,
+      "rewards/rejected": -8.128699697032571,
+      "rewards/accuracies": 0.97375,
+      "rewards/margins": 4.821457646489144,
+      "logps/chosen": -90.26756436109542,
+      "logps/rejected": -133.00522322177886,
+      "epoch": 0.46678076863233237,
+      "step": 450
+    },
+    {
+      "loss": 0.12792205810546875,
+      "grad_norm": 0.0281841903924942,
+      "learning_rate": 5.243523316062177e-06,
+      "entropy": 0.766239798675524,
+      "num_tokens": 30875568.0,
+      "logits/chosen": -1.9169137057827605,
+      "logits/rejected": -1.5443000783572756,
+      "mean_token_accuracy": 0.6505569527018815,
+      "rewards/chosen": -3.42417212592205,
+      "rewards/rejected": -8.022385044395923,
+      "rewards/accuracies": 0.95375,
+      "rewards/margins": 4.598212912380696,
+      "logps/chosen": -88.34398028373718,
+      "logps/rejected": -130.9107288122177,
+      "epoch": 0.47715367460193975,
+      "step": 460
+    },
+    {
+      "loss": 0.10438011884689331,
+      "grad_norm": 0.012769816443324089,
+      "learning_rate": 5.139896373056995e-06,
+      "entropy": 0.754261478689732,
+      "num_tokens": 31547801.0,
+      "logits/chosen": -1.9733235777624498,
+      "logits/rejected": -1.5496449682143498,
+      "mean_token_accuracy": 0.6587452093698084,
+      "rewards/chosen": -3.4046586474310607,
+      "rewards/rejected": -8.238689427375794,
+      "rewards/accuracies": 0.96,
+      "rewards/margins": 4.8340307791531085,
+      "logps/chosen": -90.22236561775208,
+      "logps/rejected": -133.67569626569747,
+      "epoch": 0.48752658057154713,
+      "step": 470
+    },
+    {
+      "loss": 0.11527338027954101,
+      "grad_norm": 0.04143095761537552,
+      "learning_rate": 5.036269430051814e-06,
+      "entropy": 0.8031201446475461,
+      "num_tokens": 32220905.0,
+      "logits/chosen": -1.8379997780593569,
+      "logits/rejected": -1.5025386930087672,
+      "mean_token_accuracy": 0.6306095580849796,
+      "rewards/chosen": -3.4276347397238713,
+      "rewards/rejected": -8.276202333420516,
+      "rewards/accuracies": 0.96,
+      "rewards/margins": 4.848567594140768,
+      "logps/chosen": -95.14371522426605,
+      "logps/rejected": -134.20426798820495,
+      "epoch": 0.4978994865411545,
+      "step": 480
+    },
+    {
+      "loss": 0.13075348138809204,
+      "grad_norm": 0.011864494532346725,
+      "learning_rate": 4.932642487046633e-06,
+      "entropy": 0.7831255796179175,
+      "num_tokens": 32894872.0,
+      "logits/chosen": -1.8320905121355995,
+      "logits/rejected": -1.4693764522144144,
+      "mean_token_accuracy": 0.6478395171277225,
+      "rewards/chosen": -3.2756898268085206,
+      "rewards/rejected": -7.780137736238539,
+      "rewards/accuracies": 0.955,
+      "rewards/margins": 4.504447904080153,
+      "logps/chosen": -91.73629876852036,
+      "logps/rejected": -129.0320829319954,
+      "epoch": 0.5082723925107618,
+      "step": 490
+    },
+    {
+      "loss": 0.12714189291000366,
+      "grad_norm": 0.02473694644868374,
+      "learning_rate": 4.829015544041451e-06,
+      "entropy": 0.7955848192726261,
+      "num_tokens": 33565691.0,
+      "logits/chosen": -1.8834938774520926,
+      "logits/rejected": -1.4981447820382991,
+      "mean_token_accuracy": 0.6382771720923484,
+      "rewards/chosen": -3.1717117047600913,
+      "rewards/rejected": -7.624671882167458,
+      "rewards/accuracies": 0.9575,
+      "rewards/margins": 4.452960164994002,
+      "logps/chosen": -87.47686364412307,
+      "logps/rejected": -125.46468779563904,
+      "epoch": 0.5186452984803692,
+      "step": 500
+    },
+    {
+      "eval_loss": 0.6964932084083557,
+      "eval_runtime": 279.8952,
+      "eval_samples_per_second": 3.573,
+      "eval_steps_per_second": 3.573,
+      "eval_entropy": 0.7739749677795916,
+      "eval_num_tokens": 33565691.0,
+      "eval_logits/chosen": -1.8410994007704304,
+      "eval_logits/rejected": -1.4860494630066636,
+      "eval_mean_token_accuracy": 0.6207611305266618,
+      "eval_rewards/chosen": -3.7967804961130023,
+      "eval_rewards/rejected": -6.769779369160533,
+      "eval_rewards/accuracies": 0.793,
+      "eval_rewards/margins": 2.9729988664388656,
+      "eval_logps/chosen": -88.14079112052917,
+      "eval_logps/rejected": -120.33766422653198,
+      "epoch": 0.5186452984803692,
+      "step": 500
+    },
+    {
+      "train_runtime": 16864.5913,
+      "train_samples_per_second": 4.573,
+      "train_steps_per_second": 0.057,
+      "total_flos": 2.7163725934432666e+17,
+      "train_loss": 0.23282903826236725,
+      "epoch": 0.5186452984803692,
+      "step": 500
+    }
+  ],
+  "validation_monitor_size": 1000,
+  "validation_monitor_selection": "fixed_seed_random_without_replacement",
+  "validation_monitor_seed": 22,
+  "validation_monitor_indices": [
+    2,
+    10,
+    14,
+    21,
+    55,
+    63,
+    66,
+    69,
+    107,
+    110,
+    142,
+    144,
+    149,
+    150,
+    151,
+    152,
+    160,
+    163,
+    166,
+    167,
+    176,
+    181,
+    206,
+    207,
+    259,
+    267,
+    281,
+    295,
+    298,
+    306,
+    307,
+    317,
+    321,
+    331,
+    336,
+    341,
+    346,
+    349,
+    351,
+    352,
+    357,
+    358,
+    369,
+    370,
+    382,
+    386,
+    391,
+    401,
+    411,
+    412,
+    432,
+    437,
+    452,
+    462,
+    465,
+    488,
+    490,
+    491,
+    492,
+    494,
+    503,
+    504,
+    508,
+    528,
+    538,
+    540,
+    551,
+    553,
+    567,
+    586,
+    605,
+    606,
+    625,
+    627,
+    661,
+    663,
+    666,
+    677,
+    685,
+    690,
+    692,
+    704,
+    708,
+    714,
+    715,
+    727,
+    728,
+    733,
+    745,
+    752,
+    753,
+    755,
+    764,
+    773,
+    779,
+    783,
+    793,
+    796,
+    799,
+    803,
+    804,
+    805,
+    813,
+    816,
+    818,
+    823,
+    827,
+    829,
+    830,
+    837,
+    842,
+    845,
+    850,
+    853,
+    856,
+    889,
+    890,
+    905,
+    915,
+    924,
+    930,
+    939,
+    951,
+    988,
+    1002,
+    1005,
+    1023,
+    1029,
+    1038,
+    1049,
+    1050,
+    1054,
+    1056,
+    1067,
+    1068,
+    1079,
+    1088,
+    1091,
+    1103,
+    1114,
+    1118,
+    1133,
+    1140,
+    1144,
+    1145,
+    1155,
+    1186,
+    1195,
+    1206,
+    1223,
+    1251,
+    1252,
+    1257,
+    1259,
+    1270,
+    1271,
+    1295,
+    1303,
+    1304,
+    1305,
+    1329,
+    1335,
+    1336,
+    1343,
+    1367,
+    1373,
+    1377,
+    1403,
+    1412,
+    1429,
+    1443,
+    1457,
+    1459,
+    1460,
+    1476,
+    1483,
+    1486,
+    1494,
+    1507,
+    1510,
+    1519,
+    1521,
+    1522,
+    1545,
+    1551,
+    1570,
+    1582,
+    1593,
+    1595,
+    1603,
+    1607,
+    1614,
+    1615,
+    1625,
+    1634,
+    1639,
+    1648,
+    1654,
+    1657,
+    1662,
+    1667,
+    1673,
+    1690,
+    1704,
+    1746,
+    1756,
+    1781,
+    1783,
+    1796,
+    1799,
+    1809,
+    1814,
+    1827,
+    1829,
+    1832,
+    1844,
+    1847,
+    1854,
+    1856,
+    1857,
+    1858,
+    1874,
+    1883,
+    1889,
+    1924,
+    1925,
+    1931,
+    1932,
+    1934,
+    1935,
+    1938,
+    1950,
+    1957,
+    1962,
+    1967,
+    1975,
+    1982,
+    1983,
+    1991,
+    1998,
+    2003,
+    2008,
+    2017,
+    2021,
+    2026,
+    2035,
+    2040,
+    2050,
+    2056,
+    2077,
+    2079,
+    2082,
+    2098,
+    2100,
+    2108,
+    2121,
+    2130,
+    2133,
+    2134,
+    2138,
+    2143,
+    2166,
+    2167,
+    2180,
+    2181,
+    2185,
+    2204,
+    2205,
+    2212,
+    2221,
+    2224,
+    2226,
+    2230,
+    2233,
+    2256,
+    2261,
+    2263,
+    2269,
+    2273,
+    2290,
+    2291,
+    2299,
+    2301,
+    2321,
+    2323,
+    2330,
+    2384,
+    2401,
+    2417,
+    2420,
+    2421,
+    2424,
+    2430,
+    2435,
+    2456,
+    2488,
+    2502,
+    2504,
+    2519,
+    2527,
+    2532,
+    2538,
+    2542,
+    2553,
+    2555,
+    2558,
+    2559,
+    2562,
+    2578,
+    2583,
+    2585,
+    2590,
+    2592,
+    2594,
+    2596,
+    2600,
+    2606,
+    2607,
+    2618,
+    2630,
+    2637,
+    2647,
+    2650,
+    2657,
+    2679,
+    2685,
+    2705,
+    2706,
+    2712,
+    2713,
+    2714,
+    2727,
+    2740,
+    2742,
+    2755,
+    2780,
+    2784,
+    2792,
+    2807,
+    2808,
+    2810,
+    2820,
+    2831,
+    2839,
+    2860,
+    2862,
+    2863,
+    2866,
+    2875,
+    2878,
+    2898,
+    2905,
+    2921,
+    2922,
+    2926,
+    2930,
+    2934,
+    2944,
+    2955,
+    2957,
+    2959,
+    2973,
+    2978,
+    2998,
+    3018,
+    3022,
+    3028,
+    3031,
+    3061,
+    3085,
+    3090,
+    3104,
+    3105,
+    3111,
+    3115,
+    3121,
+    3122,
+    3129,
+    3133,
+    3135,
+    3161,
+    3162,
+    3169,
+    3173,
+    3194,
+    3195,
+    3215,
+    3225,
+    3226,
+    3241,
+    3247,
+    3250,
+    3253,
+    3265,
+    3268,
+    3293,
+    3301,
+    3312,
+    3329,
+    3352,
+    3361,
+    3362,
+    3376,
+    3396,
+    3401,
+    3403,
+    3410,
+    3419,
+    3432,
+    3443,
+    3452,
+    3467,
+    3469,
+    3475,
+    3485,
+    3503,
+    3514,
+    3515,
+    3524,
+    3528,
+    3538,
+    3544,
+    3557,
+    3560,
+    3565,
+    3600,
+    3637,
+    3642,
+    3658,
+    3659,
+    3692,
+    3693,
+    3699,
+    3722,
+    3725,
+    3728,
+    3731,
+    3740,
+    3742,
+    3762,
+    3766,
+    3780,
+    3788,
+    3794,
+    3796,
+    3798,
+    3805,
+    3817,
+    3819,
+    3822,
+    3837,
+    3839,
+    3843,
+    3846,
+    3851,
+    3854,
+    3865,
+    3870,
+    3871,
+    3884,
+    3894,
+    3895,
+    3896,
+    3907,
+    3911,
+    3915,
+    3919,
+    3920,
+    3923,
+    3933,
+    3955,
+    3967,
+    3972,
+    3974,
+    3975,
+    3984,
+    3985,
+    3997,
+    4002,
+    4010,
+    4034,
+    4044,
+    4063,
+    4073,
+    4079,
+    4082,
+    4088,
+    4121,
+    4145,
+    4148,
+    4159,
+    4161,
+    4164,
+    4177,
+    4188,
+    4199,
+    4203,
+    4207,
+    4208,
+    4213,
+    4221,
+    4225,
+    4233,
+    4241,
+    4243,
+    4247,
+    4264,
+    4274,
+    4282,
+    4286,
+    4290,
+    4308,
+    4310,
+    4313,
+    4321,
+    4324,
+    4327,
+    4349,
+    4362,
+    4370,
+    4374,
+    4380,
+    4407,
+    4409,
+    4411,
+    4415,
+    4417,
+    4418,
+    4427,
+    4431,
+    4433,
+    4451,
+    4466,
+    4477,
+    4478,
+    4479,
+    4493,
+    4494,
+    4514,
+    4527,
+    4539,
+    4550,
+    4558,
+    4568,
+    4579,
+    4583,
+    4584,
+    4586,
+    4587,
+    4590,
+    4599,
+    4602,
+    4610,
+    4626,
+    4627,
+    4630,
+    4641,
+    4647,
+    4655,
+    4656,
+    4657,
+    4661,
+    4685,
+    4714,
+    4715,
+    4731,
+    4749,
+    4752,
+    4769,
+    4777,
+    4782,
+    4791,
+    4805,
+    4818,
+    4829,
+    4833,
+    4837,
+    4839,
+    4843,
+    4871,
+    4875,
+    4879,
+    4880,
+    4885,
+    4888,
+    4895,
+    4900,
+    4923,
+    4966,
+    4968,
+    4972,
+    4989,
+    4994,
+    4998,
+    5001,
+    5013,
+    5019,
+    5026,
+    5032,
+    5034,
+    5046,
+    5055,
+    5085,
+    5086,
+    5088,
+    5089,
+    5090,
+    5095,
+    5108,
+    5110,
+    5119,
+    5120,
+    5121,
+    5133,
+    5148,
+    5154,
+    5160,
+    5169,
+    5178,
+    5222,
+    5223,
+    5232,
+    5233,
+    5240,
+    5256,
+    5259,
+    5264,
+    5271,
+    5276,
+    5279,
+    5294,
+    5300,
+    5303,
+    5321,
+    5335,
+    5337,
+    5345,
+    5348,
+    5365,
+    5373,
+    5378,
+    5384,
+    5422,
+    5442,
+    5443,
+    5445,
+    5477,
+    5485,
+    5495,
+    5497,
+    5504,
+    5526,
+    5533,
+    5542,
+    5564,
+    5570,
+    5579,
+    5587,
+    5592,
+    5608,
+    5610,
+    5624,
+    5630,
+    5638,
+    5651,
+    5663,
+    5670,
+    5675,
+    5691,
+    5700,
+    5706,
+    5707,
+    5715,
+    5720,
+    5721,
+    5722,
+    5732,
+    5738,
+    5741,
+    5769,
+    5771,
+    5775,
+    5795,
+    5796,
+    5800,
+    5809,
+    5810,
+    5815,
+    5819,
+    5827,
+    5848,
+    5849,
+    5852,
+    5859,
+    5880,
+    5884,
+    5907,
+    5909,
+    5912,
+    5919,
+    5931,
+    5932,
+    5934,
+    5941,
+    5948,
+    5950,
+    5952,
+    5953,
+    5969,
+    5981,
+    6000,
+    6003,
+    6010,
+    6018,
+    6041,
+    6065,
+    6075,
+    6090,
+    6103,
+    6106,
+    6109,
+    6114,
+    6123,
+    6131,
+    6136,
+    6138,
+    6139,
+    6164,
+    6165,
+    6171,
+    6173,
+    6180,
+    6185,
+    6189,
+    6190,
+    6221,
+    6223,
+    6237,
+    6255,
+    6262,
+    6265,
+    6293,
+    6296,
+    6305,
+    6318,
+    6331,
+    6336,
+    6340,
+    6355,
+    6366,
+    6371,
+    6396,
+    6399,
+    6402,
+    6408,
+    6432,
+    6433,
+    6441,
+    6456,
+    6465,
+    6478,
+    6486,
+    6489,
+    6507,
+    6508,
+    6520,
+    6522,
+    6528,
+    6537,
+    6551,
+    6564,
+    6589,
+    6590,
+    6598,
+    6599,
+    6611,
+    6613,
+    6615,
+    6621,
+    6634,
+    6647,
+    6649,
+    6654,
+    6676,
+    6680,
+    6690,
+    6708,
+    6729,
+    6736,
+    6744,
+    6749,
+    6756,
+    6761,
+    6763,
+    6773,
+    6788,
+    6790,
+    6796,
+    6797,
+    6811,
+    6824,
+    6850,
+    6869,
+    6871,
+    6882,
+    6892,
+    6895,
+    6906,
+    6911,
+    6912,
+    6914,
+    6927,
+    6952,
+    6966,
+    6985,
+    7001,
+    7021,
+    7031,
+    7035,
+    7038,
+    7041,
+    7045,
+    7052,
+    7057,
+    7058,
+    7072,
+    7073,
+    7076,
+    7086,
+    7102,
+    7107,
+    7109,
+    7117,
+    7122,
+    7125,
+    7166,
+    7182,
+    7199,
+    7207,
+    7212,
+    7215,
+    7232,
+    7243,
+    7246,
+    7250,
+    7253,
+    7258,
+    7263,
+    7267,
+    7270,
+    7274,
+    7280,
+    7286,
+    7293,
+    7298,
+    7302,
+    7306,
+    7316,
+    7325,
+    7326,
+    7334,
+    7356,
+    7357,
+    7363,
+    7364,
+    7367,
+    7385,
+    7392,
+    7399,
+    7405,
+    7416,
+    7420,
+    7421,
+    7426,
+    7452,
+    7476,
+    7481,
+    7519,
+    7534,
+    7542,
+    7546,
+    7573,
+    7585,
+    7601,
+    7604,
+    7606,
+    7609,
+    7629,
+    7649,
+    7653,
+    7667,
+    7682,
+    7699,
+    7738,
+    7750,
+    7786,
+    7798,
+    7800,
+    7801,
+    7805,
+    7806,
+    7811,
+    7813,
+    7832,
+    7837,
+    7849,
+    7856,
+    7876,
+    7877,
+    7887,
+    7905,
+    7916,
+    7919,
+    7920,
+    7922,
+    7923,
+    7926,
+    7944,
+    7961,
+    7966,
+    7970,
+    7973,
+    7974,
+    7976,
+    7986,
+    7999,
+    8027,
+    8028,
+    8034,
+    8047,
+    8068,
+    8074,
+    8077,
+    8091,
+    8120,
+    8122,
+    8125,
+    8152,
+    8153,
+    8164,
+    8167,
+    8169,
+    8171,
+    8177,
+    8184,
+    8189,
+    8192,
+    8196,
+    8202,
+    8212,
+    8217,
+    8231,
+    8242,
+    8244,
+    8250,
+    8256,
+    8257,
+    8265,
+    8270,
+    8274,
+    8283,
+    8290,
+    8294,
+    8301,
+    8302,
+    8307,
+    8318,
+    8326,
+    8338,
+    8349,
+    8350,
+    8353,
+    8357,
+    8371,
+    8374,
+    8377,
+    8380,
+    8387,
+    8388,
+    8396,
+    8402,
+    8419,
+    8423,
+    8428,
+    8435,
+    8439,
+    8442,
+    8444,
+    8453,
+    8461,
+    8475,
+    8481,
+    8485,
+    8493,
+    8495,
+    8498,
+    8523,
+    8530,
+    8531,
+    8562
+  ],
+  "early_stopping_patience": 3
+}

phase1/ablations/matched_clean/README.md

@@ -0,0 +1,80 @@
+---
+library_name: peft
+model_name: phase1-qwen-matched-clean-ablation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-1.5B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+license: apache-2.0
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+pipeline_tag: text-generation
+---
+
+# Phase 1 Qwen `matched_clean` diagnostic ablation
+
+This is the Phase 1 `matched_clean` diagnostic LoRA-DPO adapter fine-tuned from
+[Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct).
+The frozen training recipe and membership hashes are included in this directory.
+
+## Quick start
+
+```python
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo_id = "geyuxu/york-milestone-project-self-traing-loop-model"
+subfolder = "phase1/ablations/matched_clean"
+base_id = "Qwen/Qwen2.5-1.5B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(repo_id, subfolder=subfolder)
+base_model = AutoModelForCausalLM.from_pretrained(
+    base_id, torch_dtype="auto", device_map="auto"
+)
+model = PeftModel.from_pretrained(base_model, repo_id, subfolder=subfolder)
+```
+
+## Training procedure
+
+
+
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 0.29.0
+- Transformers: 5.3.0
+- Pytorch: 2.10.0
+- Datasets: 4.6.1
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+    title        = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+    author       = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+    year         = 2023,
+    booktitle    = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+    url          = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+    editor       = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

phase1/ablations/matched_clean/adapter_config.json

@@ -0,0 +1,43 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "k_proj",
+    "o_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

phase1/ablations/matched_clean/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:724b1e847a25563f7fe04d27dfa05df9213b97348490c4cf5b8f1c0cbdb90b68
+size 17462432

phase1/ablations/matched_clean/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

phase1/ablations/matched_clean/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5
+size 11422166

phase1/ablations/matched_clean/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

phase1/ablations/matched_clean/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfafc3f2a531cdc31948a992577839650613ac2f09616d9263508e3e97da8555
+size 5841

phase1/ablations/matched_clean/training_recipe.json

@@ -0,0 +1,22 @@
+{
+  "format_version": 1,
+  "experiment": "confusion_ablation",
+  "ablation": "matched_clean",
+  "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+  "train_membership_sha256": "b497fa765223a9810ad784f31e1598edfa992a162146d6f94d9cdf4adcedd87d",
+  "validation_membership_sha256": "2f609467438cd6a01fd84b9d80c0fcd84c44767e3b2ed98f4f6a83d68a62c181",
+  "num_train_pairs": 77124,
+  "num_validation_pairs": 8563,
+  "transformation_seed": 22,
+  "validation_prompt_mode": "clean_rag",
+  "learning_rate": 1e-05,
+  "beta": 0.1,
+  "num_epochs": 1.0,
+  "seed": 22,
+  "max_length": 512,
+  "max_prompt_length": 384,
+  "gradient_accumulation_steps": 80,
+  "save_steps": 125,
+  "save_total_limit": 12,
+  "preference_pairs_format_version": 2
+}

phase1/ablations/matched_clean/training_summary.json

@@ -0,0 +1,2925 @@
+{
+  "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+  "resolved_model": "Qwen/Qwen2.5-1.5B-Instruct",
+  "num_train_records": 77124,
+  "num_validation_records_available": 8563,
+  "training_config": {
+    "lora_rank": 16,
+    "lora_alpha": 32,
+    "learning_rate": 1e-05,
+    "num_epochs": 1.0,
+    "per_device_batch_size": 1,
+    "per_device_eval_batch_size": 1,
+    "gradient_accumulation_steps": 80,
+    "beta": 0.1,
+    "max_length": 512,
+    "max_prompt_length": 384,
+    "seed": 22,
+    "save_steps": 125,
+    "save_total_limit": 12,
+    "resume_from_checkpoint": null,
+    "precision_dtype": "torch.bfloat16",
+    "bf16": true,
+    "fp16": false,
+    "tokenizer_add_bos_token": false
+  },
+  "global_step": 965,
+  "best_metric": 0.4090946316719055,
+  "best_model_checkpoint": "outputs/confusion_ablation_phase1_repair_v3/matched_clean/lora/checkpoint-875",
+  "train_metrics": {
+    "train_runtime": 36967.0361,
+    "train_samples_per_second": 2.086,
+    "train_steps_per_second": 0.026,
+    "total_flos": 5.2405488888805786e+17,
+    "train_loss": 0.453640728787437
+  },
+  "log_history": [
+    {
+      "loss": 0.6900316715240479,
+      "grad_norm": 0.011662614531815052,
+      "learning_rate": 9.906735751295338e-06,
+      "entropy": 0.8081745196832344,
+      "num_tokens": 669528.0,
+      "logits/chosen": -0.9878667544984245,
+      "logits/rejected": -0.8929429136538265,
+      "mean_token_accuracy": 0.7366863564029336,
+      "rewards/chosen": 0.0014809935365531145,
+      "rewards/rejected": -0.0051326466447972055,
+      "rewards/accuracies": 0.495,
+      "rewards/margins": 0.00661364022129419,
+      "logps/chosen": -48.20964523553848,
+      "logps/rejected": -54.25052870512009,
+      "epoch": 0.010372905969607386,
+      "step": 10
+    },
+    {
+      "loss": 0.6709239482879639,
+      "grad_norm": 0.01787947677075863,
+      "learning_rate": 9.803108808290156e-06,
+      "entropy": 0.7905870282277465,
+      "num_tokens": 1338759.0,
+      "logits/chosen": -0.9877784900726513,
+      "logits/rejected": -0.9284377500817239,
+      "mean_token_accuracy": 0.7397855435311794,
+      "rewards/chosen": -0.016295835754335714,
+      "rewards/rejected": -0.06304080443188653,
+      "rewards/accuracies": 0.74,
+      "rewards/margins": 0.046744968572747896,
+      "logps/chosen": -49.98522471666336,
+      "logps/rejected": -53.08380969285965,
+      "epoch": 0.02074581193921477,
+      "step": 20
+    },
+    {
+      "loss": 0.6449037075042725,
+      "grad_norm": 0.012443667277693748,
+      "learning_rate": 9.699481865284975e-06,
+      "entropy": 0.8007427603192627,
+      "num_tokens": 2009918.0,
+      "logits/chosen": -1.0123033629117577,
+      "logits/rejected": -0.9116259647175791,
+      "mean_token_accuracy": 0.7288216127734631,
+      "rewards/chosen": -0.08125005806276021,
+      "rewards/rejected": -0.18847612114805087,
+      "rewards/accuracies": 0.7525,
+      "rewards/margins": 0.10722606290772091,
+      "logps/chosen": -51.017456583976745,
+      "logps/rejected": -55.82100034952164,
+      "epoch": 0.031118717908822157,
+      "step": 30
+    },
+    {
+      "loss": 0.6104300498962403,
+      "grad_norm": 0.011565622873604298,
+      "learning_rate": 9.595854922279793e-06,
+      "entropy": 0.7688610723486636,
+      "num_tokens": 2680367.0,
+      "logits/chosen": -1.0477489025836138,
+      "logits/rejected": -0.9185756571968596,
+      "mean_token_accuracy": 0.7304606804065407,
+      "rewards/chosen": -0.18015499103314597,
+      "rewards/rejected": -0.377999417107676,
+      "rewards/accuracies": 0.76125,
+      "rewards/margins": 0.19784442565287463,
+      "logps/chosen": -52.41810529708862,
+      "logps/rejected": -56.31561070203781,
+      "epoch": 0.04149162387842954,
+      "step": 40
+    },
+    {
+      "loss": 0.5890812397003173,
+      "grad_norm": 0.011881239712238312,
+      "learning_rate": 9.492227979274612e-06,
+      "entropy": 0.767548923434224,
+      "num_tokens": 3350695.0,
+      "logits/chosen": -1.045900867553522,
+      "logits/rejected": -0.8987545206323213,
+      "mean_token_accuracy": 0.7290248272754252,
+      "rewards/chosen": -0.3571337752114414,
+      "rewards/rejected": -0.635303623637883,
+      "rewards/accuracies": 0.76125,
+      "rewards/margins": 0.27816984807141126,
+      "logps/chosen": -53.70132049441337,
+      "logps/rejected": -58.7293958568573,
+      "epoch": 0.051864529848036925,
+      "step": 50
+    },
+    {
+      "loss": 0.5672520160675049,
+      "grad_norm": 0.012122954241931438,
+      "learning_rate": 9.388601036269432e-06,
+      "entropy": 0.7509493381145876,
+      "num_tokens": 4019867.0,
+      "logits/chosen": -1.0931640275508827,
+      "logits/rejected": -0.9542402869856921,
+      "mean_token_accuracy": 0.7216887871176004,
+      "rewards/chosen": -0.5263037721261935,
+      "rewards/rejected": -0.9088172687360202,
+      "rewards/accuracies": 0.74875,
+      "rewards/margins": 0.38251349702011794,
+      "logps/chosen": -54.063412301540374,
+      "logps/rejected": -60.20355456352234,
+      "epoch": 0.062237435817644314,
+      "step": 60
+    },
+    {
+      "loss": 0.5313051700592041,
+      "grad_norm": 0.012870470061898232,
+      "learning_rate": 9.284974093264249e-06,
+      "entropy": 0.7521841926735943,
+      "num_tokens": 4689830.0,
+      "logits/chosen": -1.1636649836163147,
+      "logits/rejected": -0.9546643791863164,
+      "mean_token_accuracy": 0.7130510857887566,
+      "rewards/chosen": -0.7180539178028266,
+      "rewards/rejected": -1.2522271308361086,
+      "rewards/accuracies": 0.765,
+      "rewards/margins": 0.5341732117999345,
+      "logps/chosen": -57.92338786125183,
+      "logps/rejected": -64.00875183820725,
+      "epoch": 0.0726103417872517,
+      "step": 70
+    },
+    {
+      "loss": 0.524173355102539,
+      "grad_norm": 0.012850801460444927,
+      "learning_rate": 9.181347150259067e-06,
+      "entropy": 0.7584943031892181,
+      "num_tokens": 5359790.0,
+      "logits/chosen": -1.257342141058688,
+      "logits/rejected": -1.0016241015611054,
+      "mean_token_accuracy": 0.7126817052811384,
+      "rewards/chosen": -0.9431407463727374,
+      "rewards/rejected": -1.5808365274948302,
+      "rewards/accuracies": 0.76375,
+      "rewards/margins": 0.6376957796374336,
+      "logps/chosen": -58.12816961526871,
+      "logps/rejected": -69.45178006649017,
+      "epoch": 0.08298324775685909,
+      "step": 80
+    },
+    {
+      "loss": 0.5096397399902344,
+      "grad_norm": 0.015030446462333202,
+      "learning_rate": 9.077720207253888e-06,
+      "entropy": 0.7446463107218733,
+      "num_tokens": 6029642.0,
+      "logits/chosen": -1.3131360929541611,
+      "logits/rejected": -1.12108701233509,
+      "mean_token_accuracy": 0.6976540317758918,
+      "rewards/chosen": -1.0887427906056837,
+      "rewards/rejected": -1.8297204279349535,
+      "rewards/accuracies": 0.765,
+      "rewards/margins": 0.7409776375815272,
+      "logps/chosen": -62.075138006210324,
+      "logps/rejected": -71.068751745224,
+      "epoch": 0.09335615372646647,
+      "step": 90
+    },
+    {
+      "loss": 0.5227419376373291,
+      "grad_norm": 0.017649168148636818,
+      "learning_rate": 8.974093264248706e-06,
+      "entropy": 0.774682844899653,
+      "num_tokens": 6703750.0,
+      "logits/chosen": -1.3070892244508543,
+      "logits/rejected": -1.0911262790664193,
+      "mean_token_accuracy": 0.698255299879238,
+      "rewards/chosen": -1.2929798694234342,
+      "rewards/rejected": -2.059200689474819,
+      "rewards/accuracies": 0.755,
+      "rewards/margins": 0.7662208179756999,
+      "logps/chosen": -67.27675194501877,
+      "logps/rejected": -78.23825345277787,
+      "epoch": 0.10372905969607385,
+      "step": 100
+    },
+    {
+      "loss": 0.5076948642730713,
+      "grad_norm": 0.018913043662905693,
+      "learning_rate": 8.870466321243523e-06,
+      "entropy": 0.723871832547884,
+      "num_tokens": 7374932.0,
+      "logits/chosen": -1.3768008463722332,
+      "logits/rejected": -1.1314466745219074,
+      "mean_token_accuracy": 0.7054882103763521,
+      "rewards/chosen": -1.247365918380383,
+      "rewards/rejected": -2.083569277639035,
+      "rewards/accuracies": 0.775,
+      "rewards/margins": 0.8362033596914261,
+      "logps/chosen": -62.56643625497818,
+      "logps/rejected": -73.74503843307495,
+      "epoch": 0.11410196566568125,
+      "step": 110
+    },
+    {
+      "loss": 0.518116807937622,
+      "grad_norm": 0.014157967641949654,
+      "learning_rate": 8.766839378238343e-06,
+      "entropy": 0.7146987286728108,
+      "num_tokens": 8045025.0,
+      "logits/chosen": -1.392513233808292,
+      "logits/rejected": -1.167585582525032,
+      "mean_token_accuracy": 0.7107785574346781,
+      "rewards/chosen": -1.1991379019781017,
+      "rewards/rejected": -1.9976002290286123,
+      "rewards/accuracies": 0.76625,
+      "rewards/margins": 0.7984623239561915,
+      "logps/chosen": -63.023617169857026,
+      "logps/rejected": -70.3282685637474,
+      "epoch": 0.12447487163528863,
+      "step": 120
+    },
+    {
+      "eval_loss": 0.49094030261039734,
+      "eval_runtime": 195.9122,
+      "eval_samples_per_second": 5.104,
+      "eval_steps_per_second": 5.104,
+      "eval_entropy": 0.7422072718068957,
+      "eval_num_tokens": 8380297.0,
+      "eval_logits/chosen": -1.3606685028205767,
+      "eval_logits/rejected": -1.1272819430340941,
+      "eval_mean_token_accuracy": 0.7042824859470129,
+      "eval_rewards/chosen": -1.0899753504878027,
+      "eval_rewards/rejected": -1.9470994755723514,
+      "eval_rewards/accuracies": 0.777,
+      "eval_rewards/margins": 0.8571241236105561,
+      "eval_logps/chosen": -61.07274015617371,
+      "eval_logps/rejected": -72.11086616706848,
+      "epoch": 0.1296613246200923,
+      "step": 125
+    },
+    {
+      "loss": 0.4915438652038574,
+      "grad_norm": 0.014024928212165833,
+      "learning_rate": 8.663212435233162e-06,
+      "entropy": 0.7629431400250177,
+      "num_tokens": 8716283.0,
+      "logits/chosen": -1.3493070406504677,
+      "logits/rejected": -1.1303618494014056,
+      "mean_token_accuracy": 0.7068962034024299,
+      "rewards/chosen": -1.064063529693958,
+      "rewards/rejected": -1.9368204824090935,
+      "rewards/accuracies": 0.7825,
+      "rewards/margins": 0.8727569509390741,
+      "logps/chosen": -61.36201237678528,
+      "logps/rejected": -73.24028332233429,
+      "epoch": 0.13484777760489602,
+      "step": 130
+    },
+    {
+      "loss": 0.5040828704833984,
+      "grad_norm": 0.01371421106159687,
+      "learning_rate": 8.55958549222798e-06,
+      "entropy": 0.7165489907818846,
+      "num_tokens": 9385323.0,
+      "logits/chosen": -1.4613930482520017,
+      "logits/rejected": -1.211856862455939,
+      "mean_token_accuracy": 0.7094380159396678,
+      "rewards/chosen": -1.0889925239968579,
+      "rewards/rejected": -1.8759657187038101,
+      "rewards/accuracies": 0.77875,
+      "rewards/margins": 0.7869731936138123,
+      "logps/chosen": -57.32686332225799,
+      "logps/rejected": -71.04226065397262,
+      "epoch": 0.1452206835745034,
+      "step": 140
+    },
+    {
+      "loss": 0.45884413719177247,
+      "grad_norm": 0.01413357350975275,
+      "learning_rate": 8.455958549222799e-06,
+      "entropy": 0.701725748301833,
+      "num_tokens": 10057097.0,
+      "logits/chosen": -1.5608966569387421,
+      "logits/rejected": -1.247274744986142,
+      "mean_token_accuracy": 0.7153122393228114,
+      "rewards/chosen": -1.071364169051376,
+      "rewards/rejected": -2.1302177897136425,
+      "rewards/accuracies": 0.8075,
+      "rewards/margins": 1.0588536195317284,
+      "logps/chosen": -58.66886381149292,
+      "logps/rejected": -75.77571431159973,
+      "epoch": 0.1555935895441108,
+      "step": 150
+    },
+    {
+      "loss": 0.5141475677490235,
+      "grad_norm": 0.01554640382528305,
+      "learning_rate": 8.352331606217617e-06,
+      "entropy": 0.7456542236352106,
+      "num_tokens": 10728653.0,
+      "logits/chosen": -1.4681753702348763,
+      "logits/rejected": -1.2457289726855978,
+      "mean_token_accuracy": 0.70392991816625,
+      "rewards/chosen": -1.161004657074118,
+      "rewards/rejected": -2.000340641771909,
+      "rewards/accuracies": 0.77375,
+      "rewards/margins": 0.8393359830789268,
+      "logps/chosen": -64.61760116338729,
+      "logps/rejected": -74.73108973264694,
+      "epoch": 0.16596649551371817,
+      "step": 160
+    },
+    {
+      "loss": 0.48798179626464844,
+      "grad_norm": 0.013569124042987823,
+      "learning_rate": 8.248704663212436e-06,
+      "entropy": 0.7485733293113299,
+      "num_tokens": 11400551.0,
+      "logits/chosen": -1.469850987453924,
+      "logits/rejected": -1.2418553626350932,
+      "mean_token_accuracy": 0.7040991749428213,
+      "rewards/chosen": -1.1290287345201069,
+      "rewards/rejected": -2.089886896072421,
+      "rewards/accuracies": 0.775,
+      "rewards/margins": 0.9608581608976238,
+      "logps/chosen": -63.84155872344971,
+      "logps/rejected": -74.28893199920654,
+      "epoch": 0.17633940148332555,
+      "step": 170
+    },
+    {
+      "loss": 0.4588949203491211,
+      "grad_norm": 0.013312213122844696,
+      "learning_rate": 8.145077720207254e-06,
+      "entropy": 0.7363077022106154,
+      "num_tokens": 12070305.0,
+      "logits/chosen": -1.5014831557472343,
+      "logits/rejected": -1.253561978212398,
+      "mean_token_accuracy": 0.709209191892296,
+      "rewards/chosen": -1.082961096275685,
+      "rewards/rejected": -2.0920695228746626,
+      "rewards/accuracies": 0.80375,
+      "rewards/margins": 1.009108423003927,
+      "logps/chosen": -60.205562086105346,
+      "logps/rejected": -73.08745456457137,
+      "epoch": 0.18671230745293294,
+      "step": 180
+    },
+    {
+      "loss": 0.44720139503479006,
+      "grad_norm": 0.019025955349206924,
+      "learning_rate": 8.041450777202073e-06,
+      "entropy": 0.7329158629218,
+      "num_tokens": 12738966.0,
+      "logits/chosen": -1.6102334910947507,
+      "logits/rejected": -1.2938961518114829,
+      "mean_token_accuracy": 0.7022849645651877,
+      "rewards/chosen": -1.096955630929151,
+      "rewards/rejected": -2.191100225120317,
+      "rewards/accuracies": 0.80375,
+      "rewards/margins": 1.0941445926297455,
+      "logps/chosen": -62.84091185808182,
+      "logps/rejected": -73.56753115415573,
+      "epoch": 0.19708521342254032,
+      "step": 190
+    },
+    {
+      "loss": 0.5053049564361572,
+      "grad_norm": 0.022256096825003624,
+      "learning_rate": 7.937823834196891e-06,
+      "entropy": 0.7230936869091238,
+      "num_tokens": 13408002.0,
+      "logits/chosen": -1.6278242340657059,
+      "logits/rejected": -1.3487957538640436,
+      "mean_token_accuracy": 0.7035108367726207,
+      "rewards/chosen": -1.2608012669734308,
+      "rewards/rejected": -2.296834750812268,
+      "rewards/accuracies": 0.7525,
+      "rewards/margins": 1.036033486686647,
+      "logps/chosen": -61.396967930793764,
+      "logps/rejected": -75.2872954583168,
+      "epoch": 0.2074581193921477,
+      "step": 200
+    },
+    {
+      "loss": 0.4660433292388916,
+      "grad_norm": 0.014599275775253773,
+      "learning_rate": 7.834196891191712e-06,
+      "entropy": 0.6921609364774486,
+      "num_tokens": 14074115.0,
+      "logits/chosen": -1.7416979060664273,
+      "logits/rejected": -1.418815718305467,
+      "mean_token_accuracy": 0.7174021677859127,
+      "rewards/chosen": -1.077466004992384,
+      "rewards/rejected": -2.168546776890289,
+      "rewards/accuracies": 0.78125,
+      "rewards/margins": 1.0910807690175717,
+      "logps/chosen": -57.26282028913498,
+      "logps/rejected": -71.84579543352127,
+      "epoch": 0.2178310253617551,
+      "step": 210
+    },
+    {
+      "loss": 0.44950242042541505,
+      "grad_norm": 0.015949787572026253,
+      "learning_rate": 7.730569948186528e-06,
+      "entropy": 0.718045612655842,
+      "num_tokens": 14745903.0,
+      "logits/chosen": -1.6029292718575974,
+      "logits/rejected": -1.2788201063427131,
+      "mean_token_accuracy": 0.7201293612085283,
+      "rewards/chosen": -1.034126255120791,
+      "rewards/rejected": -2.162680851666955,
+      "rewards/accuracies": 0.8025,
+      "rewards/margins": 1.1285545978834852,
+      "logps/chosen": -58.2793731212616,
+      "logps/rejected": -75.22644976615906,
+      "epoch": 0.2282039313313625,
+      "step": 220
+    },
+    {
+      "loss": 0.43929591178894045,
+      "grad_norm": 0.013934735208749771,
+      "learning_rate": 7.626943005181348e-06,
+      "entropy": 0.7872297932501533,
+      "num_tokens": 15418570.0,
+      "logits/chosen": -1.4825574599936209,
+      "logits/rejected": -1.2401306349486072,
+      "mean_token_accuracy": 0.7127851650305093,
+      "rewards/chosen": -0.9212520667978242,
+      "rewards/rejected": -2.088999391239631,
+      "rewards/accuracies": 0.7975,
+      "rewards/margins": 1.1677473206445574,
+      "logps/chosen": -64.23240270137786,
+      "logps/rejected": -75.61436933040619,
+      "epoch": 0.23857683730096987,
+      "step": 230
+    },
+    {
+      "loss": 0.46149606704711915,
+      "grad_norm": 0.016272729262709618,
+      "learning_rate": 7.523316062176167e-06,
+      "entropy": 0.7730766562063945,
+      "num_tokens": 16089276.0,
+      "logits/chosen": -1.4998074947450468,
+      "logits/rejected": -1.2344424531786968,
+      "mean_token_accuracy": 0.7029178227111698,
+      "rewards/chosen": -0.9342721946419624,
+      "rewards/rejected": -2.0036902719875798,
+      "rewards/accuracies": 0.805,
+      "rewards/margins": 1.069418080030009,
+      "logps/chosen": -61.67156839847565,
+      "logps/rejected": -73.40640575170517,
+      "epoch": 0.24894974327057726,
+      "step": 240
+    },
+    {
+      "loss": 0.46842288970947266,
+      "grad_norm": 0.01317604910582304,
+      "learning_rate": 7.419689119170985e-06,
+      "entropy": 0.7556795614853036,
+      "num_tokens": 16760081.0,
+      "logits/chosen": -1.5251408018321724,
+      "logits/rejected": -1.2550001884745545,
+      "mean_token_accuracy": 0.7026587814092636,
+      "rewards/chosen": -0.9681878225761466,
+      "rewards/rejected": -1.9993022468869457,
+      "rewards/accuracies": 0.79625,
+      "rewards/margins": 1.031114422827959,
+      "logps/chosen": -62.68858078241348,
+      "logps/rejected": -72.8400109577179,
+      "epoch": 0.2593226492401846,
+      "step": 250
+    },
+    {
+      "eval_loss": 0.45063868165016174,
+      "eval_runtime": 228.335,
+      "eval_samples_per_second": 4.38,
+      "eval_steps_per_second": 4.38,
+      "eval_entropy": 0.7599611716154031,
+      "eval_num_tokens": 16760081.0,
+      "eval_logits/chosen": -1.5228878121420686,
+      "eval_logits/rejected": -1.2457822987193723,
+      "eval_mean_token_accuracy": 0.7089936109930277,
+      "eval_rewards/chosen": -0.9581855501902755,
+      "eval_rewards/rejected": -2.058153331893496,
+      "eval_rewards/accuracies": 0.804,
+      "eval_rewards/margins": 1.0999677814096211,
+      "eval_logps/chosen": -59.75484214401245,
+      "eval_logps/rejected": -73.22140468788147,
+      "epoch": 0.2593226492401846,
+      "step": 250
+    },
+    {
+      "loss": 0.4814570426940918,
+      "grad_norm": 0.015276545658707619,
+      "learning_rate": 7.3160621761658035e-06,
+      "entropy": 0.7907758677133825,
+      "num_tokens": 17432770.0,
+      "logits/chosen": -1.5061012207082294,
+      "logits/rejected": -1.2302418451410948,
+      "mean_token_accuracy": 0.7116011143475771,
+      "rewards/chosen": -0.9151591785412165,
+      "rewards/rejected": -2.000397002430982,
+      "rewards/accuracies": 0.78375,
+      "rewards/margins": 1.085237822062336,
+      "logps/chosen": -62.86761567592621,
+      "logps/rejected": -74.82630295515061,
+      "epoch": 0.26969555520979205,
+      "step": 260
+    },
+    {
+      "loss": 0.4540716648101807,
+      "grad_norm": 0.013052938506007195,
+      "learning_rate": 7.212435233160623e-06,
+      "entropy": 0.7655438208338455,
+      "num_tokens": 18100438.0,
+      "logits/chosen": -1.525467093384173,
+      "logits/rejected": -1.2448767667260814,
+      "mean_token_accuracy": 0.7059390800073743,
+      "rewards/chosen": -0.8308528269545059,
+      "rewards/rejected": -1.8842440211544453,
+      "rewards/accuracies": 0.79375,
+      "rewards/margins": 1.0533911933761555,
+      "logps/chosen": -56.91530933380127,
+      "logps/rejected": -70.32416380643845,
+      "epoch": 0.28006846117939943,
+      "step": 270
+    },
+    {
+      "loss": 0.44190540313720705,
+      "grad_norm": 0.014065246097743511,
+      "learning_rate": 7.108808290155441e-06,
+      "entropy": 0.7839024822821375,
+      "num_tokens": 18772061.0,
+      "logits/chosen": -1.5250317827718283,
+      "logits/rejected": -1.227351313367144,
+      "mean_token_accuracy": 0.7068624695949256,
+      "rewards/chosen": -0.8610654402987712,
+      "rewards/rejected": -2.0157007031119427,
+      "rewards/accuracies": 0.81875,
+      "rewards/margins": 1.1546352597884835,
+      "logps/chosen": -61.52414802789688,
+      "logps/rejected": -72.9297788310051,
+      "epoch": 0.2904413671490068,
+      "step": 280
+    },
+    {
+      "loss": 0.4456183910369873,
+      "grad_norm": 0.015716280788183212,
+      "learning_rate": 7.005181347150259e-06,
+      "entropy": 0.773276298265555,
+      "num_tokens": 19443506.0,
+      "logits/chosen": -1.5330978743938402,
+      "logits/rejected": -1.2171561791639127,
+      "mean_token_accuracy": 0.7161024253070355,
+      "rewards/chosen": -0.9044717278318422,
+      "rewards/rejected": -2.01870841772703,
+      "rewards/accuracies": 0.8025,
+      "rewards/margins": 1.1142366893729194,
+      "logps/chosen": -60.47999572515488,
+      "logps/rejected": -74.31323668241501,
+      "epoch": 0.3008142731186142,
+      "step": 290
+    },
+    {
+      "loss": 0.44274072647094725,
+      "grad_norm": 0.014205160550773144,
+      "learning_rate": 6.9015544041450784e-06,
+      "entropy": 0.7546117506347946,
+      "num_tokens": 20114567.0,
+      "logits/chosen": -1.6212462502874232,
+      "logits/rejected": -1.3379802284295708,
+      "mean_token_accuracy": 0.7209940696135163,
+      "rewards/chosen": -0.9589981533990612,
+      "rewards/rejected": -2.169916975693777,
+      "rewards/accuracies": 0.795,
+      "rewards/margins": 1.2109188246540725,
+      "logps/chosen": -60.32763281106949,
+      "logps/rejected": -75.60327394008637,
+      "epoch": 0.3111871790882216,
+      "step": 300
+    },
+    {
+      "loss": 0.4552000522613525,
+      "grad_norm": 0.014621925540268421,
+      "learning_rate": 6.797927461139897e-06,
+      "entropy": 0.7755605199559068,
+      "num_tokens": 20787766.0,
+      "logits/chosen": -1.612899054190608,
+      "logits/rejected": -1.3232876722515345,
+      "mean_token_accuracy": 0.7147772766277194,
+      "rewards/chosen": -0.9240679290609841,
+      "rewards/rejected": -2.1595666578254895,
+      "rewards/accuracies": 0.79,
+      "rewards/margins": 1.2354987265914679,
+      "logps/chosen": -61.32829741239548,
+      "logps/rejected": -77.28044722557068,
+      "epoch": 0.32156008505782896,
+      "step": 310
+    },
+    {
+      "loss": 0.4410384178161621,
+      "grad_norm": 0.01790008693933487,
+      "learning_rate": 6.6943005181347155e-06,
+      "entropy": 0.7489672241648077,
+      "num_tokens": 21457559.0,
+      "logits/chosen": -1.6415965131088623,
+      "logits/rejected": -1.347490645506002,
+      "mean_token_accuracy": 0.7051716044172645,
+      "rewards/chosen": -0.9709404561911651,
+      "rewards/rejected": -2.2215542565838406,
+      "rewards/accuracies": 0.7925,
+      "rewards/margins": 1.250613798997365,
+      "logps/chosen": -61.40900204181671,
+      "logps/rejected": -75.62910031557084,
+      "epoch": 0.33193299102743634,
+      "step": 320
+    },
+    {
+      "loss": 0.4336719512939453,
+      "grad_norm": 0.014728070236742496,
+      "learning_rate": 6.590673575129535e-06,
+      "entropy": 0.7923685038526309,
+      "num_tokens": 22128277.0,
+      "logits/chosen": -1.5480857679147186,
+      "logits/rejected": -1.2608543306081368,
+      "mean_token_accuracy": 0.703275697156787,
+      "rewards/chosen": -0.9459266294135409,
+      "rewards/rejected": -2.2482709953014273,
+      "rewards/accuracies": 0.80875,
+      "rewards/margins": 1.3023443661583587,
+      "logps/chosen": -60.81426377058029,
+      "logps/rejected": -77.32076614141464,
+      "epoch": 0.3423058969970437,
+      "step": 330
+    },
+    {
+      "loss": 0.4488088130950928,
+      "grad_norm": 0.01915821246802807,
+      "learning_rate": 6.487046632124353e-06,
+      "entropy": 0.7655984267906751,
+      "num_tokens": 22798429.0,
+      "logits/chosen": -1.5794928447393075,
+      "logits/rejected": -1.3028102903528602,
+      "mean_token_accuracy": 0.7063317270204424,
+      "rewards/chosen": -0.9481951130661764,
+      "rewards/rejected": -2.169792889961391,
+      "rewards/accuracies": 0.7975,
+      "rewards/margins": 1.2215977760369423,
+      "logps/chosen": -60.23724600553513,
+      "logps/rejected": -74.96227306127548,
+      "epoch": 0.3526788029666511,
+      "step": 340
+    },
+    {
+      "loss": 0.4490936756134033,
+      "grad_norm": 0.017944717779755592,
+      "learning_rate": 6.383419689119171e-06,
+      "entropy": 0.7542878538439982,
+      "num_tokens": 23470239.0,
+      "logits/chosen": -1.5727406857545416,
+      "logits/rejected": -1.276317261399869,
+      "mean_token_accuracy": 0.7159937589988112,
+      "rewards/chosen": -0.9953277947314564,
+      "rewards/rejected": -2.3134128075395712,
+      "rewards/accuracies": 0.79875,
+      "rewards/margins": 1.3180850110016764,
+      "logps/chosen": -61.82909578323364,
+      "logps/rejected": -78.39685034513474,
+      "epoch": 0.3630517089362585,
+      "step": 350
+    },
+    {
+      "loss": 0.4634250640869141,
+      "grad_norm": 0.017308367416262627,
+      "learning_rate": 6.2797927461139905e-06,
+      "entropy": 0.7325191626208835,
+      "num_tokens": 24140239.0,
+      "logits/chosen": -1.6208129462287795,
+      "logits/rejected": -1.3357805451866935,
+      "mean_token_accuracy": 0.7167608435824513,
+      "rewards/chosen": -0.9554049956912786,
+      "rewards/rejected": -2.160907984265359,
+      "rewards/accuracies": 0.7925,
+      "rewards/margins": 1.205502986907959,
+      "logps/chosen": -60.00006797075272,
+      "logps/rejected": -73.66487602233887,
+      "epoch": 0.37342461490586587,
+      "step": 360
+    },
+    {
+      "loss": 0.4462597846984863,
+      "grad_norm": 0.012704321183264256,
+      "learning_rate": 6.176165803108809e-06,
+      "entropy": 0.7730704579531448,
+      "num_tokens": 24812178.0,
+      "logits/chosen": -1.5574609696072523,
+      "logits/rejected": -1.260951689955208,
+      "mean_token_accuracy": 0.7079000303149223,
+      "rewards/chosen": -0.9217178278604843,
+      "rewards/rejected": -2.219055590032367,
+      "rewards/accuracies": 0.79875,
+      "rewards/margins": 1.2973377590533346,
+      "logps/chosen": -63.260009841918944,
+      "logps/rejected": -75.37872500896454,
+      "epoch": 0.38379752087547325,
+      "step": 370
+    },
+    {
+      "eval_loss": 0.4316043555736542,
+      "eval_runtime": 247.4497,
+      "eval_samples_per_second": 4.041,
+      "eval_steps_per_second": 4.041,
+      "eval_entropy": 0.7676939936636481,
+      "eval_num_tokens": 25147105.0,
+      "eval_logits/chosen": -1.5026115886795766,
+      "eval_logits/rejected": -1.2167814175886924,
+      "eval_mean_token_accuracy": 0.7118223344534635,
+      "eval_rewards/chosen": -0.7611177709406475,
+      "eval_rewards/rejected": -1.9837484061150754,
+      "eval_rewards/accuracies": 0.812,
+      "eval_rewards/margins": 1.2226306333914398,
+      "eval_logps/chosen": -57.784164419174196,
+      "eval_logps/rejected": -72.47735544586182,
+      "epoch": 0.38898397386027694,
+      "step": 375
+    },
+    {
+      "loss": 0.42072787284851076,
+      "grad_norm": 0.01512940414249897,
+      "learning_rate": 6.0725388601036275e-06,
+      "entropy": 0.7662378180027009,
+      "num_tokens": 25482532.0,
+      "logits/chosen": -1.4455535287130075,
+      "logits/rejected": -1.1827597378991437,
+      "mean_token_accuracy": 0.7134129768237472,
+      "rewards/chosen": -0.7430938966096438,
+      "rewards/rejected": -1.9754590333898159,
+      "rewards/accuracies": 0.81625,
+      "rewards/margins": 1.2323651346145197,
+      "logps/chosen": -59.126625483036044,
+      "logps/rejected": -71.10404143095016,
+      "epoch": 0.39417042684508063,
+      "step": 380
+    },
+    {
+      "loss": 0.4370901107788086,
+      "grad_norm": 0.013698432594537735,
+      "learning_rate": 5.968911917098445e-06,
+      "entropy": 0.7876232456968865,
+      "num_tokens": 26154266.0,
+      "logits/chosen": -1.4235855556360628,
+      "logits/rejected": -1.1775204862100577,
+      "mean_token_accuracy": 0.7055446618422866,
+      "rewards/chosen": -0.7474835715969129,
+      "rewards/rejected": -1.9929245413161698,
+      "rewards/accuracies": 0.81125,
+      "rewards/margins": 1.2454409682005645,
+      "logps/chosen": -60.49559247493744,
+      "logps/rejected": -74.38287235736847,
+      "epoch": 0.404543332814688,
+      "step": 390
+    },
+    {
+      "loss": 0.43750705718994143,
+      "grad_norm": 0.014688065275549889,
+      "learning_rate": 5.865284974093265e-06,
+      "entropy": 0.8005011603605817,
+      "num_tokens": 26825159.0,
+      "logits/chosen": -1.4366348824569963,
+      "logits/rejected": -1.1432329035500064,
+      "mean_token_accuracy": 0.7120773574337363,
+      "rewards/chosen": -0.7654138012278418,
+      "rewards/rejected": -1.9986112341369153,
+      "rewards/accuracies": 0.80625,
+      "rewards/margins": 1.2331974321743473,
+      "logps/chosen": -59.77998896837234,
+      "logps/rejected": -74.2601730799675,
+      "epoch": 0.4149162387842954,
+      "step": 400
+    },
+    {
+      "loss": 0.4483372211456299,
+      "grad_norm": 0.01656300760805607,
+      "learning_rate": 5.761658031088083e-06,
+      "entropy": 0.7781640422128839,
+      "num_tokens": 27495663.0,
+      "logits/chosen": -1.469463162115489,
+      "logits/rejected": -1.1635286970905576,
+      "mean_token_accuracy": 0.7104245729371905,
+      "rewards/chosen": -0.7733871162302784,
+      "rewards/rejected": -2.009647502012085,
+      "rewards/accuracies": 0.7775,
+      "rewards/margins": 1.236260382984765,
+      "logps/chosen": -59.314085538387296,
+      "logps/rejected": -73.74020329475402,
+      "epoch": 0.4252891447539028,
+      "step": 410
+    },
+    {
+      "loss": 0.40839343070983886,
+      "grad_norm": 0.014693827368319035,
+      "learning_rate": 5.658031088082902e-06,
+      "entropy": 0.7646654536006827,
+      "num_tokens": 28167389.0,
+      "logits/chosen": -1.5559907884752369,
+      "logits/rejected": -1.2529527889366994,
+      "mean_token_accuracy": 0.7122585397399962,
+      "rewards/chosen": -0.812723506633156,
+      "rewards/rejected": -2.2211731961916668,
+      "rewards/accuracies": 0.8325,
+      "rewards/margins": 1.4084496867936105,
+      "logps/chosen": -61.519875913858414,
+      "logps/rejected": -76.1291908788681,
+      "epoch": 0.4356620507235102,
+      "step": 420
+    },
+    {
+      "loss": 0.4050635814666748,
+      "grad_norm": 0.014569880440831184,
+      "learning_rate": 5.554404145077721e-06,
+      "entropy": 0.7813236166906427,
+      "num_tokens": 28839388.0,
+      "logits/chosen": -1.4355108492016981,
+      "logits/rejected": -1.1415682707849564,
+      "mean_token_accuracy": 0.7073199293017387,
+      "rewards/chosen": -0.8523474504226578,
+      "rewards/rejected": -2.3254094777817955,
+      "rewards/accuracies": 0.82,
+      "rewards/margins": 1.4730620284751057,
+      "logps/chosen": -60.116721353530885,
+      "logps/rejected": -76.6921053647995,
+      "epoch": 0.4460349566931176,
+      "step": 430
+    },
+    {
+      "loss": 0.40834870338439944,
+      "grad_norm": 0.01712818816304207,
+      "learning_rate": 5.4507772020725395e-06,
+      "entropy": 0.7444785924613825,
+      "num_tokens": 29510096.0,
+      "logits/chosen": -1.6494087471154035,
+      "logits/rejected": -1.2686231425397783,
+      "mean_token_accuracy": 0.7219790226966143,
+      "rewards/chosen": -0.9436784289986826,
+      "rewards/rejected": -2.380959646940464,
+      "rewards/accuracies": 0.82875,
+      "rewards/margins": 1.4372812157822772,
+      "logps/chosen": -59.11840226888657,
+      "logps/rejected": -76.61158362627029,
+      "epoch": 0.456407862662725,
+      "step": 440
+    },
+    {
+      "loss": 0.44087815284729004,
+      "grad_norm": 0.022262893617153168,
+      "learning_rate": 5.347150259067357e-06,
+      "entropy": 0.7486801335980999,
+      "num_tokens": 30181792.0,
+      "logits/chosen": -1.6525915875287975,
+      "logits/rejected": -1.292702118780194,
+      "mean_token_accuracy": 0.7041960602998734,
+      "rewards/chosen": -1.0031295236017104,
+      "rewards/rejected": -2.4976175978421815,
+      "rewards/accuracies": 0.805,
+      "rewards/margins": 1.4944880706071855,
+      "logps/chosen": -61.45527629375458,
+      "logps/rejected": -78.80332666873932,
+      "epoch": 0.46678076863233237,
+      "step": 450
+    },
+    {
+      "loss": 0.41168813705444335,
+      "grad_norm": 0.01639724150300026,
+      "learning_rate": 5.243523316062177e-06,
+      "entropy": 0.7348720514599699,
+      "num_tokens": 30851409.0,
+      "logits/chosen": -1.6804476245129396,
+      "logits/rejected": -1.3568483163933152,
+      "mean_token_accuracy": 0.7089731366932392,
+      "rewards/chosen": -0.9868619644123828,
+      "rewards/rejected": -2.5387522551673465,
+      "rewards/accuracies": 0.8125,
+      "rewards/margins": 1.5518902849871665,
+      "logps/chosen": -61.486987377405164,
+      "logps/rejected": -78.15051867246628,
+      "epoch": 0.47715367460193975,
+      "step": 460
+    },
+    {
+      "loss": 0.44434194564819335,
+      "grad_norm": 0.021561825647950172,
+      "learning_rate": 5.139896373056995e-06,
+      "entropy": 0.7410113723303948,
+      "num_tokens": 31519564.0,
+      "logits/chosen": -1.640166155420613,
+      "logits/rejected": -1.3236098037357882,
+      "mean_token_accuracy": 0.7085670954920351,
+      "rewards/chosen": -0.9577470733088558,
+      "rewards/rejected": -2.3723132372720284,
+      "rewards/accuracies": 0.80125,
+      "rewards/margins": 1.4145661635301077,
+      "logps/chosen": -59.36082991600037,
+      "logps/rejected": -74.87610804796219,
+      "epoch": 0.48752658057154713,
+      "step": 470
+    },
+    {
+      "loss": 0.4187155723571777,
+      "grad_norm": 0.015293825417757034,
+      "learning_rate": 5.036269430051814e-06,
+      "entropy": 0.819734820156591,
+      "num_tokens": 32192056.0,
+      "logits/chosen": -1.4117972345061844,
+      "logits/rejected": -1.1251919844105895,
+      "mean_token_accuracy": 0.699143321532756,
+      "rewards/chosen": -0.827813671599506,
+      "rewards/rejected": -2.1753858611593024,
+      "rewards/accuracies": 0.82125,
+      "rewards/margins": 1.347572191953659,
+      "logps/chosen": -62.310746512413026,
+      "logps/rejected": -76.46800751209258,
+      "epoch": 0.4978994865411545,
+      "step": 480
+    },
+    {
+      "loss": 0.41543288230895997,
+      "grad_norm": 0.016549421474337578,
+      "learning_rate": 4.932642487046633e-06,
+      "entropy": 0.793799487358192,
+      "num_tokens": 32864077.0,
+      "logits/chosen": -1.45461478674727,
+      "logits/rejected": -1.1643940857008355,
+      "mean_token_accuracy": 0.7187336018308997,
+      "rewards/chosen": -0.8021746008354603,
+      "rewards/rejected": -2.22868984402754,
+      "rewards/accuracies": 0.82,
+      "rewards/margins": 1.4265152450464667,
+      "logps/chosen": -58.781578176021576,
+      "logps/rejected": -78.34760097026825,
+      "epoch": 0.5082723925107618,
+      "step": 490
+    },
+    {
+      "loss": 0.4357460498809814,
+      "grad_norm": 0.015738829970359802,
+      "learning_rate": 4.829015544041451e-06,
+      "entropy": 0.7886733054189244,
+      "num_tokens": 33534279.0,
+      "logits/chosen": -1.482443230472497,
+      "logits/rejected": -1.1571150498996416,
+      "mean_token_accuracy": 0.7196742885001004,
+      "rewards/chosen": -0.7262393125944072,
+      "rewards/rejected": -2.081412044418103,
+      "rewards/accuracies": 0.80375,
+      "rewards/margins": 1.3551727333851158,
+      "logps/chosen": -56.740478208065035,
+      "logps/rejected": -74.49606896877289,
+      "epoch": 0.5186452984803692,
+      "step": 500
+    },
+    {
+      "eval_loss": 0.42026275396347046,
+      "eval_runtime": 281.7739,
+      "eval_samples_per_second": 3.549,
+      "eval_steps_per_second": 3.549,
+      "eval_entropy": 0.7882621039540972,
+      "eval_num_tokens": 33534279.0,
+      "eval_logits/chosen": -1.4541776191798497,
+      "eval_logits/rejected": -1.1470192806607813,
+      "eval_mean_token_accuracy": 0.7118717866092921,
+      "eval_rewards/chosen": -0.7887642608310853,
+      "eval_rewards/rejected": -2.164067000001669,
+      "eval_rewards/accuracies": 0.818,
+      "eval_rewards/margins": 1.3753027384020389,
+      "eval_logps/chosen": -58.06062930107117,
+      "eval_logps/rejected": -74.28054134368897,
+      "epoch": 0.5186452984803692,
+      "step": 500
+    },
+    {
+      "loss": 0.4276765823364258,
+      "grad_norm": 0.014744630083441734,
+      "learning_rate": 4.72538860103627e-06,
+      "entropy": 0.7855714731411717,
+      "num_tokens": 34205135.0,
+      "logits/chosen": -1.4180048717028029,
+      "logits/rejected": -1.1214338358385025,
+      "mean_token_accuracy": 0.7126064065098763,
+      "rewards/chosen": -0.82133626240171,
+      "rewards/rejected": -2.191293025066843,
+      "rewards/accuracies": 0.8175,
+      "rewards/margins": 1.3699567627965008,
+      "logps/chosen": -59.999370622634885,
+      "logps/rejected": -74.11055833339691,
+      "epoch": 0.5290182044499767,
+      "step": 510
+    },
+    {
+      "loss": 0.4480290412902832,
+      "grad_norm": 0.016974102705717087,
+      "learning_rate": 4.621761658031089e-06,
+      "entropy": 0.7657113060192206,
+      "num_tokens": 34873976.0,
+      "logits/chosen": -1.4652295046030377,
+      "logits/rejected": -1.1381464982050828,
+      "mean_token_accuracy": 0.7193653728254139,
+      "rewards/chosen": -0.8232743983055844,
+      "rewards/rejected": -2.115962935181451,
+      "rewards/accuracies": 0.79,
+      "rewards/margins": 1.2926885320199653,
+      "logps/chosen": -57.11219574928284,
+      "logps/rejected": -72.7778330230713,
+      "epoch": 0.5393911104195841,
+      "step": 520
+    },
+    {
+      "loss": 0.4185093879699707,
+      "grad_norm": 0.014205333776772022,
+      "learning_rate": 4.518134715025907e-06,
+      "entropy": 0.8013994769033161,
+      "num_tokens": 35544999.0,
+      "logits/chosen": -1.471904915902926,
+      "logits/rejected": -1.1525475558082925,
+      "mean_token_accuracy": 0.7086275420710444,
+      "rewards/chosen": -0.821920394521876,
+      "rewards/rejected": -2.3078401576905163,
+      "rewards/accuracies": 0.8275,
+      "rewards/margins": 1.4859197605680674,
+      "logps/chosen": -60.18735828638077,
+      "logps/rejected": -76.90893436431885,
+      "epoch": 0.5497640163891915,
+      "step": 530
+    },
+    {
+      "loss": 0.43764190673828124,
+      "grad_norm": 0.01859429106116295,
+      "learning_rate": 4.414507772020726e-06,
+      "entropy": 0.8030325107637327,
+      "num_tokens": 36215629.0,
+      "logits/chosen": -1.3828510403033027,
+      "logits/rejected": -1.1259861866767447,
+      "mean_token_accuracy": 0.7117785899341107,
+      "rewards/chosen": -0.8377752207776211,
+      "rewards/rejected": -2.164465318453731,
+      "rewards/accuracies": 0.8325,
+      "rewards/margins": 1.3266900933114811,
+      "logps/chosen": -59.40348263263702,
+      "logps/rejected": -73.95402544498444,
+      "epoch": 0.5601369223587989,
+      "step": 540
+    },
+    {
+      "loss": 0.43619637489318847,
+      "grad_norm": 0.022518545389175415,
+      "learning_rate": 4.310880829015544e-06,
+      "entropy": 0.8470213223638712,
+      "num_tokens": 36888235.0,
+      "logits/chosen": -1.3614448333016513,
+      "logits/rejected": -1.0777429963848568,
+      "mean_token_accuracy": 0.7116067171096802,
+      "rewards/chosen": -0.8179999309696723,
+      "rewards/rejected": -2.2714405926934886,
+      "rewards/accuracies": 0.8025,
+      "rewards/margins": 1.453440659949556,
+      "logps/chosen": -61.603306529521944,
+      "logps/rejected": -78.86535804271698,
+      "epoch": 0.5705098283284062,
+      "step": 550
+    },
+    {
+      "loss": 0.4372966766357422,
+      "grad_norm": 0.017818979918956757,
+      "learning_rate": 4.207253886010363e-06,
+      "entropy": 0.8097228137392085,
+      "num_tokens": 37560215.0,
+      "logits/chosen": -1.4065342416818991,
+      "logits/rejected": -1.1310520368013937,
+      "mean_token_accuracy": 0.7106514018215239,
+      "rewards/chosen": -0.7871383707236419,
+      "rewards/rejected": -2.1830000240239316,
+      "rewards/accuracies": 0.79375,
+      "rewards/margins": 1.3958616532059387,
+      "logps/chosen": -59.665771260261536,
+      "logps/rejected": -74.6952103304863,
+      "epoch": 0.5808827342980136,
+      "step": 560
+    },
+    {
+      "loss": 0.4129640102386475,
+      "grad_norm": 0.01815822720527649,
+      "learning_rate": 4.103626943005182e-06,
+      "entropy": 0.7890104844880989,
+      "num_tokens": 38229872.0,
+      "logits/chosen": -1.4210442868235984,
+      "logits/rejected": -1.1203699372972569,
+      "mean_token_accuracy": 0.7099655389133841,
+      "rewards/chosen": -0.8200407591336989,
+      "rewards/rejected": -2.2342929665278644,
+      "rewards/accuracies": 0.83,
+      "rewards/margins": 1.4142522051371633,
+      "logps/chosen": -59.6181519818306,
+      "logps/rejected": -74.29007930994034,
+      "epoch": 0.591255640267621,
+      "step": 570
+    },
+    {
+      "loss": 0.4248640537261963,
+      "grad_norm": 0.021521631628274918,
+      "learning_rate": 4.000000000000001e-06,
+      "entropy": 0.7830230033496628,
+      "num_tokens": 38898599.0,
+      "logits/chosen": -1.556829666838658,
+      "logits/rejected": -1.22248957027324,
+      "mean_token_accuracy": 0.7121115596592427,
+      "rewards/chosen": -0.8376483802072471,
+      "rewards/rejected": -2.30365946133883,
+      "rewards/accuracies": 0.82,
+      "rewards/margins": 1.466011079289019,
+      "logps/chosen": -57.19522080659866,
+      "logps/rejected": -74.76314610958099,
+      "epoch": 0.6016285462372284,
+      "step": 580
+    },
+    {
+      "loss": 0.46982684135437014,
+      "grad_norm": 0.0202327873557806,
+      "learning_rate": 3.896373056994819e-06,
+      "entropy": 0.789131090187293,
+      "num_tokens": 39570011.0,
+      "logits/chosen": -1.4979756501997874,
+      "logits/rejected": -1.2087421360795596,
+      "mean_token_accuracy": 0.7092419915273785,
+      "rewards/chosen": -0.9460587426692837,
+      "rewards/rejected": -2.2724328075315863,
+      "rewards/accuracies": 0.77125,
+      "rewards/margins": 1.3263740684324876,
+      "logps/chosen": -61.68762954473495,
+      "logps/rejected": -76.94917388081551,
+      "epoch": 0.6120014522068358,
+      "step": 590
+    },
+    {
+      "loss": 0.4296851634979248,
+      "grad_norm": 0.023099711164832115,
+      "learning_rate": 3.7927461139896377e-06,
+      "entropy": 0.7784710348435329,
+      "num_tokens": 40238459.0,
+      "logits/chosen": -1.5463305650925598,
+      "logits/rejected": -1.1855332843764201,
+      "mean_token_accuracy": 0.7213506529480219,
+      "rewards/chosen": -0.7897813039762969,
+      "rewards/rejected": -2.1785082579992014,
+      "rewards/accuracies": 0.80625,
+      "rewards/margins": 1.388726954490412,
+      "logps/chosen": -55.0355699801445,
+      "logps/rejected": -74.42590669631959,
+      "epoch": 0.6223743581764432,
+      "step": 600
+    },
+    {
+      "loss": 0.4378302574157715,
+      "grad_norm": 0.017044473439455032,
+      "learning_rate": 3.6891191709844567e-06,
+      "entropy": 0.8122139454813442,
+      "num_tokens": 40907823.0,
+      "logits/chosen": -1.4782042004502953,
+      "logits/rejected": -1.1733026566264682,
+      "mean_token_accuracy": 0.7134785779751838,
+      "rewards/chosen": -0.7893356951180612,
+      "rewards/rejected": -2.1324991090265395,
+      "rewards/accuracies": 0.81125,
+      "rewards/margins": 1.3431634148815648,
+      "logps/chosen": -56.50307439565658,
+      "logps/rejected": -74.36804901599884,
+      "epoch": 0.6327472641460505,
+      "step": 610
+    },
+    {
+      "loss": 0.42770676612854003,
+      "grad_norm": 0.017237193882465363,
+      "learning_rate": 3.5854922279792748e-06,
+      "entropy": 0.7561554417069419,
+      "num_tokens": 41576629.0,
+      "logits/chosen": -1.5855324711359189,
+      "logits/rejected": -1.2464155038601519,
+      "mean_token_accuracy": 0.7195479864627123,
+      "rewards/chosen": -0.7948263869498987,
+      "rewards/rejected": -2.2129694822058084,
+      "rewards/accuracies": 0.81375,
+      "rewards/margins": 1.4181430943030864,
+      "logps/chosen": -55.47762014627457,
+      "logps/rejected": -74.27213495016098,
+      "epoch": 0.6431201701156579,
+      "step": 620
+    },
+    {
+      "eval_loss": 0.4145970344543457,
+      "eval_runtime": 312.4458,
+      "eval_samples_per_second": 3.201,
+      "eval_steps_per_second": 3.201,
+      "eval_entropy": 0.7926596267716959,
+      "eval_num_tokens": 41911681.0,
+      "eval_logits/chosen": -1.5117800904965717,
+      "eval_logits/rejected": -1.196360660912854,
+      "eval_mean_token_accuracy": 0.7107393295019865,
+      "eval_rewards/chosen": -0.7929535148476134,
+      "eval_rewards/rejected": -2.2358115541365695,
+      "eval_rewards/accuracies": 0.82,
+      "eval_rewards/margins": 1.442858038464561,
+      "eval_logps/chosen": -58.102521854400635,
+      "eval_logps/rejected": -74.9979869003296,
+      "epoch": 0.6483066231004616,
+      "step": 625
+    },
+    {
+      "loss": 0.4174999713897705,
+      "grad_norm": 0.023992126807570457,
+      "learning_rate": 3.4818652849740937e-06,
+      "entropy": 0.7907538907864364,
+      "num_tokens": 42247859.0,
+      "logits/chosen": -1.5275107018411662,
+      "logits/rejected": -1.1729716012203115,
+      "mean_token_accuracy": 0.7168286069855094,
+      "rewards/chosen": -0.785840036013833,
+      "rewards/rejected": -2.3167090034011926,
+      "rewards/accuracies": 0.82,
+      "rewards/margins": 1.5308689690474422,
+      "logps/chosen": -56.834257082939146,
+      "logps/rejected": -78.19684539794922,
+      "epoch": 0.6534930760852653,
+      "step": 630
+    },
+    {
+      "loss": 0.4136983394622803,
+      "grad_norm": 0.017943687736988068,
+      "learning_rate": 3.3782383419689123e-06,
+      "entropy": 0.782110237882007,
+      "num_tokens": 42918160.0,
+      "logits/chosen": -1.5180649119494551,
+      "logits/rejected": -1.1926738964700518,
+      "mean_token_accuracy": 0.7150443252548575,
+      "rewards/chosen": -0.8211558357491231,
+      "rewards/rejected": -2.2642588213164707,
+      "rewards/accuracies": 0.82875,
+      "rewards/margins": 1.4431029830686748,
+      "logps/chosen": -56.69160728216171,
+      "logps/rejected": -77.15469831943511,
+      "epoch": 0.6638659820548727,
+      "step": 640
+    },
+    {
+      "loss": 0.46654863357543946,
+      "grad_norm": 0.016258180141448975,
+      "learning_rate": 3.274611398963731e-06,
+      "entropy": 0.781107221354032,
+      "num_tokens": 43587843.0,
+      "logits/chosen": -1.5242385121688045,
+      "logits/rejected": -1.2121969127570629,
+      "mean_token_accuracy": 0.7117258696630597,
+      "rewards/chosen": -0.8765105128889991,
+      "rewards/rejected": -2.300915027359915,
+      "rewards/accuracies": 0.79625,
+      "rewards/margins": 1.4244045152794569,
+      "logps/chosen": -58.10146237373352,
+      "logps/rejected": -75.90745508432389,
+      "epoch": 0.6742388880244801,
+      "step": 650
+    },
+    {
+      "loss": 0.46282315254211426,
+      "grad_norm": 0.014686384238302708,
+      "learning_rate": 3.1709844559585493e-06,
+      "entropy": 0.7962630777983577,
+      "num_tokens": 44256661.0,
+      "logits/chosen": -1.4734712758844637,
+      "logits/rejected": -1.1916934080775792,
+      "mean_token_accuracy": 0.7055912931449712,
+      "rewards/chosen": -0.7513918488781929,
+      "rewards/rejected": -2.0676744230859914,
+      "rewards/accuracies": 0.8025,
+      "rewards/margins": 1.316282574729994,
+      "logps/chosen": -57.497248324155805,
+      "logps/rejected": -72.39635850191117,
+      "epoch": 0.6846117939940874,
+      "step": 660
+    },
+    {
+      "loss": 0.38728673458099366,
+      "grad_norm": 0.012905064970254898,
+      "learning_rate": 3.0673575129533683e-06,
+      "entropy": 0.7850103608728386,
+      "num_tokens": 44928208.0,
+      "logits/chosen": -1.4906549367707287,
+      "logits/rejected": -1.1785130829765196,
+      "mean_token_accuracy": 0.7167331029660999,
+      "rewards/chosen": -0.729108623168795,
+      "rewards/rejected": -2.2293507476183003,
+      "rewards/accuracies": 0.835,
+      "rewards/margins": 1.500242124721408,
+      "logps/chosen": -57.06263860464096,
+      "logps/rejected": -75.03939726114272,
+      "epoch": 0.6949846999636948,
+      "step": 670
+    },
+    {
+      "loss": 0.42326745986938474,
+      "grad_norm": 0.020321018993854523,
+      "learning_rate": 2.963730569948187e-06,
+      "entropy": 0.8228285726247122,
+      "num_tokens": 45599959.0,
+      "logits/chosen": -1.4107692209920355,
+      "logits/rejected": -1.1415338745002617,
+      "mean_token_accuracy": 0.7098798759281635,
+      "rewards/chosen": -0.7472892588960599,
+      "rewards/rejected": -2.205724095766054,
+      "rewards/accuracies": 0.82,
+      "rewards/margins": 1.4584348343219609,
+      "logps/chosen": -60.79011604785919,
+      "logps/rejected": -76.145639398098,
+      "epoch": 0.7053576059333022,
+      "step": 680
+    },
+    {
+      "loss": 0.46504626274108884,
+      "grad_norm": 0.026528460904955864,
+      "learning_rate": 2.8601036269430053e-06,
+      "entropy": 0.8374029883276671,
+      "num_tokens": 46271100.0,
+      "logits/chosen": -1.438753673313662,
+      "logits/rejected": -1.1758532082539777,
+      "mean_token_accuracy": 0.7012219382543117,
+      "rewards/chosen": -0.8035630289492474,
+      "rewards/rejected": -2.1134093309140733,
+      "rewards/accuracies": 0.78875,
+      "rewards/margins": 1.3098462954116985,
+      "logps/chosen": -60.888806674480435,
+      "logps/rejected": -75.00253082275391,
+      "epoch": 0.7157305119029096,
+      "step": 690
+    },
+    {
+      "loss": 0.3986149072647095,
+      "grad_norm": 0.014411377720534801,
+      "learning_rate": 2.7564766839378243e-06,
+      "entropy": 0.8010915271285921,
+      "num_tokens": 46940588.0,
+      "logits/chosen": -1.4958920674077623,
+      "logits/rejected": -1.1794351626729602,
+      "mean_token_accuracy": 0.7159527142904699,
+      "rewards/chosen": -0.7098511749218233,
+      "rewards/rejected": -2.1309537043143063,
+      "rewards/accuracies": 0.83,
+      "rewards/margins": 1.4211025240272284,
+      "logps/chosen": -58.82859611272812,
+      "logps/rejected": -71.86115275144577,
+      "epoch": 0.726103417872517,
+      "step": 700
+    },
+    {
+      "loss": 0.43906688690185547,
+      "grad_norm": 0.01570284552872181,
+      "learning_rate": 2.6528497409326424e-06,
+      "entropy": 0.7988389390867087,
+      "num_tokens": 47612649.0,
+      "logits/chosen": -1.5084893760106768,
+      "logits/rejected": -1.2217334834194045,
+      "mean_token_accuracy": 0.7133530013635755,
+      "rewards/chosen": -0.7729418969854669,
+      "rewards/rejected": -2.181924787040334,
+      "rewards/accuracies": 0.795,
+      "rewards/margins": 1.4089828893507366,
+      "logps/chosen": -61.10609317064285,
+      "logps/rejected": -74.83846760034561,
+      "epoch": 0.7364763238421244,
+      "step": 710
+    },
+    {
+      "loss": 0.40947885513305665,
+      "grad_norm": 0.023827839642763138,
+      "learning_rate": 2.5492227979274614e-06,
+      "entropy": 0.7832058588406653,
+      "num_tokens": 48280316.0,
+      "logits/chosen": -1.5411506649745486,
+      "logits/rejected": -1.2429037100661278,
+      "mean_token_accuracy": 0.7112616512458771,
+      "rewards/chosen": -0.6660112939180545,
+      "rewards/rejected": -2.1130318040723797,
+      "rewards/accuracies": 0.82375,
+      "rewards/margins": 1.4470205087191426,
+      "logps/chosen": -56.754196836948395,
+      "logps/rejected": -72.01071586370468,
+      "epoch": 0.7468492298117317,
+      "step": 720
+    },
+    {
+      "loss": 0.4252438545227051,
+      "grad_norm": 0.016843726858496666,
+      "learning_rate": 2.44559585492228e-06,
+      "entropy": 0.8381482209177921,
+      "num_tokens": 48952457.0,
+      "logits/chosen": -1.4216991849843164,
+      "logits/rejected": -1.1461203116135463,
+      "mean_token_accuracy": 0.7074375116080046,
+      "rewards/chosen": -0.6695640615803495,
+      "rewards/rejected": -2.0397114378174592,
+      "rewards/accuracies": 0.81125,
+      "rewards/margins": 1.3701473774015904,
+      "logps/chosen": -58.40614979505539,
+      "logps/rejected": -75.83607840061188,
+      "epoch": 0.7572221357813391,
+      "step": 730
+    },
+    {
+      "loss": 0.42221574783325194,
+      "grad_norm": 0.016754452139139175,
+      "learning_rate": 2.3419689119170984e-06,
+      "entropy": 0.8271626771151205,
+      "num_tokens": 49623039.0,
+      "logits/chosen": -1.5209721524713908,
+      "logits/rejected": -1.1751758197782376,
+      "mean_token_accuracy": 0.713498560115695,
+      "rewards/chosen": -0.693279623198905,
+      "rewards/rejected": -2.141712246098032,
+      "rewards/accuracies": 0.8225,
+      "rewards/margins": 1.448432622421533,
+      "logps/chosen": -58.120414593219756,
+      "logps/rejected": -76.37068503379822,
+      "epoch": 0.7675950417509465,
+      "step": 740
+    },
+    {
+      "loss": 0.42609853744506837,
+      "grad_norm": 0.015460499562323093,
+      "learning_rate": 2.2383419689119174e-06,
+      "entropy": 0.8402992183688912,
+      "num_tokens": 50296449.0,
+      "logits/chosen": -1.4753382490797897,
+      "logits/rejected": -1.1819095770610415,
+      "mean_token_accuracy": 0.7086446931585669,
+      "rewards/chosen": -0.756505029114196,
+      "rewards/rejected": -2.2547341962716017,
+      "rewards/accuracies": 0.815,
+      "rewards/margins": 1.4982291649701074,
+      "logps/chosen": -61.22059069037437,
+      "logps/rejected": -78.80186176776886,
+      "epoch": 0.7779679477205539,
+      "step": 750
+    },
+    {
+      "eval_loss": 0.411579966545105,
+      "eval_runtime": 345.2282,
+      "eval_samples_per_second": 2.897,
+      "eval_steps_per_second": 2.897,
+      "eval_entropy": 0.7931306820721366,
+      "eval_num_tokens": 50296449.0,
+      "eval_logits/chosen": -1.5620700320679246,
+      "eval_logits/rejected": -1.2442960317998197,
+      "eval_mean_token_accuracy": 0.7116911947578192,
+      "eval_rewards/chosen": -0.7726080814675516,
+      "eval_rewards/rejected": -2.2532580847712236,
+      "eval_rewards/accuracies": 0.825,
+      "eval_rewards/margins": 1.4806500013172625,
+      "eval_logps/chosen": -57.89906752586365,
+      "eval_logps/rejected": -75.1724522151947,
+      "epoch": 0.7779679477205539,
+      "step": 750
+    },
+    {
+      "loss": 0.3913722515106201,
+      "grad_norm": 0.014632239006459713,
+      "learning_rate": 2.134715025906736e-06,
+      "entropy": 0.8136140358856937,
+      "num_tokens": 50967491.0,
+      "logits/chosen": -1.562353067996462,
+      "logits/rejected": -1.2513613833271493,
+      "mean_token_accuracy": 0.7093207446113229,
+      "rewards/chosen": -0.7712609519985927,
+      "rewards/rejected": -2.253486830406473,
+      "rewards/accuracies": 0.84375,
+      "rewards/margins": 1.4822258768649772,
+      "logps/chosen": -59.60826306819916,
+      "logps/rejected": -76.35254566431045,
+      "epoch": 0.7883408536901613,
+      "step": 760
+    },
+    {
+      "loss": 0.40571393966674807,
+      "grad_norm": 0.025566810742020607,
+      "learning_rate": 2.0310880829015544e-06,
+      "entropy": 0.8285544535657391,
+      "num_tokens": 51641328.0,
+      "logits/chosen": -1.522793583175992,
+      "logits/rejected": -1.1822050205320058,
+      "mean_token_accuracy": 0.7068053354136645,
+      "rewards/chosen": -0.7923616364665214,
+      "rewards/rejected": -2.37010533751054,
+      "rewards/accuracies": 0.81375,
+      "rewards/margins": 1.5777437023073435,
+      "logps/chosen": -61.9151118016243,
+      "logps/rejected": -79.94920246601104,
+      "epoch": 0.7987137596597687,
+      "step": 770
+    },
+    {
+      "loss": 0.4391521453857422,
+      "grad_norm": 0.016789298504590988,
+      "learning_rate": 1.9274611398963734e-06,
+      "entropy": 0.7836550371508929,
+      "num_tokens": 52311665.0,
+      "logits/chosen": -1.6191126859645035,
+      "logits/rejected": -1.29179419660706,
+      "mean_token_accuracy": 0.7130378339067102,
+      "rewards/chosen": -0.8251502061705105,
+      "rewards/rejected": -2.330989454947412,
+      "rewards/accuracies": 0.81375,
+      "rewards/margins": 1.5058392491098493,
+      "logps/chosen": -58.14819228887558,
+      "logps/rejected": -76.84343997001648,
+      "epoch": 0.809086665629376,
+      "step": 780
+    },
+    {
+      "loss": 0.4155034065246582,
+      "grad_norm": 0.015209637582302094,
+      "learning_rate": 1.823834196891192e-06,
+      "entropy": 0.816227734730055,
+      "num_tokens": 52982847.0,
+      "logits/chosen": -1.552296107544077,
+      "logits/rejected": -1.2655800747433517,
+      "mean_token_accuracy": 0.7077531234174966,
+      "rewards/chosen": -0.7692844783003966,
+      "rewards/rejected": -2.2729846961359725,
+      "rewards/accuracies": 0.81625,
+      "rewards/margins": 1.5037002183427104,
+      "logps/chosen": -59.88701428174973,
+      "logps/rejected": -75.7218120956421,
+      "epoch": 0.8194595715989834,
+      "step": 790
+    },
+    {
+      "loss": 0.4396702289581299,
+      "grad_norm": 0.017707331106066704,
+      "learning_rate": 1.7202072538860104e-06,
+      "entropy": 0.8157146892453602,
+      "num_tokens": 53653414.0,
+      "logits/chosen": -1.5836949831528315,
+      "logits/rejected": -1.2884593526275128,
+      "mean_token_accuracy": 0.7132340743020177,
+      "rewards/chosen": -0.7823633211505876,
+      "rewards/rejected": -2.2281043492229764,
+      "rewards/accuracies": 0.80375,
+      "rewards/margins": 1.4457410290511326,
+      "logps/chosen": -59.43502898097038,
+      "logps/rejected": -75.63278030633927,
+      "epoch": 0.8298324775685908,
+      "step": 800
+    },
+    {
+      "loss": 0.4166680335998535,
+      "grad_norm": 0.017773514613509178,
+      "learning_rate": 1.6165803108808292e-06,
+      "entropy": 0.797488763818983,
+      "num_tokens": 54324247.0,
+      "logits/chosen": -1.6215031533790736,
+      "logits/rejected": -1.3289979163820846,
+      "mean_token_accuracy": 0.7087218741327524,
+      "rewards/chosen": -0.761779917108106,
+      "rewards/rejected": -2.309470071569667,
+      "rewards/accuracies": 0.81125,
+      "rewards/margins": 1.5476901544816792,
+      "logps/chosen": -58.93250516653061,
+      "logps/rejected": -76.49902043104171,
+      "epoch": 0.8402053835381982,
+      "step": 810
+    },
+    {
+      "loss": 0.4239467144012451,
+      "grad_norm": 0.018757648766040802,
+      "learning_rate": 1.5129533678756477e-06,
+      "entropy": 0.8032379890448647,
+      "num_tokens": 54994016.0,
+      "logits/chosen": -1.578882165777075,
+      "logits/rejected": -1.2860967628921216,
+      "mean_token_accuracy": 0.7140155434235931,
+      "rewards/chosen": -0.7499237745841674,
+      "rewards/rejected": -2.2461700457381086,
+      "rewards/accuracies": 0.82375,
+      "rewards/margins": 1.496246272022836,
+      "logps/chosen": -58.89809783697128,
+      "logps/rejected": -74.08916418790817,
+      "epoch": 0.8505782895078056,
+      "step": 820
+    },
+    {
+      "loss": 0.419736909866333,
+      "grad_norm": 0.018885280936956406,
+      "learning_rate": 1.4093264248704663e-06,
+      "entropy": 0.7875165941729211,
+      "num_tokens": 55663818.0,
+      "logits/chosen": -1.6236698629206843,
+      "logits/rejected": -1.2788538558527598,
+      "mean_token_accuracy": 0.7192669866792858,
+      "rewards/chosen": -0.7543148858308633,
+      "rewards/rejected": -2.269310671926214,
+      "rewards/accuracies": 0.8075,
+      "rewards/margins": 1.5149957838421688,
+      "logps/chosen": -55.762867684364316,
+      "logps/rejected": -76.66288559436798,
+      "epoch": 0.860951195477413,
+      "step": 830
+    },
+    {
+      "loss": 0.4073533058166504,
+      "grad_norm": 0.01915486343204975,
+      "learning_rate": 1.3056994818652852e-06,
+      "entropy": 0.8327871753677027,
+      "num_tokens": 56334476.0,
+      "logits/chosen": -1.4996538615666455,
+      "logits/rejected": -1.237756981594407,
+      "mean_token_accuracy": 0.716468540597707,
+      "rewards/chosen": -0.7192282251862707,
+      "rewards/rejected": -2.26593919366569,
+      "rewards/accuracies": 0.82875,
+      "rewards/margins": 1.5467109655588864,
+      "logps/chosen": -59.84484759330749,
+      "logps/rejected": -76.10131199359894,
+      "epoch": 0.8713241014470204,
+      "step": 840
+    },
+    {
+      "loss": 0.42834720611572263,
+      "grad_norm": 0.01567392610013485,
+      "learning_rate": 1.2020725388601037e-06,
+      "entropy": 0.827671511786757,
+      "num_tokens": 57005010.0,
+      "logits/chosen": -1.5224089619857955,
+      "logits/rejected": -1.203097930514173,
+      "mean_token_accuracy": 0.7094048094563186,
+      "rewards/chosen": -0.8032524040471617,
+      "rewards/rejected": -2.2959909180374236,
+      "rewards/accuracies": 0.8,
+      "rewards/margins": 1.4927385161118583,
+      "logps/chosen": -60.616187605857846,
+      "logps/rejected": -76.86470454216004,
+      "epoch": 0.8816970074166278,
+      "step": 850
+    },
+    {
+      "loss": 0.4034994602203369,
+      "grad_norm": 0.014525278471410275,
+      "learning_rate": 1.0984455958549225e-06,
+      "entropy": 0.818393541239202,
+      "num_tokens": 57677536.0,
+      "logits/chosen": -1.543480139299166,
+      "logits/rejected": -1.2204454822419504,
+      "mean_token_accuracy": 0.7158707704581321,
+      "rewards/chosen": -0.7887829605976913,
+      "rewards/rejected": -2.3561961521262127,
+      "rewards/accuracies": 0.82125,
+      "rewards/margins": 1.5674131939606741,
+      "logps/chosen": -60.70202103614807,
+      "logps/rejected": -77.77336375951766,
+      "epoch": 0.8920699133862352,
+      "step": 860
+    },
+    {
+      "loss": 0.432753324508667,
+      "grad_norm": 0.02228359691798687,
+      "learning_rate": 9.94818652849741e-07,
+      "entropy": 0.8074402804669808,
+      "num_tokens": 58349497.0,
+      "logits/chosen": -1.558037182784836,
+      "logits/rejected": -1.2543498522866354,
+      "mean_token_accuracy": 0.703925465606153,
+      "rewards/chosen": -0.805554041202995,
+      "rewards/rejected": -2.2746161510888485,
+      "rewards/accuracies": 0.835,
+      "rewards/margins": 1.4690621069492773,
+      "logps/chosen": -62.29052616596222,
+      "logps/rejected": -75.3348803281784,
+      "epoch": 0.9024428193558426,
+      "step": 870
+    },
+    {
+      "eval_loss": 0.4090946316719055,
+      "eval_runtime": 375.2006,
+      "eval_samples_per_second": 2.665,
+      "eval_steps_per_second": 2.665,
+      "eval_entropy": 0.7924561413838528,
+      "eval_num_tokens": 58685036.0,
+      "eval_logits/chosen": -1.5811667035215011,
+      "eval_logits/rejected": -1.2610527832956517,
+      "eval_mean_token_accuracy": 0.7133496539741755,
+      "eval_rewards/chosen": -0.7897945828609518,
+      "eval_rewards/rejected": -2.3197074431839866,
+      "eval_rewards/accuracies": 0.828,
+      "eval_rewards/margins": 1.5299128606393932,
+      "eval_logps/chosen": -58.07093252563477,
+      "eval_logps/rejected": -75.83694574356079,
+      "epoch": 0.9076292723406463,
+      "step": 875
+    },
+    {
+      "loss": 0.39424493312835696,
+      "grad_norm": 0.013235539197921753,
+      "learning_rate": 8.911917098445596e-07,
+      "entropy": 0.7996839477901813,
+      "num_tokens": 59020756.0,
+      "logits/chosen": -1.5843618404439912,
+      "logits/rejected": -1.2211371427803261,
+      "mean_token_accuracy": 0.7160469963587821,
+      "rewards/chosen": -0.7639252483138261,
+      "rewards/rejected": -2.3575840306148166,
+      "rewards/accuracies": 0.82625,
+      "rewards/margins": 1.5936587833426892,
+      "logps/chosen": -57.04736487388611,
+      "logps/rejected": -79.49960944414138,
+      "epoch": 0.91281572532545,
+      "step": 880
+    },
+    {
+      "loss": 0.4502077102661133,
+      "grad_norm": 0.016691625118255615,
+      "learning_rate": 7.875647668393784e-07,
+      "entropy": 0.7824898976105032,
+      "num_tokens": 59691542.0,
+      "logits/chosen": -1.6007244896342394,
+      "logits/rejected": -1.2807950876104501,
+      "mean_token_accuracy": 0.7142804705537855,
+      "rewards/chosen": -0.8533335844849352,
+      "rewards/rejected": -2.263447732278146,
+      "rewards/accuracies": 0.80875,
+      "rewards/margins": 1.4101141486410051,
+      "logps/chosen": -58.45585764169693,
+      "logps/rejected": -76.72949911355973,
+      "epoch": 0.9231886312950573,
+      "step": 890
+    },
+    {
+      "loss": 0.40724778175354004,
+      "grad_norm": 0.019250568002462387,
+      "learning_rate": 6.839378238341969e-07,
+      "entropy": 0.819874558126321,
+      "num_tokens": 60363238.0,
+      "logits/chosen": -1.5555056418182593,
+      "logits/rejected": -1.2092615978992753,
+      "mean_token_accuracy": 0.7146024034544826,
+      "rewards/chosen": -0.8092134252478718,
+      "rewards/rejected": -2.3746094698365776,
+      "rewards/accuracies": 0.8225,
+      "rewards/margins": 1.565396042652428,
+      "logps/chosen": -58.4417240858078,
+      "logps/rejected": -80.7738204741478,
+      "epoch": 0.9335615372646647,
+      "step": 900
+    },
+    {
+      "loss": 0.4182878971099854,
+      "grad_norm": 0.02170160599052906,
+      "learning_rate": 5.803108808290156e-07,
+      "entropy": 0.8145798404130619,
+      "num_tokens": 61034470.0,
+      "logits/chosen": -1.5395043610571841,
+      "logits/rejected": -1.2172720667091816,
+      "mean_token_accuracy": 0.7149157860130072,
+      "rewards/chosen": -0.8415973563533043,
+      "rewards/rejected": -2.4253665239037945,
+      "rewards/accuracies": 0.81,
+      "rewards/margins": 1.5837691660691053,
+      "logps/chosen": -58.98787801384926,
+      "logps/rejected": -78.09846720218658,
+      "epoch": 0.9439344432342721,
+      "step": 910
+    },
+    {
+      "loss": 0.42937483787536623,
+      "grad_norm": 0.014422536827623844,
+      "learning_rate": 4.7668393782383424e-07,
+      "entropy": 0.7941974463209044,
+      "num_tokens": 61703303.0,
+      "logits/chosen": -1.5714764751371695,
+      "logits/rejected": -1.2391417194541616,
+      "mean_token_accuracy": 0.7113890647143125,
+      "rewards/chosen": -0.8099557375663425,
+      "rewards/rejected": -2.294640830200858,
+      "rewards/accuracies": 0.81125,
+      "rewards/margins": 1.4846850905031896,
+      "logps/chosen": -56.28238318800926,
+      "logps/rejected": -74.58795207023621,
+      "epoch": 0.9543073492038795,
+      "step": 920
+    },
+    {
+      "loss": 0.40997958183288574,
+      "grad_norm": 0.02402566932141781,
+      "learning_rate": 3.730569948186528e-07,
+      "entropy": 0.8154971726459916,
+      "num_tokens": 62375816.0,
+      "logits/chosen": -1.545892127215608,
+      "logits/rejected": -1.2600055196671498,
+      "mean_token_accuracy": 0.704818404596299,
+      "rewards/chosen": -0.7739588160630956,
+      "rewards/rejected": -2.308201268577832,
+      "rewards/accuracies": 0.8175,
+      "rewards/margins": 1.5342424516985194,
+      "logps/chosen": -60.89072738647461,
+      "logps/rejected": -77.38187520503998,
+      "epoch": 0.9646802551734869,
+      "step": 930
+    },
+    {
+      "loss": 0.4273488998413086,
+      "grad_norm": 0.016814034432172775,
+      "learning_rate": 2.694300518134715e-07,
+      "entropy": 0.81020716742496,
+      "num_tokens": 63047673.0,
+      "logits/chosen": -1.5172310235884794,
+      "logits/rejected": -1.1998523520459965,
+      "mean_token_accuracy": 0.7050479584932328,
+      "rewards/chosen": -0.8097914783160922,
+      "rewards/rejected": -2.262013399923744,
+      "rewards/accuracies": 0.80625,
+      "rewards/margins": 1.452221922907047,
+      "logps/chosen": -60.9624414229393,
+      "logps/rejected": -75.92849832773209,
+      "epoch": 0.9750531611430943,
+      "step": 940
+    },
+    {
+      "loss": 0.4405078411102295,
+      "grad_norm": 0.02221529930830002,
+      "learning_rate": 1.6580310880829015e-07,
+      "entropy": 0.7960373500641436,
+      "num_tokens": 63718529.0,
+      "logits/chosen": -1.5261262470344286,
+      "logits/rejected": -1.2351896031266807,
+      "mean_token_accuracy": 0.7166906687803567,
+      "rewards/chosen": -0.8054070144893921,
+      "rewards/rejected": -2.2143535910593344,
+      "rewards/accuracies": 0.79875,
+      "rewards/margins": 1.4089465801790357,
+      "logps/chosen": -58.76033858776093,
+      "logps/rejected": -75.32699841499328,
+      "epoch": 0.9854260671127016,
+      "step": 950
+    },
+    {
+      "loss": 0.4032235622406006,
+      "grad_norm": 0.015166543424129486,
+      "learning_rate": 6.217616580310881e-08,
+      "entropy": 0.8253091154253344,
+      "num_tokens": 64391541.0,
+      "logits/chosen": -1.5054467577713744,
+      "logits/rejected": -1.1940948964954847,
+      "mean_token_accuracy": 0.7102081939950585,
+      "rewards/chosen": -0.7573958630473498,
+      "rewards/rejected": -2.3041888994569306,
+      "rewards/accuracies": 0.8125,
+      "rewards/margins": 1.5467930364748463,
+      "logps/chosen": -59.873460137844084,
+      "logps/rejected": -77.67126994132995,
+      "epoch": 0.995798973082309,
+      "step": 960
+    },
+    {
+      "train_runtime": 36967.0361,
+      "train_samples_per_second": 2.086,
+      "train_steps_per_second": 0.026,
+      "total_flos": 5.2405488888805786e+17,
+      "train_loss": 0.453640728787437,
+      "entropy": 0.8554125972153952,
+      "num_tokens": 64664162.0,
+      "logits/chosen": -1.3863200845494572,
+      "logits/rejected": -1.1139205918868142,
+      "mean_token_accuracy": 0.7043190732322357,
+      "rewards/chosen": -0.8579825216625461,
+      "rewards/rejected": -2.1552414747104507,
+      "rewards/accuracies": 0.7932098765432098,
+      "rewards/margins": 1.2972589509883596,
+      "logps/chosen": -62.61760814690295,
+      "logps/rejected": -79.34557756376856,
+      "epoch": 1.0,
+      "step": 965
+    }
+  ],
+  "validation_monitor_size": 1000,
+  "validation_monitor_selection": "fixed_seed_random_without_replacement",
+  "validation_monitor_seed": 22,
+  "validation_monitor_indices": [
+    2,
+    10,
+    14,
+    21,
+    55,
+    63,
+    66,
+    69,
+    107,
+    110,
+    142,
+    144,
+    149,
+    150,
+    151,
+    152,
+    160,
+    163,
+    166,
+    167,
+    176,
+    181,
+    206,
+    207,
+    259,
+    267,
+    281,
+    295,
+    298,
+    306,
+    307,
+    317,
+    321,
+    331,
+    336,
+    341,
+    346,
+    349,
+    351,
+    352,
+    357,
+    358,
+    369,
+    370,
+    382,
+    386,
+    391,
+    401,
+    411,
+    412,
+    432,
+    437,
+    452,
+    462,
+    465,
+    488,
+    490,
+    491,
+    492,
+    494,
+    503,
+    504,
+    508,
+    528,
+    538,
+    540,
+    551,
+    553,
+    567,
+    586,
+    605,
+    606,
+    625,
+    627,
+    661,
+    663,
+    666,
+    677,
+    685,
+    690,
+    692,
+    704,
+    708,
+    714,
+    715,
+    727,
+    728,
+    733,
+    745,
+    752,
+    753,
+    755,
+    764,
+    773,
+    779,
+    783,
+    793,
+    796,
+    799,
+    803,
+    804,
+    805,
+    813,
+    816,
+    818,
+    823,
+    827,
+    829,
+    830,
+    837,
+    842,
+    845,
+    850,
+    853,
+    856,
+    889,
+    890,
+    905,
+    915,
+    924,
+    930,
+    939,
+    951,
+    988,
+    1002,
+    1005,
+    1023,
+    1029,
+    1038,
+    1049,
+    1050,
+    1054,
+    1056,
+    1067,
+    1068,
+    1079,
+    1088,
+    1091,
+    1103,
+    1114,
+    1118,
+    1133,
+    1140,
+    1144,
+    1145,
+    1155,
+    1186,
+    1195,
+    1206,
+    1223,
+    1251,
+    1252,
+    1257,
+    1259,
+    1270,
+    1271,
+    1295,
+    1303,
+    1304,
+    1305,
+    1329,
+    1335,
+    1336,
+    1343,
+    1367,
+    1373,
+    1377,
+    1403,
+    1412,
+    1429,
+    1443,
+    1457,
+    1459,
+    1460,
+    1476,
+    1483,
+    1486,
+    1494,
+    1507,
+    1510,
+    1519,
+    1521,
+    1522,
+    1545,
+    1551,
+    1570,
+    1582,
+    1593,
+    1595,
+    1603,
+    1607,
+    1614,
+    1615,
+    1625,
+    1634,
+    1639,
+    1648,
+    1654,
+    1657,
+    1662,
+    1667,
+    1673,
+    1690,
+    1704,
+    1746,
+    1756,
+    1781,
+    1783,
+    1796,
+    1799,
+    1809,
+    1814,
+    1827,
+    1829,
+    1832,
+    1844,
+    1847,
+    1854,
+    1856,
+    1857,
+    1858,
+    1874,
+    1883,
+    1889,
+    1924,
+    1925,
+    1931,
+    1932,
+    1934,
+    1935,
+    1938,
+    1950,
+    1957,
+    1962,
+    1967,
+    1975,
+    1982,
+    1983,
+    1991,
+    1998,
+    2003,
+    2008,
+    2017,
+    2021,
+    2026,
+    2035,
+    2040,
+    2050,
+    2056,
+    2077,
+    2079,
+    2082,
+    2098,
+    2100,
+    2108,
+    2121,
+    2130,
+    2133,
+    2134,
+    2138,
+    2143,
+    2166,
+    2167,
+    2180,
+    2181,
+    2185,
+    2204,
+    2205,
+    2212,
+    2221,
+    2224,
+    2226,
+    2230,
+    2233,
+    2256,
+    2261,
+    2263,
+    2269,
+    2273,
+    2290,
+    2291,
+    2299,
+    2301,
+    2321,
+    2323,
+    2330,
+    2384,
+    2401,
+    2417,
+    2420,
+    2421,
+    2424,
+    2430,
+    2435,
+    2456,
+    2488,
+    2502,
+    2504,
+    2519,
+    2527,
+    2532,
+    2538,
+    2542,
+    2553,
+    2555,
+    2558,
+    2559,
+    2562,
+    2578,
+    2583,
+    2585,
+    2590,
+    2592,
+    2594,
+    2596,
+    2600,
+    2606,
+    2607,
+    2618,
+    2630,
+    2637,
+    2647,
+    2650,
+    2657,
+    2679,
+    2685,
+    2705,
+    2706,
+    2712,
+    2713,
+    2714,
+    2727,
+    2740,
+    2742,
+    2755,
+    2780,
+    2784,
+    2792,
+    2807,
+    2808,
+    2810,
+    2820,
+    2831,
+    2839,
+    2860,
+    2862,
+    2863,
+    2866,
+    2875,
+    2878,
+    2898,
+    2905,
+    2921,
+    2922,
+    2926,
+    2930,
+    2934,
+    2944,
+    2955,
+    2957,
+    2959,
+    2973,
+    2978,
+    2998,
+    3018,
+    3022,
+    3028,
+    3031,
+    3061,
+    3085,
+    3090,
+    3104,
+    3105,
+    3111,
+    3115,
+    3121,
+    3122,
+    3129,
+    3133,
+    3135,
+    3161,
+    3162,
+    3169,
+    3173,
+    3194,
+    3195,
+    3215,
+    3225,
+    3226,
+    3241,
+    3247,
+    3250,
+    3253,
+    3265,
+    3268,
+    3293,
+    3301,
+    3312,
+    3329,
+    3352,
+    3361,
+    3362,
+    3376,
+    3396,
+    3401,
+    3403,
+    3410,
+    3419,
+    3432,
+    3443,
+    3452,
+    3467,
+    3469,
+    3475,
+    3485,
+    3503,
+    3514,
+    3515,
+    3524,
+    3528,
+    3538,
+    3544,
+    3557,
+    3560,
+    3565,
+    3600,
+    3637,
+    3642,
+    3658,
+    3659,
+    3692,
+    3693,
+    3699,
+    3722,
+    3725,
+    3728,
+    3731,
+    3740,
+    3742,
+    3762,
+    3766,
+    3780,
+    3788,
+    3794,
+    3796,
+    3798,
+    3805,
+    3817,
+    3819,
+    3822,
+    3837,
+    3839,
+    3843,
+    3846,
+    3851,
+    3854,
+    3865,
+    3870,
+    3871,
+    3884,
+    3894,
+    3895,
+    3896,
+    3907,
+    3911,
+    3915,
+    3919,
+    3920,
+    3923,
+    3933,
+    3955,
+    3967,
+    3972,
+    3974,
+    3975,
+    3984,
+    3985,
+    3997,
+    4002,
+    4010,
+    4034,
+    4044,
+    4063,
+    4073,
+    4079,
+    4082,
+    4088,
+    4121,
+    4145,
+    4148,
+    4159,
+    4161,
+    4164,
+    4177,
+    4188,
+    4199,
+    4203,
+    4207,
+    4208,
+    4213,
+    4221,
+    4225,
+    4233,
+    4241,
+    4243,
+    4247,
+    4264,
+    4274,
+    4282,
+    4286,
+    4290,
+    4308,
+    4310,
+    4313,
+    4321,
+    4324,
+    4327,
+    4349,
+    4362,
+    4370,
+    4374,
+    4380,
+    4407,
+    4409,
+    4411,
+    4415,
+    4417,
+    4418,
+    4427,
+    4431,
+    4433,
+    4451,
+    4466,
+    4477,
+    4478,
+    4479,
+    4493,
+    4494,
+    4514,
+    4527,
+    4539,
+    4550,
+    4558,
+    4568,
+    4579,
+    4583,
+    4584,
+    4586,
+    4587,
+    4590,
+    4599,
+    4602,
+    4610,
+    4626,
+    4627,
+    4630,
+    4641,
+    4647,
+    4655,
+    4656,
+    4657,
+    4661,
+    4685,
+    4714,
+    4715,
+    4731,
+    4749,
+    4752,
+    4769,
+    4777,
+    4782,
+    4791,
+    4805,
+    4818,
+    4829,
+    4833,
+    4837,
+    4839,
+    4843,
+    4871,
+    4875,
+    4879,
+    4880,
+    4885,
+    4888,
+    4895,
+    4900,
+    4923,
+    4966,
+    4968,
+    4972,
+    4989,
+    4994,
+    4998,
+    5001,
+    5013,
+    5019,
+    5026,
+    5032,
+    5034,
+    5046,
+    5055,
+    5085,
+    5086,
+    5088,
+    5089,
+    5090,
+    5095,
+    5108,
+    5110,
+    5119,
+    5120,
+    5121,
+    5133,
+    5148,
+    5154,
+    5160,
+    5169,
+    5178,
+    5222,
+    5223,
+    5232,
+    5233,
+    5240,
+    5256,
+    5259,
+    5264,
+    5271,
+    5276,
+    5279,
+    5294,
+    5300,
+    5303,
+    5321,
+    5335,
+    5337,
+    5345,
+    5348,
+    5365,
+    5373,
+    5378,
+    5384,
+    5422,
+    5442,
+    5443,
+    5445,
+    5477,
+    5485,
+    5495,
+    5497,
+    5504,
+    5526,
+    5533,
+    5542,
+    5564,
+    5570,
+    5579,
+    5587,
+    5592,
+    5608,
+    5610,
+    5624,
+    5630,
+    5638,
+    5651,
+    5663,
+    5670,
+    5675,
+    5691,
+    5700,
+    5706,
+    5707,
+    5715,
+    5720,
+    5721,
+    5722,
+    5732,
+    5738,
+    5741,
+    5769,
+    5771,
+    5775,
+    5795,
+    5796,
+    5800,
+    5809,
+    5810,
+    5815,
+    5819,
+    5827,
+    5848,
+    5849,
+    5852,
+    5859,
+    5880,
+    5884,
+    5907,
+    5909,
+    5912,
+    5919,
+    5931,
+    5932,
+    5934,
+    5941,
+    5948,
+    5950,
+    5952,
+    5953,
+    5969,
+    5981,
+    6000,
+    6003,
+    6010,
+    6018,
+    6041,
+    6065,
+    6075,
+    6090,
+    6103,
+    6106,
+    6109,
+    6114,
+    6123,
+    6131,
+    6136,
+    6138,
+    6139,
+    6164,
+    6165,
+    6171,
+    6173,
+    6180,
+    6185,
+    6189,
+    6190,
+    6221,
+    6223,
+    6237,
+    6255,
+    6262,
+    6265,
+    6293,
+    6296,
+    6305,
+    6318,
+    6331,
+    6336,
+    6340,
+    6355,
+    6366,
+    6371,
+    6396,
+    6399,
+    6402,
+    6408,
+    6432,
+    6433,
+    6441,
+    6456,
+    6465,
+    6478,
+    6486,
+    6489,
+    6507,
+    6508,
+    6520,
+    6522,
+    6528,
+    6537,
+    6551,
+    6564,
+    6589,
+    6590,
+    6598,
+    6599,
+    6611,
+    6613,
+    6615,
+    6621,
+    6634,
+    6647,
+    6649,
+    6654,
+    6676,
+    6680,
+    6690,
+    6708,
+    6729,
+    6736,
+    6744,
+    6749,
+    6756,
+    6761,
+    6763,
+    6773,
+    6788,
+    6790,
+    6796,
+    6797,
+    6811,
+    6824,
+    6850,
+    6869,
+    6871,
+    6882,
+    6892,
+    6895,
+    6906,
+    6911,
+    6912,
+    6914,
+    6927,
+    6952,
+    6966,
+    6985,
+    7001,
+    7021,
+    7031,
+    7035,
+    7038,
+    7041,
+    7045,
+    7052,
+    7057,
+    7058,
+    7072,
+    7073,
+    7076,
+    7086,
+    7102,
+    7107,
+    7109,
+    7117,
+    7122,
+    7125,
+    7166,
+    7182,
+    7199,
+    7207,
+    7212,
+    7215,
+    7232,
+    7243,
+    7246,
+    7250,
+    7253,
+    7258,
+    7263,
+    7267,
+    7270,
+    7274,
+    7280,
+    7286,
+    7293,
+    7298,
+    7302,
+    7306,
+    7316,
+    7325,
+    7326,
+    7334,
+    7356,
+    7357,
+    7363,
+    7364,
+    7367,
+    7385,
+    7392,
+    7399,
+    7405,
+    7416,
+    7420,
+    7421,
+    7426,
+    7452,
+    7476,
+    7481,
+    7519,
+    7534,
+    7542,
+    7546,
+    7573,
+    7585,
+    7601,
+    7604,
+    7606,
+    7609,
+    7629,
+    7649,
+    7653,
+    7667,
+    7682,
+    7699,
+    7738,
+    7750,
+    7786,
+    7798,
+    7800,
+    7801,
+    7805,
+    7806,
+    7811,
+    7813,
+    7832,
+    7837,
+    7849,
+    7856,
+    7876,
+    7877,
+    7887,
+    7905,
+    7916,
+    7919,
+    7920,
+    7922,
+    7923,
+    7926,
+    7944,
+    7961,
+    7966,
+    7970,
+    7973,
+    7974,
+    7976,
+    7986,
+    7999,
+    8027,
+    8028,
+    8034,
+    8047,
+    8068,
+    8074,
+    8077,
+    8091,
+    8120,
+    8122,
+    8125,
+    8152,
+    8153,
+    8164,
+    8167,
+    8169,
+    8171,
+    8177,
+    8184,
+    8189,
+    8192,
+    8196,
+    8202,
+    8212,
+    8217,
+    8231,
+    8242,
+    8244,
+    8250,
+    8256,
+    8257,
+    8265,
+    8270,
+    8274,
+    8283,
+    8290,
+    8294,
+    8301,
+    8302,
+    8307,
+    8318,
+    8326,
+    8338,
+    8349,
+    8350,
+    8353,
+    8357,
+    8371,
+    8374,
+    8377,
+    8380,
+    8387,
+    8388,
+    8396,
+    8402,
+    8419,
+    8423,
+    8428,
+    8435,
+    8439,
+    8442,
+    8444,
+    8453,
+    8461,
+    8475,
+    8481,
+    8485,
+    8493,
+    8495,
+    8498,
+    8523,
+    8530,
+    8531,
+    8562
+  ],
+  "early_stopping_patience": 3
+}

phase1/ablations/no_context/README.md

@@ -0,0 +1,80 @@
+---
+library_name: peft
+model_name: phase1-qwen-no-context-ablation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-1.5B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+license: apache-2.0
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+pipeline_tag: text-generation
+---
+
+# Phase 1 Qwen `no_context` diagnostic ablation
+
+This is the Phase 1 `no_context` diagnostic LoRA-DPO adapter fine-tuned from
+[Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct).
+The frozen training recipe and membership hashes are included in this directory.
+
+## Quick start
+
+```python
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo_id = "geyuxu/york-milestone-project-self-traing-loop-model"
+subfolder = "phase1/ablations/no_context"
+base_id = "Qwen/Qwen2.5-1.5B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(repo_id, subfolder=subfolder)
+base_model = AutoModelForCausalLM.from_pretrained(
+    base_id, torch_dtype="auto", device_map="auto"
+)
+model = PeftModel.from_pretrained(base_model, repo_id, subfolder=subfolder)
+```
+
+## Training procedure
+
+
+
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 0.29.0
+- Transformers: 5.3.0
+- Pytorch: 2.10.0
+- Datasets: 4.6.1
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+    title        = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+    author       = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+    year         = 2023,
+    booktitle    = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+    url          = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+    editor       = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

phase1/ablations/no_context/adapter_config.json

@@ -0,0 +1,43 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "k_proj",
+    "o_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

phase1/ablations/no_context/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f3a8ff3c19fd3e0ff728feab4e0a73a33c71608510f37d9dd27e53b74f97523
+size 17462432

phase1/ablations/no_context/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

phase1/ablations/no_context/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5
+size 11422166

phase1/ablations/no_context/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

phase1/ablations/no_context/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd47c7f20edcbf52ada253a96ebd7ddcb8045bdd2a75f52fdd0f51ad7bc0183e
+size 5841

phase1/ablations/no_context/training_recipe.json

@@ -0,0 +1,22 @@
+{
+  "format_version": 1,
+  "experiment": "confusion_ablation",
+  "ablation": "no_context",
+  "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+  "train_membership_sha256": "b497fa765223a9810ad784f31e1598edfa992a162146d6f94d9cdf4adcedd87d",
+  "validation_membership_sha256": "2f609467438cd6a01fd84b9d80c0fcd84c44767e3b2ed98f4f6a83d68a62c181",
+  "num_train_pairs": 77124,
+  "num_validation_pairs": 8563,
+  "transformation_seed": 22,
+  "validation_prompt_mode": "clean_rag",
+  "learning_rate": 1e-05,
+  "beta": 0.1,
+  "num_epochs": 1.0,
+  "seed": 22,
+  "max_length": 512,
+  "max_prompt_length": 384,
+  "gradient_accumulation_steps": 80,
+  "save_steps": 125,
+  "save_total_limit": 12,
+  "preference_pairs_format_version": 2
+}

phase1/ablations/no_context/training_summary.json

@@ -0,0 +1,2925 @@
+{
+  "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+  "resolved_model": "Qwen/Qwen2.5-1.5B-Instruct",
+  "num_train_records": 77124,
+  "num_validation_records_available": 8563,
+  "training_config": {
+    "lora_rank": 16,
+    "lora_alpha": 32,
+    "learning_rate": 1e-05,
+    "num_epochs": 1.0,
+    "per_device_batch_size": 1,
+    "per_device_eval_batch_size": 1,
+    "gradient_accumulation_steps": 80,
+    "beta": 0.1,
+    "max_length": 512,
+    "max_prompt_length": 384,
+    "seed": 22,
+    "save_steps": 125,
+    "save_total_limit": 12,
+    "resume_from_checkpoint": null,
+    "precision_dtype": "torch.bfloat16",
+    "bf16": true,
+    "fp16": false,
+    "tokenizer_add_bos_token": false
+  },
+  "global_step": 965,
+  "best_metric": 0.5303567051887512,
+  "best_model_checkpoint": "outputs/confusion_ablation_phase1_repair_v3/no_context/lora/checkpoint-875",
+  "train_metrics": {
+    "train_runtime": 22000.2483,
+    "train_samples_per_second": 3.506,
+    "train_steps_per_second": 0.044,
+    "total_flos": 1.2331818072769536e+17,
+    "train_loss": 0.4788281094842624
+  },
+  "log_history": [
+    {
+      "loss": 0.6900143623352051,
+      "grad_norm": 0.012825769372284412,
+      "learning_rate": 9.906735751295338e-06,
+      "entropy": 0.9465285880398006,
+      "num_tokens": 143734.0,
+      "logits/chosen": -1.5012888478025672,
+      "logits/rejected": -1.5723651012268272,
+      "mean_token_accuracy": 0.6951504178717732,
+      "rewards/chosen": 0.008079942484078515,
+      "rewards/rejected": 0.0013256204138269822,
+      "rewards/accuracies": 0.5425,
+      "rewards/margins": 0.006754322004853748,
+      "logps/chosen": -54.978611971139905,
+      "logps/rejected": -44.00920191049576,
+      "epoch": 0.010372905969607386,
+      "step": 10
+    },
+    {
+      "loss": 0.662090539932251,
+      "grad_norm": 0.01821574941277504,
+      "learning_rate": 9.803108808290156e-06,
+      "entropy": 0.9671179948747158,
+      "num_tokens": 288007.0,
+      "logits/chosen": -1.4026643295951302,
+      "logits/rejected": -1.5102725032640183,
+      "mean_token_accuracy": 0.6902420591562987,
+      "rewards/chosen": 0.051943560895306294,
+      "rewards/rejected": -0.014707293705987468,
+      "rewards/accuracies": 0.76875,
+      "rewards/margins": 0.06665085479675327,
+      "logps/chosen": -57.24453290700912,
+      "logps/rejected": -42.05887178063393,
+      "epoch": 0.02074581193921477,
+      "step": 20
+    },
+    {
+      "loss": 0.6254077911376953,
+      "grad_norm": 0.011037914082407951,
+      "learning_rate": 9.699481865284975e-06,
+      "entropy": 1.022590966722928,
+      "num_tokens": 433590.0,
+      "logits/chosen": -1.275362931486493,
+      "logits/rejected": -1.3053425668445908,
+      "mean_token_accuracy": 0.6875665122829377,
+      "rewards/chosen": 0.104925400447828,
+      "rewards/rejected": -0.05808189030554786,
+      "rewards/accuracies": 0.785,
+      "rewards/margins": 0.16300729091250105,
+      "logps/chosen": -56.563721351623535,
+      "logps/rejected": -44.831772941350934,
+      "epoch": 0.031118717908822157,
+      "step": 30
+    },
+    {
+      "loss": 0.5835778713226318,
+      "grad_norm": 0.011659706942737103,
+      "learning_rate": 9.595854922279793e-06,
+      "entropy": 1.0491496316390112,
+      "num_tokens": 579257.0,
+      "logits/chosen": -1.103721370379413,
+      "logits/rejected": -1.1007857062801798,
+      "mean_token_accuracy": 0.6836125956382603,
+      "rewards/chosen": 0.10443844198705847,
+      "rewards/rejected": -0.17612101439032812,
+      "rewards/accuracies": 0.79875,
+      "rewards/margins": 0.2805594558198936,
+      "logps/chosen": -56.31521392583847,
+      "logps/rejected": -44.80111095190048,
+      "epoch": 0.04149162387842954,
+      "step": 40
+    },
+    {
+      "loss": 0.5701297760009766,
+      "grad_norm": 0.009541669860482216,
+      "learning_rate": 9.492227979274612e-06,
+      "entropy": 1.1355412375973537,
+      "num_tokens": 724279.0,
+      "logits/chosen": -0.9254812350216316,
+      "logits/rejected": -0.9079043695626247,
+      "mean_token_accuracy": 0.6852783388644457,
+      "rewards/chosen": 0.054056347796804405,
+      "rewards/rejected": -0.30969686928990997,
+      "rewards/accuracies": 0.78,
+      "rewards/margins": 0.3637532171921339,
+      "logps/chosen": -57.41013575792313,
+      "logps/rejected": -45.747311503887175,
+      "epoch": 0.051864529848036925,
+      "step": 50
+    },
+    {
+      "loss": 0.5617631912231446,
+      "grad_norm": 0.007650681305676699,
+      "learning_rate": 9.388601036269432e-06,
+      "entropy": 1.1949225943023338,
+      "num_tokens": 868037.0,
+      "logits/chosen": -0.8281910829349749,
+      "logits/rejected": -0.7978684013816298,
+      "mean_token_accuracy": 0.6832736423797905,
+      "rewards/chosen": -0.03967508725108928,
+      "rewards/rejected": -0.47718707822641593,
+      "rewards/accuracies": 0.76625,
+      "rewards/margins": 0.4375119912764058,
+      "logps/chosen": -56.544705327749256,
+      "logps/rejected": -46.81773233771324,
+      "epoch": 0.062237435817644314,
+      "step": 60
+    },
+    {
+      "loss": 0.5275696277618408,
+      "grad_norm": 0.008909703232347965,
+      "learning_rate": 9.284974093264249e-06,
+      "entropy": 1.2164792704861611,
+      "num_tokens": 1012838.0,
+      "logits/chosen": -0.8094323563514035,
+      "logits/rejected": -0.6994889637697067,
+      "mean_token_accuracy": 0.6774187626503408,
+      "rewards/chosen": -0.13885413558671644,
+      "rewards/rejected": -0.6821774935087888,
+      "rewards/accuracies": 0.78875,
+      "rewards/margins": 0.5433233577781357,
+      "logps/chosen": -59.331356877088545,
+      "logps/rejected": -48.74847481250763,
+      "epoch": 0.0726103417872517,
+      "step": 70
+    },
+    {
+      "loss": 0.5331552982330322,
+      "grad_norm": 0.009889204055070877,
+      "learning_rate": 9.181347150259067e-06,
+      "entropy": 1.3044282801821827,
+      "num_tokens": 1157318.0,
+      "logits/chosen": -0.7322735281718121,
+      "logits/rejected": -0.641984971321019,
+      "mean_token_accuracy": 0.6695321470685304,
+      "rewards/chosen": -0.2586568782431277,
+      "rewards/rejected": -0.8568377101852093,
+      "rewards/accuracies": 0.7625,
+      "rewards/margins": 0.5981808328768239,
+      "logps/chosen": -59.761966693401334,
+      "logps/rejected": -52.77639355659485,
+      "epoch": 0.08298324775685909,
+      "step": 80
+    },
+    {
+      "loss": 0.5138424873352051,
+      "grad_norm": 0.009201128035783768,
+      "learning_rate": 9.077720207253888e-06,
+      "entropy": 1.2985483697801827,
+      "num_tokens": 1301440.0,
+      "logits/chosen": -0.7244757922082572,
+      "logits/rejected": -0.6446976643569048,
+      "mean_token_accuracy": 0.6679086892027408,
+      "rewards/chosen": -0.3228293718020723,
+      "rewards/rejected": -1.0106709606619553,
+      "rewards/accuracies": 0.78125,
+      "rewards/margins": 0.6878415882331319,
+      "logps/chosen": -61.18230092167855,
+      "logps/rejected": -53.36138055682182,
+      "epoch": 0.09335615372646647,
+      "step": 90
+    },
+    {
+      "loss": 0.5422882556915283,
+      "grad_norm": 0.011827374808490276,
+      "learning_rate": 8.974093264248706e-06,
+      "entropy": 1.3437509477324783,
+      "num_tokens": 1450582.0,
+      "logits/chosen": -0.6577579178959528,
+      "logits/rejected": -0.6052760341023756,
+      "mean_token_accuracy": 0.6652949979342521,
+      "rewards/chosen": -0.48851178389566485,
+      "rewards/rejected": -1.1487117192932055,
+      "rewards/accuracies": 0.75,
+      "rewards/margins": 0.660199935096316,
+      "logps/chosen": -66.8969352221489,
+      "logps/rejected": -59.775410163402555,
+      "epoch": 0.10372905969607385,
+      "step": 100
+    },
+    {
+      "loss": 0.4941854953765869,
+      "grad_norm": 0.009122959338128567,
+      "learning_rate": 8.870466321243523e-06,
+      "entropy": 1.3095360630284996,
+      "num_tokens": 1596102.0,
+      "logits/chosen": -0.7076684162519699,
+      "logits/rejected": -0.6176523284098799,
+      "mean_token_accuracy": 0.6634110971167684,
+      "rewards/chosen": -0.461084103367175,
+      "rewards/rejected": -1.2571547295921481,
+      "rewards/accuracies": 0.805,
+      "rewards/margins": 0.7960706273838878,
+      "logps/chosen": -62.40037791490555,
+      "logps/rejected": -55.013062043190004,
+      "epoch": 0.11410196566568125,
+      "step": 110
+    },
+    {
+      "loss": 0.5182926654815674,
+      "grad_norm": 0.009255604818463326,
+      "learning_rate": 8.766839378238343e-06,
+      "entropy": 1.3175755373714493,
+      "num_tokens": 1741007.0,
+      "logits/chosen": -0.7542329802157357,
+      "logits/rejected": -0.6569479564448296,
+      "mean_token_accuracy": 0.6686466364003718,
+      "rewards/chosen": -0.5461745312937274,
+      "rewards/rejected": -1.3049446252000052,
+      "rewards/accuracies": 0.78125,
+      "rewards/margins": 0.7587700937641785,
+      "logps/chosen": -63.756407718658444,
+      "logps/rejected": -53.10148733973503,
+      "epoch": 0.12447487163528863,
+      "step": 120
+    },
+    {
+      "eval_loss": 0.6102487444877625,
+      "eval_runtime": 187.9736,
+      "eval_samples_per_second": 5.32,
+      "eval_steps_per_second": 5.32,
+      "eval_entropy": 0.9602043167352676,
+      "eval_num_tokens": 1813585.0,
+      "eval_logits/chosen": -0.35928736165014746,
+      "eval_logits/rejected": -0.2653593227174459,
+      "eval_mean_token_accuracy": 0.71722535353899,
+      "eval_rewards/chosen": -0.11001014763419516,
+      "eval_rewards/rejected": -0.37161904421189684,
+      "eval_rewards/accuracies": 0.674,
+      "eval_rewards/margins": 0.2616088960794732,
+      "eval_logps/chosen": -51.27308830451965,
+      "eval_logps/rejected": -56.35606210327148,
+      "epoch": 0.1296613246200923,
+      "step": 125
+    },
+    {
+      "loss": 0.5195056438446045,
+      "grad_norm": 0.00852493941783905,
+      "learning_rate": 8.663212435233162e-06,
+      "entropy": 1.3568350885761902,
+      "num_tokens": 1887149.0,
+      "logits/chosen": -0.6995870647573145,
+      "logits/rejected": -0.6066075725004298,
+      "mean_token_accuracy": 0.6595153860282152,
+      "rewards/chosen": -0.5691824928086499,
+      "rewards/rejected": -1.3477147495548707,
+      "rewards/accuracies": 0.77125,
+      "rewards/margins": 0.7785322573361918,
+      "logps/chosen": -63.50056747078896,
+      "logps/rejected": -57.747277438640594,
+      "epoch": 0.13484777760489602,
+      "step": 130
+    },
+    {
+      "loss": 0.5127462863922119,
+      "grad_norm": 0.010999510996043682,
+      "learning_rate": 8.55958549222798e-06,
+      "entropy": 1.2911182015296072,
+      "num_tokens": 2030779.0,
+      "logits/chosen": -0.8425986763113174,
+      "logits/rejected": -0.6790966632280194,
+      "mean_token_accuracy": 0.6699926462769509,
+      "rewards/chosen": -0.5714914060512092,
+      "rewards/rejected": -1.3701531535839604,
+      "rewards/accuracies": 0.79,
+      "rewards/margins": 0.798661746904254,
+      "logps/chosen": -58.21274412512779,
+      "logps/rejected": -57.35458103775978,
+      "epoch": 0.1452206835745034,
+      "step": 140
+    },
+    {
+      "loss": 0.4771514892578125,
+      "grad_norm": 0.007932349108159542,
+      "learning_rate": 8.455958549222799e-06,
+      "entropy": 1.3540867683663964,
+      "num_tokens": 2177571.0,
+      "logits/chosen": -0.7752437049236788,
+      "logits/rejected": -0.6567138403536552,
+      "mean_token_accuracy": 0.6547455007676035,
+      "rewards/chosen": -0.5410610925909713,
+      "rewards/rejected": -1.4754458899376914,
+      "rewards/accuracies": 0.7975,
+      "rewards/margins": 0.9343847993016243,
+      "logps/chosen": -62.85497339010239,
+      "logps/rejected": -59.90050791025162,
+      "epoch": 0.1555935895441108,
+      "step": 150
+    },
+    {
+      "loss": 0.5065268516540528,
+      "grad_norm": 0.009417089633643627,
+      "learning_rate": 8.352331606217617e-06,
+      "entropy": 1.3703182196151464,
+      "num_tokens": 2323969.0,
+      "logits/chosen": -0.7446562811347729,
+      "logits/rejected": -0.6594706622933276,
+      "mean_token_accuracy": 0.6509130381792784,
+      "rewards/chosen": -0.6582208207750955,
+      "rewards/rejected": -1.4940199976743316,
+      "rewards/accuracies": 0.79125,
+      "rewards/margins": 0.8357991751469672,
+      "logps/chosen": -67.1081642484665,
+      "logps/rejected": -60.26060558557511,
+      "epoch": 0.16596649551371817,
+      "step": 160
+    },
+    {
+      "loss": 0.49721107482910154,
+      "grad_norm": 0.009800152853131294,
+      "learning_rate": 8.248704663212436e-06,
+      "entropy": 1.366801006840542,
+      "num_tokens": 2470885.0,
+      "logits/chosen": -0.6928292309660968,
+      "logits/rejected": -0.6209869620155596,
+      "mean_token_accuracy": 0.6535427515022456,
+      "rewards/chosen": -0.6162330767056119,
+      "rewards/rejected": -1.5209841228489678,
+      "rewards/accuracies": 0.78875,
+      "rewards/margins": 0.9047510461229831,
+      "logps/chosen": -66.02899884462357,
+      "logps/rejected": -58.60980107545853,
+      "epoch": 0.17633940148332555,
+      "step": 170
+    },
+    {
+      "loss": 0.4829860210418701,
+      "grad_norm": 0.008786365389823914,
+      "learning_rate": 8.145077720207254e-06,
+      "entropy": 1.3717705434653908,
+      "num_tokens": 2615121.0,
+      "logits/chosen": -0.6580562375952098,
+      "logits/rejected": -0.5878139433574051,
+      "mean_token_accuracy": 0.6564870945271104,
+      "rewards/chosen": -0.5556664189803269,
+      "rewards/rejected": -1.4979612402966085,
+      "rewards/accuracies": 0.7825,
+      "rewards/margins": 0.9422948203328997,
+      "logps/chosen": -62.85998333334923,
+      "logps/rejected": -57.214853674173355,
+      "epoch": 0.18671230745293294,
+      "step": 180
+    },
+    {
+      "loss": 0.4684004783630371,
+      "grad_norm": 0.009316110983490944,
+      "learning_rate": 8.041450777202073e-06,
+      "entropy": 1.4150286172702908,
+      "num_tokens": 2758874.0,
+      "logits/chosen": -0.6699597359601204,
+      "logits/rejected": -0.5473851034553693,
+      "mean_token_accuracy": 0.6443419794179499,
+      "rewards/chosen": -0.503764379483473,
+      "rewards/rejected": -1.4732603453175397,
+      "rewards/accuracies": 0.7825,
+      "rewards/margins": 0.9694959655869753,
+      "logps/chosen": -65.26039588212967,
+      "logps/rejected": -55.79923560500145,
+      "epoch": 0.19708521342254032,
+      "step": 190
+    },
+    {
+      "loss": 0.5344902038574219,
+      "grad_norm": 0.013661396689713001,
+      "learning_rate": 7.937823834196891e-06,
+      "entropy": 1.3751253792829812,
+      "num_tokens": 2902718.0,
+      "logits/chosen": -0.65461918654035,
+      "logits/rejected": -0.5536890222464702,
+      "mean_token_accuracy": 0.6590519631467759,
+      "rewards/chosen": -0.5584196869007791,
+      "rewards/rejected": -1.4234223538136574,
+      "rewards/accuracies": 0.77125,
+      "rewards/margins": 0.8650026666652412,
+      "logps/chosen": -61.13262881875038,
+      "logps/rejected": -56.61175240635872,
+      "epoch": 0.2074581193921477,
+      "step": 200
+    },
+    {
+      "loss": 0.4941723823547363,
+      "grad_norm": 0.008777705021202564,
+      "learning_rate": 7.834196891191712e-06,
+      "entropy": 1.3652805363852532,
+      "num_tokens": 3043885.0,
+      "logits/chosen": -0.7452820956668422,
+      "logits/rejected": -0.6034335554689337,
+      "mean_token_accuracy": 0.6580210606381297,
+      "rewards/chosen": -0.45352055150666276,
+      "rewards/rejected": -1.3747392913223302,
+      "rewards/accuracies": 0.78125,
+      "rewards/margins": 0.9212187370774336,
+      "logps/chosen": -58.416047328710555,
+      "logps/rejected": -53.947862002849575,
+      "epoch": 0.2178310253617551,
+      "step": 210
+    },
+    {
+      "loss": 0.47504258155822754,
+      "grad_norm": 0.009036123752593994,
+      "learning_rate": 7.730569948186528e-06,
+      "entropy": 1.4040400226414205,
+      "num_tokens": 3190499.0,
+      "logits/chosen": -0.6346807976930225,
+      "logits/rejected": -0.5184029459658583,
+      "mean_token_accuracy": 0.6466581939533352,
+      "rewards/chosen": -0.536972903214264,
+      "rewards/rejected": -1.5215354163426673,
+      "rewards/accuracies": 0.8025,
+      "rewards/margins": 0.984562511825934,
+      "logps/chosen": -62.78493340015412,
+      "logps/rejected": -58.09246099233627,
+      "epoch": 0.2282039313313625,
+      "step": 220
+    },
+    {
+      "loss": 0.45198073387146,
+      "grad_norm": 0.007527140900492668,
+      "learning_rate": 7.626943005181348e-06,
+      "entropy": 1.4020196551457047,
+      "num_tokens": 3337768.0,
+      "logits/chosen": -0.6210873045715655,
+      "logits/rejected": -0.5822921518676821,
+      "mean_token_accuracy": 0.6555538633279503,
+      "rewards/chosen": -0.4542729445986333,
+      "rewards/rejected": -1.5461742853268516,
+      "rewards/accuracies": 0.8175,
+      "rewards/margins": 1.091901341448538,
+      "logps/chosen": -67.90270529866218,
+      "logps/rejected": -58.65799897551537,
+      "epoch": 0.23857683730096987,
+      "step": 230
+    },
+    {
+      "loss": 0.4732102870941162,
+      "grad_norm": 0.016743697226047516,
+      "learning_rate": 7.523316062176167e-06,
+      "entropy": 1.408635692326352,
+      "num_tokens": 3483520.0,
+      "logits/chosen": -0.5801546989494524,
+      "logits/rejected": -0.5212051543725787,
+      "mean_token_accuracy": 0.6469839760102332,
+      "rewards/chosen": -0.5587615105054283,
+      "rewards/rejected": -1.562202037232928,
+      "rewards/accuracies": 0.8,
+      "rewards/margins": 1.003440523883328,
+      "logps/chosen": -65.64231182932853,
+      "logps/rejected": -58.95616222620011,
+      "epoch": 0.24894974327057726,
+      "step": 240
+    },
+    {
+      "loss": 0.48830270767211914,
+      "grad_norm": 0.010347820818424225,
+      "learning_rate": 7.419689119170985e-06,
+      "entropy": 1.3876401880290359,
+      "num_tokens": 3629401.0,
+      "logits/chosen": -0.6944931846154596,
+      "logits/rejected": -0.5525909304478883,
+      "mean_token_accuracy": 0.6595196689106524,
+      "rewards/chosen": -0.5708247896161629,
+      "rewards/rejected": -1.5276743739199445,
+      "rewards/accuracies": 0.7825,
+      "rewards/margins": 0.9568495863489807,
+      "logps/chosen": -65.0908039200306,
+      "logps/rejected": -59.28663974046707,
+      "epoch": 0.2593226492401846,
+      "step": 250
+    },
+    {
+      "eval_loss": 0.586716890335083,
+      "eval_runtime": 218.1079,
+      "eval_samples_per_second": 4.585,
+      "eval_steps_per_second": 4.585,
+      "eval_entropy": 0.9768514372222126,
+      "eval_num_tokens": 3629401.0,
+      "eval_logits/chosen": -0.288027065089679,
+      "eval_logits/rejected": -0.1875060971678181,
+      "eval_mean_token_accuracy": 0.715995571538806,
+      "eval_rewards/chosen": -0.13773577162688888,
+      "eval_rewards/rejected": -0.4831975232921541,
+      "eval_rewards/accuracies": 0.7,
+      "eval_rewards/margins": 0.3454617517972365,
+      "eval_logps/chosen": -51.550344535827634,
+      "eval_logps/rejected": -57.47184686470032,
+      "epoch": 0.2593226492401846,
+      "step": 250
+    },
+    {
+      "loss": 0.5234602928161621,
+      "grad_norm": 0.01417848002165556,
+      "learning_rate": 7.3160621761658035e-06,
+      "entropy": 1.4126720386464149,
+      "num_tokens": 3776772.0,
+      "logits/chosen": -0.6336535534010754,
+      "logits/rejected": -0.5590520939183056,
+      "mean_token_accuracy": 0.6559896361455322,
+      "rewards/chosen": -0.607539504897577,
+      "rewards/rejected": -1.5568346106001991,
+      "rewards/accuracies": 0.76625,
+      "rewards/margins": 0.9492951015941798,
+      "logps/chosen": -67.75836271762847,
+      "logps/rejected": -60.43963164687157,
+      "epoch": 0.26969555520979205,
+      "step": 260
+    },
+    {
+      "loss": 0.45171551704406737,
+      "grad_norm": 0.008155270479619503,
+      "learning_rate": 7.212435233160623e-06,
+      "entropy": 1.386252193665132,
+      "num_tokens": 3918594.0,
+      "logits/chosen": -0.692322886394876,
+      "logits/rejected": -0.5863808643424501,
+      "mean_token_accuracy": 0.6557753992639482,
+      "rewards/chosen": -0.4844140848268034,
+      "rewards/rejected": -1.5001980412582634,
+      "rewards/accuracies": 0.8075,
+      "rewards/margins": 1.0157839558646082,
+      "logps/chosen": -60.645376416444776,
+      "logps/rejected": -56.63234967470169,
+      "epoch": 0.28006846117939943,
+      "step": 270
+    },
+    {
+      "loss": 0.4645240306854248,
+      "grad_norm": 0.010044134221971035,
+      "learning_rate": 7.108808290155441e-06,
+      "entropy": 1.4262400729209184,
+      "num_tokens": 4064591.0,
+      "logits/chosen": -0.6778223895913023,
+      "logits/rejected": -0.5782403406452822,
+      "mean_token_accuracy": 0.6480579270608723,
+      "rewards/chosen": -0.4780267339639249,
+      "rewards/rejected": -1.5043474953982514,
+      "rewards/accuracies": 0.81,
+      "rewards/margins": 1.0263207618752495,
+      "logps/chosen": -66.15268970012664,
+      "logps/rejected": -56.97477602481842,
+      "epoch": 0.2904413671490068,
+      "step": 280
+    },
+    {
+      "loss": 0.48730764389038084,
+      "grad_norm": 0.011102184653282166,
+      "learning_rate": 7.005181347150259e-06,
+      "entropy": 1.3913073570653796,
+      "num_tokens": 4210880.0,
+      "logits/chosen": -0.6611820471311323,
+      "logits/rejected": -0.5711755343264715,
+      "mean_token_accuracy": 0.6609016039222478,
+      "rewards/chosen": -0.5330563006985176,
+      "rewards/rejected": -1.4805517839052482,
+      "rewards/accuracies": 0.79125,
+      "rewards/margins": 0.9474954811111093,
+      "logps/chosen": -64.55881326556205,
+      "logps/rejected": -58.55792887210846,
+      "epoch": 0.3008142731186142,
+      "step": 290
+    },
+    {
+      "loss": 0.4731907367706299,
+      "grad_norm": 0.008548606187105179,
+      "learning_rate": 6.9015544041450784e-06,
+      "entropy": 1.3624146432522684,
+      "num_tokens": 4356383.0,
+      "logits/chosen": -0.6928838046169237,
+      "logits/rejected": -0.6042798964782825,
+      "mean_token_accuracy": 0.6615170135349036,
+      "rewards/chosen": -0.562419148215231,
+      "rewards/rejected": -1.5397747305584197,
+      "rewards/accuracies": 0.8075,
+      "rewards/margins": 0.9773555782041512,
+      "logps/chosen": -63.553271045684816,
+      "logps/rejected": -58.97647937297821,
+      "epoch": 0.3111871790882216,
+      "step": 300
+    },
+    {
+      "loss": 0.49598259925842286,
+      "grad_norm": 0.007993742823600769,
+      "learning_rate": 6.797927461139897e-06,
+      "entropy": 1.3700548317469656,
+      "num_tokens": 4504654.0,
+      "logits/chosen": -0.644779540875669,
+      "logits/rejected": -0.5648210397573508,
+      "mean_token_accuracy": 0.6573794655874372,
+      "rewards/chosen": -0.5243248527575634,
+      "rewards/rejected": -1.5287844626943115,
+      "rewards/accuracies": 0.78625,
+      "rewards/margins": 1.00445960723795,
+      "logps/chosen": -64.00666294693947,
+      "logps/rejected": -61.50243379831314,
+      "epoch": 0.32156008505782896,
+      "step": 310
+    },
+    {
+      "loss": 0.4658979415893555,
+      "grad_norm": 0.010582108981907368,
+      "learning_rate": 6.6943005181347155e-06,
+      "entropy": 1.377428816948086,
+      "num_tokens": 4649289.0,
+      "logits/chosen": -0.6946231373321079,
+      "logits/rejected": -0.615736163527076,
+      "mean_token_accuracy": 0.6496203068085015,
+      "rewards/chosen": -0.5053178700394346,
+      "rewards/rejected": -1.5253387601411668,
+      "rewards/accuracies": 0.7975,
+      "rewards/margins": 1.020020889963489,
+      "logps/chosen": -65.05610402703286,
+      "logps/rejected": -58.5456183898449,
+      "epoch": 0.33193299102743634,
+      "step": 320
+    },
+    {
+      "loss": 0.4793886661529541,
+      "grad_norm": 0.010722431354224682,
+      "learning_rate": 6.590673575129535e-06,
+      "entropy": 1.4115223482623696,
+      "num_tokens": 4794149.0,
+      "logits/chosen": -0.640069723947673,
+      "logits/rejected": -0.5809591451997832,
+      "mean_token_accuracy": 0.6453945213742555,
+      "rewards/chosen": -0.495894172671542,
+      "rewards/rejected": -1.5554326001886511,
+      "rewards/accuracies": 0.77875,
+      "rewards/margins": 1.059538428708911,
+      "logps/chosen": -64.34471502423287,
+      "logps/rejected": -60.021323671340944,
+      "epoch": 0.3423058969970437,
+      "step": 330
+    },
+    {
+      "loss": 0.4895618438720703,
+      "grad_norm": 0.012254049070179462,
+      "learning_rate": 6.487046632124353e-06,
+      "entropy": 1.341764758527279,
+      "num_tokens": 4938961.0,
+      "logits/chosen": -0.6740448118186494,
+      "logits/rejected": -0.6176664939637678,
+      "mean_token_accuracy": 0.6506805537641048,
+      "rewards/chosen": -0.5430054054186622,
+      "rewards/rejected": -1.5190805373876355,
+      "rewards/accuracies": 0.78125,
+      "rewards/margins": 0.9760751294251532,
+      "logps/chosen": -62.98277623295784,
+      "logps/rejected": -58.627308850288394,
+      "epoch": 0.3526788029666511,
+      "step": 340
+    },
+    {
+      "loss": 0.4906949996948242,
+      "grad_norm": 0.012603401206433773,
+      "learning_rate": 6.383419689119171e-06,
+      "entropy": 1.335228986721486,
+      "num_tokens": 5085185.0,
+      "logits/chosen": -0.6529947110376427,
+      "logits/rejected": -0.6009205883950112,
+      "mean_token_accuracy": 0.6537076928000897,
+      "rewards/chosen": -0.5141149228686117,
+      "rewards/rejected": -1.5397958215233667,
+      "rewards/accuracies": 0.78375,
+      "rewards/margins": 1.0256808973429725,
+      "logps/chosen": -65.80777725458145,
+      "logps/rejected": -60.204533588886264,
+      "epoch": 0.3630517089362585,
+      "step": 350
+    },
+    {
+      "loss": 0.4906170845031738,
+      "grad_norm": 0.013215990737080574,
+      "learning_rate": 6.2797927461139905e-06,
+      "entropy": 1.2594143666606397,
+      "num_tokens": 5230359.0,
+      "logits/chosen": -0.7363216904006744,
+      "logits/rejected": -0.6437616409973009,
+      "mean_token_accuracy": 0.6658474483340978,
+      "rewards/chosen": -0.5248335571045755,
+      "rewards/rejected": -1.505687792309327,
+      "rewards/accuracies": 0.78,
+      "rewards/margins": 0.9808542363531888,
+      "logps/chosen": -61.92662429690361,
+      "logps/rejected": -57.71264513731003,
+      "epoch": 0.37342461490586587,
+      "step": 360
+    },
+    {
+      "loss": 0.47084531784057615,
+      "grad_norm": 0.010505126789212227,
+      "learning_rate": 6.176165803108809e-06,
+      "entropy": 1.3553831833507866,
+      "num_tokens": 5376488.0,
+      "logits/chosen": -0.6449173292890904,
+      "logits/rejected": -0.5551785184102145,
+      "mean_token_accuracy": 0.6537860439531505,
+      "rewards/chosen": -0.4608478151052259,
+      "rewards/rejected": -1.484516432384262,
+      "rewards/accuracies": 0.79125,
+      "rewards/margins": 1.0236686167120934,
+      "logps/chosen": -66.82504806637763,
+      "logps/rejected": -58.22365792274475,
+      "epoch": 0.38379752087547325,
+      "step": 370
+    },
+    {
+      "eval_loss": 0.5689035654067993,
+      "eval_runtime": 250.9045,
+      "eval_samples_per_second": 3.986,
+      "eval_steps_per_second": 3.986,
+      "eval_entropy": 0.9553070103861392,
+      "eval_num_tokens": 5448735.0,
+      "eval_logits/chosen": -0.2716756952684425,
+      "eval_logits/rejected": -0.16629699776825788,
+      "eval_mean_token_accuracy": 0.7177043009251356,
+      "eval_rewards/chosen": -0.11320603249309352,
+      "eval_rewards/rejected": -0.5053918305169282,
+      "eval_rewards/accuracies": 0.725,
+      "eval_rewards/margins": 0.392185798952356,
+      "eval_logps/chosen": -51.3050471496582,
+      "eval_logps/rejected": -57.69378993988037,
+      "epoch": 0.38898397386027694,
+      "step": 375
+    },
+    {
+      "loss": 0.45557408332824706,
+      "grad_norm": 0.010905127041041851,
+      "learning_rate": 6.0725388601036275e-06,
+      "entropy": 1.3120742352865635,
+      "num_tokens": 5521548.0,
+      "logits/chosen": -0.6177198584454004,
+      "logits/rejected": -0.5450247570549444,
+      "mean_token_accuracy": 0.6573154540918767,
+      "rewards/chosen": -0.3887985857592139,
+      "rewards/rejected": -1.4611027611684404,
+      "rewards/accuracies": 0.815,
+      "rewards/margins": 1.0723041738849133,
+      "logps/chosen": -62.481010044813154,
+      "logps/rejected": -56.344697498083114,
+      "epoch": 0.39417042684508063,
+      "step": 380
+    },
+    {
+      "loss": 0.4662487983703613,
+      "grad_norm": 0.010954899713397026,
+      "learning_rate": 5.968911917098445e-06,
+      "entropy": 1.345905083543621,
+      "num_tokens": 5667800.0,
+      "logits/chosen": -0.5844921973924755,
+      "logits/rejected": -0.48380765844974577,
+      "mean_token_accuracy": 0.6540606117062271,
+      "rewards/chosen": -0.390090519907244,
+      "rewards/rejected": -1.462948713658261,
+      "rewards/accuracies": 0.8025,
+      "rewards/margins": 1.07285819449462,
+      "logps/chosen": -64.31483766436577,
+      "logps/rejected": -59.39098523259163,
+      "epoch": 0.404543332814688,
+      "step": 390
+    },
+    {
+      "loss": 0.46701774597167967,
+      "grad_norm": 0.010829386301338673,
+      "learning_rate": 5.865284974093265e-06,
+      "entropy": 1.3651179377734661,
+      "num_tokens": 5813205.0,
+      "logits/chosen": -0.5822485740098883,
+      "logits/rejected": -0.5166833227123047,
+      "mean_token_accuracy": 0.6551306374650449,
+      "rewards/chosen": -0.35834275034227175,
+      "rewards/rejected": -1.365874043785443,
+      "rewards/accuracies": 0.78875,
+      "rewards/margins": 1.0075312922801822,
+      "logps/chosen": -63.65349508166313,
+      "logps/rejected": -58.2333509349823,
+      "epoch": 0.4149162387842954,
+      "step": 400
+    },
+    {
+      "loss": 0.46943206787109376,
+      "grad_norm": 0.012473917566239834,
+      "learning_rate": 5.761658031088083e-06,
+      "entropy": 1.3225441289972515,
+      "num_tokens": 5959181.0,
+      "logits/chosen": -0.6183900032329287,
+      "logits/rejected": -0.5336649034548987,
+      "mean_token_accuracy": 0.6569803632609547,
+      "rewards/chosen": -0.35909413290792147,
+      "rewards/rejected": -1.3612432872604223,
+      "rewards/accuracies": 0.795,
+      "rewards/margins": 1.002149153780192,
+      "logps/chosen": -62.85942430019379,
+      "logps/rejected": -57.029898535013196,
+      "epoch": 0.4252891447539028,
+      "step": 410
+    },
+    {
+      "loss": 0.43849844932556153,
+      "grad_norm": 0.008844634518027306,
+      "learning_rate": 5.658031088082902e-06,
+      "entropy": 1.3128622455336154,
+      "num_tokens": 6104961.0,
+      "logits/chosen": -0.6210691174054768,
+      "logits/rejected": -0.576289690726631,
+      "mean_token_accuracy": 0.6577657607197761,
+      "rewards/chosen": -0.3327711314160842,
+      "rewards/rejected": -1.4455869932868517,
+      "rewards/accuracies": 0.825,
+      "rewards/margins": 1.1128158613247796,
+      "logps/chosen": -64.98676935911179,
+      "logps/rejected": -57.15970268011093,
+      "epoch": 0.4356620507235102,
+      "step": 420
+    },
+    {
+      "loss": 0.43562884330749513,
+      "grad_norm": 0.00888186227530241,
+      "learning_rate": 5.554404145077721e-06,
+      "entropy": 1.3373534345161169,
+      "num_tokens": 6251772.0,
+      "logits/chosen": -0.5417993838933208,
+      "logits/rejected": -0.4885455063213445,
+      "mean_token_accuracy": 0.6521896417625248,
+      "rewards/chosen": -0.39902110468392493,
+      "rewards/rejected": -1.5569669758284,
+      "rewards/accuracies": 0.83625,
+      "rewards/margins": 1.1579458705382422,
+      "logps/chosen": -63.90601393699646,
+      "logps/rejected": -58.24345481872559,
+      "epoch": 0.4460349566931176,
+      "step": 430
+    },
+    {
+      "loss": 0.4367820262908936,
+      "grad_norm": 0.011835568584501743,
+      "learning_rate": 5.4507772020725395e-06,
+      "entropy": 1.302687416060362,
+      "num_tokens": 6397498.0,
+      "logits/chosen": -0.6515899833172267,
+      "logits/rejected": -0.5672783053215388,
+      "mean_token_accuracy": 0.6643678575474768,
+      "rewards/chosen": -0.4223745361570036,
+      "rewards/rejected": -1.5821643100841902,
+      "rewards/accuracies": 0.8225,
+      "rewards/margins": 1.1597897751070558,
+      "logps/chosen": -62.087783161401745,
+      "logps/rejected": -58.09040701627731,
+      "epoch": 0.456407862662725,
+      "step": 440
+    },
+    {
+      "loss": 0.4615288257598877,
+      "grad_norm": 0.011570082977414131,
+      "learning_rate": 5.347150259067357e-06,
+      "entropy": 1.3379024799168109,
+      "num_tokens": 6544292.0,
+      "logits/chosen": -0.6416285881723627,
+      "logits/rejected": -0.5313172360798328,
+      "mean_token_accuracy": 0.6538208884559572,
+      "rewards/chosen": -0.43649624643381685,
+      "rewards/rejected": -1.6286552884728007,
+      "rewards/accuracies": 0.79875,
+      "rewards/margins": 1.192159042903222,
+      "logps/chosen": -62.45707392811775,
+      "logps/rejected": -60.70825082540512,
+      "epoch": 0.46678076863233237,
+      "step": 450
+    },
+    {
+      "loss": 0.4522082328796387,
+      "grad_norm": 0.01289224810898304,
+      "learning_rate": 5.243523316062177e-06,
+      "entropy": 1.2857942930911668,
+      "num_tokens": 6688703.0,
+      "logits/chosen": -0.6984393716749847,
+      "logits/rejected": -0.616657341014624,
+      "mean_token_accuracy": 0.6569575572758913,
+      "rewards/chosen": -0.4177262162156694,
+      "rewards/rejected": -1.5946978869343729,
+      "rewards/accuracies": 0.82125,
+      "rewards/margins": 1.1769716703612358,
+      "logps/chosen": -62.88765250682831,
+      "logps/rejected": -57.941710426807404,
+      "epoch": 0.47715367460193975,
+      "step": 460
+    },
+    {
+      "loss": 0.4684887886047363,
+      "grad_norm": 0.013750020414590836,
+      "learning_rate": 5.139896373056995e-06,
+      "entropy": 1.2873771674977617,
+      "num_tokens": 6831976.0,
+      "logits/chosen": -0.7360928165548556,
+      "logits/rejected": -0.6532006934035273,
+      "mean_token_accuracy": 0.649969874471426,
+      "rewards/chosen": -0.45782311129412845,
+      "rewards/rejected": -1.5659245843085228,
+      "rewards/accuracies": 0.7975,
+      "rewards/margins": 1.1081014727987348,
+      "logps/chosen": -62.05344919681549,
+      "logps/rejected": -57.16824733495712,
+      "epoch": 0.48752658057154713,
+      "step": 470
+    },
+    {
+      "loss": 0.4470407962799072,
+      "grad_norm": 0.010775277391076088,
+      "learning_rate": 5.036269430051814e-06,
+      "entropy": 1.3075012436602265,
+      "num_tokens": 6979562.0,
+      "logits/chosen": -0.6950100893212269,
+      "logits/rejected": -0.6174856653398093,
+      "mean_token_accuracy": 0.6546831333450973,
+      "rewards/chosen": -0.4612816582483902,
+      "rewards/rejected": -1.6060521737975069,
+      "rewards/accuracies": 0.79625,
+      "rewards/margins": 1.1447705142386257,
+      "logps/chosen": -65.90533972740174,
+      "logps/rejected": -60.80634157061577,
+      "epoch": 0.4978994865411545,
+      "step": 480
+    },
+    {
+      "loss": 0.44890718460083007,
+      "grad_norm": 0.010780111886560917,
+      "learning_rate": 4.932642487046633e-06,
+      "entropy": 1.2887511976994575,
+      "num_tokens": 7126641.0,
+      "logits/chosen": -0.6761976126203127,
+      "logits/rejected": -0.6278953113861633,
+      "mean_token_accuracy": 0.6634333984181285,
+      "rewards/chosen": -0.4675208572049451,
+      "rewards/rejected": -1.6545327439898392,
+      "rewards/accuracies": 0.82375,
+      "rewards/margins": 1.1870118879154326,
+      "logps/chosen": -63.26052482247353,
+      "logps/rejected": -61.286433795690535,
+      "epoch": 0.5082723925107618,
+      "step": 490
+    },
+    {
+      "loss": 0.4814422607421875,
+      "grad_norm": 0.012935275211930275,
+      "learning_rate": 4.829015544041451e-06,
+      "entropy": 1.27828567199409,
+      "num_tokens": 7271511.0,
+      "logits/chosen": -0.7259033669003568,
+      "logits/rejected": -0.6351857639096569,
+      "mean_token_accuracy": 0.6566927696019411,
+      "rewards/chosen": -0.43330920343832985,
+      "rewards/rejected": -1.5484318724216428,
+      "rewards/accuracies": 0.78875,
+      "rewards/margins": 1.1151226695766672,
+      "logps/chosen": -61.74719344258308,
+      "logps/rejected": -59.84465143203735,
+      "epoch": 0.5186452984803692,
+      "step": 500
+    },
+    {
+      "eval_loss": 0.5487673282623291,
+      "eval_runtime": 282.9922,
+      "eval_samples_per_second": 3.534,
+      "eval_steps_per_second": 3.534,
+      "eval_entropy": 0.9428232955671847,
+      "eval_num_tokens": 7271511.0,
+      "eval_logits/chosen": -0.28059689932805515,
+      "eval_logits/rejected": -0.16151944836492374,
+      "eval_mean_token_accuracy": 0.7168366620391607,
+      "eval_rewards/chosen": -0.1563246784962248,
+      "eval_rewards/rejected": -0.6193268298726762,
+      "eval_rewards/accuracies": 0.754,
+      "eval_rewards/margins": 0.4630021521952003,
+      "eval_logps/chosen": -51.73623360443115,
+      "eval_logps/rejected": -58.83313990974426,
+      "epoch": 0.5186452984803692,
+      "step": 500
+    },
+    {
+      "loss": 0.4656700134277344,
+      "grad_norm": 0.01303118746727705,
+      "learning_rate": 4.72538860103627e-06,
+      "entropy": 1.2692697253311054,
+      "num_tokens": 7417281.0,
+      "logits/chosen": -0.6733742115379525,
+      "logits/rejected": -0.6193994454969072,
+      "mean_token_accuracy": 0.6586294612661004,
+      "rewards/chosen": -0.43541748865740376,
+      "rewards/rejected": -1.5032758536882467,
+      "rewards/accuracies": 0.815,
+      "rewards/margins": 1.0678583633713423,
+      "logps/chosen": -64.05021148204804,
+      "logps/rejected": -56.13307309627533,
+      "epoch": 0.5290182044499767,
+      "step": 510
+    },
+    {
+      "loss": 0.4864551067352295,
+      "grad_norm": 0.01242531556636095,
+      "learning_rate": 4.621761658031089e-06,
+      "entropy": 1.2640815615979955,
+      "num_tokens": 7560432.0,
+      "logits/chosen": -0.7002353169810469,
+      "logits/rejected": -0.6140356327426939,
+      "mean_token_accuracy": 0.6589579802565276,
+      "rewards/chosen": -0.38904498486765077,
+      "rewards/rejected": -1.4170939752570848,
+      "rewards/accuracies": 0.7925,
+      "rewards/margins": 1.0280489912256598,
+      "logps/chosen": -60.270782203674315,
+      "logps/rejected": -56.58300987482071,
+      "epoch": 0.5393911104195841,
+      "step": 520
+    },
+    {
+      "loss": 0.4569260597229004,
+      "grad_norm": 0.01068835612386465,
+      "learning_rate": 4.518134715025907e-06,
+      "entropy": 1.3159204521216452,
+      "num_tokens": 7705977.0,
+      "logits/chosen": -0.6491517350356284,
+      "logits/rejected": -0.5585772213969517,
+      "mean_token_accuracy": 0.6497207802906633,
+      "rewards/chosen": -0.3849831897905824,
+      "rewards/rejected": -1.5199620288111328,
+      "rewards/accuracies": 0.82125,
+      "rewards/margins": 1.1349788387073203,
+      "logps/chosen": -63.44159636735916,
+      "logps/rejected": -59.03181514501571,
+      "epoch": 0.5497640163891915,
+      "step": 530
+    },
+    {
+      "loss": 0.4587429046630859,
+      "grad_norm": 0.013803384266793728,
+      "learning_rate": 4.414507772020726e-06,
+      "entropy": 1.291237823315896,
+      "num_tokens": 7852079.0,
+      "logits/chosen": -0.6414628461310453,
+      "logits/rejected": -0.5613248039563115,
+      "mean_token_accuracy": 0.6623153394274414,
+      "rewards/chosen": -0.4346328362337226,
+      "rewards/rejected": -1.540268263059552,
+      "rewards/accuracies": 0.8125,
+      "rewards/margins": 1.1056354277441278,
+      "logps/chosen": -62.11032348752022,
+      "logps/rejected": -58.996895933151244,
+      "epoch": 0.5601369223587989,
+      "step": 540
+    },
+    {
+      "loss": 0.46028594970703124,
+      "grad_norm": 0.013468707911670208,
+      "learning_rate": 4.310880829015544e-06,
+      "entropy": 1.3394903557561337,
+      "num_tokens": 7999115.0,
+      "logits/chosen": -0.5828356510219183,
+      "logits/rejected": -0.5372745959336506,
+      "mean_token_accuracy": 0.6568853073753417,
+      "rewards/chosen": -0.4290946354267362,
+      "rewards/rejected": -1.5929214715841227,
+      "rewards/accuracies": 0.80625,
+      "rewards/margins": 1.1638268361473456,
+      "logps/chosen": -65.59857477068901,
+      "logps/rejected": -62.133251576423646,
+      "epoch": 0.5705098283284062,
+      "step": 550
+    },
+    {
+      "loss": 0.45781307220458983,
+      "grad_norm": 0.014243297278881073,
+      "learning_rate": 4.207253886010363e-06,
+      "entropy": 1.2652439445722847,
+      "num_tokens": 8145639.0,
+      "logits/chosen": -0.6962334313624986,
+      "logits/rejected": -0.6396616762561939,
+      "mean_token_accuracy": 0.6539523831009865,
+      "rewards/chosen": -0.42706657521594027,
+      "rewards/rejected": -1.5893809160019736,
+      "rewards/accuracies": 0.795,
+      "rewards/margins": 1.16231434000656,
+      "logps/chosen": -63.68221395015716,
+      "logps/rejected": -58.812913880348205,
+      "epoch": 0.5808827342980136,
+      "step": 560
+    },
+    {
+      "loss": 0.45180602073669435,
+      "grad_norm": 0.011750273406505585,
+      "learning_rate": 4.103626943005182e-06,
+      "entropy": 1.271689679301344,
+      "num_tokens": 8289932.0,
+      "logits/chosen": -0.6709009276653929,
+      "logits/rejected": -0.6110994225113662,
+      "mean_token_accuracy": 0.6528454353101552,
+      "rewards/chosen": -0.43575992634396243,
+      "rewards/rejected": -1.544044768281892,
+      "rewards/accuracies": 0.81875,
+      "rewards/margins": 1.1082848401460796,
+      "logps/chosen": -63.166086630821226,
+      "logps/rejected": -56.468742752075194,
+      "epoch": 0.591255640267621,
+      "step": 570
+    },
+    {
+      "loss": 0.46036481857299805,
+      "grad_norm": 0.014452456496655941,
+      "learning_rate": 4.000000000000001e-06,
+      "entropy": 1.254093002313748,
+      "num_tokens": 8433751.0,
+      "logits/chosen": -0.7479031936089587,
+      "logits/rejected": -0.660857903954597,
+      "mean_token_accuracy": 0.66091203879565,
+      "rewards/chosen": -0.4369000616581616,
+      "rewards/rejected": -1.5926436452555937,
+      "rewards/accuracies": 0.80375,
+      "rewards/margins": 1.1557435841672123,
+      "logps/chosen": -60.516501158475876,
+      "logps/rejected": -57.770808889865876,
+      "epoch": 0.6016285462372284,
+      "step": 580
+    },
+    {
+      "loss": 0.4958089828491211,
+      "grad_norm": 0.014941485598683357,
+      "learning_rate": 3.896373056994819e-06,
+      "entropy": 1.2861195527855307,
+      "num_tokens": 8580235.0,
+      "logits/chosen": -0.7134623334720516,
+      "logits/rejected": -0.6337572470141326,
+      "mean_token_accuracy": 0.6624758186563849,
+      "rewards/chosen": -0.48507625065831234,
+      "rewards/rejected": -1.5025616684707348,
+      "rewards/accuracies": 0.7675,
+      "rewards/margins": 1.0174854172300547,
+      "logps/chosen": -64.41009007811546,
+      "logps/rejected": -59.815929347276686,
+      "epoch": 0.6120014522068358,
+      "step": 590
+    },
+    {
+      "loss": 0.47800679206848146,
+      "grad_norm": 0.01430908776819706,
+      "learning_rate": 3.7927461139896377e-06,
+      "entropy": 1.2737568323127926,
+      "num_tokens": 8723363.0,
+      "logits/chosen": -0.6718185966589912,
+      "logits/rejected": -0.5801828286442322,
+      "mean_token_accuracy": 0.6622481289878488,
+      "rewards/chosen": -0.38251684666481195,
+      "rewards/rejected": -1.4414349760363712,
+      "rewards/accuracies": 0.78375,
+      "rewards/margins": 1.0589181298250332,
+      "logps/chosen": -59.11278188347816,
+      "logps/rejected": -57.41570061087609,
+      "epoch": 0.6223743581764432,
+      "step": 600
+    },
+    {
+      "loss": 0.4607038974761963,
+      "grad_norm": 0.016786962747573853,
+      "learning_rate": 3.6891191709844567e-06,
+      "entropy": 1.3031947114598006,
+      "num_tokens": 8867843.0,
+      "logits/chosen": -0.660966853554005,
+      "logits/rejected": -0.582685814465534,
+      "mean_token_accuracy": 0.6579678988829255,
+      "rewards/chosen": -0.3749055393272283,
+      "rewards/rejected": -1.4482067039191133,
+      "rewards/accuracies": 0.815,
+      "rewards/margins": 1.0733011648245157,
+      "logps/chosen": -59.77094477295876,
+      "logps/rejected": -58.26709199547768,
+      "epoch": 0.6327472641460505,
+      "step": 610
+    },
+    {
+      "loss": 0.4435013771057129,
+      "grad_norm": 0.013243520632386208,
+      "learning_rate": 3.5854922279792748e-06,
+      "entropy": 1.2442782195843756,
+      "num_tokens": 9012429.0,
+      "logits/chosen": -0.7155245964218712,
+      "logits/rejected": -0.62228141826014,
+      "mean_token_accuracy": 0.6603258750028909,
+      "rewards/chosen": -0.35220212864573114,
+      "rewards/rejected": -1.49272357024136,
+      "rewards/accuracies": 0.8175,
+      "rewards/margins": 1.1405214420426637,
+      "logps/chosen": -58.541049842834475,
+      "logps/rejected": -57.54325157284737,
+      "epoch": 0.6431201701156579,
+      "step": 620
+    },
+    {
+      "eval_loss": 0.5402435660362244,
+      "eval_runtime": 293.8352,
+      "eval_samples_per_second": 3.403,
+      "eval_steps_per_second": 3.403,
+      "eval_entropy": 0.9412421704679728,
+      "eval_num_tokens": 9084893.0,
+      "eval_logits/chosen": -0.2831247920504241,
+      "eval_logits/rejected": -0.15805534941173774,
+      "eval_mean_token_accuracy": 0.7173487603366375,
+      "eval_rewards/chosen": -0.163662012138695,
+      "eval_rewards/rejected": -0.6573172951535962,
+      "eval_rewards/accuracies": 0.763,
+      "eval_rewards/margins": 0.49365528268739584,
+      "eval_logps/chosen": -51.8096069393158,
+      "eval_logps/rejected": -59.21304455375672,
+      "epoch": 0.6483066231004616,
+      "step": 625
+    },
+    {
+      "loss": 0.45146970748901366,
+      "grad_norm": 0.011234515346586704,
+      "learning_rate": 3.4818652849740937e-06,
+      "entropy": 1.2889114275900646,
+      "num_tokens": 9158703.0,
+      "logits/chosen": -0.6542699654733424,
+      "logits/rejected": -0.5822428945211472,
+      "mean_token_accuracy": 0.6584817282296718,
+      "rewards/chosen": -0.3719235458994808,
+      "rewards/rejected": -1.5630993095623853,
+      "rewards/accuracies": 0.8,
+      "rewards/margins": 1.1911757623543964,
+      "logps/chosen": -61.62524257659912,
+      "logps/rejected": -59.77060217618942,
+      "epoch": 0.6534930760852653,
+      "step": 630
+    },
+    {
+      "loss": 0.45473732948303225,
+      "grad_norm": 0.01427957508713007,
+      "learning_rate": 3.3782383419689123e-06,
+      "entropy": 1.270301983555546,
+      "num_tokens": 9303628.0,
+      "logits/chosen": -0.6727831000063272,
+      "logits/rejected": -0.5958556125497929,
+      "mean_token_accuracy": 0.6592282411269843,
+      "rewards/chosen": -0.39630664114709363,
+      "rewards/rejected": -1.5085867938393493,
+      "rewards/accuracies": 0.81125,
+      "rewards/margins": 1.112280152433086,
+      "logps/chosen": -60.25154410600662,
+      "logps/rejected": -59.81609448671341,
+      "epoch": 0.6638659820548727,
+      "step": 640
+    },
+    {
+      "loss": 0.4724769115447998,
+      "grad_norm": 0.011104694567620754,
+      "learning_rate": 3.274611398963731e-06,
+      "entropy": 1.2825965376850217,
+      "num_tokens": 9447859.0,
+      "logits/chosen": -0.6886272934207637,
+      "logits/rejected": -0.6191571248326155,
+      "mean_token_accuracy": 0.657224724534899,
+      "rewards/chosen": -0.39751313600787397,
+      "rewards/rejected": -1.501351127484304,
+      "rewards/accuracies": 0.80375,
+      "rewards/margins": 1.1038379945326597,
+      "logps/chosen": -61.231552537679676,
+      "logps/rejected": -57.24312862277031,
+      "epoch": 0.6742388880244801,
+      "step": 650
+    },
+    {
+      "loss": 0.48994994163513184,
+      "grad_norm": 0.015749365091323853,
+      "learning_rate": 3.1709844559585493e-06,
+      "entropy": 1.2718282664287834,
+      "num_tokens": 9591275.0,
+      "logits/chosen": -0.7000141689252796,
+      "logits/rejected": -0.6344551697912493,
+      "mean_token_accuracy": 0.6556407183595002,
+      "rewards/chosen": -0.3735031143521701,
+      "rewards/rejected": -1.4077299095626221,
+      "rewards/accuracies": 0.8075,
+      "rewards/margins": 1.034226797488518,
+      "logps/chosen": -60.32739723324776,
+      "logps/rejected": -56.30893276691437,
+      "epoch": 0.6846117939940874,
+      "step": 660
+    },
+    {
+      "loss": 0.42113280296325684,
+      "grad_norm": 0.011937125585973263,
+      "learning_rate": 3.0673575129533683e-06,
+      "entropy": 1.27422906415537,
+      "num_tokens": 9737336.0,
+      "logits/chosen": -0.6675869420516058,
+      "logits/rejected": -0.6171588368781001,
+      "mean_token_accuracy": 0.6570126633532345,
+      "rewards/chosen": -0.3044447978468088,
+      "rewards/rejected": -1.5031647271365,
+      "rewards/accuracies": 0.8175,
+      "rewards/margins": 1.1987199306860565,
+      "logps/chosen": -62.614556882381436,
+      "logps/rejected": -57.67475826740265,
+      "epoch": 0.6949846999636948,
+      "step": 670
+    },
+    {
+      "loss": 0.4509533405303955,
+      "grad_norm": 0.014425016939640045,
+      "learning_rate": 2.963730569948187e-06,
+      "entropy": 1.2917946016066708,
+      "num_tokens": 9884203.0,
+      "logits/chosen": -0.6209272877296821,
+      "logits/rejected": -0.5547300822758102,
+      "mean_token_accuracy": 0.6614484623633325,
+      "rewards/chosen": -0.34687376230140216,
+      "rewards/rejected": -1.5009351192926987,
+      "rewards/accuracies": 0.8025,
+      "rewards/margins": 1.1540613523963839,
+      "logps/chosen": -63.675368639230726,
+      "logps/rejected": -59.546725879907605,
+      "epoch": 0.7053576059333022,
+      "step": 680
+    },
+    {
+      "loss": 0.49210338592529296,
+      "grad_norm": 0.01453041099011898,
+      "learning_rate": 2.8601036269430053e-06,
+      "entropy": 1.318970390278846,
+      "num_tokens": 10030490.0,
+      "logits/chosen": -0.6129511188439006,
+      "logits/rejected": -0.5470711603742071,
+      "mean_token_accuracy": 0.6580428531672805,
+      "rewards/chosen": -0.35959291283600125,
+      "rewards/rejected": -1.394242706346704,
+      "rewards/accuracies": 0.80625,
+      "rewards/margins": 1.0346497943252324,
+      "logps/chosen": -64.7718941605091,
+      "logps/rejected": -57.55288832068443,
+      "epoch": 0.7157305119029096,
+      "step": 690
+    },
+    {
+      "loss": 0.4248403549194336,
+      "grad_norm": 0.012130402959883213,
+      "learning_rate": 2.7564766839378243e-06,
+      "entropy": 1.2868076485674829,
+      "num_tokens": 10175134.0,
+      "logits/chosen": -0.6680388671827403,
+      "logits/rejected": -0.5829757325025796,
+      "mean_token_accuracy": 0.6597682436835021,
+      "rewards/chosen": -0.30255757274717326,
+      "rewards/rejected": -1.4632275151461362,
+      "rewards/accuracies": 0.82875,
+      "rewards/margins": 1.1606699449336155,
+      "logps/chosen": -62.52647110342979,
+      "logps/rejected": -54.934954075813295,
+      "epoch": 0.726103417872517,
+      "step": 700
+    },
+    {
+      "loss": 0.46398053169250486,
+      "grad_norm": 0.010899660177528858,
+      "learning_rate": 2.6528497409326424e-06,
+      "entropy": 1.3012964005907997,
+      "num_tokens": 10321605.0,
+      "logits/chosen": -0.6793537159810091,
+      "logits/rejected": -0.5945389351521791,
+      "mean_token_accuracy": 0.6582902568951249,
+      "rewards/chosen": -0.3586843095816721,
+      "rewards/rejected": -1.5040888605307554,
+      "rewards/accuracies": 0.78625,
+      "rewards/margins": 1.1454045504983514,
+      "logps/chosen": -66.0418261361122,
+      "logps/rejected": -57.98255445957184,
+      "epoch": 0.7364763238421244,
+      "step": 710
+    },
+    {
+      "loss": 0.45627450942993164,
+      "grad_norm": 0.015107187442481518,
+      "learning_rate": 2.5492227979274614e-06,
+      "entropy": 1.2992314287554472,
+      "num_tokens": 10464696.0,
+      "logits/chosen": -0.6788668626078029,
+      "logits/rejected": -0.598369878754869,
+      "mean_token_accuracy": 0.6590372899640351,
+      "rewards/chosen": -0.32335921899306413,
+      "rewards/rejected": -1.4580287650012178,
+      "rewards/accuracies": 0.80375,
+      "rewards/margins": 1.1346695464709773,
+      "logps/chosen": -60.56386103153229,
+      "logps/rejected": -55.05920248985291,
+      "epoch": 0.7468492298117317,
+      "step": 720
+    },
+    {
+      "loss": 0.45744032859802247,
+      "grad_norm": 0.015074568800628185,
+      "learning_rate": 2.44559585492228e-06,
+      "entropy": 1.3187106101540849,
+      "num_tokens": 10611681.0,
+      "logits/chosen": -0.6193487567657021,
+      "logits/rejected": -0.5510364978587164,
+      "mean_token_accuracy": 0.6565000848285854,
+      "rewards/chosen": -0.3346571850046166,
+      "rewards/rejected": -1.5139837610156974,
+      "rewards/accuracies": 0.7975,
+      "rewards/margins": 1.1793265779595823,
+      "logps/chosen": -62.09045646429062,
+      "logps/rejected": -61.43362274646759,
+      "epoch": 0.7572221357813391,
+      "step": 730
+    },
+    {
+      "loss": 0.44286222457885743,
+      "grad_norm": 0.01326123159378767,
+      "learning_rate": 2.3419689119170984e-06,
+      "entropy": 1.327558269179426,
+      "num_tokens": 10756633.0,
+      "logits/chosen": -0.6719882830268841,
+      "logits/rejected": -0.5616402227001511,
+      "mean_token_accuracy": 0.6571272361278534,
+      "rewards/chosen": -0.33738715873330877,
+      "rewards/rejected": -1.5023427059937966,
+      "rewards/accuracies": 0.82125,
+      "rewards/margins": 1.1649555454589426,
+      "logps/chosen": -62.31321774363518,
+      "logps/rejected": -59.3358825302124,
+      "epoch": 0.7675950417509465,
+      "step": 740
+    },
+    {
+      "loss": 0.4650571823120117,
+      "grad_norm": 0.0126119963824749,
+      "learning_rate": 2.2383419689119174e-06,
+      "entropy": 1.29965307561215,
+      "num_tokens": 10905175.0,
+      "logits/chosen": -0.6533030424492793,
+      "logits/rejected": -0.5875300805405916,
+      "mean_token_accuracy": 0.6576582338660956,
+      "rewards/chosen": -0.40247532632221467,
+      "rewards/rejected": -1.5875308242870414,
+      "rewards/accuracies": 0.79,
+      "rewards/margins": 1.1850554993841798,
+      "logps/chosen": -65.17352761864662,
+      "logps/rejected": -62.21493496775627,
+      "epoch": 0.7779679477205539,
+      "step": 750
+    },
+    {
+      "eval_loss": 0.5359359383583069,
+      "eval_runtime": 320.8935,
+      "eval_samples_per_second": 3.116,
+      "eval_steps_per_second": 3.116,
+      "eval_entropy": 0.9404631896130741,
+      "eval_num_tokens": 10905175.0,
+      "eval_logits/chosen": -0.2688203306854898,
+      "eval_logits/rejected": -0.1410600388922386,
+      "eval_mean_token_accuracy": 0.7175761694908143,
+      "eval_rewards/chosen": -0.1531370894421125,
+      "eval_rewards/rejected": -0.6597482364820425,
+      "eval_rewards/accuracies": 0.764,
+      "eval_rewards/margins": 0.5066111467555166,
+      "eval_logps/chosen": -51.70435771179199,
+      "eval_logps/rejected": -59.23735397338867,
+      "epoch": 0.7779679477205539,
+      "step": 750
+    },
+    {
+      "loss": 0.44441781044006345,
+      "grad_norm": 0.01466372236609459,
+      "learning_rate": 2.134715025906736e-06,
+      "entropy": 1.28976634433493,
+      "num_tokens": 11051293.0,
+      "logits/chosen": -0.7179963626211149,
+      "logits/rejected": -0.6301743637240872,
+      "mean_token_accuracy": 0.6519320147298276,
+      "rewards/chosen": -0.35903158226523374,
+      "rewards/rejected": -1.4595573616991169,
+      "rewards/accuracies": 0.805,
+      "rewards/margins": 1.100525778569281,
+      "logps/chosen": -62.626072100400926,
+      "logps/rejected": -58.10015594720841,
+      "epoch": 0.7883408536901613,
+      "step": 760
+    },
+    {
+      "loss": 0.4427034854888916,
+      "grad_norm": 0.012671389617025852,
+      "learning_rate": 2.0310880829015544e-06,
+      "entropy": 1.3044293180131354,
+      "num_tokens": 11200144.0,
+      "logits/chosen": -0.6538815830450716,
+      "logits/rejected": -0.5728917452531731,
+      "mean_token_accuracy": 0.6632946115918458,
+      "rewards/chosen": -0.37740983954085094,
+      "rewards/rejected": -1.6017498846648959,
+      "rewards/accuracies": 0.82375,
+      "rewards/margins": 1.2243400452192872,
+      "logps/chosen": -65.87450021147728,
+      "logps/rejected": -62.50144905328751,
+      "epoch": 0.7987137596597687,
+      "step": 770
+    },
+    {
+      "loss": 0.47789411544799804,
+      "grad_norm": 0.014209717512130737,
+      "learning_rate": 1.9274611398963734e-06,
+      "entropy": 1.2642307026335038,
+      "num_tokens": 11345001.0,
+      "logits/chosen": -0.7598325245824121,
+      "logits/rejected": -0.6345409980160763,
+      "mean_token_accuracy": 0.6561008535698056,
+      "rewards/chosen": -0.4101038860027188,
+      "rewards/rejected": -1.5055885665896493,
+      "rewards/accuracies": 0.7925,
+      "rewards/margins": 1.09548467958346,
+      "logps/chosen": -60.939151479005815,
+      "logps/rejected": -59.33771441936493,
+      "epoch": 0.809086665629376,
+      "step": 780
+    },
+    {
+      "loss": 0.45084495544433595,
+      "grad_norm": 0.01159591879695654,
+      "learning_rate": 1.823834196891192e-06,
+      "entropy": 1.2825998100219294,
+      "num_tokens": 11490957.0,
+      "logits/chosen": -0.6798757077356125,
+      "logits/rejected": -0.5974159600128078,
+      "mean_token_accuracy": 0.6551897264830768,
+      "rewards/chosen": -0.37878839314620566,
+      "rewards/rejected": -1.565106635761622,
+      "rewards/accuracies": 0.81125,
+      "rewards/margins": 1.1863182406220585,
+      "logps/chosen": -62.11761864900589,
+      "logps/rejected": -58.61645050764084,
+      "epoch": 0.8194595715989834,
+      "step": 790
+    },
+    {
+      "loss": 0.4757333278656006,
+      "grad_norm": 0.01929500699043274,
+      "learning_rate": 1.7202072538860104e-06,
+      "entropy": 1.3137290544318967,
+      "num_tokens": 11635952.0,
+      "logits/chosen": -0.7116650574710064,
+      "logits/rejected": -0.6434065148170602,
+      "mean_token_accuracy": 0.6526090941950679,
+      "rewards/chosen": -0.39931287897430595,
+      "rewards/rejected": -1.51923489038134,
+      "rewards/accuracies": 0.80625,
+      "rewards/margins": 1.119922012281604,
+      "logps/chosen": -63.77980515360832,
+      "logps/rejected": -58.51232698440552,
+      "epoch": 0.8298324775685908,
+      "step": 800
+    },
+    {
+      "loss": 0.4533553600311279,
+      "grad_norm": 0.011488179676234722,
+      "learning_rate": 1.6165803108808292e-06,
+      "entropy": 1.2781752744410186,
+      "num_tokens": 11781419.0,
+      "logits/chosen": -0.7322858283680784,
+      "logits/rejected": -0.6704393794115492,
+      "mean_token_accuracy": 0.6515787079930305,
+      "rewards/chosen": -0.3753399283361523,
+      "rewards/rejected": -1.5520857451565098,
+      "rewards/accuracies": 0.805,
+      "rewards/margins": 1.1767458136426285,
+      "logps/chosen": -62.6512784576416,
+      "logps/rejected": -59.1226301574707,
+      "epoch": 0.8402053835381982,
+      "step": 810
+    },
+    {
+      "loss": 0.459058952331543,
+      "grad_norm": 0.010664924047887325,
+      "learning_rate": 1.5129533678756477e-06,
+      "entropy": 1.2839164751721546,
+      "num_tokens": 11926234.0,
+      "logits/chosen": -0.668114556411545,
+      "logits/rejected": -0.6314948882239511,
+      "mean_token_accuracy": 0.6545133054628969,
+      "rewards/chosen": -0.3676813648158713,
+      "rewards/rejected": -1.5132245452463393,
+      "rewards/accuracies": 0.815,
+      "rewards/margins": 1.145543181069661,
+      "logps/chosen": -63.80598108768463,
+      "logps/rejected": -57.3190500497818,
+      "epoch": 0.8505782895078056,
+      "step": 820
+    },
+    {
+      "loss": 0.45433964729309084,
+      "grad_norm": 0.013902204111218452,
+      "learning_rate": 1.4093264248704663e-06,
+      "entropy": 1.2906162818288431,
+      "num_tokens": 12071032.0,
+      "logits/chosen": -0.6819364086266573,
+      "logits/rejected": -0.5817802480567199,
+      "mean_token_accuracy": 0.6650253617018461,
+      "rewards/chosen": -0.3574298277559137,
+      "rewards/rejected": -1.5355529067799216,
+      "rewards/accuracies": 0.82125,
+      "rewards/margins": 1.1781230779876932,
+      "logps/chosen": -59.579396767616274,
+      "logps/rejected": -58.291543385982514,
+      "epoch": 0.860951195477413,
+      "step": 830
+    },
+    {
+      "loss": 0.43956761360168456,
+      "grad_norm": 0.017366426065564156,
+      "learning_rate": 1.3056994818652852e-06,
+      "entropy": 1.3188584124017506,
+      "num_tokens": 12216650.0,
+      "logits/chosen": -0.6219026821907607,
+      "logits/rejected": -0.5560462281839198,
+      "mean_token_accuracy": 0.6576781215891242,
+      "rewards/chosen": -0.3260511834644603,
+      "rewards/rejected": -1.558354403564008,
+      "rewards/accuracies": 0.80625,
+      "rewards/margins": 1.2323032197169959,
+      "logps/chosen": -64.06994863152504,
+      "logps/rejected": -58.97870760083199,
+      "epoch": 0.8713241014470204,
+      "step": 840
+    },
+    {
+      "loss": 0.46459689140319826,
+      "grad_norm": 0.012618121691048145,
+      "learning_rate": 1.2020725388601037e-06,
+      "entropy": 1.315548148807138,
+      "num_tokens": 12362070.0,
+      "logits/chosen": -0.6251195627372872,
+      "logits/rejected": -0.5604575453557197,
+      "mean_token_accuracy": 0.6604597151651979,
+      "rewards/chosen": -0.3555148102169187,
+      "rewards/rejected": -1.5313407544395887,
+      "rewards/accuracies": 0.79625,
+      "rewards/margins": 1.1758259430155158,
+      "logps/chosen": -62.78060804247856,
+      "logps/rejected": -58.929887549877165,
+      "epoch": 0.8816970074166278,
+      "step": 850
+    },
+    {
+      "loss": 0.44550557136535646,
+      "grad_norm": 0.013532118871808052,
+      "learning_rate": 1.0984455958549225e-06,
+      "entropy": 1.3122669545235113,
+      "num_tokens": 12509338.0,
+      "logits/chosen": -0.6126990686398949,
+      "logits/rejected": -0.5539581104426821,
+      "mean_token_accuracy": 0.6569988044165075,
+      "rewards/chosen": -0.38766197150049264,
+      "rewards/rejected": -1.5939512548525818,
+      "rewards/accuracies": 0.81875,
+      "rewards/margins": 1.2062892844853923,
+      "logps/chosen": -65.91424354076385,
+      "logps/rejected": -59.84309137582779,
+      "epoch": 0.8920699133862352,
+      "step": 860
+    },
+    {
+      "loss": 0.44367341995239257,
+      "grad_norm": 0.014622218906879425,
+      "learning_rate": 9.94818652849741e-07,
+      "entropy": 1.29490221851971,
+      "num_tokens": 12656013.0,
+      "logits/chosen": -0.6367152223575993,
+      "logits/rejected": -0.5619754576335303,
+      "mean_token_accuracy": 0.6582811014540494,
+      "rewards/chosen": -0.3575555875984719,
+      "rewards/rejected": -1.485135663910769,
+      "rewards/accuracies": 0.81625,
+      "rewards/margins": 1.1275800754828378,
+      "logps/chosen": -64.74992589473725,
+      "logps/rejected": -58.28154501080513,
+      "epoch": 0.9024428193558426,
+      "step": 870
+    },
+    {
+      "eval_loss": 0.5303567051887512,
+      "eval_runtime": 351.9964,
+      "eval_samples_per_second": 2.841,
+      "eval_steps_per_second": 2.841,
+      "eval_entropy": 0.9427960855923593,
+      "eval_num_tokens": 12728578.0,
+      "eval_logits/chosen": -0.26863109543646074,
+      "eval_logits/rejected": -0.13725877180195847,
+      "eval_mean_token_accuracy": 0.7159041211903096,
+      "eval_rewards/chosen": -0.17772461017644672,
+      "eval_rewards/rejected": -0.7113549838000909,
+      "eval_rewards/accuracies": 0.763,
+      "eval_rewards/margins": 0.5336303729685024,
+      "eval_logps/chosen": -51.95023292160034,
+      "eval_logps/rejected": -59.75342144203186,
+      "epoch": 0.9076292723406463,
+      "step": 875
+    },
+    {
+      "loss": 0.42188777923583987,
+      "grad_norm": 0.014083835296332836,
+      "learning_rate": 8.911917098445596e-07,
+      "entropy": 1.2842787204217165,
+      "num_tokens": 12801884.0,
+      "logits/chosen": -0.6543839594717639,
+      "logits/rejected": -0.5576565805204583,
+      "mean_token_accuracy": 0.6647658421378583,
+      "rewards/chosen": -0.32246251756288985,
+      "rewards/rejected": -1.561580713846779,
+      "rewards/accuracies": 0.825,
+      "rewards/margins": 1.2391181947570293,
+      "logps/chosen": -60.62016844153404,
+      "logps/rejected": -60.981985919475555,
+      "epoch": 0.91281572532545,
+      "step": 880
+    },
+    {
+      "loss": 0.4861095905303955,
+      "grad_norm": 0.01509900763630867,
+      "learning_rate": 7.875647668393784e-07,
+      "entropy": 1.2736156480619685,
+      "num_tokens": 12946912.0,
+      "logits/chosen": -0.7128123134940787,
+      "logits/rejected": -0.6054497727959169,
+      "mean_token_accuracy": 0.6572731367498637,
+      "rewards/chosen": -0.4675049069095985,
+      "rewards/rejected": -1.5612919216800947,
+      "rewards/accuracies": 0.7825,
+      "rewards/margins": 1.0937870144005866,
+      "logps/chosen": -61.985627712011336,
+      "logps/rejected": -60.579013855457305,
+      "epoch": 0.9231886312950573,
+      "step": 890
+    },
+    {
+      "loss": 0.43419761657714845,
+      "grad_norm": 0.014685068279504776,
+      "learning_rate": 6.839378238341969e-07,
+      "entropy": 1.2819190438790247,
+      "num_tokens": 13093558.0,
+      "logits/chosen": -0.6502587498062717,
+      "logits/rejected": -0.5614254849582602,
+      "mean_token_accuracy": 0.6592135391384363,
+      "rewards/chosen": -0.36472693517142035,
+      "rewards/rejected": -1.6288802374559601,
+      "rewards/accuracies": 0.82,
+      "rewards/margins": 1.2641532999722402,
+      "logps/chosen": -61.57865165352821,
+      "logps/rejected": -62.50596989512444,
+      "epoch": 0.9335615372646647,
+      "step": 900
+    },
+    {
+      "loss": 0.4454813003540039,
+      "grad_norm": 0.011209873482584953,
+      "learning_rate": 5.803108808290156e-07,
+      "entropy": 1.3184868184709921,
+      "num_tokens": 13239740.0,
+      "logits/chosen": -0.6909169377714659,
+      "logits/rejected": -0.601211010607464,
+      "mean_token_accuracy": 0.6575440725311636,
+      "rewards/chosen": -0.36972430324880406,
+      "rewards/rejected": -1.6006858524744165,
+      "rewards/accuracies": 0.82875,
+      "rewards/margins": 1.2309615502599627,
+      "logps/chosen": -63.300528687238696,
+      "logps/rejected": -60.001617946624755,
+      "epoch": 0.9439344432342721,
+      "step": 910
+    },
+    {
+      "loss": 0.4595484733581543,
+      "grad_norm": 0.010628749616444111,
+      "learning_rate": 4.7668393782383424e-07,
+      "entropy": 1.271923498995602,
+      "num_tokens": 13383045.0,
+      "logits/chosen": -0.7072264696856618,
+      "logits/rejected": -0.6024060889326301,
+      "mean_token_accuracy": 0.6569805011339486,
+      "rewards/chosen": -0.3887278968687133,
+      "rewards/rejected": -1.5501526693382766,
+      "rewards/accuracies": 0.815,
+      "rewards/margins": 1.1614247707859613,
+      "logps/chosen": -60.07761764883995,
+      "logps/rejected": -56.95813281774521,
+      "epoch": 0.9543073492038795,
+      "step": 920
+    },
+    {
+      "loss": 0.4529257297515869,
+      "grad_norm": 0.014489009976387024,
+      "learning_rate": 3.730569948186528e-07,
+      "entropy": 1.2988391564786435,
+      "num_tokens": 13530054.0,
+      "logits/chosen": -0.7166704098766242,
+      "logits/rejected": -0.6650401865166429,
+      "mean_token_accuracy": 0.6520283976197243,
+      "rewards/chosen": -0.4043681784943692,
+      "rewards/rejected": -1.5586467561577593,
+      "rewards/accuracies": 0.81875,
+      "rewards/margins": 1.1542785764392465,
+      "logps/chosen": -64.28691901445389,
+      "logps/rejected": -60.51436370611191,
+      "epoch": 0.9646802551734869,
+      "step": 930
+    },
+    {
+      "loss": 0.4493688106536865,
+      "grad_norm": 0.0122548658400774,
+      "learning_rate": 2.694300518134715e-07,
+      "entropy": 1.293265828914009,
+      "num_tokens": 13676771.0,
+      "logits/chosen": -0.638114123144128,
+      "logits/rejected": -0.6033606771543887,
+      "mean_token_accuracy": 0.6584486217796802,
+      "rewards/chosen": -0.353517619818158,
+      "rewards/rejected": -1.4791943001397885,
+      "rewards/accuracies": 0.82625,
+      "rewards/margins": 1.1256766823492945,
+      "logps/chosen": -63.997545466423034,
+      "logps/rejected": -58.525367016792295,
+      "epoch": 0.9750531611430943,
+      "step": 940
+    },
+    {
+      "loss": 0.46651930809020997,
+      "grad_norm": 0.01745172217488289,
+      "learning_rate": 1.6580310880829015e-07,
+      "entropy": 1.2602451098989695,
+      "num_tokens": 13822091.0,
+      "logits/chosen": -0.7271677677940491,
+      "logits/rejected": -0.6540065888305867,
+      "mean_token_accuracy": 0.6601362200826406,
+      "rewards/chosen": -0.40040454823116306,
+      "rewards/rejected": -1.5421940996023478,
+      "rewards/accuracies": 0.80125,
+      "rewards/margins": 1.141789550515823,
+      "logps/chosen": -61.94225021719932,
+      "logps/rejected": -58.28781437397003,
+      "epoch": 0.9854260671127016,
+      "step": 950
+    },
+    {
+      "loss": 0.46761231422424315,
+      "grad_norm": 0.012348760850727558,
+      "learning_rate": 6.217616580310881e-08,
+      "entropy": 1.2971244535176083,
+      "num_tokens": 13969713.0,
+      "logits/chosen": -0.6472266813048246,
+      "logits/rejected": -0.5813531516986534,
+      "mean_token_accuracy": 0.6541070010513067,
+      "rewards/chosen": -0.4221547889366411,
+      "rewards/rejected": -1.5586294095951598,
+      "rewards/accuracies": 0.7925,
+      "rewards/margins": 1.1364746230933815,
+      "logps/chosen": -64.67578131318092,
+      "logps/rejected": -59.52407863378525,
+      "epoch": 0.995798973082309,
+      "step": 960
+    },
+    {
+      "train_runtime": 22000.2483,
+      "train_samples_per_second": 3.506,
+      "train_steps_per_second": 0.044,
+      "total_flos": 1.2331818072769536e+17,
+      "train_loss": 0.4788281094842624,
+      "entropy": 1.3463219977616343,
+      "num_tokens": 14029916.0,
+      "logits/chosen": -0.5587557057517026,
+      "logits/rejected": -0.5043508288937856,
+      "mean_token_accuracy": 0.6551804588072829,
+      "rewards/chosen": -0.42263167897738046,
+      "rewards/rejected": -1.5862010600171799,
+      "rewards/accuracies": 0.8148148148148148,
+      "rewards/margins": 1.163569376240542,
+      "logps/chosen": -66.20575960182849,
+      "logps/rejected": -63.83122500666865,
+      "epoch": 1.0,
+      "step": 965
+    }
+  ],
+  "validation_monitor_size": 1000,
+  "validation_monitor_selection": "fixed_seed_random_without_replacement",
+  "validation_monitor_seed": 22,
+  "validation_monitor_indices": [
+    2,
+    10,
+    14,
+    21,
+    55,
+    63,
+    66,
+    69,
+    107,
+    110,
+    142,
+    144,
+    149,
+    150,
+    151,
+    152,
+    160,
+    163,
+    166,
+    167,
+    176,
+    181,
+    206,
+    207,
+    259,
+    267,
+    281,
+    295,
+    298,
+    306,
+    307,
+    317,
+    321,
+    331,
+    336,
+    341,
+    346,
+    349,
+    351,
+    352,
+    357,
+    358,
+    369,
+    370,
+    382,
+    386,
+    391,
+    401,
+    411,
+    412,
+    432,
+    437,
+    452,
+    462,
+    465,
+    488,
+    490,
+    491,
+    492,
+    494,
+    503,
+    504,
+    508,
+    528,
+    538,
+    540,
+    551,
+    553,
+    567,
+    586,
+    605,
+    606,
+    625,
+    627,
+    661,
+    663,
+    666,
+    677,
+    685,
+    690,
+    692,
+    704,
+    708,
+    714,
+    715,
+    727,
+    728,
+    733,
+    745,
+    752,
+    753,
+    755,
+    764,
+    773,
+    779,
+    783,
+    793,
+    796,
+    799,
+    803,
+    804,
+    805,
+    813,
+    816,
+    818,
+    823,
+    827,
+    829,
+    830,
+    837,
+    842,
+    845,
+    850,
+    853,
+    856,
+    889,
+    890,
+    905,
+    915,
+    924,
+    930,
+    939,
+    951,
+    988,
+    1002,
+    1005,
+    1023,
+    1029,
+    1038,
+    1049,
+    1050,
+    1054,
+    1056,
+    1067,
+    1068,
+    1079,
+    1088,
+    1091,
+    1103,
+    1114,
+    1118,
+    1133,
+    1140,
+    1144,
+    1145,
+    1155,
+    1186,
+    1195,
+    1206,
+    1223,
+    1251,
+    1252,
+    1257,
+    1259,
+    1270,
+    1271,
+    1295,
+    1303,
+    1304,
+    1305,
+    1329,
+    1335,
+    1336,
+    1343,
+    1367,
+    1373,
+    1377,
+    1403,
+    1412,
+    1429,
+    1443,
+    1457,
+    1459,
+    1460,
+    1476,
+    1483,
+    1486,
+    1494,
+    1507,
+    1510,
+    1519,
+    1521,
+    1522,
+    1545,
+    1551,
+    1570,
+    1582,
+    1593,
+    1595,
+    1603,
+    1607,
+    1614,
+    1615,
+    1625,
+    1634,
+    1639,
+    1648,
+    1654,
+    1657,
+    1662,
+    1667,
+    1673,
+    1690,
+    1704,
+    1746,
+    1756,
+    1781,
+    1783,
+    1796,
+    1799,
+    1809,
+    1814,
+    1827,
+    1829,
+    1832,
+    1844,
+    1847,
+    1854,
+    1856,
+    1857,
+    1858,
+    1874,
+    1883,
+    1889,
+    1924,
+    1925,
+    1931,
+    1932,
+    1934,
+    1935,
+    1938,
+    1950,
+    1957,
+    1962,
+    1967,
+    1975,
+    1982,
+    1983,
+    1991,
+    1998,
+    2003,
+    2008,
+    2017,
+    2021,
+    2026,
+    2035,
+    2040,
+    2050,
+    2056,
+    2077,
+    2079,
+    2082,
+    2098,
+    2100,
+    2108,
+    2121,
+    2130,
+    2133,
+    2134,
+    2138,
+    2143,
+    2166,
+    2167,
+    2180,
+    2181,
+    2185,
+    2204,
+    2205,
+    2212,
+    2221,
+    2224,
+    2226,
+    2230,
+    2233,
+    2256,
+    2261,
+    2263,
+    2269,
+    2273,
+    2290,
+    2291,
+    2299,
+    2301,
+    2321,
+    2323,
+    2330,
+    2384,
+    2401,
+    2417,
+    2420,
+    2421,
+    2424,
+    2430,
+    2435,
+    2456,
+    2488,
+    2502,
+    2504,
+    2519,
+    2527,
+    2532,
+    2538,
+    2542,
+    2553,
+    2555,
+    2558,
+    2559,
+    2562,
+    2578,
+    2583,
+    2585,
+    2590,
+    2592,
+    2594,
+    2596,
+    2600,
+    2606,
+    2607,
+    2618,
+    2630,
+    2637,
+    2647,
+    2650,
+    2657,
+    2679,
+    2685,
+    2705,
+    2706,
+    2712,
+    2713,
+    2714,
+    2727,
+    2740,
+    2742,
+    2755,
+    2780,
+    2784,
+    2792,
+    2807,
+    2808,
+    2810,
+    2820,
+    2831,
+    2839,
+    2860,
+    2862,
+    2863,
+    2866,
+    2875,
+    2878,
+    2898,
+    2905,
+    2921,
+    2922,
+    2926,
+    2930,
+    2934,
+    2944,
+    2955,
+    2957,
+    2959,
+    2973,
+    2978,
+    2998,
+    3018,
+    3022,
+    3028,
+    3031,
+    3061,
+    3085,
+    3090,
+    3104,
+    3105,
+    3111,
+    3115,
+    3121,
+    3122,
+    3129,
+    3133,
+    3135,
+    3161,
+    3162,
+    3169,
+    3173,
+    3194,
+    3195,
+    3215,
+    3225,
+    3226,
+    3241,
+    3247,
+    3250,
+    3253,
+    3265,
+    3268,
+    3293,
+    3301,
+    3312,
+    3329,
+    3352,
+    3361,
+    3362,
+    3376,
+    3396,
+    3401,
+    3403,
+    3410,
+    3419,
+    3432,
+    3443,
+    3452,
+    3467,
+    3469,
+    3475,
+    3485,
+    3503,
+    3514,
+    3515,
+    3524,
+    3528,
+    3538,
+    3544,
+    3557,
+    3560,
+    3565,
+    3600,
+    3637,
+    3642,
+    3658,
+    3659,
+    3692,
+    3693,
+    3699,
+    3722,
+    3725,
+    3728,
+    3731,
+    3740,
+    3742,
+    3762,
+    3766,
+    3780,
+    3788,
+    3794,
+    3796,
+    3798,
+    3805,
+    3817,
+    3819,
+    3822,
+    3837,
+    3839,
+    3843,
+    3846,
+    3851,
+    3854,
+    3865,
+    3870,
+    3871,
+    3884,
+    3894,
+    3895,
+    3896,
+    3907,
+    3911,
+    3915,
+    3919,
+    3920,
+    3923,
+    3933,
+    3955,
+    3967,
+    3972,
+    3974,
+    3975,
+    3984,
+    3985,
+    3997,
+    4002,
+    4010,
+    4034,
+    4044,
+    4063,
+    4073,
+    4079,
+    4082,
+    4088,
+    4121,
+    4145,
+    4148,
+    4159,
+    4161,
+    4164,
+    4177,
+    4188,
+    4199,
+    4203,
+    4207,
+    4208,
+    4213,
+    4221,
+    4225,
+    4233,
+    4241,
+    4243,
+    4247,
+    4264,
+    4274,
+    4282,
+    4286,
+    4290,
+    4308,
+    4310,
+    4313,
+    4321,
+    4324,
+    4327,
+    4349,
+    4362,
+    4370,
+    4374,
+    4380,
+    4407,
+    4409,
+    4411,
+    4415,
+    4417,
+    4418,
+    4427,
+    4431,
+    4433,
+    4451,
+    4466,
+    4477,
+    4478,
+    4479,
+    4493,
+    4494,
+    4514,
+    4527,
+    4539,
+    4550,
+    4558,
+    4568,
+    4579,
+    4583,
+    4584,
+    4586,
+    4587,
+    4590,
+    4599,
+    4602,
+    4610,
+    4626,
+    4627,
+    4630,
+    4641,
+    4647,
+    4655,
+    4656,
+    4657,
+    4661,
+    4685,
+    4714,
+    4715,
+    4731,
+    4749,
+    4752,
+    4769,
+    4777,
+    4782,
+    4791,
+    4805,
+    4818,
+    4829,
+    4833,
+    4837,
+    4839,
+    4843,
+    4871,
+    4875,
+    4879,
+    4880,
+    4885,
+    4888,
+    4895,
+    4900,
+    4923,
+    4966,
+    4968,
+    4972,
+    4989,
+    4994,
+    4998,
+    5001,
+    5013,
+    5019,
+    5026,
+    5032,
+    5034,
+    5046,
+    5055,
+    5085,
+    5086,
+    5088,
+    5089,
+    5090,
+    5095,
+    5108,
+    5110,
+    5119,
+    5120,
+    5121,
+    5133,
+    5148,
+    5154,
+    5160,
+    5169,
+    5178,
+    5222,
+    5223,
+    5232,
+    5233,
+    5240,
+    5256,
+    5259,
+    5264,
+    5271,
+    5276,
+    5279,
+    5294,
+    5300,
+    5303,
+    5321,
+    5335,
+    5337,
+    5345,
+    5348,
+    5365,
+    5373,
+    5378,
+    5384,
+    5422,
+    5442,
+    5443,
+    5445,
+    5477,
+    5485,
+    5495,
+    5497,
+    5504,
+    5526,
+    5533,
+    5542,
+    5564,
+    5570,
+    5579,
+    5587,
+    5592,
+    5608,
+    5610,
+    5624,
+    5630,
+    5638,
+    5651,
+    5663,
+    5670,
+    5675,
+    5691,
+    5700,
+    5706,
+    5707,
+    5715,
+    5720,
+    5721,
+    5722,
+    5732,
+    5738,
+    5741,
+    5769,
+    5771,
+    5775,
+    5795,
+    5796,
+    5800,
+    5809,
+    5810,
+    5815,
+    5819,
+    5827,
+    5848,
+    5849,
+    5852,
+    5859,
+    5880,
+    5884,
+    5907,
+    5909,
+    5912,
+    5919,
+    5931,
+    5932,
+    5934,
+    5941,
+    5948,
+    5950,
+    5952,
+    5953,
+    5969,
+    5981,
+    6000,
+    6003,
+    6010,
+    6018,
+    6041,
+    6065,
+    6075,
+    6090,
+    6103,
+    6106,
+    6109,
+    6114,
+    6123,
+    6131,
+    6136,
+    6138,
+    6139,
+    6164,
+    6165,
+    6171,
+    6173,
+    6180,
+    6185,
+    6189,
+    6190,
+    6221,
+    6223,
+    6237,
+    6255,
+    6262,
+    6265,
+    6293,
+    6296,
+    6305,
+    6318,
+    6331,
+    6336,
+    6340,
+    6355,
+    6366,
+    6371,
+    6396,
+    6399,
+    6402,
+    6408,
+    6432,
+    6433,
+    6441,
+    6456,
+    6465,
+    6478,
+    6486,
+    6489,
+    6507,
+    6508,
+    6520,
+    6522,
+    6528,
+    6537,
+    6551,
+    6564,
+    6589,
+    6590,
+    6598,
+    6599,
+    6611,
+    6613,
+    6615,
+    6621,
+    6634,
+    6647,
+    6649,
+    6654,
+    6676,
+    6680,
+    6690,
+    6708,
+    6729,
+    6736,
+    6744,
+    6749,
+    6756,
+    6761,
+    6763,
+    6773,
+    6788,
+    6790,
+    6796,
+    6797,
+    6811,
+    6824,
+    6850,
+    6869,
+    6871,
+    6882,
+    6892,
+    6895,
+    6906,
+    6911,
+    6912,
+    6914,
+    6927,
+    6952,
+    6966,
+    6985,
+    7001,
+    7021,
+    7031,
+    7035,
+    7038,
+    7041,
+    7045,
+    7052,
+    7057,
+    7058,
+    7072,
+    7073,
+    7076,
+    7086,
+    7102,
+    7107,
+    7109,
+    7117,
+    7122,
+    7125,
+    7166,
+    7182,
+    7199,
+    7207,
+    7212,
+    7215,
+    7232,
+    7243,
+    7246,
+    7250,
+    7253,
+    7258,
+    7263,
+    7267,
+    7270,
+    7274,
+    7280,
+    7286,
+    7293,
+    7298,
+    7302,
+    7306,
+    7316,
+    7325,
+    7326,
+    7334,
+    7356,
+    7357,
+    7363,
+    7364,
+    7367,
+    7385,
+    7392,
+    7399,
+    7405,
+    7416,
+    7420,
+    7421,
+    7426,
+    7452,
+    7476,
+    7481,
+    7519,
+    7534,
+    7542,
+    7546,
+    7573,
+    7585,
+    7601,
+    7604,
+    7606,
+    7609,
+    7629,
+    7649,
+    7653,
+    7667,
+    7682,
+    7699,
+    7738,
+    7750,
+    7786,
+    7798,
+    7800,
+    7801,
+    7805,
+    7806,
+    7811,
+    7813,
+    7832,
+    7837,
+    7849,
+    7856,
+    7876,
+    7877,
+    7887,
+    7905,
+    7916,
+    7919,
+    7920,
+    7922,
+    7923,
+    7926,
+    7944,
+    7961,
+    7966,
+    7970,
+    7973,
+    7974,
+    7976,
+    7986,
+    7999,
+    8027,
+    8028,
+    8034,
+    8047,
+    8068,
+    8074,
+    8077,
+    8091,
+    8120,
+    8122,
+    8125,
+    8152,
+    8153,
+    8164,
+    8167,
+    8169,
+    8171,
+    8177,
+    8184,
+    8189,
+    8192,
+    8196,
+    8202,
+    8212,
+    8217,
+    8231,
+    8242,
+    8244,
+    8250,
+    8256,
+    8257,
+    8265,
+    8270,
+    8274,
+    8283,
+    8290,
+    8294,
+    8301,
+    8302,
+    8307,
+    8318,
+    8326,
+    8338,
+    8349,
+    8350,
+    8353,
+    8357,
+    8371,
+    8374,
+    8377,
+    8380,
+    8387,
+    8388,
+    8396,
+    8402,
+    8419,
+    8423,
+    8428,
+    8435,
+    8439,
+    8442,
+    8444,
+    8453,
+    8461,
+    8475,
+    8481,
+    8485,
+    8493,
+    8495,
+    8498,
+    8523,
+    8530,
+    8531,
+    8562
+  ],
+  "early_stopping_patience": 3
+}

phase1/ablations/random_labels/README.md

@@ -0,0 +1,80 @@
+---
+library_name: peft
+model_name: phase1-qwen-random-labels-ablation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-1.5B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+license: apache-2.0
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+pipeline_tag: text-generation
+---
+
+# Phase 1 Qwen `random_labels` diagnostic ablation
+
+This is the Phase 1 `random_labels` diagnostic LoRA-DPO adapter fine-tuned from
+[Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct).
+The frozen training recipe and membership hashes are included in this directory.
+
+## Quick start
+
+```python
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo_id = "geyuxu/york-milestone-project-self-traing-loop-model"
+subfolder = "phase1/ablations/random_labels"
+base_id = "Qwen/Qwen2.5-1.5B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(repo_id, subfolder=subfolder)
+base_model = AutoModelForCausalLM.from_pretrained(
+    base_id, torch_dtype="auto", device_map="auto"
+)
+model = PeftModel.from_pretrained(base_model, repo_id, subfolder=subfolder)
+```
+
+## Training procedure
+
+
+
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 0.29.0
+- Transformers: 5.3.0
+- Pytorch: 2.10.0
+- Datasets: 4.6.1
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+    title        = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+    author       = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+    year         = 2023,
+    booktitle    = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+    url          = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+    editor       = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

phase1/ablations/random_labels/adapter_config.json

@@ -0,0 +1,43 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "k_proj",
+    "o_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

phase1/ablations/random_labels/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bebded9e10f63603bfd9d6a135dda87c8bcac25b4a43b365e2b1ce882bf4c93
+size 17462432

phase1/ablations/random_labels/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

phase1/ablations/random_labels/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b4360dd6a184650ffc48056c2569bc603f896c5adfe94b10f1c79f809638aa5
+size 11422166

phase1/ablations/random_labels/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

phase1/ablations/random_labels/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad84a00370d0c977fcd38a1fce033b130494f3fffd4a3c0cbc44a9dd43c0bdbf
+size 5841

phase1/ablations/random_labels/training_recipe.json

@@ -0,0 +1,22 @@
+{
+  "format_version": 1,
+  "experiment": "confusion_ablation",
+  "ablation": "random_labels",
+  "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+  "train_membership_sha256": "b497fa765223a9810ad784f31e1598edfa992a162146d6f94d9cdf4adcedd87d",
+  "validation_membership_sha256": "2f609467438cd6a01fd84b9d80c0fcd84c44767e3b2ed98f4f6a83d68a62c181",
+  "num_train_pairs": 77124,
+  "num_validation_pairs": 8563,
+  "transformation_seed": 22,
+  "validation_prompt_mode": "clean_rag",
+  "learning_rate": 1e-05,
+  "beta": 0.1,
+  "num_epochs": 1.0,
+  "seed": 22,
+  "max_length": 512,
+  "max_prompt_length": 384,
+  "gradient_accumulation_steps": 80,
+  "save_steps": 125,
+  "save_total_limit": 12,
+  "preference_pairs_format_version": 2
+}

phase1/ablations/random_labels/training_summary.json

@@ -0,0 +1,2517 @@
+{
+  "model_name": "Qwen/Qwen2.5-1.5B-Instruct",
+  "resolved_model": "Qwen/Qwen2.5-1.5B-Instruct",
+  "num_train_records": 77124,
+  "num_validation_records_available": 8563,
+  "training_config": {
+    "lora_rank": 16,
+    "lora_alpha": 32,
+    "learning_rate": 1e-05,
+    "num_epochs": 1.0,
+    "per_device_batch_size": 1,
+    "per_device_eval_batch_size": 1,
+    "gradient_accumulation_steps": 80,
+    "beta": 0.1,
+    "max_length": 512,
+    "max_prompt_length": 384,
+    "seed": 22,
+    "save_steps": 125,
+    "save_total_limit": 12,
+    "resume_from_checkpoint": null,
+    "precision_dtype": "torch.bfloat16",
+    "bf16": true,
+    "fp16": false,
+    "tokenizer_add_bos_token": false
+  },
+  "global_step": 750,
+  "best_metric": 0.6905523538589478,
+  "best_model_checkpoint": "outputs/confusion_ablation_phase1_repair_v3/random_labels/lora/checkpoint-375",
+  "train_metrics": {
+    "train_runtime": 27561.1083,
+    "train_samples_per_second": 2.798,
+    "train_steps_per_second": 0.035,
+    "total_flos": 4.075839588886364e+17,
+    "train_loss": 0.6933243910471598
+  },
+  "log_history": [
+    {
+      "loss": 0.6934387683868408,
+      "grad_norm": 0.009266729466617107,
+      "learning_rate": 9.906735751295338e-06,
+      "entropy": 0.8080450683785603,
+      "num_tokens": 669528.0,
+      "logits/chosen": -0.9213643590964665,
+      "logits/rejected": -0.9683298173272368,
+      "mean_token_accuracy": 0.7228609627112746,
+      "rewards/chosen": 0.0007450232512769616,
+      "rewards/rejected": 0.001045633094436198,
+      "rewards/accuracies": 0.44,
+      "rewards/margins": -0.0003006098522382672,
+      "logps/chosen": -51.68650775671005,
+      "logps/rejected": -50.719243061542514,
+      "epoch": 0.010372905969607386,
+      "step": 10
+    },
+    {
+      "loss": 0.6930126190185547,
+      "grad_norm": 0.009303063154220581,
+      "learning_rate": 9.803108808290156e-06,
+      "entropy": 0.7915201940061525,
+      "num_tokens": 1338759.0,
+      "logits/chosen": -0.9846059676325559,
+      "logits/rejected": -0.9714732527595322,
+      "mean_token_accuracy": 0.7229587784409524,
+      "rewards/chosen": -0.00012686825164792025,
+      "rewards/rejected": -0.0007469337083875871,
+      "rewards/accuracies": 0.48625,
+      "rewards/margins": 0.0006200655033171643,
+      "logps/chosen": -50.57927699804306,
+      "logps/rejected": -51.70512897491455,
+      "epoch": 0.02074581193921477,
+      "step": 20
+    },
+    {
+      "loss": 0.693855619430542,
+      "grad_norm": 0.009770886041224003,
+      "learning_rate": 9.699481865284975e-06,
+      "entropy": 0.8051797628332861,
+      "num_tokens": 2009918.0,
+      "logits/chosen": -1.0153866272103405,
+      "logits/rejected": -0.9948517174473516,
+      "mean_token_accuracy": 0.7255033986549825,
+      "rewards/chosen": 0.000551955679097773,
+      "rewards/rejected": 0.0015872831797685194,
+      "rewards/accuracies": 0.495,
+      "rewards/margins": -0.0010353274924636935,
+      "logps/chosen": -51.339062130451204,
+      "logps/rejected": -52.78074060201645,
+      "epoch": 0.031118717908822157,
+      "step": 30
+    },
+    {
+      "loss": 0.69316725730896,
+      "grad_norm": 0.008510654792189598,
+      "learning_rate": 9.595854922279793e-06,
+      "entropy": 0.7799565396877006,
+      "num_tokens": 2680367.0,
+      "logits/chosen": -1.0400407581404876,
+      "logits/rejected": -0.9926833103096355,
+      "mean_token_accuracy": 0.7316536229476333,
+      "rewards/chosen": 0.0013948245377036982,
+      "rewards/rejected": 0.001106869012758125,
+      "rewards/accuracies": 0.4775,
+      "rewards/margins": 0.0002879555685285595,
+      "logps/chosen": -50.836872832775114,
+      "logps/rejected": -52.290282304286954,
+      "epoch": 0.04149162387842954,
+      "step": 40
+    },
+    {
+      "loss": 0.6937148094177246,
+      "grad_norm": 0.00940113328397274,
+      "learning_rate": 9.492227979274612e-06,
+      "entropy": 0.784734519557096,
+      "num_tokens": 3350695.0,
+      "logits/chosen": -1.0085622581752787,
+      "logits/rejected": -0.9717813104276766,
+      "mean_token_accuracy": 0.7277172536216676,
+      "rewards/chosen": 0.0007200987445082774,
+      "rewards/rejected": 0.001504770267456479,
+      "rewards/accuracies": 0.495,
+      "rewards/margins": -0.0007846715349296573,
+      "logps/chosen": -50.88885704755783,
+      "logps/rejected": -51.595236620903016,
+      "epoch": 0.051864529848036925,
+      "step": 50
+    },
+    {
+      "loss": 0.6925368785858155,
+      "grad_norm": 0.01000447291880846,
+      "learning_rate": 9.388601036269432e-06,
+      "entropy": 0.7792391402181238,
+      "num_tokens": 4019867.0,
+      "logits/chosen": -0.989283063151833,
+      "logits/rejected": -0.9758414097701983,
+      "mean_token_accuracy": 0.7253596894256771,
+      "rewards/chosen": 0.00419369634595455,
+      "rewards/rejected": 0.00261573103512319,
+      "rewards/accuracies": 0.515,
+      "rewards/margins": 0.0015779653389472515,
+      "logps/chosen": -50.12135502576828,
+      "logps/rejected": -49.72630759477615,
+      "epoch": 0.062237435817644314,
+      "step": 60
+    },
+    {
+      "loss": 0.6928633213043213,
+      "grad_norm": 0.008523018099367619,
+      "learning_rate": 9.284974093264249e-06,
+      "entropy": 0.7859808096196502,
+      "num_tokens": 4689830.0,
+      "logits/chosen": -0.9542620254422428,
+      "logits/rejected": -0.9723457128166635,
+      "mean_token_accuracy": 0.7191485194861889,
+      "rewards/chosen": 0.006607790327220755,
+      "rewards/rejected": 0.005695816857155478,
+      "rewards/accuracies": 0.50625,
+      "rewards/margins": 0.0009119734441992478,
+      "logps/chosen": -52.69678068399429,
+      "logps/rejected": -49.40951271772385,
+      "epoch": 0.0726103417872517,
+      "step": 70
+    },
+    {
+      "loss": 0.6931606769561768,
+      "grad_norm": 0.009254093281924725,
+      "learning_rate": 9.181347150259067e-06,
+      "entropy": 0.8076216962421313,
+      "num_tokens": 5359790.0,
+      "logits/chosen": -0.9293808590029637,
+      "logits/rejected": -0.9847294370574589,
+      "mean_token_accuracy": 0.7200651096645743,
+      "rewards/chosen": 0.011739555572351037,
+      "rewards/rejected": 0.011363767318175632,
+      "rewards/accuracies": 0.4875,
+      "rewards/margins": 0.00037578820330963936,
+      "logps/chosen": -51.38067405223846,
+      "logps/rejected": -50.72847024202347,
+      "epoch": 0.08298324775685909,
+      "step": 80
+    },
+    {
+      "loss": 0.6942155361175537,
+      "grad_norm": 0.008851570077240467,
+      "learning_rate": 9.077720207253888e-06,
+      "entropy": 0.8039566927566193,
+      "num_tokens": 6029642.0,
+      "logits/chosen": -0.9478755777728227,
+      "logits/rejected": -0.9763882057133852,
+      "mean_token_accuracy": 0.7233742567896843,
+      "rewards/chosen": 0.01409353660361603,
+      "rewards/rejected": 0.015802892130523674,
+      "rewards/accuracies": 0.48375,
+      "rewards/margins": -0.001709355540224351,
+      "logps/chosen": -51.33848252058029,
+      "logps/rejected": -52.321811312437056,
+      "epoch": 0.09335615372646647,
+      "step": 90
+    },
+    {
+      "loss": 0.6936118602752686,
+      "grad_norm": 0.009475240483880043,
+      "learning_rate": 8.974093264248706e-06,
+      "entropy": 0.8324551365058869,
+      "num_tokens": 6703750.0,
+      "logits/chosen": -0.9270073968773294,
+      "logits/rejected": -0.9359207155608903,
+      "mean_token_accuracy": 0.7229399124160409,
+      "rewards/chosen": 0.014994417404241176,
+      "rewards/rejected": 0.015462831489760163,
+      "rewards/accuracies": 0.49625,
+      "rewards/margins": -0.0004684141512552742,
+      "logps/chosen": -55.6720170044899,
+      "logps/rejected": -56.016610951423644,
+      "epoch": 0.10372905969607385,
+      "step": 100
+    },
+    {
+      "loss": 0.6934245586395263,
+      "grad_norm": 0.008504034951329231,
+      "learning_rate": 8.870466321243523e-06,
+      "entropy": 0.7861935871373862,
+      "num_tokens": 7374932.0,
+      "logits/chosen": -0.965592457466306,
+      "logits/rejected": -0.9543162182972067,
+      "mean_token_accuracy": 0.7248496604338288,
+      "rewards/chosen": 0.016234664240764685,
+      "rewards/rejected": 0.016324117717140468,
+      "rewards/accuracies": 0.495,
+      "rewards/margins": -8.945348919951357e-05,
+      "logps/chosen": -50.975947844982144,
+      "logps/rejected": -51.70058757305145,
+      "epoch": 0.11410196566568125,
+      "step": 110
+    },
+    {
+      "loss": 0.6940177917480469,
+      "grad_norm": 0.009204786270856857,
+      "learning_rate": 8.766839378238343e-06,
+      "entropy": 0.7727768113883212,
+      "num_tokens": 8045025.0,
+      "logits/chosen": -1.03487960348445,
+      "logits/rejected": -0.9815775502270385,
+      "mean_token_accuracy": 0.7311673213727772,
+      "rewards/chosen": 0.01172756811563886,
+      "rewards/rejected": 0.01306509348746431,
+      "rewards/accuracies": 0.47875,
+      "rewards/margins": -0.0013375253186222835,
+      "logps/chosen": -49.14816425204277,
+      "logps/rejected": -51.98841401100159,
+      "epoch": 0.12447487163528863,
+      "step": 120
+    },
+    {
+      "eval_loss": 0.6955941319465637,
+      "eval_runtime": 196.5871,
+      "eval_samples_per_second": 5.087,
+      "eval_steps_per_second": 5.087,
+      "eval_entropy": 0.8034458611719311,
+      "eval_num_tokens": 8380297.0,
+      "eval_logits/chosen": -0.991135582788011,
+      "eval_logits/rejected": -0.950536454501934,
+      "eval_mean_token_accuracy": 0.7337288541048765,
+      "eval_rewards/chosen": 0.008104163538169359,
+      "eval_rewards/rejected": 0.01258701845750079,
+      "eval_rewards/accuracies": 0.437,
+      "eval_rewards/margins": -0.0044828549473313615,
+      "eval_logps/chosen": -50.09194520950317,
+      "eval_logps/rejected": -52.51400153923035,
+      "epoch": 0.1296613246200923,
+      "step": 125
+    },
+    {
+      "loss": 0.6930018901824951,
+      "grad_norm": 0.00838699284940958,
+      "learning_rate": 8.663212435233162e-06,
+      "entropy": 0.8142909701773897,
+      "num_tokens": 8716283.0,
+      "logits/chosen": -0.971143901841604,
+      "logits/rejected": -1.0017587172299427,
+      "mean_token_accuracy": 0.729599024951458,
+      "rewards/chosen": 0.009170935571445398,
+      "rewards/rejected": 0.008475723866073964,
+      "rewards/accuracies": 0.52375,
+      "rewards/margins": 0.0006952117068431108,
+      "logps/chosen": -51.31078794002533,
+      "logps/rejected": -53.1062014913559,
+      "epoch": 0.13484777760489602,
+      "step": 130
+    },
+    {
+      "loss": 0.6927727699279785,
+      "grad_norm": 0.009330841712653637,
+      "learning_rate": 8.55958549222798e-06,
+      "entropy": 0.7786517775012181,
+      "num_tokens": 9385323.0,
+      "logits/chosen": -1.0336392573752233,
+      "logits/rejected": -1.019804302735747,
+      "mean_token_accuracy": 0.7317706294171512,
+      "rewards/chosen": 0.004249347767993186,
+      "rewards/rejected": 0.0031492145435004205,
+      "rewards/accuracies": 0.51875,
+      "rewards/margins": 0.001100133234722307,
+      "logps/chosen": -48.82712723731995,
+      "logps/rejected": -49.81842919588089,
+      "epoch": 0.1452206835745034,
+      "step": 140
+    },
+    {
+      "loss": 0.6928228378295899,
+      "grad_norm": 0.009226374328136444,
+      "learning_rate": 8.455958549222799e-06,
+      "entropy": 0.7707585748913698,
+      "num_tokens": 10057097.0,
+      "logits/chosen": -1.0021628340641857,
+      "logits/rejected": -1.0272578388562763,
+      "mean_token_accuracy": 0.7250917989574373,
+      "rewards/chosen": 0.0021179335123679265,
+      "rewards/rejected": 0.0010407016340150221,
+      "rewards/accuracies": 0.51875,
+      "rewards/margins": 0.0010772318854651531,
+      "logps/chosen": -50.82353290557862,
+      "logps/rejected": -51.57363982439041,
+      "epoch": 0.1555935895441108,
+      "step": 150
+    },
+    {
+      "loss": 0.6935010433197022,
+      "grad_norm": 0.008547156117856503,
+      "learning_rate": 8.352331606217617e-06,
+      "entropy": 0.8095202751364559,
+      "num_tokens": 10728653.0,
+      "logits/chosen": -0.9613099360765643,
+      "logits/rejected": -0.9753560247735856,
+      "mean_token_accuracy": 0.726740376483649,
+      "rewards/chosen": 0.003875704376255271,
+      "rewards/rejected": 0.004182930276788284,
+      "rewards/accuracies": 0.48375,
+      "rewards/margins": -0.00030722589199285724,
+      "logps/chosen": -53.913278470039366,
+      "logps/rejected": -53.74137356996536,
+      "epoch": 0.16596649551371817,
+      "step": 160
+    },
+    {
+      "loss": 0.6931381702423096,
+      "grad_norm": 0.008811748586595058,
+      "learning_rate": 8.248704663212436e-06,
+      "entropy": 0.8115301547432319,
+      "num_tokens": 11400551.0,
+      "logits/chosen": -0.9568430709542813,
+      "logits/rejected": -0.9675442470577922,
+      "mean_token_accuracy": 0.7244884933158755,
+      "rewards/chosen": 0.003357977192756607,
+      "rewards/rejected": 0.0028972053575580504,
+      "rewards/accuracies": 0.4925,
+      "rewards/margins": 0.00046077182138105856,
+      "logps/chosen": -53.2519864654541,
+      "logps/rejected": -52.62679670572281,
+      "epoch": 0.17633940148332555,
+      "step": 170
+    },
+    {
+      "loss": 0.6925437927246094,
+      "grad_norm": 0.00891165155917406,
+      "learning_rate": 8.145077720207254e-06,
+      "entropy": 0.79428493458312,
+      "num_tokens": 12070305.0,
+      "logits/chosen": -0.9719879220022611,
+      "logits/rejected": -0.9580149375927722,
+      "mean_token_accuracy": 0.7303908407129347,
+      "rewards/chosen": 0.002974712664896515,
+      "rewards/rejected": 0.0013871385961903115,
+      "rewards/accuracies": 0.5275,
+      "rewards/margins": 0.0015875740901537937,
+      "logps/chosen": -52.03131418466568,
+      "logps/rejected": -49.467778403759006,
+      "epoch": 0.18671230745293294,
+      "step": 180
+    },
+    {
+      "loss": 0.6946820259094239,
+      "grad_norm": 0.011048010550439358,
+      "learning_rate": 8.041450777202073e-06,
+      "entropy": 0.801795727997087,
+      "num_tokens": 12738966.0,
+      "logits/chosen": -0.9541037205291851,
+      "logits/rejected": -0.9584797920583412,
+      "mean_token_accuracy": 0.7201638549566269,
+      "rewards/chosen": 0.004079287395195053,
+      "rewards/rejected": 0.006725511867425666,
+      "rewards/accuracies": 0.47,
+      "rewards/margins": -0.0026462244295544224,
+      "logps/chosen": -51.030084483623504,
+      "logps/rejected": -52.38975264072418,
+      "epoch": 0.19708521342254032,
+      "step": 190
+    },
+    {
+      "loss": 0.6930304050445557,
+      "grad_norm": 0.009633849374949932,
+      "learning_rate": 7.937823834196891e-06,
+      "entropy": 0.7963748186221347,
+      "num_tokens": 13408002.0,
+      "logits/chosen": -0.9894887990110749,
+      "logits/rejected": -0.9980301170848317,
+      "mean_token_accuracy": 0.7193645982630551,
+      "rewards/chosen": 0.0047459728431940105,
+      "rewards/rejected": 0.004120458228130701,
+      "rewards/accuracies": 0.48625,
+      "rewards/margins": 0.000625514651183039,
+      "logps/chosen": -50.538159133195876,
+      "logps/rejected": -50.48108030319214,
+      "epoch": 0.2074581193921477,
+      "step": 200
+    },
+    {
+      "loss": 0.6936575412750244,
+      "grad_norm": 0.008548871614038944,
+      "learning_rate": 7.834196891191712e-06,
+      "entropy": 0.7598572969343513,
+      "num_tokens": 14074115.0,
+      "logits/chosen": -1.045821409259606,
+      "logits/rejected": -0.999601151613033,
+      "mean_token_accuracy": 0.7357131129689515,
+      "rewards/chosen": 0.005255431280912717,
+      "rewards/rejected": 0.005940479632897677,
+      "rewards/accuracies": 0.47,
+      "rewards/margins": -0.0006850483406742569,
+      "logps/chosen": -46.40404602527619,
+      "logps/rejected": -50.13248352050781,
+      "epoch": 0.2178310253617551,
+      "step": 210
+    },
+    {
+      "loss": 0.6938835144042969,
+      "grad_norm": 0.009261125698685646,
+      "learning_rate": 7.730569948186528e-06,
+      "entropy": 0.767732237551827,
+      "num_tokens": 14745903.0,
+      "logits/chosen": -1.0030631508707841,
+      "logits/rejected": -1.0114240661200429,
+      "mean_token_accuracy": 0.7297109466791153,
+      "rewards/chosen": 0.004763001757364691,
+      "rewards/rejected": 0.005841993642383727,
+      "rewards/accuracies": 0.49125,
+      "rewards/margins": -0.0010789919237868162,
+      "logps/chosen": -51.14548614025116,
+      "logps/rejected": -50.2862161898613,
+      "epoch": 0.2282039313313625,
+      "step": 220
+    },
+    {
+      "loss": 0.6946812152862549,
+      "grad_norm": 0.009788029827177525,
+      "learning_rate": 7.626943005181348e-06,
+      "entropy": 0.8289732250571251,
+      "num_tokens": 15418570.0,
+      "logits/chosen": -0.9488809894056489,
+      "logits/rejected": -1.0089717702518306,
+      "mean_token_accuracy": 0.7215882153064013,
+      "rewards/chosen": 0.006585003775003315,
+      "rewards/rejected": 0.009181454044451129,
+      "rewards/accuracies": 0.47625,
+      "rewards/margins": -0.0025964502768329113,
+      "logps/chosen": -56.4708499455452,
+      "logps/rejected": -53.11574334859848,
+      "epoch": 0.23857683730096987,
+      "step": 230
+    },
+    {
+      "loss": 0.6926135063171387,
+      "grad_norm": 0.009469996206462383,
+      "learning_rate": 7.523316062176167e-06,
+      "entropy": 0.8170388517202809,
+      "num_tokens": 16089276.0,
+      "logits/chosen": -0.9783852286633136,
+      "logits/rejected": -0.9603607895142418,
+      "mean_token_accuracy": 0.7241931633278728,
+      "rewards/chosen": 0.00405766910489433,
+      "rewards/rejected": 0.0026022558279692022,
+      "rewards/accuracies": 0.5175,
+      "rewards/margins": 0.00145541326062812,
+      "logps/chosen": -51.54288928151131,
+      "logps/rejected": -54.08886151790619,
+      "epoch": 0.24894974327057726,
+      "step": 240
+    },
+    {
+      "loss": 0.6937024593353271,
+      "grad_norm": 0.009117631241679192,
+      "learning_rate": 7.419689119170985e-06,
+      "entropy": 0.8002800923399627,
+      "num_tokens": 16760081.0,
+      "logits/chosen": -1.0175496722318613,
+      "logits/rejected": -0.9702955326900312,
+      "mean_token_accuracy": 0.7222046358138323,
+      "rewards/chosen": 0.0016811337291983363,
+      "rewards/rejected": 0.0023915858741713693,
+      "rewards/accuracies": 0.50625,
+      "rewards/margins": -0.0007104521640576422,
+      "logps/chosen": -51.49065291881561,
+      "logps/rejected": -54.3223114323616,
+      "epoch": 0.2593226492401846,
+      "step": 250
+    },
+    {
+      "eval_loss": 0.6968227624893188,
+      "eval_runtime": 223.7572,
+      "eval_samples_per_second": 4.469,
+      "eval_steps_per_second": 4.469,
+      "eval_entropy": 0.8013349784091115,
+      "eval_num_tokens": 16760081.0,
+      "eval_logits/chosen": -1.0127526828657614,
+      "eval_logits/rejected": -0.9702886589714581,
+      "eval_mean_token_accuracy": 0.7333163165301084,
+      "eval_rewards/chosen": 0.0008883264788382803,
+      "eval_rewards/rejected": 0.007837165784684316,
+      "eval_rewards/accuracies": 0.415,
+      "eval_rewards/margins": -0.00694883929831849,
+      "eval_logps/chosen": -50.164103578567506,
+      "eval_logps/rejected": -52.56150006484985,
+      "epoch": 0.2593226492401846,
+      "step": 250
+    },
+    {
+      "loss": 0.6938982009887695,
+      "grad_norm": 0.008658971637487411,
+      "learning_rate": 7.3160621761658035e-06,
+      "entropy": 0.8275363711640239,
+      "num_tokens": 17432770.0,
+      "logits/chosen": -0.9894071595486238,
+      "logits/rejected": -1.021770131425495,
+      "mean_token_accuracy": 0.7267817178182304,
+      "rewards/chosen": -6.325603737082019e-05,
+      "rewards/rejected": 0.0009481332914026552,
+      "rewards/accuracies": 0.5,
+      "rewards/margins": -0.0010113894138703472,
+      "logps/chosen": -53.769077570438384,
+      "logps/rejected": -54.760430970191955,
+      "epoch": 0.26969555520979205,
+      "step": 260
+    },
+    {
+      "loss": 0.6941106796264649,
+      "grad_norm": 0.009195917285978794,
+      "learning_rate": 7.212435233160623e-06,
+      "entropy": 0.8036660263128579,
+      "num_tokens": 18100438.0,
+      "logits/chosen": -1.0066855113664344,
+      "logits/rejected": -1.0124089541250572,
+      "mean_token_accuracy": 0.717040218450129,
+      "rewards/chosen": -0.003151719864085862,
+      "rewards/rejected": -0.00164005833167721,
+      "rewards/accuracies": 0.475,
+      "rewards/margins": -0.001511661496660963,
+      "logps/chosen": -49.700445964336396,
+      "logps/rejected": -50.435976998806,
+      "epoch": 0.28006846117939943,
+      "step": 270
+    },
+    {
+      "loss": 0.6933230876922607,
+      "grad_norm": 0.009418987669050694,
+      "learning_rate": 7.108808290155441e-06,
+      "entropy": 0.8158398796850815,
+      "num_tokens": 18772061.0,
+      "logits/chosen": -1.0278496914643969,
+      "logits/rejected": -1.0390610888345118,
+      "mean_token_accuracy": 0.724490509070456,
+      "rewards/chosen": -0.005928723971054524,
+      "rewards/rejected": -0.006031536482905722,
+      "rewards/accuracies": 0.52,
+      "rewards/margins": 0.0001028125207812991,
+      "logps/chosen": -53.07417652249336,
+      "logps/rejected": -52.73169189453125,
+      "epoch": 0.2904413671490068,
+      "step": 280
+    },
+    {
+      "loss": 0.6924946308135986,
+      "grad_norm": 0.008996859192848206,
+      "learning_rate": 7.005181347150259e-06,
+      "entropy": 0.801404341221787,
+      "num_tokens": 19443506.0,
+      "logits/chosen": -1.020471675081665,
+      "logits/rejected": -1.0060470756374977,
+      "mean_token_accuracy": 0.7225255416892469,
+      "rewards/chosen": -0.008651466769783837,
+      "rewards/rejected": -0.010357174093505818,
+      "rewards/accuracies": 0.5175,
+      "rewards/margins": 0.0017057072926036198,
+      "logps/chosen": -52.63102213382721,
+      "logps/rejected": -53.120495743751526,
+      "epoch": 0.3008142731186142,
+      "step": 290
+    },
+    {
+      "loss": 0.6929198741912842,
+      "grad_norm": 0.00904047954827547,
+      "learning_rate": 6.9015544041450784e-06,
+      "entropy": 0.7905970112606883,
+      "num_tokens": 20114567.0,
+      "logits/chosen": -1.0243293848557635,
+      "logits/rejected": -1.058803992605315,
+      "mean_token_accuracy": 0.7341359755769372,
+      "rewards/chosen": -0.012200816824647518,
+      "rewards/rejected": -0.01307745838902747,
+      "rewards/accuracies": 0.52125,
+      "rewards/margins": 0.000876641578397539,
+      "logps/chosen": -52.0853961968422,
+      "logps/rejected": -52.80914256095886,
+      "epoch": 0.3111871790882216,
+      "step": 300
+    },
+    {
+      "loss": 0.6928537845611572,
+      "grad_norm": 0.009816721081733704,
+      "learning_rate": 6.797927461139897e-06,
+      "entropy": 0.80872525641229,
+      "num_tokens": 20787766.0,
+      "logits/chosen": -0.9839863753234047,
+      "logits/rejected": -1.0009211278188388,
+      "mean_token_accuracy": 0.724884872585535,
+      "rewards/chosen": -0.012630892752328008,
+      "rewards/rejected": -0.01370205567994617,
+      "rewards/accuracies": 0.51125,
+      "rewards/margins": 0.0010711629761499353,
+      "logps/chosen": -54.78661850452423,
+      "logps/rejected": -53.24911024570465,
+      "epoch": 0.32156008505782896,
+      "step": 310
+    },
+    {
+      "loss": 0.6942753314971923,
+      "grad_norm": 0.009577946737408638,
+      "learning_rate": 6.6943005181347155e-06,
+      "entropy": 0.8015230012265966,
+      "num_tokens": 21457559.0,
+      "logits/chosen": -0.9992799805599515,
+      "logits/rejected": -1.0047482614113592,
+      "mean_token_accuracy": 0.7206378514692188,
+      "rewards/chosen": -0.015205535748583543,
+      "rewards/rejected": -0.013390423674327395,
+      "rewards/accuracies": 0.49625,
+      "rewards/margins": -0.0018151119937829209,
+      "logps/chosen": -52.25986159801483,
+      "logps/rejected": -53.13925377726555,
+      "epoch": 0.33193299102743634,
+      "step": 320
+    },
+    {
+      "loss": 0.6924636840820313,
+      "grad_norm": 0.009727108292281628,
+      "learning_rate": 6.590673575129535e-06,
+      "entropy": 0.8345451738173142,
+      "num_tokens": 22128277.0,
+      "logits/chosen": -0.971072161782628,
+      "logits/rejected": -0.9826513089025171,
+      "mean_token_accuracy": 0.7212898347340524,
+      "rewards/chosen": -0.016975908809049544,
+      "rewards/rejected": -0.018840813715414698,
+      "rewards/accuracies": 0.51625,
+      "rewards/margins": 0.001864904925096198,
+      "logps/chosen": -52.514592845439914,
+      "logps/rejected": -54.03662874937058,
+      "epoch": 0.3423058969970437,
+      "step": 330
+    },
+    {
+      "loss": 0.6921487808227539,
+      "grad_norm": 0.009848830290138721,
+      "learning_rate": 6.487046632124353e-06,
+      "entropy": 0.8036045160889626,
+      "num_tokens": 22798429.0,
+      "logits/chosen": -0.986366250980096,
+      "logits/rejected": -1.0395464497059017,
+      "mean_token_accuracy": 0.7169532440230251,
+      "rewards/chosen": -0.017634948804889063,
+      "rewards/rejected": -0.02014962821162044,
+      "rewards/accuracies": 0.54,
+      "rewards/margins": 0.002514679412379337,
+      "logps/chosen": -53.712123572826385,
+      "logps/rejected": -50.68536148548126,
+      "epoch": 0.3526788029666511,
+      "step": 340
+    },
+    {
+      "loss": 0.6948633670806885,
+      "grad_norm": 0.009547681547701359,
+      "learning_rate": 6.383419689119171e-06,
+      "entropy": 0.7954194891778752,
+      "num_tokens": 23470239.0,
+      "logits/chosen": -1.0014035524600693,
+      "logits/rejected": -0.9692854696785687,
+      "mean_token_accuracy": 0.7198419809341431,
+      "rewards/chosen": -0.019529370157788436,
+      "rewards/rejected": -0.016619607036818706,
+      "rewards/accuracies": 0.5,
+      "rewards/margins": -0.002909763151255902,
+      "logps/chosen": -54.53537731409073,
+      "logps/rejected": -52.964653131961825,
+      "epoch": 0.3630517089362585,
+      "step": 350
+    },
+    {
+      "loss": 0.6930148601531982,
+      "grad_norm": 0.00898800976574421,
+      "learning_rate": 6.2797927461139905e-06,
+      "entropy": 0.7794413399184122,
+      "num_tokens": 24140239.0,
+      "logits/chosen": -1.0375333631207078,
+      "logits/rejected": -1.0100693145671888,
+      "mean_token_accuracy": 0.737273293249309,
+      "rewards/chosen": -0.016578698544473695,
+      "rewards/rejected": -0.017291930461324226,
+      "rewards/accuracies": 0.4975,
+      "rewards/margins": 0.000713231952759088,
+      "logps/chosen": -49.863780752420425,
+      "logps/rejected": -52.97674039840698,
+      "epoch": 0.37342461490586587,
+      "step": 360
+    },
+    {
+      "loss": 0.6929784297943116,
+      "grad_norm": 0.008984371088445187,
+      "learning_rate": 6.176165803108809e-06,
+      "entropy": 0.8129843348357827,
+      "num_tokens": 24812178.0,
+      "logits/chosen": -0.9893493454110454,
+      "logits/rejected": -0.9891808390055474,
+      "mean_token_accuracy": 0.727164792958647,
+      "rewards/chosen": -0.018017661249964477,
+      "rewards/rejected": -0.01882471950921399,
+      "rewards/accuracies": 0.4975,
+      "rewards/margins": 0.0008070582516302239,
+      "logps/chosen": -53.014821940660475,
+      "logps/rejected": -54.58460282325745,
+      "epoch": 0.38379752087547325,
+      "step": 370
+    },
+    {
+      "eval_loss": 0.6905523538589478,
+      "eval_runtime": 245.3265,
+      "eval_samples_per_second": 4.076,
+      "eval_steps_per_second": 4.076,
+      "eval_entropy": 0.7969645112715662,
+      "eval_num_tokens": 25147105.0,
+      "eval_logits/chosen": -1.0253711017755525,
+      "eval_logits/rejected": -0.9797990125942502,
+      "eval_mean_token_accuracy": 0.7333250712603331,
+      "eval_rewards/chosen": -0.015263017903128458,
+      "eval_rewards/rejected": -0.020903651809865552,
+      "eval_rewards/accuracies": 0.55,
+      "eval_rewards/margins": 0.005640633857896319,
+      "eval_logps/chosen": -50.32561701965332,
+      "eval_logps/rejected": -52.84890823554993,
+      "epoch": 0.38898397386027694,
+      "step": 375
+    },
+    {
+      "loss": 0.6925588607788086,
+      "grad_norm": 0.009255084209144115,
+      "learning_rate": 6.0725388601036275e-06,
+      "entropy": 0.7894979556603358,
+      "num_tokens": 25482532.0,
+      "logits/chosen": -1.0074889789440618,
+      "logits/rejected": -0.9709891010763303,
+      "mean_token_accuracy": 0.7268249686434864,
+      "rewards/chosen": -0.017496899721906517,
+      "rewards/rejected": -0.019117686107287854,
+      "rewards/accuracies": 0.52625,
+      "rewards/margins": 0.001620786436178605,
+      "logps/chosen": -51.8191835963726,
+      "logps/rejected": -51.592100315093994,
+      "epoch": 0.39417042684508063,
+      "step": 380
+    },
+    {
+      "loss": 0.6923740386962891,
+      "grad_norm": 0.009324838407337666,
+      "learning_rate": 5.968911917098445e-06,
+      "entropy": 0.8094338296726346,
+      "num_tokens": 26154266.0,
+      "logits/chosen": -0.9966710708958524,
+      "logits/rejected": -0.9879505808785741,
+      "mean_token_accuracy": 0.7231129581481218,
+      "rewards/chosen": -0.017760099685847307,
+      "rewards/rejected": -0.019829784269199992,
+      "rewards/accuracies": 0.51875,
+      "rewards/margins": 0.0020696846699866,
+      "logps/chosen": -53.66926085233688,
+      "logps/rejected": -54.18102238893509,
+      "epoch": 0.404543332814688,
+      "step": 390
+    },
+    {
+      "loss": 0.6929931163787841,
+      "grad_norm": 0.010886781848967075,
+      "learning_rate": 5.865284974093265e-06,
+      "entropy": 0.8130081182997674,
+      "num_tokens": 26825159.0,
+      "logits/chosen": -1.026189590043526,
+      "logits/rejected": -0.980070075731585,
+      "mean_token_accuracy": 0.7226042114198208,
+      "rewards/chosen": -0.019220582297640475,
+      "rewards/rejected": -0.020057831880958474,
+      "rewards/accuracies": 0.48125,
+      "rewards/margins": 0.0008372495489311404,
+      "logps/chosen": -52.43204548358917,
+      "logps/rejected": -54.36065078496933,
+      "epoch": 0.4149162387842954,
+      "step": 400
+    },
+    {
+      "loss": 0.693073320388794,
+      "grad_norm": 0.009156073443591595,
+      "learning_rate": 5.761658031088083e-06,
+      "entropy": 0.7959862072952092,
+      "num_tokens": 27495663.0,
+      "logits/chosen": -1.0194205629554802,
+      "logits/rejected": -1.008366009739435,
+      "mean_token_accuracy": 0.7243296534568071,
+      "rewards/chosen": -0.01809768541224912,
+      "rewards/rejected": -0.01875489959079914,
+      "rewards/accuracies": 0.5025,
+      "rewards/margins": 0.0006572141727883718,
+      "logps/chosen": -52.169967291355135,
+      "logps/rejected": -53.42250164747238,
+      "epoch": 0.4252891447539028,
+      "step": 410
+    },
+    {
+      "loss": 0.6924574375152588,
+      "grad_norm": 0.010202614590525627,
+      "learning_rate": 5.658031088082902e-06,
+      "entropy": 0.7918436706252396,
+      "num_tokens": 28167389.0,
+      "logits/chosen": -1.0232014742112978,
+      "logits/rejected": -1.0440852867532966,
+      "mean_token_accuracy": 0.7258787673525512,
+      "rewards/chosen": -0.015947307585583416,
+      "rewards/rejected": -0.017838909620567733,
+      "rewards/accuracies": 0.5075,
+      "rewards/margins": 0.0018916020590404514,
+      "logps/chosen": -54.433519496917725,
+      "logps/rejected": -53.2144430232048,
+      "epoch": 0.4356620507235102,
+      "step": 420
+    },
+    {
+      "loss": 0.6926542282104492,
+      "grad_norm": 0.01011913362890482,
+      "learning_rate": 5.554404145077721e-06,
+      "entropy": 0.8117843008134514,
+      "num_tokens": 28839388.0,
+      "logits/chosen": -0.9718024988912555,
+      "logits/rejected": -0.9786808761441761,
+      "mean_token_accuracy": 0.7248365879803896,
+      "rewards/chosen": -0.014674390776453947,
+      "rewards/rejected": -0.016148133582082665,
+      "rewards/accuracies": 0.52875,
+      "rewards/margins": 0.001473742745729396,
+      "logps/chosen": -51.187984328269955,
+      "logps/rejected": -54.15149886369705,
+      "epoch": 0.4460349566931176,
+      "step": 430
+    },
+    {
+      "loss": 0.6937287330627442,
+      "grad_norm": 0.008680191822350025,
+      "learning_rate": 5.4507772020725395e-06,
+      "entropy": 0.7826772296521812,
+      "num_tokens": 29510096.0,
+      "logits/chosen": -1.038226736136573,
+      "logits/rejected": -1.0398130834010744,
+      "mean_token_accuracy": 0.7318821278214455,
+      "rewards/chosen": -0.01869894893682158,
+      "rewards/rejected": -0.01806240452982365,
+      "rewards/accuracies": 0.47625,
+      "rewards/margins": -0.0006365444045513868,
+      "logps/chosen": -51.81987749576569,
+      "logps/rejected": -51.031341784000396,
+      "epoch": 0.456407862662725,
+      "step": 440
+    },
+    {
+      "loss": 0.6937658309936523,
+      "grad_norm": 0.009057857096195221,
+      "learning_rate": 5.347150259067357e-06,
+      "entropy": 0.7959510098583996,
+      "num_tokens": 30181792.0,
+      "logits/chosen": -1.0239556820922306,
+      "logits/rejected": -1.0394439887711346,
+      "mean_token_accuracy": 0.7198793402686715,
+      "rewards/chosen": -0.01936899814491426,
+      "rewards/rejected": -0.01868944700636348,
+      "rewards/accuracies": 0.50375,
+      "rewards/margins": -0.0006795510534720961,
+      "logps/chosen": -54.870300602912906,
+      "logps/rejected": -50.76141629695892,
+      "epoch": 0.46678076863233237,
+      "step": 450
+    },
+    {
+      "loss": 0.694075345993042,
+      "grad_norm": 0.009986916556954384,
+      "learning_rate": 5.243523316062177e-06,
+      "entropy": 0.7800668972823769,
+      "num_tokens": 30851409.0,
+      "logits/chosen": -1.0086595854613267,
+      "logits/rejected": -1.0357231874675175,
+      "mean_token_accuracy": 0.7262966002896428,
+      "rewards/chosen": -0.020605784776303152,
+      "rewards/rejected": -0.019295487125091312,
+      "rewards/accuracies": 0.48125,
+      "rewards/margins": -0.0013102977239759638,
+      "logps/chosen": -53.48096129179001,
+      "logps/rejected": -51.299415925741194,
+      "epoch": 0.47715367460193975,
+      "step": 460
+    },
+    {
+      "loss": 0.6933310985565185,
+      "grad_norm": 0.007825742475688457,
+      "learning_rate": 5.139896373056995e-06,
+      "entropy": 0.7754787660343573,
+      "num_tokens": 31519564.0,
+      "logits/chosen": -1.0726965671758735,
+      "logits/rejected": -1.003480362234256,
+      "mean_token_accuracy": 0.7201942896470428,
+      "rewards/chosen": -0.017236333386230172,
+      "rewards/rejected": -0.017384318440863352,
+      "rewards/accuracies": 0.4825,
+      "rewards/margins": 0.00014798504006648726,
+      "logps/chosen": -49.58583438396454,
+      "logps/rejected": -51.69670748949051,
+      "epoch": 0.48752658057154713,
+      "step": 470
+    },
+    {
+      "loss": 0.6922287940979004,
+      "grad_norm": 0.008367015048861504,
+      "learning_rate": 5.036269430051814e-06,
+      "entropy": 0.8279160013142973,
+      "num_tokens": 32192056.0,
+      "logits/chosen": -0.9712868713229077,
+      "logits/rejected": -0.9765040762093227,
+      "mean_token_accuracy": 0.7188576187193394,
+      "rewards/chosen": -0.01717338605103805,
+      "rewards/rejected": -0.019571030664633327,
+      "rewards/accuracies": 0.51125,
+      "rewards/margins": 0.0023976446047163334,
+      "logps/chosen": -54.479641914367676,
+      "logps/rejected": -54.63456130981445,
+      "epoch": 0.4978994865411545,
+      "step": 480
+    },
+    {
+      "loss": 0.6939034461975098,
+      "grad_norm": 0.008626270107924938,
+      "learning_rate": 4.932642487046633e-06,
+      "entropy": 0.8010151171591133,
+      "num_tokens": 32864077.0,
+      "logits/chosen": -1.0169706789174378,
+      "logits/rejected": -1.0165859789377836,
+      "mean_token_accuracy": 0.7208031569048763,
+      "rewards/chosen": -0.01987447723532,
+      "rewards/rejected": -0.018864784324341598,
+      "rewards/accuracies": 0.495,
+      "rewards/margins": -0.0010096929259452735,
+      "logps/chosen": -53.33563009738922,
+      "logps/rejected": -53.87229782342911,
+      "epoch": 0.5082723925107618,
+      "step": 490
+    },
+    {
+      "loss": 0.6938543319702148,
+      "grad_norm": 0.010269825346767902,
+      "learning_rate": 4.829015544041451e-06,
+      "entropy": 0.7891198094375431,
+      "num_tokens": 33534279.0,
+      "logits/chosen": -1.033360492470276,
+      "logits/rejected": -1.0354731196495317,
+      "mean_token_accuracy": 0.7333813977241516,
+      "rewards/chosen": -0.019480555365971666,
+      "rewards/rejected": -0.01867740810344003,
+      "rewards/accuracies": 0.51,
+      "rewards/margins": -0.0008031472803850193,
+      "logps/chosen": -52.630670845508575,
+      "logps/rejected": -50.910942976474765,
+      "epoch": 0.5186452984803692,
+      "step": 500
+    },
+    {
+      "eval_loss": 0.6942050457000732,
+      "eval_runtime": 276.9162,
+      "eval_samples_per_second": 3.611,
+      "eval_steps_per_second": 3.611,
+      "eval_entropy": 0.7935083331502975,
+      "eval_num_tokens": 33534279.0,
+      "eval_logits/chosen": -1.0600360631491812,
+      "eval_logits/rejected": -1.0166348149416253,
+      "eval_mean_token_accuracy": 0.7330459494143724,
+      "eval_rewards/chosen": -0.021802856784763208,
+      "eval_rewards/rejected": -0.02036227696077913,
+      "eval_rewards/accuracies": 0.477,
+      "eval_rewards/margins": -0.0014405797821236774,
+      "eval_logps/chosen": -50.39101540756226,
+      "eval_logps/rejected": -52.84349448776245,
+      "epoch": 0.5186452984803692,
+      "step": 500
+    },
+    {
+      "loss": 0.6938997745513916,
+      "grad_norm": 0.00896107591688633,
+      "learning_rate": 4.72538860103627e-06,
+      "entropy": 0.7882147227507085,
+      "num_tokens": 34205135.0,
+      "logits/chosen": -1.0076570653558603,
+      "logits/rejected": -1.0416575961228518,
+      "mean_token_accuracy": 0.7137932823970914,
+      "rewards/chosen": -0.018858550680542974,
+      "rewards/rejected": -0.017997891291797713,
+      "rewards/accuracies": 0.495,
+      "rewards/margins": -0.0008606594161392423,
+      "logps/chosen": -54.12554481744766,
+      "logps/rejected": -50.22665623903274,
+      "epoch": 0.5290182044499767,
+      "step": 510
+    },
+    {
+      "loss": 0.6940505504608154,
+      "grad_norm": 0.009844881482422352,
+      "learning_rate": 4.621761658031089e-06,
+      "entropy": 0.7678585767536424,
+      "num_tokens": 34873976.0,
+      "logits/chosen": -1.0794209366681549,
+      "logits/rejected": -1.0414292466399215,
+      "mean_token_accuracy": 0.7310516293905676,
+      "rewards/chosen": -0.013213514556548489,
+      "rewards/rejected": -0.011983713133254241,
+      "rewards/accuracies": 0.515,
+      "rewards/margins": -0.0012298014553380199,
+      "logps/chosen": -51.02845492362976,
+      "logps/rejected": -49.72117327451706,
+      "epoch": 0.5393911104195841,
+      "step": 520
+    },
+    {
+      "loss": 0.6937035083770752,
+      "grad_norm": 0.009832088835537434,
+      "learning_rate": 4.518134715025907e-06,
+      "entropy": 0.8068192195380106,
+      "num_tokens": 35544999.0,
+      "logits/chosen": -1.0203095909710327,
+      "logits/rejected": -1.0544799987360511,
+      "mean_token_accuracy": 0.7208610328473151,
+      "rewards/chosen": -0.013758049447855001,
+      "rewards/rejected": -0.013221281499590986,
+      "rewards/accuracies": 0.50375,
+      "rewards/margins": -0.0005367679687333293,
+      "logps/chosen": -53.891741864681244,
+      "logps/rejected": -52.176739025115964,
+      "epoch": 0.5497640163891915,
+      "step": 530
+    },
+    {
+      "loss": 0.693358039855957,
+      "grad_norm": 0.009179627522826195,
+      "learning_rate": 4.414507772020726e-06,
+      "entropy": 0.8059323144424707,
+      "num_tokens": 36215629.0,
+      "logits/chosen": -1.0271287046641675,
+      "logits/rejected": -1.0058586337462303,
+      "mean_token_accuracy": 0.7286789271980524,
+      "rewards/chosen": -0.01192244978853978,
+      "rewards/rejected": -0.011959875774573447,
+      "rewards/accuracies": 0.495,
+      "rewards/margins": 3.742602792044636e-05,
+      "logps/chosen": -50.73393731594086,
+      "logps/rejected": -52.839989113807675,
+      "epoch": 0.5601369223587989,
+      "step": 540
+    },
+    {
+      "loss": 0.693217134475708,
+      "grad_norm": 0.008687409572303295,
+      "learning_rate": 4.310880829015544e-06,
+      "entropy": 0.8455263479612768,
+      "num_tokens": 36888235.0,
+      "logits/chosen": -0.9577911917232584,
+      "logits/rejected": -0.9647921425538288,
+      "mean_token_accuracy": 0.7241826535388828,
+      "rewards/chosen": -0.012230233296829738,
+      "rewards/rejected": -0.012580791678369679,
+      "rewards/accuracies": 0.52,
+      "rewards/margins": 0.00035055841028224676,
+      "logps/chosen": -53.09883669376373,
+      "logps/rejected": -56.72353342294693,
+      "epoch": 0.5705098283284062,
+      "step": 550
+    },
+    {
+      "loss": 0.6929863929748535,
+      "grad_norm": 0.008931291289627552,
+      "learning_rate": 4.207253886010363e-06,
+      "entropy": 0.8086647934932262,
+      "num_tokens": 37560215.0,
+      "logits/chosen": -1.0313529810131257,
+      "logits/rejected": -1.0182036791475924,
+      "mean_token_accuracy": 0.7183289608359337,
+      "rewards/chosen": -0.013141918864030232,
+      "rewards/rejected": -0.013932020409274628,
+      "rewards/accuracies": 0.505,
+      "rewards/margins": 0.0007901015544848633,
+      "logps/chosen": -52.67770826101303,
+      "logps/rejected": -52.252629327774045,
+      "epoch": 0.5808827342980136,
+      "step": 560
+    },
+    {
+      "loss": 0.6919141292572022,
+      "grad_norm": 0.00928713008761406,
+      "learning_rate": 4.103626943005182e-06,
+      "entropy": 0.795804328629747,
+      "num_tokens": 38229872.0,
+      "logits/chosen": -1.0115969525038464,
+      "logits/rejected": -0.9389554266112111,
+      "mean_token_accuracy": 0.7316008464246988,
+      "rewards/chosen": -0.011151957947622576,
+      "rewards/rejected": -0.014132183314768555,
+      "rewards/accuracies": 0.54375,
+      "rewards/margins": 0.0029802253525849664,
+      "logps/chosen": -50.29158478021622,
+      "logps/rejected": -53.326151065826416,
+      "epoch": 0.591255640267621,
+      "step": 570
+    },
+    {
+      "loss": 0.6936386585235595,
+      "grad_norm": 0.009219864383339882,
+      "learning_rate": 4.000000000000001e-06,
+      "entropy": 0.7849282765295357,
+      "num_tokens": 38898599.0,
+      "logits/chosen": -1.0318535424181245,
+      "logits/rejected": -1.0693508828519998,
+      "mean_token_accuracy": 0.7279249535314739,
+      "rewards/chosen": -0.015236615104731755,
+      "rewards/rejected": -0.01477163603422241,
+      "rewards/accuracies": 0.49375,
+      "rewards/margins": -0.00046497906170770873,
+      "logps/chosen": -50.606445105075835,
+      "logps/rejected": -50.23892628192902,
+      "epoch": 0.6016285462372284,
+      "step": 580
+    },
+    {
+      "loss": 0.6924667358398438,
+      "grad_norm": 0.009871033020317554,
+      "learning_rate": 3.896373056994819e-06,
+      "entropy": 0.7915234410995617,
+      "num_tokens": 39570011.0,
+      "logits/chosen": -1.0704431655924542,
+      "logits/rejected": -1.0262311271415063,
+      "mean_token_accuracy": 0.7341923769563437,
+      "rewards/chosen": -0.013985364213192497,
+      "rewards/rejected": -0.015847310858209767,
+      "rewards/accuracies": 0.53625,
+      "rewards/margins": 0.0018619466816016939,
+      "logps/chosen": -53.37289155721665,
+      "logps/rejected": -53.37732348442078,
+      "epoch": 0.6120014522068358,
+      "step": 590
+    },
+    {
+      "loss": 0.6934967041015625,
+      "grad_norm": 0.008916029706597328,
+      "learning_rate": 3.7927461139896377e-06,
+      "entropy": 0.7781280868733301,
+      "num_tokens": 40238459.0,
+      "logits/chosen": -1.031845585303488,
+      "logits/rejected": -1.0679663812358249,
+      "mean_token_accuracy": 0.7267250791564583,
+      "rewards/chosen": -0.015354626149126034,
+      "rewards/rejected": -0.015155612989574366,
+      "rewards/accuracies": 0.495,
+      "rewards/margins": -0.0001990132535865996,
+      "logps/chosen": -50.91896806716919,
+      "logps/rejected": -49.1647160077095,
+      "epoch": 0.6223743581764432,
+      "step": 600
+    },
+    {
+      "loss": 0.6943734169006348,
+      "grad_norm": 0.009726111777126789,
+      "learning_rate": 3.6891191709844567e-06,
+      "entropy": 0.8024784850515425,
+      "num_tokens": 40907823.0,
+      "logits/chosen": -1.0033099905488874,
+      "logits/rejected": -1.004673033873931,
+      "mean_token_accuracy": 0.7266425576806068,
+      "rewards/chosen": -0.016839503781302483,
+      "rewards/rejected": -0.014924811632938316,
+      "rewards/accuracies": 0.485,
+      "rewards/margins": -0.0019146921102219494,
+      "logps/chosen": -51.37045606851578,
+      "logps/rejected": -50.59996291279793,
+      "epoch": 0.6327472641460505,
+      "step": 610
+    },
+    {
+      "loss": 0.693054485321045,
+      "grad_norm": 0.00853179581463337,
+      "learning_rate": 3.5854922279792748e-06,
+      "entropy": 0.7575920634204522,
+      "num_tokens": 41576629.0,
+      "logits/chosen": -1.0128204565402743,
+      "logits/rejected": -1.0291723665089934,
+      "mean_token_accuracy": 0.7279130771569907,
+      "rewards/chosen": -0.014212547984803337,
+      "rewards/rejected": -0.014947609209768871,
+      "rewards/accuracies": 0.48875,
+      "rewards/margins": 0.000735061179366312,
+      "logps/chosen": -50.57350071668625,
+      "logps/rejected": -49.389897744655606,
+      "epoch": 0.6431201701156579,
+      "step": 620
+    },
+    {
+      "eval_loss": 0.6942050457000732,
+      "eval_runtime": 307.8984,
+      "eval_samples_per_second": 3.248,
+      "eval_steps_per_second": 3.248,
+      "eval_entropy": 0.7959122954644263,
+      "eval_num_tokens": 41911681.0,
+      "eval_logits/chosen": -1.048361439070786,
+      "eval_logits/rejected": -1.0055296550561397,
+      "eval_mean_token_accuracy": 0.7335138544887304,
+      "eval_rewards/chosen": -0.014208579457705128,
+      "eval_rewards/rejected": -0.01261782132985536,
+      "eval_rewards/accuracies": 0.478,
+      "eval_rewards/margins": -0.0015907581194769592,
+      "eval_logps/chosen": -50.31507263565064,
+      "eval_logps/rejected": -52.76604993247986,
+      "epoch": 0.6483066231004616,
+      "step": 625
+    },
+    {
+      "loss": 0.6939579486846924,
+      "grad_norm": 0.009789089672267437,
+      "learning_rate": 3.4818652849740937e-06,
+      "entropy": 0.7933417669357732,
+      "num_tokens": 42247859.0,
+      "logits/chosen": -1.0118922806884323,
+      "logits/rejected": -1.0382741999872491,
+      "mean_token_accuracy": 0.726150699108839,
+      "rewards/chosen": -0.014976464999999734,
+      "rewards/rejected": -0.013932963377121722,
+      "rewards/accuracies": 0.47375,
+      "rewards/margins": -0.0010435017172130757,
+      "logps/chosen": -53.83100793600082,
+      "logps/rejected": -50.46369902849197,
+      "epoch": 0.6534930760852653,
+      "step": 630
+    },
+    {
+      "loss": 0.694810152053833,
+      "grad_norm": 0.009261916391551495,
+      "learning_rate": 3.3782383419689123e-06,
+      "entropy": 0.7883505333820358,
+      "num_tokens": 42918160.0,
+      "logits/chosen": -1.017383539174631,
+      "logits/rejected": -1.054998851475991,
+      "mean_token_accuracy": 0.7168397939577699,
+      "rewards/chosen": -0.015089196973789569,
+      "rewards/rejected": -0.012242867652641962,
+      "rewards/accuracies": 0.46,
+      "rewards/margins": -0.0028463293392269407,
+      "logps/chosen": -54.35950309753418,
+      "logps/rejected": -48.9059769487381,
+      "epoch": 0.6638659820548727,
+      "step": 640
+    },
+    {
+      "loss": 0.6925888061523438,
+      "grad_norm": 0.00910037662833929,
+      "learning_rate": 3.274611398963731e-06,
+      "entropy": 0.790731361717917,
+      "num_tokens": 43587843.0,
+      "logits/chosen": -0.9965395541483734,
+      "logits/rejected": -1.0179589161213116,
+      "mean_token_accuracy": 0.7306452417373657,
+      "rewards/chosen": -0.010891549229872908,
+      "rewards/rejected": -0.012513885581938666,
+      "rewards/accuracies": 0.52875,
+      "rewards/margins": 0.0016223363954850356,
+      "logps/chosen": -51.07387138366699,
+      "logps/rejected": -51.394845466613766,
+      "epoch": 0.6742388880244801,
+      "step": 650
+    },
+    {
+      "loss": 0.693898057937622,
+      "grad_norm": 0.009298712946474552,
+      "learning_rate": 3.1709844559585493e-06,
+      "entropy": 0.7892762251244858,
+      "num_tokens": 44256661.0,
+      "logits/chosen": -1.040483162193093,
+      "logits/rejected": -1.0218257141848732,
+      "mean_token_accuracy": 0.7231168592534959,
+      "rewards/chosen": -0.012846070201399017,
+      "rewards/rejected": -0.01177774412741428,
+      "rewards/accuracies": 0.495,
+      "rewards/margins": -0.001068326040294778,
+      "logps/chosen": -50.02721215367317,
+      "logps/rejected": -51.92197064161301,
+      "epoch": 0.6846117939940874,
+      "step": 660
+    },
+    {
+      "loss": 0.6929262161254883,
+      "grad_norm": 0.009983985684812069,
+      "learning_rate": 3.0673575129533683e-06,
+      "entropy": 0.7727112902654335,
+      "num_tokens": 44928208.0,
+      "logits/chosen": -1.033872497474533,
+      "logits/rejected": -1.0355189189593426,
+      "mean_token_accuracy": 0.7299729858152568,
+      "rewards/chosen": -0.012007646745373676,
+      "rewards/rejected": -0.012911729082511557,
+      "rewards/accuracies": 0.49625,
+      "rewards/margins": 0.0009040823050418112,
+      "logps/chosen": -52.13899112701416,
+      "logps/rejected": -50.62764532446861,
+      "epoch": 0.6949846999636948,
+      "step": 670
+    },
+    {
+      "loss": 0.6946879863739014,
+      "grad_norm": 0.009130201302468777,
+      "learning_rate": 2.963730569948187e-06,
+      "entropy": 0.8138119697524234,
+      "num_tokens": 45599959.0,
+      "logits/chosen": -0.966092846678868,
+      "logits/rejected": -0.9774829642341187,
+      "mean_token_accuracy": 0.7230793483182788,
+      "rewards/chosen": -0.015864108831710837,
+      "rewards/rejected": -0.013314598076967742,
+      "rewards/accuracies": 0.4675,
+      "rewards/margins": -0.0025495107232927695,
+      "logps/chosen": -54.92970582008362,
+      "logps/rejected": -52.76770358800888,
+      "epoch": 0.7053576059333022,
+      "step": 680
+    },
+    {
+      "loss": 0.6929837703704834,
+      "grad_norm": 0.009250026196241379,
+      "learning_rate": 2.8601036269430053e-06,
+      "entropy": 0.82899127332028,
+      "num_tokens": 46271100.0,
+      "logits/chosen": -0.9898684929018607,
+      "logits/rejected": -0.9973996765394122,
+      "mean_token_accuracy": 0.719442187603563,
+      "rewards/chosen": -0.010820029947699367,
+      "rewards/rejected": -0.01162173981451815,
+      "rewards/accuracies": 0.515,
+      "rewards/margins": 0.0008017099273274653,
+      "logps/chosen": -51.98621440887451,
+      "logps/rejected": -54.959817726612094,
+      "epoch": 0.7157305119029096,
+      "step": 690
+    },
+    {
+      "loss": 0.6941079139709473,
+      "grad_norm": 0.009462058544158936,
+      "learning_rate": 2.7564766839378243e-06,
+      "entropy": 0.7970602755853906,
+      "num_tokens": 46940588.0,
+      "logits/chosen": -0.990150519364322,
+      "logits/rejected": -0.9981860405217275,
+      "mean_token_accuracy": 0.7297776956856251,
+      "rewards/chosen": -0.012557482681352213,
+      "rewards/rejected": -0.011030125059482998,
+      "rewards/accuracies": 0.51,
+      "rewards/margins": -0.0015273576516483445,
+      "logps/chosen": -49.37967163324356,
+      "logps/rejected": -53.13790496587753,
+      "epoch": 0.726103417872517,
+      "step": 700
+    },
+    {
+      "loss": 0.6923154830932617,
+      "grad_norm": 0.008459668606519699,
+      "learning_rate": 2.6528497409326424e-06,
+      "entropy": 0.7973380678170361,
+      "num_tokens": 47612649.0,
+      "logits/chosen": -1.0290440507413436,
+      "logits/rejected": -1.0325854925972269,
+      "mean_token_accuracy": 0.7238865295890718,
+      "rewards/chosen": -0.012139474128680376,
+      "rewards/rejected": -0.014260805117913602,
+      "rewards/accuracies": 0.52875,
+      "rewards/margins": 0.0021213308708684054,
+      "logps/chosen": -52.259123842716214,
+      "logps/rejected": -54.400773532390595,
+      "epoch": 0.7364763238421244,
+      "step": 710
+    },
+    {
+      "loss": 0.6927196025848389,
+      "grad_norm": 0.008741075173020363,
+      "learning_rate": 2.5492227979274614e-06,
+      "entropy": 0.776990000186488,
+      "num_tokens": 48280316.0,
+      "logits/chosen": -1.02981366786685,
+      "logits/rejected": -1.062369960988718,
+      "mean_token_accuracy": 0.728738241456449,
+      "rewards/chosen": -0.0141699135041722,
+      "rewards/rejected": -0.01543546461060032,
+      "rewards/accuracies": 0.52,
+      "rewards/margins": 0.0012655511016782838,
+      "logps/chosen": -50.75450169801712,
+      "logps/rejected": -50.51603441476822,
+      "epoch": 0.7468492298117317,
+      "step": 720
+    },
+    {
+      "loss": 0.6939631938934326,
+      "grad_norm": 0.008915440179407597,
+      "learning_rate": 2.44559585492228e-06,
+      "entropy": 0.8170583094470203,
+      "num_tokens": 48952457.0,
+      "logits/chosen": -1.003780851996667,
+      "logits/rejected": -0.9864155673137232,
+      "mean_token_accuracy": 0.7165420038998127,
+      "rewards/chosen": -0.013467036513036419,
+      "rewards/rejected": -0.012332614251897666,
+      "rewards/accuracies": 0.495,
+      "rewards/margins": -0.0011344222621119116,
+      "logps/chosen": -54.76776822566986,
+      "logps/rejected": -52.639701968431474,
+      "epoch": 0.7572221357813391,
+      "step": 730
+    },
+    {
+      "loss": 0.6925458908081055,
+      "grad_norm": 0.01004419568926096,
+      "learning_rate": 2.3419689119170984e-06,
+      "entropy": 0.8180013949144631,
+      "num_tokens": 49623039.0,
+      "logits/chosen": -1.0169417811830719,
+      "logits/rejected": -1.0213671464834864,
+      "mean_token_accuracy": 0.7187614095583558,
+      "rewards/chosen": -0.012766179290275658,
+      "rewards/rejected": -0.014406545395818284,
+      "rewards/accuracies": 0.52625,
+      "rewards/margins": 0.001640366151041235,
+      "logps/chosen": -52.5052701497078,
+      "logps/rejected": -53.90763850927353,
+      "epoch": 0.7675950417509465,
+      "step": 740
+    },
+    {
+      "loss": 0.6922706127166748,
+      "grad_norm": 0.009859760291874409,
+      "learning_rate": 2.2383419689119174e-06,
+      "entropy": 0.8285305345850066,
+      "num_tokens": 50296449.0,
+      "logits/chosen": -0.9801905178679585,
+      "logits/rejected": -0.9738778555663805,
+      "mean_token_accuracy": 0.723729298338294,
+      "rewards/chosen": -0.009565252497509391,
+      "rewards/rejected": -0.01175256536917459,
+      "rewards/accuracies": 0.5175,
+      "rewards/margins": 0.0021873128600418566,
+      "logps/chosen": -54.38914824962616,
+      "logps/rejected": -55.734090762138365,
+      "epoch": 0.7779679477205539,
+      "step": 750
+    },
+    {
+      "eval_loss": 0.6926036477088928,
+      "eval_runtime": 339.6814,
+      "eval_samples_per_second": 2.944,
+      "eval_steps_per_second": 2.944,
+      "eval_entropy": 0.7968713178224862,
+      "eval_num_tokens": 50296449.0,
+      "eval_logits/chosen": -1.0377698264720323,
+      "eval_logits/rejected": -0.9943464753298086,
+      "eval_mean_token_accuracy": 0.7334770380109549,
+      "eval_rewards/chosen": -0.012114850055493434,
+      "eval_rewards/rejected": -0.013621491047078962,
+      "eval_rewards/accuracies": 0.504,
+      "eval_rewards/margins": 0.0015066410247527528,
+      "eval_logps/chosen": -50.294135341644285,
+      "eval_logps/rejected": -52.77608662986756,
+      "epoch": 0.7779679477205539,
+      "step": 750
+    },
+    {
+      "train_runtime": 27561.1083,
+      "train_samples_per_second": 2.798,
+      "train_steps_per_second": 0.035,
+      "total_flos": 4.075839588886364e+17,
+      "train_loss": 0.6933243910471598,
+      "epoch": 0.7779679477205539,
+      "step": 750
+    }
+  ],
+  "validation_monitor_size": 1000,
+  "validation_monitor_selection": "fixed_seed_random_without_replacement",
+  "validation_monitor_seed": 22,
+  "validation_monitor_indices": [
+    2,
+    10,
+    14,
+    21,
+    55,
+    63,
+    66,
+    69,
+    107,
+    110,
+    142,
+    144,
+    149,
+    150,
+    151,
+    152,
+    160,
+    163,
+    166,
+    167,
+    176,
+    181,
+    206,
+    207,
+    259,
+    267,
+    281,
+    295,
+    298,
+    306,
+    307,
+    317,
+    321,
+    331,
+    336,
+    341,
+    346,
+    349,
+    351,
+    352,
+    357,
+    358,
+    369,
+    370,
+    382,
+    386,
+    391,
+    401,
+    411,
+    412,
+    432,
+    437,
+    452,
+    462,
+    465,
+    488,
+    490,
+    491,
+    492,
+    494,
+    503,
+    504,
+    508,
+    528,
+    538,
+    540,
+    551,
+    553,
+    567,
+    586,
+    605,
+    606,
+    625,
+    627,
+    661,
+    663,
+    666,
+    677,
+    685,
+    690,
+    692,
+    704,
+    708,
+    714,
+    715,
+    727,
+    728,
+    733,
+    745,
+    752,
+    753,
+    755,
+    764,
+    773,
+    779,
+    783,
+    793,
+    796,
+    799,
+    803,
+    804,
+    805,
+    813,
+    816,
+    818,
+    823,
+    827,
+    829,
+    830,
+    837,
+    842,
+    845,
+    850,
+    853,
+    856,
+    889,
+    890,
+    905,
+    915,
+    924,
+    930,
+    939,
+    951,
+    988,
+    1002,
+    1005,
+    1023,
+    1029,
+    1038,
+    1049,
+    1050,
+    1054,
+    1056,
+    1067,
+    1068,
+    1079,
+    1088,
+    1091,
+    1103,
+    1114,
+    1118,
+    1133,
+    1140,
+    1144,
+    1145,
+    1155,
+    1186,
+    1195,
+    1206,
+    1223,
+    1251,
+    1252,
+    1257,
+    1259,
+    1270,
+    1271,
+    1295,
+    1303,
+    1304,
+    1305,
+    1329,
+    1335,
+    1336,
+    1343,
+    1367,
+    1373,
+    1377,
+    1403,
+    1412,
+    1429,
+    1443,
+    1457,
+    1459,
+    1460,
+    1476,
+    1483,
+    1486,
+    1494,
+    1507,
+    1510,
+    1519,
+    1521,
+    1522,
+    1545,
+    1551,
+    1570,
+    1582,
+    1593,
+    1595,
+    1603,
+    1607,
+    1614,
+    1615,
+    1625,
+    1634,
+    1639,
+    1648,
+    1654,
+    1657,
+    1662,
+    1667,
+    1673,
+    1690,
+    1704,
+    1746,
+    1756,
+    1781,
+    1783,
+    1796,
+    1799,
+    1809,
+    1814,
+    1827,
+    1829,
+    1832,
+    1844,
+    1847,
+    1854,
+    1856,
+    1857,
+    1858,
+    1874,
+    1883,
+    1889,
+    1924,
+    1925,
+    1931,
+    1932,
+    1934,
+    1935,
+    1938,
+    1950,
+    1957,
+    1962,
+    1967,
+    1975,
+    1982,
+    1983,
+    1991,
+    1998,
+    2003,
+    2008,
+    2017,
+    2021,
+    2026,
+    2035,
+    2040,
+    2050,
+    2056,
+    2077,
+    2079,
+    2082,
+    2098,
+    2100,
+    2108,
+    2121,
+    2130,
+    2133,
+    2134,
+    2138,
+    2143,
+    2166,
+    2167,
+    2180,
+    2181,
+    2185,
+    2204,
+    2205,
+    2212,
+    2221,
+    2224,
+    2226,
+    2230,
+    2233,
+    2256,
+    2261,
+    2263,
+    2269,
+    2273,
+    2290,
+    2291,
+    2299,
+    2301,
+    2321,
+    2323,
+    2330,
+    2384,
+    2401,
+    2417,
+    2420,
+    2421,
+    2424,
+    2430,
+    2435,
+    2456,
+    2488,
+    2502,
+    2504,
+    2519,
+    2527,
+    2532,
+    2538,
+    2542,
+    2553,
+    2555,
+    2558,
+    2559,
+    2562,
+    2578,
+    2583,
+    2585,
+    2590,
+    2592,
+    2594,
+    2596,
+    2600,
+    2606,
+    2607,
+    2618,
+    2630,
+    2637,
+    2647,
+    2650,
+    2657,
+    2679,
+    2685,
+    2705,
+    2706,
+    2712,
+    2713,
+    2714,
+    2727,
+    2740,
+    2742,
+    2755,
+    2780,
+    2784,
+    2792,
+    2807,
+    2808,
+    2810,
+    2820,
+    2831,
+    2839,
+    2860,
+    2862,
+    2863,
+    2866,
+    2875,
+    2878,
+    2898,
+    2905,
+    2921,
+    2922,
+    2926,
+    2930,
+    2934,
+    2944,
+    2955,
+    2957,
+    2959,
+    2973,
+    2978,
+    2998,
+    3018,
+    3022,
+    3028,
+    3031,
+    3061,
+    3085,
+    3090,
+    3104,
+    3105,
+    3111,
+    3115,
+    3121,
+    3122,
+    3129,
+    3133,
+    3135,
+    3161,
+    3162,
+    3169,
+    3173,
+    3194,
+    3195,
+    3215,
+    3225,
+    3226,
+    3241,
+    3247,
+    3250,
+    3253,
+    3265,
+    3268,
+    3293,
+    3301,
+    3312,
+    3329,
+    3352,
+    3361,
+    3362,
+    3376,
+    3396,
+    3401,
+    3403,
+    3410,
+    3419,
+    3432,
+    3443,
+    3452,
+    3467,
+    3469,
+    3475,
+    3485,
+    3503,
+    3514,
+    3515,
+    3524,
+    3528,
+    3538,
+    3544,
+    3557,
+    3560,
+    3565,
+    3600,
+    3637,
+    3642,
+    3658,
+    3659,
+    3692,
+    3693,
+    3699,
+    3722,
+    3725,
+    3728,
+    3731,
+    3740,
+    3742,
+    3762,
+    3766,
+    3780,
+    3788,
+    3794,
+    3796,
+    3798,
+    3805,
+    3817,
+    3819,
+    3822,
+    3837,
+    3839,
+    3843,
+    3846,
+    3851,
+    3854,
+    3865,
+    3870,
+    3871,
+    3884,
+    3894,
+    3895,
+    3896,
+    3907,
+    3911,
+    3915,
+    3919,
+    3920,
+    3923,
+    3933,
+    3955,
+    3967,
+    3972,
+    3974,
+    3975,
+    3984,
+    3985,
+    3997,
+    4002,
+    4010,
+    4034,
+    4044,
+    4063,
+    4073,
+    4079,
+    4082,
+    4088,
+    4121,
+    4145,
+    4148,
+    4159,
+    4161,
+    4164,
+    4177,
+    4188,
+    4199,
+    4203,
+    4207,
+    4208,
+    4213,
+    4221,
+    4225,
+    4233,
+    4241,
+    4243,
+    4247,
+    4264,
+    4274,
+    4282,
+    4286,
+    4290,
+    4308,
+    4310,
+    4313,
+    4321,
+    4324,
+    4327,
+    4349,
+    4362,
+    4370,
+    4374,
+    4380,
+    4407,
+    4409,
+    4411,
+    4415,
+    4417,
+    4418,
+    4427,
+    4431,
+    4433,
+    4451,
+    4466,
+    4477,
+    4478,
+    4479,
+    4493,
+    4494,
+    4514,
+    4527,
+    4539,
+    4550,
+    4558,
+    4568,
+    4579,
+    4583,
+    4584,
+    4586,
+    4587,
+    4590,
+    4599,
+    4602,
+    4610,
+    4626,
+    4627,
+    4630,
+    4641,
+    4647,
+    4655,
+    4656,
+    4657,
+    4661,
+    4685,
+    4714,
+    4715,
+    4731,
+    4749,
+    4752,
+    4769,
+    4777,
+    4782,
+    4791,
+    4805,
+    4818,
+    4829,
+    4833,
+    4837,
+    4839,
+    4843,
+    4871,
+    4875,
+    4879,
+    4880,
+    4885,
+    4888,
+    4895,
+    4900,
+    4923,
+    4966,
+    4968,
+    4972,
+    4989,
+    4994,
+    4998,
+    5001,
+    5013,
+    5019,
+    5026,
+    5032,
+    5034,
+    5046,
+    5055,
+    5085,
+    5086,
+    5088,
+    5089,
+    5090,
+    5095,
+    5108,
+    5110,
+    5119,
+    5120,
+    5121,
+    5133,
+    5148,
+    5154,
+    5160,
+    5169,
+    5178,
+    5222,
+    5223,
+    5232,
+    5233,
+    5240,
+    5256,
+    5259,
+    5264,
+    5271,
+    5276,
+    5279,
+    5294,
+    5300,
+    5303,
+    5321,
+    5335,
+    5337,
+    5345,
+    5348,
+    5365,
+    5373,
+    5378,
+    5384,
+    5422,
+    5442,
+    5443,
+    5445,
+    5477,
+    5485,
+    5495,
+    5497,
+    5504,
+    5526,
+    5533,
+    5542,
+    5564,
+    5570,
+    5579,
+    5587,
+    5592,
+    5608,
+    5610,
+    5624,
+    5630,
+    5638,
+    5651,
+    5663,
+    5670,
+    5675,
+    5691,
+    5700,
+    5706,
+    5707,
+    5715,
+    5720,
+    5721,
+    5722,
+    5732,
+    5738,
+    5741,
+    5769,
+    5771,
+    5775,
+    5795,
+    5796,
+    5800,
+    5809,
+    5810,
+    5815,
+    5819,
+    5827,
+    5848,
+    5849,
+    5852,
+    5859,
+    5880,
+    5884,
+    5907,
+    5909,
+    5912,
+    5919,
+    5931,
+    5932,
+    5934,
+    5941,
+    5948,
+    5950,
+    5952,
+    5953,
+    5969,
+    5981,
+    6000,
+    6003,
+    6010,
+    6018,
+    6041,
+    6065,
+    6075,
+    6090,
+    6103,
+    6106,
+    6109,
+    6114,
+    6123,
+    6131,
+    6136,
+    6138,
+    6139,
+    6164,
+    6165,
+    6171,
+    6173,
+    6180,
+    6185,
+    6189,
+    6190,
+    6221,
+    6223,
+    6237,
+    6255,
+    6262,
+    6265,
+    6293,
+    6296,
+    6305,
+    6318,
+    6331,
+    6336,
+    6340,
+    6355,
+    6366,
+    6371,
+    6396,
+    6399,
+    6402,
+    6408,
+    6432,
+    6433,
+    6441,
+    6456,
+    6465,
+    6478,
+    6486,
+    6489,
+    6507,
+    6508,
+    6520,
+    6522,
+    6528,
+    6537,
+    6551,
+    6564,
+    6589,
+    6590,
+    6598,
+    6599,
+    6611,
+    6613,
+    6615,
+    6621,
+    6634,
+    6647,
+    6649,
+    6654,
+    6676,
+    6680,
+    6690,
+    6708,
+    6729,
+    6736,
+    6744,
+    6749,
+    6756,
+    6761,
+    6763,
+    6773,
+    6788,
+    6790,
+    6796,
+    6797,
+    6811,
+    6824,
+    6850,
+    6869,
+    6871,
+    6882,
+    6892,
+    6895,
+    6906,
+    6911,
+    6912,
+    6914,
+    6927,
+    6952,
+    6966,
+    6985,
+    7001,
+    7021,
+    7031,
+    7035,
+    7038,
+    7041,
+    7045,
+    7052,
+    7057,
+    7058,
+    7072,
+    7073,
+    7076,
+    7086,
+    7102,
+    7107,
+    7109,
+    7117,
+    7122,
+    7125,
+    7166,
+    7182,
+    7199,
+    7207,
+    7212,
+    7215,
+    7232,
+    7243,
+    7246,
+    7250,
+    7253,
+    7258,
+    7263,
+    7267,
+    7270,
+    7274,
+    7280,
+    7286,
+    7293,
+    7298,
+    7302,
+    7306,
+    7316,
+    7325,
+    7326,
+    7334,
+    7356,
+    7357,
+    7363,
+    7364,
+    7367,
+    7385,
+    7392,
+    7399,
+    7405,
+    7416,
+    7420,
+    7421,
+    7426,
+    7452,
+    7476,
+    7481,
+    7519,
+    7534,
+    7542,
+    7546,
+    7573,
+    7585,
+    7601,
+    7604,
+    7606,
+    7609,
+    7629,
+    7649,
+    7653,
+    7667,
+    7682,
+    7699,
+    7738,
+    7750,
+    7786,
+    7798,
+    7800,
+    7801,
+    7805,
+    7806,
+    7811,
+    7813,
+    7832,
+    7837,
+    7849,
+    7856,
+    7876,
+    7877,
+    7887,
+    7905,
+    7916,
+    7919,
+    7920,
+    7922,
+    7923,
+    7926,
+    7944,
+    7961,
+    7966,
+    7970,
+    7973,
+    7974,
+    7976,
+    7986,
+    7999,
+    8027,
+    8028,
+    8034,
+    8047,
+    8068,
+    8074,
+    8077,
+    8091,
+    8120,
+    8122,
+    8125,
+    8152,
+    8153,
+    8164,
+    8167,
+    8169,
+    8171,
+    8177,
+    8184,
+    8189,
+    8192,
+    8196,
+    8202,
+    8212,
+    8217,
+    8231,
+    8242,
+    8244,
+    8250,
+    8256,
+    8257,
+    8265,
+    8270,
+    8274,
+    8283,
+    8290,
+    8294,
+    8301,
+    8302,
+    8307,
+    8318,
+    8326,
+    8338,
+    8349,
+    8350,
+    8353,
+    8357,
+    8371,
+    8374,
+    8377,
+    8380,
+    8387,
+    8388,
+    8396,
+    8402,
+    8419,
+    8423,
+    8428,
+    8435,
+    8439,
+    8442,
+    8444,
+    8453,
+    8461,
+    8475,
+    8481,
+    8485,
+    8493,
+    8495,
+    8498,
+    8523,
+    8530,
+    8531,
+    8562
+  ],
+  "early_stopping_patience": 3
+}

phase1/ablations/shuffled_documents/README.md

@@ -0,0 +1,81 @@
+---
+library_name: peft
+model_name: phase1-qwen-shuffled-documents-ablation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-1.5B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+license: apache-2.0
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+pipeline_tag: text-generation
+---
+
+# Phase 1 Qwen `shuffled_documents` diagnostic ablation
+
+This is the Phase 1 `shuffled_documents` diagnostic LoRA-DPO adapter fine-tuned
+from
+[Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct).
+The frozen training recipe and membership hashes are included in this directory.
+
+## Quick start
+
+```python
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo_id = "geyuxu/york-milestone-project-self-traing-loop-model"
+subfolder = "phase1/ablations/shuffled_documents"
+base_id = "Qwen/Qwen2.5-1.5B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(repo_id, subfolder=subfolder)
+base_model = AutoModelForCausalLM.from_pretrained(
+    base_id, torch_dtype="auto", device_map="auto"
+)
+model = PeftModel.from_pretrained(base_model, repo_id, subfolder=subfolder)
+```
+
+## Training procedure
+
+
+
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 0.29.0
+- Transformers: 5.3.0
+- Pytorch: 2.10.0
+- Datasets: 4.6.1
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+    title        = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+    author       = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+    year         = 2023,
+    booktitle    = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+    url          = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+    editor       = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

phase1/ablations/shuffled_documents/adapter_config.json

@@ -0,0 +1,43 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "k_proj",
+    "o_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}