Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

NOTICE

@@ -0,0 +1,3 @@
+Kakeya OProver LoRA derivative, 2026 FluffyAIcode.
+Base model: m-a-p/OProver-8B, Apache-2.0, revision cd9ffd383b584d95bf00e04b88b35b05928b211c.
+Training material derives from leanprover-community/mathlib4, Apache-2.0, revision 360da6fa66c1273b76b6b2d8c5666fd5ac2e3b56.

README.md

@@ -0,0 +1,93 @@
+---
+license: apache-2.0
+base_model: m-a-p/OProver-8B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- lean
+- lean4
+- theorem-proving
+- lora
+- peft
+---
+
+# Kakeya OProver Stage 3 — DPO
+
+> **Status: NO-GO REGRESSION**
+
+## Summary
+
+This is an independent PEFT LoRA checkpoint in the Kakeya OProver training journey. It is a proof-advisor research artifact, not an autonomous proof or mathematical truth claim.
+
+## Training Data
+
+Public data config: [`dpo-2000`](https://huggingface.co/datasets/FluffyAIcode/Kakeya-OProver-Training). Artifact SHA-256: `2de80cfbb1c5e14c89611810e02b9f432ec2643a203e4c983cd48b1e6ad919a3`. Only Mathlib Apache-2.0 material and explicitly derived training negatives are included; provenance-incomplete OProofs mixtures were excluded.
+
+## Method
+
+DPO on 1,400 training pairs and 200 validation pairs; 400 holdout pairs were excluded from training.
+
+## Strict Lean Evaluation
+
+All matched Stage2–Stage4 evaluations used 22 tasks (18 holdout + 4 canary), one frozen task order, seed, temperature, token budget, and context policy. Parser rate was 22/22 and unsafe generations were 0 for matched evaluations.
+
+```json
+{
+  "status": "NO-GO",
+  "all22": {
+    "corrected_exact": 4,
+    "n": 22,
+    "parser": 22,
+    "unsafe": 0
+  },
+  "holdout18": {
+    "corrected_exact": 1,
+    "n": 18
+  },
+  "canary4": {
+    "corrected_exact": 3,
+    "n": 4
+  },
+  "audit_sha256": "aeb63952d78a132cbe24f4c9238b1c305737c243a51e337c67d5ec08150fe391",
+  "comparison_payload_sha256": "837360593b1fdee9205166cf59d7d091b5992c33a8488d013b3283f6da5b7b8d",
+  "comparison_signature": {
+    "algorithm": "Ed25519",
+    "public_key_base64": "VEQy27ukhgP7vMlIFwRBVYBgXM0RvFXPrOmQ2cnLOmU=",
+    "signature_base64": "ZEVKmswS+H8sLkrVO3ZFi/OftAUACZNn7h4qH7F2EK2cGzniGMisvAtH/n8wziW5xf40/HylwBEh4HqNGrcmDg=="
+  },
+  "paired_stage2": {
+    "all22_mcnemar_exact_two_sided_p": 0.0625,
+    "holdout18_mcnemar_exact_two_sided_p": 0.125,
+    "interpretation": "Practically material and directionally strong regression, but not conclusive at alpha 0.05."
+  }
+}
+```
+
+## Safety
+
+No private chain-of-thought, production prompts, Jensen candidate proof bodies, credentials, or raw candidate-bearing signed audits are published. Stage status constrains use; Stage4 is **PROOF-ADVISOR-ONLY**.
+
+## Known Failures
+
+Corrected strict validity regressed to 4/22 overall and 1/18 holdout. The paired Stage2 comparison was directionally strong but not significant at alpha 0.05.
+
+## Intended Use
+
+Research on Lean proof suggestion, evaluation, and repair. Every generated proof must be compiled and independently reviewed. Not intended for unsupervised theorem claims or production deployment without separate authorization.
+
+## Reproducibility
+
+- Base: [`m-a-p/OProver-8B`](https://huggingface.co/m-a-p/OProver-8B) at `cd9ffd383b584d95bf00e04b88b35b05928b211c`
+- Immutable source revision: `c3b684ce7729ec94305deac4fbc5fd506c7f79a6`
+- Adapter SHA-256: `064fb996230352ae12fd326944e8c5696edc22c36d34fc710805cb5aa2a3f9e3`
+- Dataset artifact SHA-256: `2de80cfbb1c5e14c89611810e02b9f432ec2643a203e4c983cd48b1e6ad919a3`
+- Sanitized machine-readable files: `evaluation/results.json`, `training/metrics.jsonl`, `training/trainer_state.json`, `provenance/manifest.json`
+- Evaluation/audit digest references are in `evaluation/results.json`; raw candidate-bearing artifacts remain private.
+
+## Limitations
+
+Small, single-seed strict suites cannot establish broad theorem-proving capability. Exact Lean rates have wide confidence intervals. Stage4 did not significantly outperform Stage2 in the matched paired test. Jensen remained `SEARCH_EXHAUSTED` with 0/6 Lean-valid candidates. **RH PROVED: NO.**
+
+## Attribution
+
+Base model: m-a-p/OProver-8B (Apache-2.0). Training data derives from Mathlib at `360da6fa66c1273b76b6b2d8c5666fd5ac2e3b56` (Apache-2.0). See `NOTICE`.

adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "m-a-p/OProver-8B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.20.0",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "q_proj",
+    "up_proj",
+    "v_proj",
+    "down_proj",
+    "o_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:064fb996230352ae12fd326944e8c5696edc22c36d34fc710805cb5aa2a3f9e3
+size 349243752

added_tokens.json

@@ -0,0 +1,57 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<dream>": 151682,
+  "<dream_end>": 151684,
+  "<dream_start>": 151683,
+  "<im_end>": 151681,
+  "<im_patch>": 151679,
+  "<im_start>": 151680,
+  "<patch_end>": 151690,
+  "<patch_newline>": 151691,
+  "<patch_start>": 151689,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<video_end>": 151688,
+  "<video_start>": 151687,
+  "<|BOT|>": 151670,
+  "<|CALL_END|>": 151672,
+  "<|CALL_START|>": 151671,
+  "<|EOT|>": 151669,
+  "<|IMG_END|>": 151676,
+  "<|IMG_START|>": 151675,
+  "<|MASK_1e69f|>": 151685,
+  "<|META_END|>": 151678,
+  "<|META_START|>": 151677,
+  "<|THINK_END|>": 151674,
+  "<|THINK_START|>": 151673,
+  "<|UNMASK_1e69f|>": 151686,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652,
+  "<｜begin▁of▁mask｜>": 151693,
+  "<｜begin▁of▁sentence｜>": 151692,
+  "<｜end▁of▁mask｜>": 151694,
+  "<｜end▁of▁sentence｜>": 151697,
+  "<｜fim▁begin｜>": 151695,
+  "<｜fim▁hole｜>": 151696
+}

chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

evaluation/results.json

@@ -0,0 +1,29 @@
+{
+  "status": "NO-GO",
+  "all22": {
+    "corrected_exact": 4,
+    "n": 22,
+    "parser": 22,
+    "unsafe": 0
+  },
+  "holdout18": {
+    "corrected_exact": 1,
+    "n": 18
+  },
+  "canary4": {
+    "corrected_exact": 3,
+    "n": 4
+  },
+  "audit_sha256": "aeb63952d78a132cbe24f4c9238b1c305737c243a51e337c67d5ec08150fe391",
+  "comparison_payload_sha256": "837360593b1fdee9205166cf59d7d091b5992c33a8488d013b3283f6da5b7b8d",
+  "comparison_signature": {
+    "algorithm": "Ed25519",
+    "public_key_base64": "VEQy27ukhgP7vMlIFwRBVYBgXM0RvFXPrOmQ2cnLOmU=",
+    "signature_base64": "ZEVKmswS+H8sLkrVO3ZFi/OftAUACZNn7h4qH7F2EK2cGzniGMisvAtH/n8wziW5xf40/HylwBEh4HqNGrcmDg=="
+  },
+  "paired_stage2": {
+    "all22_mcnemar_exact_two_sided_p": 0.0625,
+    "holdout18_mcnemar_exact_two_sided_p": 0.125,
+    "interpretation": "Practically material and directionally strong regression, but not conclusive at alpha 0.05."
+  }
+}

provenance/manifest.json

@@ -0,0 +1,26 @@
+{
+  "schema_version": 1,
+  "public_repo": "FluffyAIcode/Kakeya-OProver-Stage3-DPO",
+  "base_model": "m-a-p/OProver-8B",
+  "base_revision": "cd9ffd383b584d95bf00e04b88b35b05928b211c",
+  "source_private_repo": "FluffyAIcode/oprover-8b-mathlib-lora-stage1",
+  "source_immutable_revision": "c3b684ce7729ec94305deac4fbc5fd506c7f79a6",
+  "adapter_sha256": "064fb996230352ae12fd326944e8c5696edc22c36d34fc710805cb5aa2a3f9e3",
+  "dataset_artifact_sha256": "2de80cfbb1c5e14c89611810e02b9f432ec2643a203e4c983cd48b1e6ad919a3",
+  "public_private_boundary": {
+    "public": [
+      "adapter",
+      "tokenizer",
+      "sanitized metrics/trainer state",
+      "aggregate strict evaluation",
+      "hashes/signatures/config"
+    ],
+    "private": [
+      "chain-of-thought",
+      "production prompts",
+      "Jensen candidate proof bodies",
+      "raw candidate-bearing signed audits",
+      "optimizer and RNG checkpoints"
+    ]
+  }
+}

special_tokens_map.json

@@ -0,0 +1,47 @@
+{
+  "additional_special_tokens": [
+    "<|EOT|>",
+    "<|BOT|>",
+    "<|CALL_START|>",
+    "<|CALL_END|>",
+    "<|THINK_START|>",
+    "<|THINK_END|>",
+    "<|IMG_START|>",
+    "<|IMG_END|>",
+    "<|META_START|>",
+    "<|META_END|>",
+    "<im_patch>",
+    "<im_start>",
+    "<im_end>",
+    "<dream>",
+    "<dream_start>",
+    "<dream_end>",
+    "<|MASK_1e69f|>",
+    "<|UNMASK_1e69f|>",
+    "<video_start>",
+    "<video_end>",
+    "<patch_start>",
+    "<patch_end>",
+    "<patch_newline>",
+    "<｜begin▁of▁sentence｜>",
+    "<｜begin▁of▁mask｜>",
+    "<｜end▁of▁mask｜>",
+    "<｜fim▁begin｜>",
+    "<｜fim▁hole｜>",
+    "<｜end▁of▁sentence｜>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:217e7242c0cafb50790f703bb14e062966b997a1bc8dc980e11240f5f1d06d27
+size 11428198

tokenizer_config.json

@@ -0,0 +1,487 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151669": {
+      "content": "<|EOT|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "<|BOT|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "<|CALL_START|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "<|CALL_END|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "<|THINK_START|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151674": {
+      "content": "<|THINK_END|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151675": {
+      "content": "<|IMG_START|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151676": {
+      "content": "<|IMG_END|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151677": {
+      "content": "<|META_START|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151678": {
+      "content": "<|META_END|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151679": {
+      "content": "<im_patch>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151680": {
+      "content": "<im_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151681": {
+      "content": "<im_end>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151682": {
+      "content": "<dream>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151683": {
+      "content": "<dream_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151684": {
+      "content": "<dream_end>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151685": {
+      "content": "<|MASK_1e69f|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151686": {
+      "content": "<|UNMASK_1e69f|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151687": {
+      "content": "<video_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151688": {
+      "content": "<video_end>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151689": {
+      "content": "<patch_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151690": {
+      "content": "<patch_end>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151691": {
+      "content": "<patch_newline>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151692": {
+      "content": "<｜begin▁of▁sentence｜>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151693": {
+      "content": "<｜begin▁of▁mask｜>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151694": {
+      "content": "<｜end▁of▁mask｜>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151695": {
+      "content": "<｜fim▁begin｜>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151696": {
+      "content": "<｜fim▁hole｜>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151697": {
+      "content": "<｜end▁of▁sentence｜>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|EOT|>",
+    "<|BOT|>",
+    "<|CALL_START|>",
+    "<|CALL_END|>",
+    "<|THINK_START|>",
+    "<|THINK_END|>",
+    "<|IMG_START|>",
+    "<|IMG_END|>",
+    "<|META_START|>",
+    "<|META_END|>",
+    "<im_patch>",
+    "<im_start>",
+    "<im_end>",
+    "<dream>",
+    "<dream_start>",
+    "<dream_end>",
+    "<|MASK_1e69f|>",
+    "<|UNMASK_1e69f|>",
+    "<video_start>",
+    "<video_end>",
+    "<patch_start>",
+    "<patch_end>",
+    "<patch_newline>",
+    "<｜begin▁of▁sentence｜>",
+    "<｜begin▁of▁mask｜>",
+    "<｜end▁of▁mask｜>",
+    "<｜fim▁begin｜>",
+    "<｜fim▁hole｜>",
+    "<｜end▁of▁sentence｜>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

training/metrics.jsonl

@@ -0,0 +1,93 @@
+{"epoch": 0.011428571428571429, "grad_norm": 5.672101974487305, "learning_rate": 0.0, "logits/chosen": -0.003827691078186035, "logits/rejected": 1.6622543334960938, "logps/chosen": -53.823280334472656, "logps/rejected": -83.19792175292969, "loss": 0.6931, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/rejected": 0.0, "step": 1}
+{"epoch": 0.022857142857142857, "grad_norm": 5.676645755767822, "learning_rate": 1.0000000000000002e-06, "logits/chosen": -0.23317879438400269, "logits/rejected": 1.487736701965332, "logps/chosen": -105.3038101196289, "logps/rejected": -72.62786102294922, "loss": 0.6931, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/rejected": 0.0, "step": 2}
+{"epoch": 0.03428571428571429, "grad_norm": 5.494569778442383, "learning_rate": 2.0000000000000003e-06, "logits/chosen": -0.34497007727622986, "logits/rejected": 1.3572413921356201, "logps/chosen": -78.18315124511719, "logps/rejected": -92.42572021484375, "loss": 0.6925, "rewards/accuracies": 0.5, "rewards/chosen": -0.010104288347065449, "rewards/margins": 0.001475572120398283, "rewards/rejected": -0.011579860001802444, "step": 3}
+{"epoch": 0.045714285714285714, "grad_norm": 5.875666618347168, "learning_rate": 3e-06, "logits/chosen": 0.0768582671880722, "logits/rejected": 1.793702244758606, "logps/chosen": -49.45244598388672, "logps/rejected": -70.34400939941406, "loss": 0.6824, "rewards/accuracies": 0.75, "rewards/chosen": -0.006157493684440851, "rewards/margins": 0.02201489359140396, "rewards/rejected": -0.028172386810183525, "step": 4}
+{"epoch": 0.05714285714285714, "grad_norm": 4.4969987869262695, "learning_rate": 4.000000000000001e-06, "logits/chosen": 0.29756978154182434, "logits/rejected": 1.5599043369293213, "logps/chosen": -38.330902099609375, "logps/rejected": -58.7880859375, "loss": 0.6602, "rewards/accuracies": 1.0, "rewards/chosen": 0.0054547907784581184, "rewards/margins": 0.06765428930521011, "rewards/rejected": -0.06219949945807457, "step": 5}
+{"epoch": 0.06857142857142857, "grad_norm": 5.193802356719971, "learning_rate": 5e-06, "logits/chosen": -0.097133569419384, "logits/rejected": 1.5903254747390747, "logps/chosen": -71.06304168701172, "logps/rejected": -69.80386352539062, "loss": 0.6118, "rewards/accuracies": 0.9375, "rewards/chosen": 0.02022118680179119, "rewards/margins": 0.1755855232477188, "rewards/rejected": -0.15536434948444366, "step": 6}
+{"epoch": 0.08, "grad_norm": 4.568131446838379, "learning_rate": 4.998209387040829e-06, "logits/chosen": 0.22878801822662354, "logits/rejected": 1.2406954765319824, "logps/chosen": -65.77464294433594, "logps/rejected": -70.03345489501953, "loss": 0.5809, "rewards/accuracies": 1.0, "rewards/chosen": 0.008165514096617699, "rewards/margins": 0.2519247233867645, "rewards/rejected": -0.2437591850757599, "step": 7}
+{"epoch": 0.09142857142857143, "grad_norm": 3.668226480484009, "learning_rate": 4.992840113199131e-06, "logits/chosen": -0.06199551001191139, "logits/rejected": 1.0075984001159668, "logps/chosen": -31.54962158203125, "logps/rejected": -48.84445571899414, "loss": 0.5925, "rewards/accuracies": 1.0, "rewards/chosen": 0.006166815757751465, "rewards/margins": 0.21949976682662964, "rewards/rejected": -0.21333293616771698, "step": 8}
+{"epoch": 0.10285714285714286, "grad_norm": 4.007819652557373, "learning_rate": 4.983899869907963e-06, "logits/chosen": 0.04794704169034958, "logits/rejected": 1.0474152565002441, "logps/chosen": -74.90632629394531, "logps/rejected": -68.87115478515625, "loss": 0.5221, "rewards/accuracies": 1.0, "rewards/chosen": 0.017194844782352448, "rewards/margins": 0.40241336822509766, "rewards/rejected": -0.3852185010910034, "step": 9}
+{"epoch": 0.11428571428571428, "grad_norm": 4.3215508460998535, "learning_rate": 4.971401463979722e-06, "logits/chosen": 0.11494935303926468, "logits/rejected": 1.1323603391647339, "logps/chosen": -57.83014678955078, "logps/rejected": -69.66509246826172, "loss": 0.4632, "rewards/accuracies": 1.0, "rewards/chosen": 0.035677470266819, "rewards/margins": 0.5738754868507385, "rewards/rejected": -0.5381979942321777, "step": 10}
+{"epoch": 0.12571428571428572, "grad_norm": 3.5642457008361816, "learning_rate": 4.955362799260507e-06, "logits/chosen": -0.14375551044940948, "logits/rejected": 0.6537873148918152, "logps/chosen": -53.47944259643555, "logps/rejected": -81.25194549560547, "loss": 0.4828, "rewards/accuracies": 1.0, "rewards/chosen": 0.019211266189813614, "rewards/margins": 0.5275158882141113, "rewards/rejected": -0.5083046555519104, "step": 11}
+{"epoch": 0.13714285714285715, "grad_norm": 3.5064291954040527, "learning_rate": 4.935806850983034e-06, "logits/chosen": -0.02809108793735504, "logits/rejected": 1.3847907781600952, "logps/chosen": -54.27452850341797, "logps/rejected": -84.8036117553711, "loss": 0.3669, "rewards/accuracies": 1.0, "rewards/chosen": 0.038996659219264984, "rewards/margins": 0.9113224744796753, "rewards/rejected": -0.8723257184028625, "step": 12}
+{"epoch": 0.14857142857142858, "grad_norm": 4.0220561027526855, "learning_rate": 4.912761632854834e-06, "logits/chosen": -0.40826213359832764, "logits/rejected": 1.0781131982803345, "logps/chosen": -80.10173797607422, "logps/rejected": -64.88600158691406, "loss": 0.4242, "rewards/accuracies": 1.0, "rewards/chosen": 0.028980137780308723, "rewards/margins": 0.7560938000679016, "rewards/rejected": -0.727113664150238, "step": 13}
+{"epoch": 0.16, "grad_norm": 3.641127347946167, "learning_rate": 4.8862601569288885e-06, "logits/chosen": -0.3284621238708496, "logits/rejected": 0.6769169569015503, "logps/chosen": -126.9131088256836, "logps/rejected": -108.0735855102539, "loss": 0.378, "rewards/accuracies": 1.0, "rewards/chosen": 0.10002796351909637, "rewards/margins": 0.9067593812942505, "rewards/rejected": -0.8067314028739929, "step": 14}
+{"epoch": 0.17142857142857143, "grad_norm": 3.1787471771240234, "learning_rate": 4.8563403863141825e-06, "logits/chosen": -0.23578333854675293, "logits/rejected": 0.8872382044792175, "logps/chosen": -77.02982330322266, "logps/rejected": -91.4344482421875, "loss": 0.3247, "rewards/accuracies": 1.0, "rewards/chosen": -0.0036322600208222866, "rewards/margins": 1.1259247064590454, "rewards/rejected": -1.1295570135116577, "step": 15}
+{"epoch": 0.18285714285714286, "grad_norm": 3.3664069175720215, "learning_rate": 4.823045180793914e-06, "logits/chosen": -0.33357855677604675, "logits/rejected": 0.9027751088142395, "logps/chosen": -57.791385650634766, "logps/rejected": -71.88220977783203, "loss": 0.3925, "rewards/accuracies": 1.0, "rewards/chosen": 0.03762559965252876, "rewards/margins": 0.8615567684173584, "rewards/rejected": -0.823931097984314, "step": 16}
+{"epoch": 0.19428571428571428, "grad_norm": 2.321681261062622, "learning_rate": 4.786422235429269e-06, "logits/chosen": -0.2371266484260559, "logits/rejected": 0.4381519556045532, "logps/chosen": -45.621620178222656, "logps/rejected": -89.46638488769531, "loss": 0.3503, "rewards/accuracies": 1.0, "rewards/chosen": 0.028645562008023262, "rewards/margins": 1.0791183710098267, "rewards/rejected": -1.0504727363586426, "step": 17}
+{"epoch": 0.2057142857142857, "grad_norm": 2.6169960498809814, "learning_rate": 4.746524012236706e-06, "logits/chosen": -0.1888342797756195, "logits/rejected": 0.44934579730033875, "logps/chosen": -69.54957580566406, "logps/rejected": -59.541568756103516, "loss": 0.4125, "rewards/accuracies": 1.0, "rewards/chosen": 0.0358089953660965, "rewards/margins": 0.8032007813453674, "rewards/rejected": -0.7673917412757874, "step": 18}
+{"epoch": 0.21714285714285714, "grad_norm": 2.923261880874634, "learning_rate": 4.703407665036622e-06, "logits/chosen": -0.20129157602787018, "logits/rejected": 0.6290000677108765, "logps/chosen": -56.686153411865234, "logps/rejected": -87.55162048339844, "loss": 0.264, "rewards/accuracies": 1.0, "rewards/chosen": 0.054906971752643585, "rewards/margins": 1.5454007387161255, "rewards/rejected": -1.4904940128326416, "step": 19}
+{"epoch": 0.22857142857142856, "grad_norm": 2.864265203475952, "learning_rate": 4.657134957581057e-06, "logits/chosen": -0.3821311295032501, "logits/rejected": 0.48754024505615234, "logps/chosen": -62.24723815917969, "logps/rejected": -60.93783950805664, "loss": 0.3685, "rewards/accuracies": 1.0, "rewards/chosen": 0.06970775872468948, "rewards/margins": 1.0107057094573975, "rewards/rejected": -0.9409979581832886, "step": 20}
+{"epoch": 0.24, "grad_norm": 1.9049395322799683, "learning_rate": 4.607772175077712e-06, "logits/chosen": -0.5025634765625, "logits/rejected": 0.4401867389678955, "logps/chosen": -57.89384460449219, "logps/rejected": -79.51171112060547, "loss": 0.3142, "rewards/accuracies": 1.0, "rewards/chosen": 0.049478232860565186, "rewards/margins": 1.3050354719161987, "rewards/rejected": -1.2555571794509888, "step": 21}
+{"epoch": 0.25142857142857145, "grad_norm": 2.7351036071777344, "learning_rate": 4.555390029237026e-06, "logits/chosen": -0.36178550124168396, "logits/rejected": 0.8178424835205078, "logps/chosen": -65.52536010742188, "logps/rejected": -100.3760986328125, "loss": 0.2818, "rewards/accuracies": 0.9375, "rewards/chosen": -0.06696957349777222, "rewards/margins": 1.5236375331878662, "rewards/rejected": -1.5906071662902832, "step": 22}
+{"epoch": 0.25142857142857145, "eval_logits/chosen": -0.36531969904899597, "eval_logits/rejected": 0.3482947051525116, "eval_logps/chosen": -87.77478790283203, "eval_logps/rejected": -92.11418914794922, "eval_loss": 0.30896151065826416, "eval_rewards/accuracies": 0.9800000190734863, "eval_rewards/chosen": -0.20244216918945312, "eval_rewards/margins": 1.3565798997879028, "eval_rewards/rejected": -1.559022068977356, "eval_runtime": 209.0014, "eval_samples_per_second": 0.957, "eval_steps_per_second": 0.957, "step": 22}
+{"epoch": 0.26285714285714284, "grad_norm": 2.936436653137207, "learning_rate": 4.5000635569783365e-06, "logits/chosen": -0.278434157371521, "logits/rejected": 0.1438252478837967, "logps/chosen": -70.02806091308594, "logps/rejected": -57.32536697387695, "loss": 0.3441, "rewards/accuracies": 1.0, "rewards/chosen": 0.005648649297654629, "rewards/margins": 1.069925308227539, "rewards/rejected": -1.0642765760421753, "step": 23}
+{"epoch": 0.2742857142857143, "grad_norm": 1.480675458908081, "learning_rate": 4.4418720129402145e-06, "logits/chosen": -0.6613065004348755, "logits/rejected": 0.20875723659992218, "logps/chosen": -43.0631217956543, "logps/rejected": -113.96063995361328, "loss": 0.2264, "rewards/accuracies": 1.0, "rewards/chosen": 0.007623173296451569, "rewards/margins": 1.854811191558838, "rewards/rejected": -1.8471879959106445, "step": 24}
+{"epoch": 0.2857142857142857, "grad_norm": 2.0406107902526855, "learning_rate": 4.3808987559489536e-06, "logits/chosen": -0.4824203848838806, "logits/rejected": 0.370841383934021, "logps/chosen": -39.89043426513672, "logps/rejected": -89.99382781982422, "loss": 0.2224, "rewards/accuracies": 1.0, "rewards/chosen": 0.0602618083357811, "rewards/margins": 1.6918809413909912, "rewards/rejected": -1.6316192150115967, "step": 25}
+{"epoch": 0.29714285714285715, "grad_norm": 2.2385094165802, "learning_rate": 4.317231129607859e-06, "logits/chosen": -0.5961613655090332, "logits/rejected": 0.14968276023864746, "logps/chosen": -76.45050048828125, "logps/rejected": -99.48614501953125, "loss": 0.2123, "rewards/accuracies": 1.0, "rewards/chosen": 0.07128332555294037, "rewards/margins": 1.763782024383545, "rewards/rejected": -1.6924986839294434, "step": 26}
+{"epoch": 0.30857142857142855, "grad_norm": 1.6301268339157104, "learning_rate": 4.2509603371783776e-06, "logits/chosen": -0.3383774161338806, "logits/rejected": 0.15638643503189087, "logps/chosen": -58.246971130371094, "logps/rejected": -79.45714569091797, "loss": 0.2009, "rewards/accuracies": 1.0, "rewards/chosen": -0.04440131038427353, "rewards/margins": 1.9561799764633179, "rewards/rejected": -2.0005815029144287, "step": 27}
+{"epoch": 0.32, "grad_norm": 2.236377716064453, "learning_rate": 4.1821813109322975e-06, "logits/chosen": -0.647399365901947, "logits/rejected": 0.22453227639198303, "logps/chosen": -88.88107299804688, "logps/rejected": -96.83210754394531, "loss": 0.2263, "rewards/accuracies": 1.0, "rewards/chosen": 0.040186457335948944, "rewards/margins": 1.824659824371338, "rewards/rejected": -1.7844732999801636, "step": 28}
+{"epoch": 0.3314285714285714, "grad_norm": 2.0542852878570557, "learning_rate": 4.110992576162193e-06, "logits/chosen": -0.5227777361869812, "logits/rejected": -0.2961636781692505, "logps/chosen": -78.49622344970703, "logps/rejected": -78.24589538574219, "loss": 0.2121, "rewards/accuracies": 1.0, "rewards/chosen": 0.10791393369436264, "rewards/margins": 1.7918672561645508, "rewards/rejected": -1.6839532852172852, "step": 29}
+{"epoch": 0.34285714285714286, "grad_norm": 1.9906846284866333, "learning_rate": 4.037496110044885e-06, "logits/chosen": -0.44146519899368286, "logits/rejected": 0.24826864898204803, "logps/chosen": -89.6866683959961, "logps/rejected": -122.11793518066406, "loss": 0.1621, "rewards/accuracies": 1.0, "rewards/chosen": -0.04959860444068909, "rewards/margins": 2.5837507247924805, "rewards/rejected": -2.6333494186401367, "step": 30}
+{"epoch": 0.35428571428571426, "grad_norm": 2.3201839923858643, "learning_rate": 3.961797195560118e-06, "logits/chosen": -0.37571054697036743, "logits/rejected": -0.12684349715709686, "logps/chosen": -62.36381530761719, "logps/rejected": -68.8594741821289, "loss": 0.2406, "rewards/accuracies": 1.0, "rewards/chosen": 0.013891173526644707, "rewards/margins": 1.7061822414398193, "rewards/rejected": -1.6922911405563354, "step": 31}
+{"epoch": 0.3657142857142857, "grad_norm": 1.7459288835525513, "learning_rate": 3.884004270673711e-06, "logits/chosen": -0.4904371500015259, "logits/rejected": -0.09516231715679169, "logps/chosen": -51.67000198364258, "logps/rejected": -87.77179718017578, "loss": 0.1739, "rewards/accuracies": 1.0, "rewards/chosen": 0.05766277387738228, "rewards/margins": 2.167266845703125, "rewards/rejected": -2.1096038818359375, "step": 32}
+{"epoch": 0.37714285714285717, "grad_norm": 1.9478814601898193, "learning_rate": 3.8042287730012117e-06, "logits/chosen": -0.7833026051521301, "logits/rejected": -0.14795458316802979, "logps/chosen": -49.5107307434082, "logps/rejected": -77.62210845947266, "loss": 0.1861, "rewards/accuracies": 1.0, "rewards/chosen": 0.02107466384768486, "rewards/margins": 1.9514578580856323, "rewards/rejected": -1.9303834438323975, "step": 33}
+{"epoch": 0.38857142857142857, "grad_norm": 2.1133761405944824, "learning_rate": 3.7225849801745835e-06, "logits/chosen": -0.8189583420753479, "logits/rejected": -0.24318619072437286, "logps/chosen": -75.10845947265625, "logps/rejected": -111.31871032714844, "loss": 0.1489, "rewards/accuracies": 1.0, "rewards/chosen": 0.029190005734562874, "rewards/margins": 2.6350479125976562, "rewards/rejected": -2.6058578491210938, "step": 34}
+{"epoch": 0.4, "grad_norm": 2.116342306137085, "learning_rate": 3.6391898461406045e-06, "logits/chosen": -0.4198293685913086, "logits/rejected": -0.3276508152484894, "logps/chosen": -63.513519287109375, "logps/rejected": -93.56440734863281, "loss": 0.2266, "rewards/accuracies": 1.0, "rewards/chosen": 0.09839610755443573, "rewards/margins": 1.798091173171997, "rewards/rejected": -1.6996949911117554, "step": 35}
+{"epoch": 0.4114285714285714, "grad_norm": 1.6935662031173706, "learning_rate": 3.55416283362546e-06, "logits/chosen": -0.8055887222290039, "logits/rejected": 0.15487802028656006, "logps/chosen": -46.510643005371094, "logps/rejected": -105.87352752685547, "loss": 0.1607, "rewards/accuracies": 1.0, "rewards/chosen": 0.033490121364593506, "rewards/margins": 2.649106025695801, "rewards/rejected": -2.6156160831451416, "step": 36}
+{"epoch": 0.4228571428571429, "grad_norm": 2.2720091342926025, "learning_rate": 3.4676257430055438e-06, "logits/chosen": -0.8104305267333984, "logits/rejected": 0.04826318100094795, "logps/chosen": -77.22013854980469, "logps/rejected": -98.19013977050781, "loss": 0.1415, "rewards/accuracies": 1.0, "rewards/chosen": 0.03917883336544037, "rewards/margins": 2.6867618560791016, "rewards/rejected": -2.6475830078125, "step": 37}
+{"epoch": 0.4342857142857143, "grad_norm": 1.546297550201416, "learning_rate": 3.3797025378295826e-06, "logits/chosen": -0.8403704166412354, "logits/rejected": 0.052355289459228516, "logps/chosen": -56.61090087890625, "logps/rejected": -119.7930679321289, "loss": 0.1658, "rewards/accuracies": 1.0, "rewards/chosen": 0.022541070356965065, "rewards/margins": 2.8231887817382812, "rewards/rejected": -2.800647735595703, "step": 38}
+{"epoch": 0.44571428571428573, "grad_norm": 2.0681819915771484, "learning_rate": 3.29051916724206e-06, "logits/chosen": -0.6301568746566772, "logits/rejected": -0.32246649265289307, "logps/chosen": -66.40396118164062, "logps/rejected": -81.09312438964844, "loss": 0.1799, "rewards/accuracies": 1.0, "rewards/chosen": 0.009487343952059746, "rewards/margins": 2.2801120281219482, "rewards/rejected": -2.2706246376037598, "step": 39}
+{"epoch": 0.45714285714285713, "grad_norm": 1.4052331447601318, "learning_rate": 3.2002033855622683e-06, "logits/chosen": -0.8533369302749634, "logits/rejected": -0.4283853769302368, "logps/chosen": -47.345802307128906, "logps/rejected": -93.77117919921875, "loss": 0.1517, "rewards/accuracies": 1.0, "rewards/chosen": -0.026743369176983833, "rewards/margins": 2.6302499771118164, "rewards/rejected": -2.6569931507110596, "step": 40}
+{"epoch": 0.4685714285714286, "grad_norm": 2.0809950828552246, "learning_rate": 3.1088845692774798e-06, "logits/chosen": -0.6270914077758789, "logits/rejected": -0.6155582070350647, "logps/chosen": -88.60835266113281, "logps/rejected": -67.6736831665039, "loss": 0.1825, "rewards/accuracies": 1.0, "rewards/chosen": 0.04146631807088852, "rewards/margins": 2.0336129665374756, "rewards/rejected": -1.9921468496322632, "step": 41}
+{"epoch": 0.48, "grad_norm": 1.1893919706344604, "learning_rate": 3.0166935317123824e-06, "logits/chosen": -1.0347564220428467, "logits/rejected": -0.4240405261516571, "logps/chosen": -56.76289367675781, "logps/rejected": -103.66142272949219, "loss": 0.1124, "rewards/accuracies": 1.0, "rewards/chosen": -0.057413943111896515, "rewards/margins": 3.0061051845550537, "rewards/rejected": -3.06351900100708, "step": 42}
+{"epoch": 0.49142857142857144, "grad_norm": 1.1706154346466064, "learning_rate": 2.9237623356402423e-06, "logits/chosen": -1.0618159770965576, "logits/rejected": -0.327092707157135, "logps/chosen": -32.207794189453125, "logps/rejected": -111.8665771484375, "loss": 0.0903, "rewards/accuracies": 1.0, "rewards/chosen": -0.027263116091489792, "rewards/margins": 3.3231871128082275, "rewards/rejected": -3.350450277328491, "step": 43}
+{"epoch": 0.5028571428571429, "grad_norm": 1.079911470413208, "learning_rate": 2.8302241041042564e-06, "logits/chosen": -1.1862759590148926, "logits/rejected": -0.7669480443000793, "logps/chosen": -89.4259262084961, "logps/rejected": -127.56314086914062, "loss": 0.0967, "rewards/accuracies": 1.0, "rewards/chosen": 0.14472071826457977, "rewards/margins": 3.0603933334350586, "rewards/rejected": -2.915672540664673, "step": 44}
+{"epoch": 0.5028571428571429, "eval_logits/chosen": -0.8571964502334595, "eval_logits/rejected": -0.6531373858451843, "eval_logps/chosen": -90.10453033447266, "eval_logps/rejected": -104.75491333007812, "eval_loss": 0.17935897409915924, "eval_rewards/accuracies": 0.9700000286102295, "eval_rewards/chosen": -0.4354174733161926, "eval_rewards/margins": 2.387676954269409, "eval_rewards/rejected": -2.8230948448181152, "eval_runtime": 208.3973, "eval_samples_per_second": 0.96, "eval_steps_per_second": 0.96, "step": 44}
+{"epoch": 0.5142857142857142, "grad_norm": 1.5324379205703735, "learning_rate": 2.7362128297200784e-06, "logits/chosen": -0.7203244566917419, "logits/rejected": -0.5576686859130859, "logps/chosen": -65.52645874023438, "logps/rejected": -95.54450988769531, "loss": 0.1323, "rewards/accuracies": 1.0, "rewards/chosen": 0.1149684339761734, "rewards/margins": 2.6911990642547607, "rewards/rejected": -2.576230525970459, "step": 45}
+{"epoch": 0.5257142857142857, "grad_norm": 1.2247161865234375, "learning_rate": 2.6418631827326857e-06, "logits/chosen": -0.9313367009162903, "logits/rejected": -0.8070792555809021, "logps/chosen": -55.876502990722656, "logps/rejected": -119.91787719726562, "loss": 0.1285, "rewards/accuracies": 1.0, "rewards/chosen": 0.06400280445814133, "rewards/margins": 2.901925802230835, "rewards/rejected": -2.837923288345337, "step": 46}
+{"epoch": 0.5371428571428571, "grad_norm": 1.3425116539001465, "learning_rate": 2.547310318102548e-06, "logits/chosen": -0.9409236907958984, "logits/rejected": -0.7352248430252075, "logps/chosen": -50.5601806640625, "logps/rejected": -86.23294830322266, "loss": 0.1466, "rewards/accuracies": 1.0, "rewards/chosen": 0.09888257831335068, "rewards/margins": 2.470425844192505, "rewards/rejected": -2.3715431690216064, "step": 47}
+{"epoch": 0.5485714285714286, "grad_norm": 1.4391740560531616, "learning_rate": 2.4526896818974534e-06, "logits/chosen": -0.8374643921852112, "logits/rejected": -0.47213953733444214, "logps/chosen": -50.190616607666016, "logps/rejected": -91.12651824951172, "loss": 0.1023, "rewards/accuracies": 1.0, "rewards/chosen": -0.009993518702685833, "rewards/margins": 2.796506881713867, "rewards/rejected": -2.8065006732940674, "step": 48}
+{"epoch": 0.56, "grad_norm": 1.5294134616851807, "learning_rate": 2.358136817267315e-06, "logits/chosen": -0.8648539185523987, "logits/rejected": -0.736384391784668, "logps/chosen": -72.85952758789062, "logps/rejected": -83.08409881591797, "loss": 0.1516, "rewards/accuracies": 1.0, "rewards/chosen": 0.09306713193655014, "rewards/margins": 2.5247979164123535, "rewards/rejected": -2.4317305088043213, "step": 49}
+{"epoch": 0.5714285714285714, "grad_norm": 1.6909253597259521, "learning_rate": 2.263787170279922e-06, "logits/chosen": -1.3599584102630615, "logits/rejected": -1.2676033973693848, "logps/chosen": -78.65673065185547, "logps/rejected": -69.09491729736328, "loss": 0.1557, "rewards/accuracies": 1.0, "rewards/chosen": 0.00439932756125927, "rewards/margins": 2.1822092533111572, "rewards/rejected": -2.177809953689575, "step": 50}
+{"epoch": 0.5828571428571429, "grad_norm": 1.3136837482452393, "learning_rate": 2.169775895895745e-06, "logits/chosen": -0.7750363349914551, "logits/rejected": -0.7582042217254639, "logps/chosen": -48.27521514892578, "logps/rejected": -105.779541015625, "loss": 0.0944, "rewards/accuracies": 1.0, "rewards/chosen": 0.060845933854579926, "rewards/margins": 3.2103078365325928, "rewards/rejected": -3.1494617462158203, "step": 51}
+{"epoch": 0.5942857142857143, "grad_norm": 1.304816722869873, "learning_rate": 2.0762376643597586e-06, "logits/chosen": -0.7206559777259827, "logits/rejected": -0.7368943691253662, "logps/chosen": -49.27516174316406, "logps/rejected": -99.28387451171875, "loss": 0.1227, "rewards/accuracies": 1.0, "rewards/chosen": 0.08159955590963364, "rewards/margins": 2.943441390991211, "rewards/rejected": -2.861841917037964, "step": 52}
+{"epoch": 0.6057142857142858, "grad_norm": 1.1686677932739258, "learning_rate": 1.9833064682876175e-06, "logits/chosen": -0.7927972674369812, "logits/rejected": -0.8789339065551758, "logps/chosen": -62.97304916381836, "logps/rejected": -93.07855224609375, "loss": 0.097, "rewards/accuracies": 1.0, "rewards/chosen": 0.10683320462703705, "rewards/margins": 2.952019453048706, "rewards/rejected": -2.8451859951019287, "step": 53}
+{"epoch": 0.6171428571428571, "grad_norm": 0.7459912300109863, "learning_rate": 1.8911154307225204e-06, "logits/chosen": -0.7670217752456665, "logits/rejected": -0.7017982006072998, "logps/chosen": -31.30267333984375, "logps/rejected": -110.78553771972656, "loss": 0.0593, "rewards/accuracies": 1.0, "rewards/chosen": 0.003862532787024975, "rewards/margins": 3.5174167156219482, "rewards/rejected": -3.513554096221924, "step": 54}
+{"epoch": 0.6285714285714286, "grad_norm": 1.5418952703475952, "learning_rate": 1.7997966144377328e-06, "logits/chosen": -0.986474871635437, "logits/rejected": -1.1061036586761475, "logps/chosen": -47.49861526489258, "logps/rejected": -67.53656005859375, "loss": 0.1543, "rewards/accuracies": 1.0, "rewards/chosen": -0.04688534885644913, "rewards/margins": 2.227695941925049, "rewards/rejected": -2.27458119392395, "step": 55}
+{"epoch": 0.64, "grad_norm": 1.0748093128204346, "learning_rate": 1.7094808327579401e-06, "logits/chosen": -1.324556589126587, "logits/rejected": -0.8223966360092163, "logps/chosen": -36.40914535522461, "logps/rejected": -105.08839416503906, "loss": 0.084, "rewards/accuracies": 1.0, "rewards/chosen": -0.07963553071022034, "rewards/margins": 3.377859354019165, "rewards/rejected": -3.4574952125549316, "step": 56}
+{"epoch": 0.6514285714285715, "grad_norm": 0.9280484318733215, "learning_rate": 1.6202974621704176e-06, "logits/chosen": -1.450647234916687, "logits/rejected": -0.8040248155593872, "logps/chosen": -68.76046752929688, "logps/rejected": -125.3299331665039, "loss": 0.0635, "rewards/accuracies": 1.0, "rewards/chosen": 0.06499414145946503, "rewards/margins": 3.9410018920898438, "rewards/rejected": -3.876007318496704, "step": 57}
+{"epoch": 0.6628571428571428, "grad_norm": 0.8896822333335876, "learning_rate": 1.5323742569944573e-06, "logits/chosen": -1.0071394443511963, "logits/rejected": -1.0022106170654297, "logps/chosen": -45.09584426879883, "logps/rejected": -109.64781951904297, "loss": 0.0802, "rewards/accuracies": 1.0, "rewards/chosen": 0.034890901297330856, "rewards/margins": 3.5162737369537354, "rewards/rejected": -3.4813833236694336, "step": 58}
+{"epoch": 0.6742857142857143, "grad_norm": 0.7682043313980103, "learning_rate": 1.44583716637454e-06, "logits/chosen": -1.0028694868087769, "logits/rejected": -0.813642144203186, "logps/chosen": -66.2764663696289, "logps/rejected": -112.77369689941406, "loss": 0.0576, "rewards/accuracies": 1.0, "rewards/chosen": -0.0501595139503479, "rewards/margins": 3.7721147537231445, "rewards/rejected": -3.822274684906006, "step": 59}
+{"epoch": 0.6857142857142857, "grad_norm": 1.0216327905654907, "learning_rate": 1.3608101538593965e-06, "logits/chosen": -1.1250649690628052, "logits/rejected": -0.9627130627632141, "logps/chosen": -42.440757751464844, "logps/rejected": -100.12843322753906, "loss": 0.0804, "rewards/accuracies": 1.0, "rewards/chosen": 0.11411059647798538, "rewards/margins": 3.393651247024536, "rewards/rejected": -3.2795403003692627, "step": 60}
+{"epoch": 0.6971428571428572, "grad_norm": 0.7271348834037781, "learning_rate": 1.277415019825417e-06, "logits/chosen": -0.7807080745697021, "logits/rejected": -0.7796292304992676, "logps/chosen": -33.5434455871582, "logps/rejected": -112.23175048828125, "loss": 0.0711, "rewards/accuracies": 1.0, "rewards/chosen": -0.06058162823319435, "rewards/margins": 3.8745694160461426, "rewards/rejected": -3.9351511001586914, "step": 61}
+{"epoch": 0.7085714285714285, "grad_norm": 1.7274105548858643, "learning_rate": 1.195771226998789e-06, "logits/chosen": -0.9823087453842163, "logits/rejected": -1.2148367166519165, "logps/chosen": -72.55377960205078, "logps/rejected": -75.32920837402344, "loss": 0.1183, "rewards/accuracies": 1.0, "rewards/chosen": -0.12965497374534607, "rewards/margins": 2.7459590435028076, "rewards/rejected": -2.8756141662597656, "step": 62}
+{"epoch": 0.72, "grad_norm": 0.9623571038246155, "learning_rate": 1.1159957293262888e-06, "logits/chosen": -1.0808844566345215, "logits/rejected": -0.8635554909706116, "logps/chosen": -49.637996673583984, "logps/rejected": -116.99398803710938, "loss": 0.09, "rewards/accuracies": 1.0, "rewards/chosen": 0.02851293236017227, "rewards/margins": 3.4344334602355957, "rewards/rejected": -3.4059205055236816, "step": 63}
+{"epoch": 0.7314285714285714, "grad_norm": 1.0066514015197754, "learning_rate": 1.0382028044398823e-06, "logits/chosen": -1.197101354598999, "logits/rejected": -1.1306017637252808, "logps/chosen": -50.98615264892578, "logps/rejected": -120.8044662475586, "loss": 0.0899, "rewards/accuracies": 1.0, "rewards/chosen": -0.035914015024900436, "rewards/margins": 3.1908507347106934, "rewards/rejected": -3.2267649173736572, "step": 64}
+{"epoch": 0.7428571428571429, "grad_norm": 1.1005553007125854, "learning_rate": 9.625038899551162e-07, "logits/chosen": -1.22350013256073, "logits/rejected": -1.1208860874176025, "logps/chosen": -66.32962036132812, "logps/rejected": -103.34649658203125, "loss": 0.0709, "rewards/accuracies": 1.0, "rewards/chosen": 0.18690410256385803, "rewards/margins": 3.4826159477233887, "rewards/rejected": -3.2957119941711426, "step": 65}
+{"epoch": 0.7542857142857143, "grad_norm": 0.954483687877655, "learning_rate": 8.890074238378074e-07, "logits/chosen": -0.9665268659591675, "logits/rejected": -0.8029574751853943, "logps/chosen": -43.259857177734375, "logps/rejected": -104.5782241821289, "loss": 0.0993, "rewards/accuracies": 1.0, "rewards/chosen": -0.044777870178222656, "rewards/margins": 3.1982903480529785, "rewards/rejected": -3.243067979812622, "step": 66}
+{"epoch": 0.7542857142857143, "eval_logits/chosen": -1.1089446544647217, "eval_logits/rejected": -1.1692878007888794, "eval_logps/chosen": -91.48172760009766, "eval_logps/rejected": -110.86624145507812, "eval_loss": 0.14132851362228394, "eval_rewards/accuracies": 0.9700000286102295, "eval_rewards/chosen": -0.573137104511261, "eval_rewards/margins": 2.8610901832580566, "eval_rewards/rejected": -3.4342269897460938, "eval_runtime": 208.8549, "eval_samples_per_second": 0.958, "eval_steps_per_second": 0.958, "step": 66}
+{"epoch": 0.7657142857142857, "grad_norm": 0.9773061871528625, "learning_rate": 8.178186890677029e-07, "logits/chosen": -1.02699875831604, "logits/rejected": -1.1385068893432617, "logps/chosen": -80.82888793945312, "logps/rejected": -100.25572967529297, "loss": 0.072, "rewards/accuracies": 1.0, "rewards/chosen": 0.0796530619263649, "rewards/margins": 3.1960816383361816, "rewards/rejected": -3.116428852081299, "step": 67}
+{"epoch": 0.7771428571428571, "grad_norm": 1.2940922975540161, "learning_rate": 7.490396628216237e-07, "logits/chosen": -1.2090729475021362, "logits/rejected": -1.2437564134597778, "logps/chosen": -87.85923767089844, "logps/rejected": -120.87287902832031, "loss": 0.1183, "rewards/accuracies": 1.0, "rewards/chosen": 0.018895722925662994, "rewards/margins": 2.969498634338379, "rewards/rejected": -2.9506030082702637, "step": 68}
+{"epoch": 0.7885714285714286, "grad_norm": 0.7793561220169067, "learning_rate": 6.827688703921407e-07, "logits/chosen": -1.1516605615615845, "logits/rejected": -1.295521855354309, "logps/chosen": -52.94973373413086, "logps/rejected": -84.64179229736328, "loss": 0.067, "rewards/accuracies": 1.0, "rewards/chosen": 0.18537920713424683, "rewards/margins": 3.4953463077545166, "rewards/rejected": -3.309967041015625, "step": 69}
+{"epoch": 0.8, "grad_norm": 1.4742248058319092, "learning_rate": 6.191012440510469e-07, "logits/chosen": -1.3784356117248535, "logits/rejected": -1.3366318941116333, "logps/chosen": -57.7204475402832, "logps/rejected": -90.2862548828125, "loss": 0.1197, "rewards/accuracies": 1.0, "rewards/chosen": -0.044269490987062454, "rewards/margins": 2.759215831756592, "rewards/rejected": -2.80348539352417, "step": 70}
+{"epoch": 0.8114285714285714, "grad_norm": 0.7249897718429565, "learning_rate": 5.581279870597866e-07, "logits/chosen": -1.1907292604446411, "logits/rejected": -0.6756631731987, "logps/chosen": -32.33828353881836, "logps/rejected": -125.02144622802734, "loss": 0.0525, "rewards/accuracies": 1.0, "rewards/chosen": -0.06472505629062653, "rewards/margins": 4.0654168128967285, "rewards/rejected": -4.130141258239746, "step": 71}
+{"epoch": 0.8228571428571428, "grad_norm": 1.0847841501235962, "learning_rate": 4.999364430216639e-07, "logits/chosen": -1.237339735031128, "logits/rejected": -1.5806419849395752, "logps/chosen": -52.456146240234375, "logps/rejected": -94.31544494628906, "loss": 0.0992, "rewards/accuracies": 1.0, "rewards/chosen": 0.10363951325416565, "rewards/margins": 2.9890761375427246, "rewards/rejected": -2.885436534881592, "step": 72}
+{"epoch": 0.8342857142857143, "grad_norm": 1.278956651687622, "learning_rate": 4.4460997076297504e-07, "logits/chosen": -1.0774935483932495, "logits/rejected": -1.4760003089904785, "logps/chosen": -43.07012939453125, "logps/rejected": -74.25629425048828, "loss": 0.1131, "rewards/accuracies": 1.0, "rewards/chosen": 0.06534770131111145, "rewards/margins": 2.8206787109375, "rewards/rejected": -2.75533127784729, "step": 73}
+{"epoch": 0.8457142857142858, "grad_norm": 0.7833404541015625, "learning_rate": 3.922278249222894e-07, "logits/chosen": -1.1311602592468262, "logits/rejected": -1.0242271423339844, "logps/chosen": -53.541568756103516, "logps/rejected": -105.48152923583984, "loss": 0.0774, "rewards/accuracies": 1.0, "rewards/chosen": 0.04792798310518265, "rewards/margins": 3.678433418273926, "rewards/rejected": -3.6305058002471924, "step": 74}
+{"epoch": 0.8571428571428571, "grad_norm": 1.03605318069458, "learning_rate": 3.4286504241894283e-07, "logits/chosen": -0.9688073992729187, "logits/rejected": -1.2700920104980469, "logps/chosen": -38.00965118408203, "logps/rejected": -92.08466339111328, "loss": 0.1049, "rewards/accuracies": 1.0, "rewards/chosen": -0.01876715011894703, "rewards/margins": 3.16585636138916, "rewards/rejected": -3.1846237182617188, "step": 75}
+{"epoch": 0.8685714285714285, "grad_norm": 1.319770336151123, "learning_rate": 2.965923349633779e-07, "logits/chosen": -1.3658266067504883, "logits/rejected": -1.3608825206756592, "logps/chosen": -49.00103759765625, "logps/rejected": -117.49503326416016, "loss": 0.0818, "rewards/accuracies": 1.0, "rewards/chosen": -0.09428122639656067, "rewards/margins": 3.61112642288208, "rewards/rejected": -3.7054080963134766, "step": 76}
+{"epoch": 0.88, "grad_norm": 0.9297468662261963, "learning_rate": 2.53475987763295e-07, "logits/chosen": -1.0338634252548218, "logits/rejected": -1.1930313110351562, "logps/chosen": -67.0118179321289, "logps/rejected": -106.61903381347656, "loss": 0.0727, "rewards/accuracies": 1.0, "rewards/chosen": -0.13288229703903198, "rewards/margins": 3.487334728240967, "rewards/rejected": -3.6202168464660645, "step": 77}
+{"epoch": 0.8914285714285715, "grad_norm": 1.1066629886627197, "learning_rate": 2.135777645707318e-07, "logits/chosen": -1.384450078010559, "logits/rejected": -1.3552913665771484, "logps/chosen": -52.43301010131836, "logps/rejected": -101.87602996826172, "loss": 0.0989, "rewards/accuracies": 1.0, "rewards/chosen": 0.009377628564834595, "rewards/margins": 3.135222911834717, "rewards/rejected": -3.125845432281494, "step": 78}
+{"epoch": 0.9028571428571428, "grad_norm": 1.0631781816482544, "learning_rate": 1.7695481920608716e-07, "logits/chosen": -1.1844675540924072, "logits/rejected": -1.0223643779754639, "logps/chosen": -45.87510299682617, "logps/rejected": -102.84295654296875, "loss": 0.0862, "rewards/accuracies": 1.0, "rewards/chosen": -0.05180816724896431, "rewards/margins": 3.2761740684509277, "rewards/rejected": -3.3279824256896973, "step": 79}
+{"epoch": 0.9142857142857143, "grad_norm": 1.2742042541503906, "learning_rate": 1.4365961368581844e-07, "logits/chosen": -1.2797547578811646, "logits/rejected": -1.6902124881744385, "logps/chosen": -79.47929382324219, "logps/rejected": -78.93382263183594, "loss": 0.1025, "rewards/accuracies": 1.0, "rewards/chosen": -0.0567280538380146, "rewards/margins": 2.9206228256225586, "rewards/rejected": -2.977350950241089, "step": 80}
+{"epoch": 0.9257142857142857, "grad_norm": 0.8278792500495911, "learning_rate": 1.137398430711123e-07, "logits/chosen": -1.119363784790039, "logits/rejected": -0.9186141490936279, "logps/chosen": -29.899991989135742, "logps/rejected": -105.7519302368164, "loss": 0.0648, "rewards/accuracies": 1.0, "rewards/chosen": 0.031892240047454834, "rewards/margins": 3.5144004821777344, "rewards/rejected": -3.4825081825256348, "step": 81}
+{"epoch": 0.9371428571428572, "grad_norm": 1.0955795049667358, "learning_rate": 8.723836714516681e-08, "logits/chosen": -1.2281020879745483, "logits/rejected": -1.3817219734191895, "logps/chosen": -73.53196716308594, "logps/rejected": -103.11698913574219, "loss": 0.0811, "rewards/accuracies": 1.0, "rewards/chosen": -0.06516408920288086, "rewards/margins": 3.2024097442626953, "rewards/rejected": -3.267573833465576, "step": 82}
+{"epoch": 0.9485714285714286, "grad_norm": 0.6436601877212524, "learning_rate": 6.419314901696671e-08, "logits/chosen": -1.2053653001785278, "logits/rejected": -1.208804965019226, "logps/chosen": -28.06381607055664, "logps/rejected": -100.99942016601562, "loss": 0.0586, "rewards/accuracies": 1.0, "rewards/chosen": -0.027196241542696953, "rewards/margins": 3.633328437805176, "rewards/rejected": -3.660524845123291, "step": 83}
+{"epoch": 0.96, "grad_norm": 0.9888738393783569, "learning_rate": 4.4637200739493514e-08, "logits/chosen": -1.231987714767456, "logits/rejected": -1.143172025680542, "logps/chosen": -57.63927459716797, "logps/rejected": -96.07897186279297, "loss": 0.1077, "rewards/accuracies": 1.0, "rewards/chosen": -0.05573497712612152, "rewards/margins": 3.1897521018981934, "rewards/rejected": -3.2454872131347656, "step": 84}
+{"epoch": 0.9714285714285714, "grad_norm": 1.0415937900543213, "learning_rate": 2.8598536020278678e-08, "logits/chosen": -1.221508264541626, "logits/rejected": -1.4119690656661987, "logps/chosen": -48.45242691040039, "logps/rejected": -84.4826431274414, "loss": 0.0827, "rewards/accuracies": 1.0, "rewards/chosen": 0.04423363134264946, "rewards/margins": 3.218625545501709, "rewards/rejected": -3.174391984939575, "step": 85}
+{"epoch": 0.9828571428571429, "grad_norm": 1.0360701084136963, "learning_rate": 1.6100130092037704e-08, "logits/chosen": -1.3318215608596802, "logits/rejected": -1.6143568754196167, "logps/chosen": -45.959354400634766, "logps/rejected": -79.45930480957031, "loss": 0.0892, "rewards/accuracies": 1.0, "rewards/chosen": -0.08850360661745071, "rewards/margins": 2.9804043769836426, "rewards/rejected": -3.068908214569092, "step": 86}
+{"epoch": 0.9942857142857143, "grad_norm": 0.7161802053451538, "learning_rate": 7.159886800869875e-09, "logits/chosen": -1.1106700897216797, "logits/rejected": -0.9769763946533203, "logps/chosen": -85.0906982421875, "logps/rejected": -132.0802459716797, "loss": 0.0524, "rewards/accuracies": 1.0, "rewards/chosen": -0.0784958153963089, "rewards/margins": 4.037193298339844, "rewards/rejected": -4.115689277648926, "step": 87}
+{"epoch": 1.0, "grad_norm": 2.9429450035095215, "learning_rate": 1.7906129591713228e-09, "logits/chosen": -1.2418022155761719, "logits/rejected": -0.8798779249191284, "logps/chosen": -90.4301986694336, "logps/rejected": -115.04806518554688, "loss": 0.0987, "rewards/accuracies": 1.0, "rewards/chosen": -0.08904826641082764, "rewards/margins": 3.531545639038086, "rewards/rejected": -3.620594024658203, "step": 88}
+{"epoch": 1.0, "eval_logits/chosen": -1.1477130651474, "eval_logits/rejected": -1.244410514831543, "eval_logps/chosen": -91.70614624023438, "eval_logps/rejected": -111.82247161865234, "eval_loss": 0.1359921395778656, "eval_rewards/accuracies": 0.9700000286102295, "eval_rewards/chosen": -0.5955771207809448, "eval_rewards/margins": 2.9342737197875977, "eval_rewards/rejected": -3.529850959777832, "eval_runtime": 209.3361, "eval_samples_per_second": 0.955, "eval_steps_per_second": 0.955, "step": 88}
+{"epoch": 1.0, "step": 88, "total_flos": 0.0, "train_loss": 0.2132695221172815, "train_runtime": 4416.7921, "train_samples_per_second": 0.317, "train_steps_per_second": 0.02}

training/trainer_state.json

@@ -0,0 +1,1423 @@
+{
+  "max_steps": 88,
+  "save_steps": 22,
+  "is_world_process_zero": true,
+  "train_batch_size": 1,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "is_hyper_param_search": false,
+  "num_input_tokens_seen": 0,
+  "log_history": [
+    {
+      "epoch": 0.011428571428571429,
+      "grad_norm": 5.672101974487305,
+      "learning_rate": 0.0,
+      "logits/chosen": -0.003827691078186035,
+      "logits/rejected": 1.6622543334960938,
+      "logps/chosen": -53.823280334472656,
+      "logps/rejected": -83.19792175292969,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.022857142857142857,
+      "grad_norm": 5.676645755767822,
+      "learning_rate": 1.0000000000000002e-06,
+      "logits/chosen": -0.23317879438400269,
+      "logits/rejected": 1.487736701965332,
+      "logps/chosen": -105.3038101196289,
+      "logps/rejected": -72.62786102294922,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 2
+    },
+    {
+      "epoch": 0.03428571428571429,
+      "grad_norm": 5.494569778442383,
+      "learning_rate": 2.0000000000000003e-06,
+      "logits/chosen": -0.34497007727622986,
+      "logits/rejected": 1.3572413921356201,
+      "logps/chosen": -78.18315124511719,
+      "logps/rejected": -92.42572021484375,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.010104288347065449,
+      "rewards/margins": 0.001475572120398283,
+      "rewards/rejected": -0.011579860001802444,
+      "step": 3
+    },
+    {
+      "epoch": 0.045714285714285714,
+      "grad_norm": 5.875666618347168,
+      "learning_rate": 3e-06,
+      "logits/chosen": 0.0768582671880722,
+      "logits/rejected": 1.793702244758606,
+      "logps/chosen": -49.45244598388672,
+      "logps/rejected": -70.34400939941406,
+      "loss": 0.6824,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.006157493684440851,
+      "rewards/margins": 0.02201489359140396,
+      "rewards/rejected": -0.028172386810183525,
+      "step": 4
+    },
+    {
+      "epoch": 0.05714285714285714,
+      "grad_norm": 4.4969987869262695,
+      "learning_rate": 4.000000000000001e-06,
+      "logits/chosen": 0.29756978154182434,
+      "logits/rejected": 1.5599043369293213,
+      "logps/chosen": -38.330902099609375,
+      "logps/rejected": -58.7880859375,
+      "loss": 0.6602,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.0054547907784581184,
+      "rewards/margins": 0.06765428930521011,
+      "rewards/rejected": -0.06219949945807457,
+      "step": 5
+    },
+    {
+      "epoch": 0.06857142857142857,
+      "grad_norm": 5.193802356719971,
+      "learning_rate": 5e-06,
+      "logits/chosen": -0.097133569419384,
+      "logits/rejected": 1.5903254747390747,
+      "logps/chosen": -71.06304168701172,
+      "logps/rejected": -69.80386352539062,
+      "loss": 0.6118,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.02022118680179119,
+      "rewards/margins": 0.1755855232477188,
+      "rewards/rejected": -0.15536434948444366,
+      "step": 6
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 4.568131446838379,
+      "learning_rate": 4.998209387040829e-06,
+      "logits/chosen": 0.22878801822662354,
+      "logits/rejected": 1.2406954765319824,
+      "logps/chosen": -65.77464294433594,
+      "logps/rejected": -70.03345489501953,
+      "loss": 0.5809,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.008165514096617699,
+      "rewards/margins": 0.2519247233867645,
+      "rewards/rejected": -0.2437591850757599,
+      "step": 7
+    },
+    {
+      "epoch": 0.09142857142857143,
+      "grad_norm": 3.668226480484009,
+      "learning_rate": 4.992840113199131e-06,
+      "logits/chosen": -0.06199551001191139,
+      "logits/rejected": 1.0075984001159668,
+      "logps/chosen": -31.54962158203125,
+      "logps/rejected": -48.84445571899414,
+      "loss": 0.5925,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.006166815757751465,
+      "rewards/margins": 0.21949976682662964,
+      "rewards/rejected": -0.21333293616771698,
+      "step": 8
+    },
+    {
+      "epoch": 0.10285714285714286,
+      "grad_norm": 4.007819652557373,
+      "learning_rate": 4.983899869907963e-06,
+      "logits/chosen": 0.04794704169034958,
+      "logits/rejected": 1.0474152565002441,
+      "logps/chosen": -74.90632629394531,
+      "logps/rejected": -68.87115478515625,
+      "loss": 0.5221,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.017194844782352448,
+      "rewards/margins": 0.40241336822509766,
+      "rewards/rejected": -0.3852185010910034,
+      "step": 9
+    },
+    {
+      "epoch": 0.11428571428571428,
+      "grad_norm": 4.3215508460998535,
+      "learning_rate": 4.971401463979722e-06,
+      "logits/chosen": 0.11494935303926468,
+      "logits/rejected": 1.1323603391647339,
+      "logps/chosen": -57.83014678955078,
+      "logps/rejected": -69.66509246826172,
+      "loss": 0.4632,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.035677470266819,
+      "rewards/margins": 0.5738754868507385,
+      "rewards/rejected": -0.5381979942321777,
+      "step": 10
+    },
+    {
+      "epoch": 0.12571428571428572,
+      "grad_norm": 3.5642457008361816,
+      "learning_rate": 4.955362799260507e-06,
+      "logits/chosen": -0.14375551044940948,
+      "logits/rejected": 0.6537873148918152,
+      "logps/chosen": -53.47944259643555,
+      "logps/rejected": -81.25194549560547,
+      "loss": 0.4828,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.019211266189813614,
+      "rewards/margins": 0.5275158882141113,
+      "rewards/rejected": -0.5083046555519104,
+      "step": 11
+    },
+    {
+      "epoch": 0.13714285714285715,
+      "grad_norm": 3.5064291954040527,
+      "learning_rate": 4.935806850983034e-06,
+      "logits/chosen": -0.02809108793735504,
+      "logits/rejected": 1.3847907781600952,
+      "logps/chosen": -54.27452850341797,
+      "logps/rejected": -84.8036117553711,
+      "loss": 0.3669,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.038996659219264984,
+      "rewards/margins": 0.9113224744796753,
+      "rewards/rejected": -0.8723257184028625,
+      "step": 12
+    },
+    {
+      "epoch": 0.14857142857142858,
+      "grad_norm": 4.0220561027526855,
+      "learning_rate": 4.912761632854834e-06,
+      "logits/chosen": -0.40826213359832764,
+      "logits/rejected": 1.0781131982803345,
+      "logps/chosen": -80.10173797607422,
+      "logps/rejected": -64.88600158691406,
+      "loss": 0.4242,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.028980137780308723,
+      "rewards/margins": 0.7560938000679016,
+      "rewards/rejected": -0.727113664150238,
+      "step": 13
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 3.641127347946167,
+      "learning_rate": 4.8862601569288885e-06,
+      "logits/chosen": -0.3284621238708496,
+      "logits/rejected": 0.6769169569015503,
+      "logps/chosen": -126.9131088256836,
+      "logps/rejected": -108.0735855102539,
+      "loss": 0.378,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.10002796351909637,
+      "rewards/margins": 0.9067593812942505,
+      "rewards/rejected": -0.8067314028739929,
+      "step": 14
+    },
+    {
+      "epoch": 0.17142857142857143,
+      "grad_norm": 3.1787471771240234,
+      "learning_rate": 4.8563403863141825e-06,
+      "logits/chosen": -0.23578333854675293,
+      "logits/rejected": 0.8872382044792175,
+      "logps/chosen": -77.02982330322266,
+      "logps/rejected": -91.4344482421875,
+      "loss": 0.3247,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0036322600208222866,
+      "rewards/margins": 1.1259247064590454,
+      "rewards/rejected": -1.1295570135116577,
+      "step": 15
+    },
+    {
+      "epoch": 0.18285714285714286,
+      "grad_norm": 3.3664069175720215,
+      "learning_rate": 4.823045180793914e-06,
+      "logits/chosen": -0.33357855677604675,
+      "logits/rejected": 0.9027751088142395,
+      "logps/chosen": -57.791385650634766,
+      "logps/rejected": -71.88220977783203,
+      "loss": 0.3925,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.03762559965252876,
+      "rewards/margins": 0.8615567684173584,
+      "rewards/rejected": -0.823931097984314,
+      "step": 16
+    },
+    {
+      "epoch": 0.19428571428571428,
+      "grad_norm": 2.321681261062622,
+      "learning_rate": 4.786422235429269e-06,
+      "logits/chosen": -0.2371266484260559,
+      "logits/rejected": 0.4381519556045532,
+      "logps/chosen": -45.621620178222656,
+      "logps/rejected": -89.46638488769531,
+      "loss": 0.3503,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.028645562008023262,
+      "rewards/margins": 1.0791183710098267,
+      "rewards/rejected": -1.0504727363586426,
+      "step": 17
+    },
+    {
+      "epoch": 0.2057142857142857,
+      "grad_norm": 2.6169960498809814,
+      "learning_rate": 4.746524012236706e-06,
+      "logits/chosen": -0.1888342797756195,
+      "logits/rejected": 0.44934579730033875,
+      "logps/chosen": -69.54957580566406,
+      "logps/rejected": -59.541568756103516,
+      "loss": 0.4125,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.0358089953660965,
+      "rewards/margins": 0.8032007813453674,
+      "rewards/rejected": -0.7673917412757874,
+      "step": 18
+    },
+    {
+      "epoch": 0.21714285714285714,
+      "grad_norm": 2.923261880874634,
+      "learning_rate": 4.703407665036622e-06,
+      "logits/chosen": -0.20129157602787018,
+      "logits/rejected": 0.6290000677108765,
+      "logps/chosen": -56.686153411865234,
+      "logps/rejected": -87.55162048339844,
+      "loss": 0.264,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.054906971752643585,
+      "rewards/margins": 1.5454007387161255,
+      "rewards/rejected": -1.4904940128326416,
+      "step": 19
+    },
+    {
+      "epoch": 0.22857142857142856,
+      "grad_norm": 2.864265203475952,
+      "learning_rate": 4.657134957581057e-06,
+      "logits/chosen": -0.3821311295032501,
+      "logits/rejected": 0.48754024505615234,
+      "logps/chosen": -62.24723815917969,
+      "logps/rejected": -60.93783950805664,
+      "loss": 0.3685,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.06970775872468948,
+      "rewards/margins": 1.0107057094573975,
+      "rewards/rejected": -0.9409979581832886,
+      "step": 20
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 1.9049395322799683,
+      "learning_rate": 4.607772175077712e-06,
+      "logits/chosen": -0.5025634765625,
+      "logits/rejected": 0.4401867389678955,
+      "logps/chosen": -57.89384460449219,
+      "logps/rejected": -79.51171112060547,
+      "loss": 0.3142,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.049478232860565186,
+      "rewards/margins": 1.3050354719161987,
+      "rewards/rejected": -1.2555571794509888,
+      "step": 21
+    },
+    {
+      "epoch": 0.25142857142857145,
+      "grad_norm": 2.7351036071777344,
+      "learning_rate": 4.555390029237026e-06,
+      "logits/chosen": -0.36178550124168396,
+      "logits/rejected": 0.8178424835205078,
+      "logps/chosen": -65.52536010742188,
+      "logps/rejected": -100.3760986328125,
+      "loss": 0.2818,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.06696957349777222,
+      "rewards/margins": 1.5236375331878662,
+      "rewards/rejected": -1.5906071662902832,
+      "step": 22
+    },
+    {
+      "epoch": 0.25142857142857145,
+      "eval_logits/chosen": -0.36531969904899597,
+      "eval_logits/rejected": 0.3482947051525116,
+      "eval_logps/chosen": -87.77478790283203,
+      "eval_logps/rejected": -92.11418914794922,
+      "eval_loss": 0.30896151065826416,
+      "eval_rewards/accuracies": 0.9800000190734863,
+      "eval_rewards/chosen": -0.20244216918945312,
+      "eval_rewards/margins": 1.3565798997879028,
+      "eval_rewards/rejected": -1.559022068977356,
+      "eval_runtime": 209.0014,
+      "eval_samples_per_second": 0.957,
+      "eval_steps_per_second": 0.957,
+      "step": 22
+    },
+    {
+      "epoch": 0.26285714285714284,
+      "grad_norm": 2.936436653137207,
+      "learning_rate": 4.5000635569783365e-06,
+      "logits/chosen": -0.278434157371521,
+      "logits/rejected": 0.1438252478837967,
+      "logps/chosen": -70.02806091308594,
+      "logps/rejected": -57.32536697387695,
+      "loss": 0.3441,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.005648649297654629,
+      "rewards/margins": 1.069925308227539,
+      "rewards/rejected": -1.0642765760421753,
+      "step": 23
+    },
+    {
+      "epoch": 0.2742857142857143,
+      "grad_norm": 1.480675458908081,
+      "learning_rate": 4.4418720129402145e-06,
+      "logits/chosen": -0.6613065004348755,
+      "logits/rejected": 0.20875723659992218,
+      "logps/chosen": -43.0631217956543,
+      "logps/rejected": -113.96063995361328,
+      "loss": 0.2264,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.007623173296451569,
+      "rewards/margins": 1.854811191558838,
+      "rewards/rejected": -1.8471879959106445,
+      "step": 24
+    },
+    {
+      "epoch": 0.2857142857142857,
+      "grad_norm": 2.0406107902526855,
+      "learning_rate": 4.3808987559489536e-06,
+      "logits/chosen": -0.4824203848838806,
+      "logits/rejected": 0.370841383934021,
+      "logps/chosen": -39.89043426513672,
+      "logps/rejected": -89.99382781982422,
+      "loss": 0.2224,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.0602618083357811,
+      "rewards/margins": 1.6918809413909912,
+      "rewards/rejected": -1.6316192150115967,
+      "step": 25
+    },
+    {
+      "epoch": 0.29714285714285715,
+      "grad_norm": 2.2385094165802,
+      "learning_rate": 4.317231129607859e-06,
+      "logits/chosen": -0.5961613655090332,
+      "logits/rejected": 0.14968276023864746,
+      "logps/chosen": -76.45050048828125,
+      "logps/rejected": -99.48614501953125,
+      "loss": 0.2123,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.07128332555294037,
+      "rewards/margins": 1.763782024383545,
+      "rewards/rejected": -1.6924986839294434,
+      "step": 26
+    },
+    {
+      "epoch": 0.30857142857142855,
+      "grad_norm": 1.6301268339157104,
+      "learning_rate": 4.2509603371783776e-06,
+      "logits/chosen": -0.3383774161338806,
+      "logits/rejected": 0.15638643503189087,
+      "logps/chosen": -58.246971130371094,
+      "logps/rejected": -79.45714569091797,
+      "loss": 0.2009,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.04440131038427353,
+      "rewards/margins": 1.9561799764633179,
+      "rewards/rejected": -2.0005815029144287,
+      "step": 27
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 2.236377716064453,
+      "learning_rate": 4.1821813109322975e-06,
+      "logits/chosen": -0.647399365901947,
+      "logits/rejected": 0.22453227639198303,
+      "logps/chosen": -88.88107299804688,
+      "logps/rejected": -96.83210754394531,
+      "loss": 0.2263,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.040186457335948944,
+      "rewards/margins": 1.824659824371338,
+      "rewards/rejected": -1.7844732999801636,
+      "step": 28
+    },
+    {
+      "epoch": 0.3314285714285714,
+      "grad_norm": 2.0542852878570557,
+      "learning_rate": 4.110992576162193e-06,
+      "logits/chosen": -0.5227777361869812,
+      "logits/rejected": -0.2961636781692505,
+      "logps/chosen": -78.49622344970703,
+      "logps/rejected": -78.24589538574219,
+      "loss": 0.2121,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.10791393369436264,
+      "rewards/margins": 1.7918672561645508,
+      "rewards/rejected": -1.6839532852172852,
+      "step": 29
+    },
+    {
+      "epoch": 0.34285714285714286,
+      "grad_norm": 1.9906846284866333,
+      "learning_rate": 4.037496110044885e-06,
+      "logits/chosen": -0.44146519899368286,
+      "logits/rejected": 0.24826864898204803,
+      "logps/chosen": -89.6866683959961,
+      "logps/rejected": -122.11793518066406,
+      "loss": 0.1621,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.04959860444068909,
+      "rewards/margins": 2.5837507247924805,
+      "rewards/rejected": -2.6333494186401367,
+      "step": 30
+    },
+    {
+      "epoch": 0.35428571428571426,
+      "grad_norm": 2.3201839923858643,
+      "learning_rate": 3.961797195560118e-06,
+      "logits/chosen": -0.37571054697036743,
+      "logits/rejected": -0.12684349715709686,
+      "logps/chosen": -62.36381530761719,
+      "logps/rejected": -68.8594741821289,
+      "loss": 0.2406,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.013891173526644707,
+      "rewards/margins": 1.7061822414398193,
+      "rewards/rejected": -1.6922911405563354,
+      "step": 31
+    },
+    {
+      "epoch": 0.3657142857142857,
+      "grad_norm": 1.7459288835525513,
+      "learning_rate": 3.884004270673711e-06,
+      "logits/chosen": -0.4904371500015259,
+      "logits/rejected": -0.09516231715679169,
+      "logps/chosen": -51.67000198364258,
+      "logps/rejected": -87.77179718017578,
+      "loss": 0.1739,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.05766277387738228,
+      "rewards/margins": 2.167266845703125,
+      "rewards/rejected": -2.1096038818359375,
+      "step": 32
+    },
+    {
+      "epoch": 0.37714285714285717,
+      "grad_norm": 1.9478814601898193,
+      "learning_rate": 3.8042287730012117e-06,
+      "logits/chosen": -0.7833026051521301,
+      "logits/rejected": -0.14795458316802979,
+      "logps/chosen": -49.5107307434082,
+      "logps/rejected": -77.62210845947266,
+      "loss": 0.1861,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.02107466384768486,
+      "rewards/margins": 1.9514578580856323,
+      "rewards/rejected": -1.9303834438323975,
+      "step": 33
+    },
+    {
+      "epoch": 0.38857142857142857,
+      "grad_norm": 2.1133761405944824,
+      "learning_rate": 3.7225849801745835e-06,
+      "logits/chosen": -0.8189583420753479,
+      "logits/rejected": -0.24318619072437286,
+      "logps/chosen": -75.10845947265625,
+      "logps/rejected": -111.31871032714844,
+      "loss": 0.1489,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.029190005734562874,
+      "rewards/margins": 2.6350479125976562,
+      "rewards/rejected": -2.6058578491210938,
+      "step": 34
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 2.116342306137085,
+      "learning_rate": 3.6391898461406045e-06,
+      "logits/chosen": -0.4198293685913086,
+      "logits/rejected": -0.3276508152484894,
+      "logps/chosen": -63.513519287109375,
+      "logps/rejected": -93.56440734863281,
+      "loss": 0.2266,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.09839610755443573,
+      "rewards/margins": 1.798091173171997,
+      "rewards/rejected": -1.6996949911117554,
+      "step": 35
+    },
+    {
+      "epoch": 0.4114285714285714,
+      "grad_norm": 1.6935662031173706,
+      "learning_rate": 3.55416283362546e-06,
+      "logits/chosen": -0.8055887222290039,
+      "logits/rejected": 0.15487802028656006,
+      "logps/chosen": -46.510643005371094,
+      "logps/rejected": -105.87352752685547,
+      "loss": 0.1607,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.033490121364593506,
+      "rewards/margins": 2.649106025695801,
+      "rewards/rejected": -2.6156160831451416,
+      "step": 36
+    },
+    {
+      "epoch": 0.4228571428571429,
+      "grad_norm": 2.2720091342926025,
+      "learning_rate": 3.4676257430055438e-06,
+      "logits/chosen": -0.8104305267333984,
+      "logits/rejected": 0.04826318100094795,
+      "logps/chosen": -77.22013854980469,
+      "logps/rejected": -98.19013977050781,
+      "loss": 0.1415,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.03917883336544037,
+      "rewards/margins": 2.6867618560791016,
+      "rewards/rejected": -2.6475830078125,
+      "step": 37
+    },
+    {
+      "epoch": 0.4342857142857143,
+      "grad_norm": 1.546297550201416,
+      "learning_rate": 3.3797025378295826e-06,
+      "logits/chosen": -0.8403704166412354,
+      "logits/rejected": 0.052355289459228516,
+      "logps/chosen": -56.61090087890625,
+      "logps/rejected": -119.7930679321289,
+      "loss": 0.1658,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.022541070356965065,
+      "rewards/margins": 2.8231887817382812,
+      "rewards/rejected": -2.800647735595703,
+      "step": 38
+    },
+    {
+      "epoch": 0.44571428571428573,
+      "grad_norm": 2.0681819915771484,
+      "learning_rate": 3.29051916724206e-06,
+      "logits/chosen": -0.6301568746566772,
+      "logits/rejected": -0.32246649265289307,
+      "logps/chosen": -66.40396118164062,
+      "logps/rejected": -81.09312438964844,
+      "loss": 0.1799,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.009487343952059746,
+      "rewards/margins": 2.2801120281219482,
+      "rewards/rejected": -2.2706246376037598,
+      "step": 39
+    },
+    {
+      "epoch": 0.45714285714285713,
+      "grad_norm": 1.4052331447601318,
+      "learning_rate": 3.2002033855622683e-06,
+      "logits/chosen": -0.8533369302749634,
+      "logits/rejected": -0.4283853769302368,
+      "logps/chosen": -47.345802307128906,
+      "logps/rejected": -93.77117919921875,
+      "loss": 0.1517,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.026743369176983833,
+      "rewards/margins": 2.6302499771118164,
+      "rewards/rejected": -2.6569931507110596,
+      "step": 40
+    },
+    {
+      "epoch": 0.4685714285714286,
+      "grad_norm": 2.0809950828552246,
+      "learning_rate": 3.1088845692774798e-06,
+      "logits/chosen": -0.6270914077758789,
+      "logits/rejected": -0.6155582070350647,
+      "logps/chosen": -88.60835266113281,
+      "logps/rejected": -67.6736831665039,
+      "loss": 0.1825,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.04146631807088852,
+      "rewards/margins": 2.0336129665374756,
+      "rewards/rejected": -1.9921468496322632,
+      "step": 41
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 1.1893919706344604,
+      "learning_rate": 3.0166935317123824e-06,
+      "logits/chosen": -1.0347564220428467,
+      "logits/rejected": -0.4240405261516571,
+      "logps/chosen": -56.76289367675781,
+      "logps/rejected": -103.66142272949219,
+      "loss": 0.1124,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.057413943111896515,
+      "rewards/margins": 3.0061051845550537,
+      "rewards/rejected": -3.06351900100708,
+      "step": 42
+    },
+    {
+      "epoch": 0.49142857142857144,
+      "grad_norm": 1.1706154346466064,
+      "learning_rate": 2.9237623356402423e-06,
+      "logits/chosen": -1.0618159770965576,
+      "logits/rejected": -0.327092707157135,
+      "logps/chosen": -32.207794189453125,
+      "logps/rejected": -111.8665771484375,
+      "loss": 0.0903,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.027263116091489792,
+      "rewards/margins": 3.3231871128082275,
+      "rewards/rejected": -3.350450277328491,
+      "step": 43
+    },
+    {
+      "epoch": 0.5028571428571429,
+      "grad_norm": 1.079911470413208,
+      "learning_rate": 2.8302241041042564e-06,
+      "logits/chosen": -1.1862759590148926,
+      "logits/rejected": -0.7669480443000793,
+      "logps/chosen": -89.4259262084961,
+      "logps/rejected": -127.56314086914062,
+      "loss": 0.0967,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.14472071826457977,
+      "rewards/margins": 3.0603933334350586,
+      "rewards/rejected": -2.915672540664673,
+      "step": 44
+    },
+    {
+      "epoch": 0.5028571428571429,
+      "eval_logits/chosen": -0.8571964502334595,
+      "eval_logits/rejected": -0.6531373858451843,
+      "eval_logps/chosen": -90.10453033447266,
+      "eval_logps/rejected": -104.75491333007812,
+      "eval_loss": 0.17935897409915924,
+      "eval_rewards/accuracies": 0.9700000286102295,
+      "eval_rewards/chosen": -0.4354174733161926,
+      "eval_rewards/margins": 2.387676954269409,
+      "eval_rewards/rejected": -2.8230948448181152,
+      "eval_runtime": 208.3973,
+      "eval_samples_per_second": 0.96,
+      "eval_steps_per_second": 0.96,
+      "step": 44
+    },
+    {
+      "epoch": 0.5142857142857142,
+      "grad_norm": 1.5324379205703735,
+      "learning_rate": 2.7362128297200784e-06,
+      "logits/chosen": -0.7203244566917419,
+      "logits/rejected": -0.5576686859130859,
+      "logps/chosen": -65.52645874023438,
+      "logps/rejected": -95.54450988769531,
+      "loss": 0.1323,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1149684339761734,
+      "rewards/margins": 2.6911990642547607,
+      "rewards/rejected": -2.576230525970459,
+      "step": 45
+    },
+    {
+      "epoch": 0.5257142857142857,
+      "grad_norm": 1.2247161865234375,
+      "learning_rate": 2.6418631827326857e-06,
+      "logits/chosen": -0.9313367009162903,
+      "logits/rejected": -0.8070792555809021,
+      "logps/chosen": -55.876502990722656,
+      "logps/rejected": -119.91787719726562,
+      "loss": 0.1285,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.06400280445814133,
+      "rewards/margins": 2.901925802230835,
+      "rewards/rejected": -2.837923288345337,
+      "step": 46
+    },
+    {
+      "epoch": 0.5371428571428571,
+      "grad_norm": 1.3425116539001465,
+      "learning_rate": 2.547310318102548e-06,
+      "logits/chosen": -0.9409236907958984,
+      "logits/rejected": -0.7352248430252075,
+      "logps/chosen": -50.5601806640625,
+      "logps/rejected": -86.23294830322266,
+      "loss": 0.1466,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.09888257831335068,
+      "rewards/margins": 2.470425844192505,
+      "rewards/rejected": -2.3715431690216064,
+      "step": 47
+    },
+    {
+      "epoch": 0.5485714285714286,
+      "grad_norm": 1.4391740560531616,
+      "learning_rate": 2.4526896818974534e-06,
+      "logits/chosen": -0.8374643921852112,
+      "logits/rejected": -0.47213953733444214,
+      "logps/chosen": -50.190616607666016,
+      "logps/rejected": -91.12651824951172,
+      "loss": 0.1023,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.009993518702685833,
+      "rewards/margins": 2.796506881713867,
+      "rewards/rejected": -2.8065006732940674,
+      "step": 48
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 1.5294134616851807,
+      "learning_rate": 2.358136817267315e-06,
+      "logits/chosen": -0.8648539185523987,
+      "logits/rejected": -0.736384391784668,
+      "logps/chosen": -72.85952758789062,
+      "logps/rejected": -83.08409881591797,
+      "loss": 0.1516,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.09306713193655014,
+      "rewards/margins": 2.5247979164123535,
+      "rewards/rejected": -2.4317305088043213,
+      "step": 49
+    },
+    {
+      "epoch": 0.5714285714285714,
+      "grad_norm": 1.6909253597259521,
+      "learning_rate": 2.263787170279922e-06,
+      "logits/chosen": -1.3599584102630615,
+      "logits/rejected": -1.2676033973693848,
+      "logps/chosen": -78.65673065185547,
+      "logps/rejected": -69.09491729736328,
+      "loss": 0.1557,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.00439932756125927,
+      "rewards/margins": 2.1822092533111572,
+      "rewards/rejected": -2.177809953689575,
+      "step": 50
+    },
+    {
+      "epoch": 0.5828571428571429,
+      "grad_norm": 1.3136837482452393,
+      "learning_rate": 2.169775895895745e-06,
+      "logits/chosen": -0.7750363349914551,
+      "logits/rejected": -0.7582042217254639,
+      "logps/chosen": -48.27521514892578,
+      "logps/rejected": -105.779541015625,
+      "loss": 0.0944,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.060845933854579926,
+      "rewards/margins": 3.2103078365325928,
+      "rewards/rejected": -3.1494617462158203,
+      "step": 51
+    },
+    {
+      "epoch": 0.5942857142857143,
+      "grad_norm": 1.304816722869873,
+      "learning_rate": 2.0762376643597586e-06,
+      "logits/chosen": -0.7206559777259827,
+      "logits/rejected": -0.7368943691253662,
+      "logps/chosen": -49.27516174316406,
+      "logps/rejected": -99.28387451171875,
+      "loss": 0.1227,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.08159955590963364,
+      "rewards/margins": 2.943441390991211,
+      "rewards/rejected": -2.861841917037964,
+      "step": 52
+    },
+    {
+      "epoch": 0.6057142857142858,
+      "grad_norm": 1.1686677932739258,
+      "learning_rate": 1.9833064682876175e-06,
+      "logits/chosen": -0.7927972674369812,
+      "logits/rejected": -0.8789339065551758,
+      "logps/chosen": -62.97304916381836,
+      "logps/rejected": -93.07855224609375,
+      "loss": 0.097,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.10683320462703705,
+      "rewards/margins": 2.952019453048706,
+      "rewards/rejected": -2.8451859951019287,
+      "step": 53
+    },
+    {
+      "epoch": 0.6171428571428571,
+      "grad_norm": 0.7459912300109863,
+      "learning_rate": 1.8911154307225204e-06,
+      "logits/chosen": -0.7670217752456665,
+      "logits/rejected": -0.7017982006072998,
+      "logps/chosen": -31.30267333984375,
+      "logps/rejected": -110.78553771972656,
+      "loss": 0.0593,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.003862532787024975,
+      "rewards/margins": 3.5174167156219482,
+      "rewards/rejected": -3.513554096221924,
+      "step": 54
+    },
+    {
+      "epoch": 0.6285714285714286,
+      "grad_norm": 1.5418952703475952,
+      "learning_rate": 1.7997966144377328e-06,
+      "logits/chosen": -0.986474871635437,
+      "logits/rejected": -1.1061036586761475,
+      "logps/chosen": -47.49861526489258,
+      "logps/rejected": -67.53656005859375,
+      "loss": 0.1543,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.04688534885644913,
+      "rewards/margins": 2.227695941925049,
+      "rewards/rejected": -2.27458119392395,
+      "step": 55
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 1.0748093128204346,
+      "learning_rate": 1.7094808327579401e-06,
+      "logits/chosen": -1.324556589126587,
+      "logits/rejected": -0.8223966360092163,
+      "logps/chosen": -36.40914535522461,
+      "logps/rejected": -105.08839416503906,
+      "loss": 0.084,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.07963553071022034,
+      "rewards/margins": 3.377859354019165,
+      "rewards/rejected": -3.4574952125549316,
+      "step": 56
+    },
+    {
+      "epoch": 0.6514285714285715,
+      "grad_norm": 0.9280484318733215,
+      "learning_rate": 1.6202974621704176e-06,
+      "logits/chosen": -1.450647234916687,
+      "logits/rejected": -0.8040248155593872,
+      "logps/chosen": -68.76046752929688,
+      "logps/rejected": -125.3299331665039,
+      "loss": 0.0635,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.06499414145946503,
+      "rewards/margins": 3.9410018920898438,
+      "rewards/rejected": -3.876007318496704,
+      "step": 57
+    },
+    {
+      "epoch": 0.6628571428571428,
+      "grad_norm": 0.8896822333335876,
+      "learning_rate": 1.5323742569944573e-06,
+      "logits/chosen": -1.0071394443511963,
+      "logits/rejected": -1.0022106170654297,
+      "logps/chosen": -45.09584426879883,
+      "logps/rejected": -109.64781951904297,
+      "loss": 0.0802,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.034890901297330856,
+      "rewards/margins": 3.5162737369537354,
+      "rewards/rejected": -3.4813833236694336,
+      "step": 58
+    },
+    {
+      "epoch": 0.6742857142857143,
+      "grad_norm": 0.7682043313980103,
+      "learning_rate": 1.44583716637454e-06,
+      "logits/chosen": -1.0028694868087769,
+      "logits/rejected": -0.813642144203186,
+      "logps/chosen": -66.2764663696289,
+      "logps/rejected": -112.77369689941406,
+      "loss": 0.0576,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0501595139503479,
+      "rewards/margins": 3.7721147537231445,
+      "rewards/rejected": -3.822274684906006,
+      "step": 59
+    },
+    {
+      "epoch": 0.6857142857142857,
+      "grad_norm": 1.0216327905654907,
+      "learning_rate": 1.3608101538593965e-06,
+      "logits/chosen": -1.1250649690628052,
+      "logits/rejected": -0.9627130627632141,
+      "logps/chosen": -42.440757751464844,
+      "logps/rejected": -100.12843322753906,
+      "loss": 0.0804,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.11411059647798538,
+      "rewards/margins": 3.393651247024536,
+      "rewards/rejected": -3.2795403003692627,
+      "step": 60
+    },
+    {
+      "epoch": 0.6971428571428572,
+      "grad_norm": 0.7271348834037781,
+      "learning_rate": 1.277415019825417e-06,
+      "logits/chosen": -0.7807080745697021,
+      "logits/rejected": -0.7796292304992676,
+      "logps/chosen": -33.5434455871582,
+      "logps/rejected": -112.23175048828125,
+      "loss": 0.0711,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.06058162823319435,
+      "rewards/margins": 3.8745694160461426,
+      "rewards/rejected": -3.9351511001586914,
+      "step": 61
+    },
+    {
+      "epoch": 0.7085714285714285,
+      "grad_norm": 1.7274105548858643,
+      "learning_rate": 1.195771226998789e-06,
+      "logits/chosen": -0.9823087453842163,
+      "logits/rejected": -1.2148367166519165,
+      "logps/chosen": -72.55377960205078,
+      "logps/rejected": -75.32920837402344,
+      "loss": 0.1183,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.12965497374534607,
+      "rewards/margins": 2.7459590435028076,
+      "rewards/rejected": -2.8756141662597656,
+      "step": 62
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 0.9623571038246155,
+      "learning_rate": 1.1159957293262888e-06,
+      "logits/chosen": -1.0808844566345215,
+      "logits/rejected": -0.8635554909706116,
+      "logps/chosen": -49.637996673583984,
+      "logps/rejected": -116.99398803710938,
+      "loss": 0.09,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.02851293236017227,
+      "rewards/margins": 3.4344334602355957,
+      "rewards/rejected": -3.4059205055236816,
+      "step": 63
+    },
+    {
+      "epoch": 0.7314285714285714,
+      "grad_norm": 1.0066514015197754,
+      "learning_rate": 1.0382028044398823e-06,
+      "logits/chosen": -1.197101354598999,
+      "logits/rejected": -1.1306017637252808,
+      "logps/chosen": -50.98615264892578,
+      "logps/rejected": -120.8044662475586,
+      "loss": 0.0899,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.035914015024900436,
+      "rewards/margins": 3.1908507347106934,
+      "rewards/rejected": -3.2267649173736572,
+      "step": 64
+    },
+    {
+      "epoch": 0.7428571428571429,
+      "grad_norm": 1.1005553007125854,
+      "learning_rate": 9.625038899551162e-07,
+      "logits/chosen": -1.22350013256073,
+      "logits/rejected": -1.1208860874176025,
+      "logps/chosen": -66.32962036132812,
+      "logps/rejected": -103.34649658203125,
+      "loss": 0.0709,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.18690410256385803,
+      "rewards/margins": 3.4826159477233887,
+      "rewards/rejected": -3.2957119941711426,
+      "step": 65
+    },
+    {
+      "epoch": 0.7542857142857143,
+      "grad_norm": 0.954483687877655,
+      "learning_rate": 8.890074238378074e-07,
+      "logits/chosen": -0.9665268659591675,
+      "logits/rejected": -0.8029574751853943,
+      "logps/chosen": -43.259857177734375,
+      "logps/rejected": -104.5782241821289,
+      "loss": 0.0993,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.044777870178222656,
+      "rewards/margins": 3.1982903480529785,
+      "rewards/rejected": -3.243067979812622,
+      "step": 66
+    },
+    {
+      "epoch": 0.7542857142857143,
+      "eval_logits/chosen": -1.1089446544647217,
+      "eval_logits/rejected": -1.1692878007888794,
+      "eval_logps/chosen": -91.48172760009766,
+      "eval_logps/rejected": -110.86624145507812,
+      "eval_loss": 0.14132851362228394,
+      "eval_rewards/accuracies": 0.9700000286102295,
+      "eval_rewards/chosen": -0.573137104511261,
+      "eval_rewards/margins": 2.8610901832580566,
+      "eval_rewards/rejected": -3.4342269897460938,
+      "eval_runtime": 208.8549,
+      "eval_samples_per_second": 0.958,
+      "eval_steps_per_second": 0.958,
+      "step": 66
+    },
+    {
+      "epoch": 0.7657142857142857,
+      "grad_norm": 0.9773061871528625,
+      "learning_rate": 8.178186890677029e-07,
+      "logits/chosen": -1.02699875831604,
+      "logits/rejected": -1.1385068893432617,
+      "logps/chosen": -80.82888793945312,
+      "logps/rejected": -100.25572967529297,
+      "loss": 0.072,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.0796530619263649,
+      "rewards/margins": 3.1960816383361816,
+      "rewards/rejected": -3.116428852081299,
+      "step": 67
+    },
+    {
+      "epoch": 0.7771428571428571,
+      "grad_norm": 1.2940922975540161,
+      "learning_rate": 7.490396628216237e-07,
+      "logits/chosen": -1.2090729475021362,
+      "logits/rejected": -1.2437564134597778,
+      "logps/chosen": -87.85923767089844,
+      "logps/rejected": -120.87287902832031,
+      "loss": 0.1183,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.018895722925662994,
+      "rewards/margins": 2.969498634338379,
+      "rewards/rejected": -2.9506030082702637,
+      "step": 68
+    },
+    {
+      "epoch": 0.7885714285714286,
+      "grad_norm": 0.7793561220169067,
+      "learning_rate": 6.827688703921407e-07,
+      "logits/chosen": -1.1516605615615845,
+      "logits/rejected": -1.295521855354309,
+      "logps/chosen": -52.94973373413086,
+      "logps/rejected": -84.64179229736328,
+      "loss": 0.067,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.18537920713424683,
+      "rewards/margins": 3.4953463077545166,
+      "rewards/rejected": -3.309967041015625,
+      "step": 69
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 1.4742248058319092,
+      "learning_rate": 6.191012440510469e-07,
+      "logits/chosen": -1.3784356117248535,
+      "logits/rejected": -1.3366318941116333,
+      "logps/chosen": -57.7204475402832,
+      "logps/rejected": -90.2862548828125,
+      "loss": 0.1197,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.044269490987062454,
+      "rewards/margins": 2.759215831756592,
+      "rewards/rejected": -2.80348539352417,
+      "step": 70
+    },
+    {
+      "epoch": 0.8114285714285714,
+      "grad_norm": 0.7249897718429565,
+      "learning_rate": 5.581279870597866e-07,
+      "logits/chosen": -1.1907292604446411,
+      "logits/rejected": -0.6756631731987,
+      "logps/chosen": -32.33828353881836,
+      "logps/rejected": -125.02144622802734,
+      "loss": 0.0525,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.06472505629062653,
+      "rewards/margins": 4.0654168128967285,
+      "rewards/rejected": -4.130141258239746,
+      "step": 71
+    },
+    {
+      "epoch": 0.8228571428571428,
+      "grad_norm": 1.0847841501235962,
+      "learning_rate": 4.999364430216639e-07,
+      "logits/chosen": -1.237339735031128,
+      "logits/rejected": -1.5806419849395752,
+      "logps/chosen": -52.456146240234375,
+      "logps/rejected": -94.31544494628906,
+      "loss": 0.0992,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.10363951325416565,
+      "rewards/margins": 2.9890761375427246,
+      "rewards/rejected": -2.885436534881592,
+      "step": 72
+    },
+    {
+      "epoch": 0.8342857142857143,
+      "grad_norm": 1.278956651687622,
+      "learning_rate": 4.4460997076297504e-07,
+      "logits/chosen": -1.0774935483932495,
+      "logits/rejected": -1.4760003089904785,
+      "logps/chosen": -43.07012939453125,
+      "logps/rejected": -74.25629425048828,
+      "loss": 0.1131,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.06534770131111145,
+      "rewards/margins": 2.8206787109375,
+      "rewards/rejected": -2.75533127784729,
+      "step": 73
+    },
+    {
+      "epoch": 0.8457142857142858,
+      "grad_norm": 0.7833404541015625,
+      "learning_rate": 3.922278249222894e-07,
+      "logits/chosen": -1.1311602592468262,
+      "logits/rejected": -1.0242271423339844,
+      "logps/chosen": -53.541568756103516,
+      "logps/rejected": -105.48152923583984,
+      "loss": 0.0774,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.04792798310518265,
+      "rewards/margins": 3.678433418273926,
+      "rewards/rejected": -3.6305058002471924,
+      "step": 74
+    },
+    {
+      "epoch": 0.8571428571428571,
+      "grad_norm": 1.03605318069458,
+      "learning_rate": 3.4286504241894283e-07,
+      "logits/chosen": -0.9688073992729187,
+      "logits/rejected": -1.2700920104980469,
+      "logps/chosen": -38.00965118408203,
+      "logps/rejected": -92.08466339111328,
+      "loss": 0.1049,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.01876715011894703,
+      "rewards/margins": 3.16585636138916,
+      "rewards/rejected": -3.1846237182617188,
+      "step": 75
+    },
+    {
+      "epoch": 0.8685714285714285,
+      "grad_norm": 1.319770336151123,
+      "learning_rate": 2.965923349633779e-07,
+      "logits/chosen": -1.3658266067504883,
+      "logits/rejected": -1.3608825206756592,
+      "logps/chosen": -49.00103759765625,
+      "logps/rejected": -117.49503326416016,
+      "loss": 0.0818,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.09428122639656067,
+      "rewards/margins": 3.61112642288208,
+      "rewards/rejected": -3.7054080963134766,
+      "step": 76
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 0.9297468662261963,
+      "learning_rate": 2.53475987763295e-07,
+      "logits/chosen": -1.0338634252548218,
+      "logits/rejected": -1.1930313110351562,
+      "logps/chosen": -67.0118179321289,
+      "logps/rejected": -106.61903381347656,
+      "loss": 0.0727,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.13288229703903198,
+      "rewards/margins": 3.487334728240967,
+      "rewards/rejected": -3.6202168464660645,
+      "step": 77
+    },
+    {
+      "epoch": 0.8914285714285715,
+      "grad_norm": 1.1066629886627197,
+      "learning_rate": 2.135777645707318e-07,
+      "logits/chosen": -1.384450078010559,
+      "logits/rejected": -1.3552913665771484,
+      "logps/chosen": -52.43301010131836,
+      "logps/rejected": -101.87602996826172,
+      "loss": 0.0989,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.009377628564834595,
+      "rewards/margins": 3.135222911834717,
+      "rewards/rejected": -3.125845432281494,
+      "step": 78
+    },
+    {
+      "epoch": 0.9028571428571428,
+      "grad_norm": 1.0631781816482544,
+      "learning_rate": 1.7695481920608716e-07,
+      "logits/chosen": -1.1844675540924072,
+      "logits/rejected": -1.0223643779754639,
+      "logps/chosen": -45.87510299682617,
+      "logps/rejected": -102.84295654296875,
+      "loss": 0.0862,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.05180816724896431,
+      "rewards/margins": 3.2761740684509277,
+      "rewards/rejected": -3.3279824256896973,
+      "step": 79
+    },
+    {
+      "epoch": 0.9142857142857143,
+      "grad_norm": 1.2742042541503906,
+      "learning_rate": 1.4365961368581844e-07,
+      "logits/chosen": -1.2797547578811646,
+      "logits/rejected": -1.6902124881744385,
+      "logps/chosen": -79.47929382324219,
+      "logps/rejected": -78.93382263183594,
+      "loss": 0.1025,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0567280538380146,
+      "rewards/margins": 2.9206228256225586,
+      "rewards/rejected": -2.977350950241089,
+      "step": 80
+    },
+    {
+      "epoch": 0.9257142857142857,
+      "grad_norm": 0.8278792500495911,
+      "learning_rate": 1.137398430711123e-07,
+      "logits/chosen": -1.119363784790039,
+      "logits/rejected": -0.9186141490936279,
+      "logps/chosen": -29.899991989135742,
+      "logps/rejected": -105.7519302368164,
+      "loss": 0.0648,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.031892240047454834,
+      "rewards/margins": 3.5144004821777344,
+      "rewards/rejected": -3.4825081825256348,
+      "step": 81
+    },
+    {
+      "epoch": 0.9371428571428572,
+      "grad_norm": 1.0955795049667358,
+      "learning_rate": 8.723836714516681e-08,
+      "logits/chosen": -1.2281020879745483,
+      "logits/rejected": -1.3817219734191895,
+      "logps/chosen": -73.53196716308594,
+      "logps/rejected": -103.11698913574219,
+      "loss": 0.0811,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.06516408920288086,
+      "rewards/margins": 3.2024097442626953,
+      "rewards/rejected": -3.267573833465576,
+      "step": 82
+    },
+    {
+      "epoch": 0.9485714285714286,
+      "grad_norm": 0.6436601877212524,
+      "learning_rate": 6.419314901696671e-08,
+      "logits/chosen": -1.2053653001785278,
+      "logits/rejected": -1.208804965019226,
+      "logps/chosen": -28.06381607055664,
+      "logps/rejected": -100.99942016601562,
+      "loss": 0.0586,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.027196241542696953,
+      "rewards/margins": 3.633328437805176,
+      "rewards/rejected": -3.660524845123291,
+      "step": 83
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 0.9888738393783569,
+      "learning_rate": 4.4637200739493514e-08,
+      "logits/chosen": -1.231987714767456,
+      "logits/rejected": -1.143172025680542,
+      "logps/chosen": -57.63927459716797,
+      "logps/rejected": -96.07897186279297,
+      "loss": 0.1077,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.05573497712612152,
+      "rewards/margins": 3.1897521018981934,
+      "rewards/rejected": -3.2454872131347656,
+      "step": 84
+    },
+    {
+      "epoch": 0.9714285714285714,
+      "grad_norm": 1.0415937900543213,
+      "learning_rate": 2.8598536020278678e-08,
+      "logits/chosen": -1.221508264541626,
+      "logits/rejected": -1.4119690656661987,
+      "logps/chosen": -48.45242691040039,
+      "logps/rejected": -84.4826431274414,
+      "loss": 0.0827,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.04423363134264946,
+      "rewards/margins": 3.218625545501709,
+      "rewards/rejected": -3.174391984939575,
+      "step": 85
+    },
+    {
+      "epoch": 0.9828571428571429,
+      "grad_norm": 1.0360701084136963,
+      "learning_rate": 1.6100130092037704e-08,
+      "logits/chosen": -1.3318215608596802,
+      "logits/rejected": -1.6143568754196167,
+      "logps/chosen": -45.959354400634766,
+      "logps/rejected": -79.45930480957031,
+      "loss": 0.0892,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.08850360661745071,
+      "rewards/margins": 2.9804043769836426,
+      "rewards/rejected": -3.068908214569092,
+      "step": 86
+    },
+    {
+      "epoch": 0.9942857142857143,
+      "grad_norm": 0.7161802053451538,
+      "learning_rate": 7.159886800869875e-09,
+      "logits/chosen": -1.1106700897216797,
+      "logits/rejected": -0.9769763946533203,
+      "logps/chosen": -85.0906982421875,
+      "logps/rejected": -132.0802459716797,
+      "loss": 0.0524,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0784958153963089,
+      "rewards/margins": 4.037193298339844,
+      "rewards/rejected": -4.115689277648926,
+      "step": 87
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 2.9429450035095215,
+      "learning_rate": 1.7906129591713228e-09,
+      "logits/chosen": -1.2418022155761719,
+      "logits/rejected": -0.8798779249191284,
+      "logps/chosen": -90.4301986694336,
+      "logps/rejected": -115.04806518554688,
+      "loss": 0.0987,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.08904826641082764,
+      "rewards/margins": 3.531545639038086,
+      "rewards/rejected": -3.620594024658203,
+      "step": 88
+    },
+    {
+      "epoch": 1.0,
+      "eval_logits/chosen": -1.1477130651474,
+      "eval_logits/rejected": -1.244410514831543,
+      "eval_logps/chosen": -91.70614624023438,
+      "eval_logps/rejected": -111.82247161865234,
+      "eval_loss": 0.1359921395778656,
+      "eval_rewards/accuracies": 0.9700000286102295,
+      "eval_rewards/chosen": -0.5955771207809448,
+      "eval_rewards/margins": 2.9342737197875977,
+      "eval_rewards/rejected": -3.529850959777832,
+      "eval_runtime": 209.3361,
+      "eval_samples_per_second": 0.955,
+      "eval_steps_per_second": 0.955,
+      "step": 88
+    },
+    {
+      "epoch": 1.0,
+      "step": 88,
+      "total_flos": 0.0,
+      "train_loss": 0.2132695221172815,
+      "train_runtime": 4416.7921,
+      "train_samples_per_second": 0.317,
+      "train_steps_per_second": 0.02
+    }
+  ],
+  "trial_params": null,
+  "global_step": 88,
+  "is_local_process_zero": true,
+  "total_flos": 0.0,
+  "num_train_epochs": 1,
+  "epoch": 1.0,
+  "best_metric": null,
+  "trial_name": null
+}