Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,26 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_10k/MDPO_10k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_10k/lora/checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_10k/lora/checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_10k/lora/checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_10k/lora/checkpoint-2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_10k/lora/checkpoint-3000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_10k/lora/checkpoint-3500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_10k/lora/checkpoint-3750/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_10k/lora/checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_1k/MDPO_1k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_1k/lora/checkpoint-375/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_5k/MDPO_5k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_5k/lora/checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_5k/lora/checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_5k/lora/checkpoint-1875/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_5k/lora/checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/ORPO/ORPO_1k/MORPO_1k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/ORPO/ORPO_1k/lora/checkpoint-375/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/ORPO/ORPO_5k/MORPO_5k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/ORPO/ORPO_5k/lora/checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/ORPO/ORPO_5k/lora/checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/ORPO/ORPO_5k/lora/checkpoint-1875/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/ORPO/ORPO_5k/lora/checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text

v3/Base/gen-output/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efabd6f3e982da43a3b3d18d95b36f512aee97801c2e980bce071c90f75c41d3
+size 982376

v3/Base/gen-output/dataset_info.json

@@ -0,0 +1,34 @@
+{
+  "citation": "",
+  "description": "",
+  "features": {
+    "prompt": {
+      "feature": {
+        "content": {
+          "dtype": "string",
+          "_type": "Value"
+        },
+        "role": {
+          "dtype": "string",
+          "_type": "Value"
+        }
+      },
+      "_type": "List"
+    },
+    "generated_text": {
+      "feature": {
+        "content": {
+          "dtype": "string",
+          "_type": "Value"
+        },
+        "role": {
+          "dtype": "string",
+          "_type": "Value"
+        }
+      },
+      "_type": "List"
+    }
+  },
+  "homepage": "",
+  "license": ""
+}

v3/Base/gen-output/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "f319c5a85dc7aa54",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

v3/DPO/DPO_10k/DPO_10k/README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

v3/DPO/DPO_10k/DPO_10k/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "gate_proj",
+    "down_proj",
+    "k_proj",
+    "o_proj",
+    "up_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

v3/DPO/DPO_10k/DPO_10k/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27a5e5a70889c639646183be14875b89ae81e677265281b25eadec5735c154e0
+size 180385008

v3/DPO/DPO_10k/MDPO_10k/chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

v3/DPO/DPO_10k/MDPO_10k/config.json

@@ -0,0 +1,40 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "float16",
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 8,
+  "pad_token_id": null,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.2.0",
+  "use_cache": true,
+  "vocab_size": 128256
+}

v3/DPO/DPO_10k/MDPO_10k/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.2.0"
+}

v3/DPO/DPO_10k/MDPO_10k/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85c9d82fdf7dbcecd629e2cc9efd75e800f619818033fb81d1f7884887a67cc7
+size 2471645464

v3/DPO/DPO_10k/MDPO_10k/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

v3/DPO/DPO_10k/MDPO_10k/tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

v3/DPO/DPO_10k/lora/README.md

@@ -0,0 +1,68 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: transformers
+model_name: lora
+tags:
+- generated_from_trainer
+- trl
+- dpo
+licence: license
+---
+
+# Model Card for lora
+
+This model is a fine-tuned version of [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/sea-rod/huggingface/runs/9nfqa09t) 
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- TRL: 0.28.0
+- Transformers: 5.2.0
+- Pytorch: 2.10.0
+- Datasets: 4.5.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+    title        = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+    author       = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+    year         = 2023,
+    booktitle    = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+    url          = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+    editor       = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

v3/DPO/DPO_10k/lora/checkpoint-1000/README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

v3/DPO/DPO_10k/lora/checkpoint-1000/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "gate_proj",
+    "down_proj",
+    "k_proj",
+    "o_proj",
+    "up_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

v3/DPO/DPO_10k/lora/checkpoint-1000/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27a5e5a70889c639646183be14875b89ae81e677265281b25eadec5735c154e0
+size 180385008

v3/DPO/DPO_10k/lora/checkpoint-1000/chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

v3/DPO/DPO_10k/lora/checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d87beff974577f720d7e8125b66164907ad5b681a0dedce34f2ee0ab3d344976
+size 360902475

v3/DPO/DPO_10k/lora/checkpoint-1000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea11996454b5587fcf33ae0ab5cf14b2031bf5f53f8c2ed5a48e87de31e29c84
+size 14645

v3/DPO/DPO_10k/lora/checkpoint-1000/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14ae2a2128444abab378aa06c09a61a84665f758fcc19fc46f5789b0bc1b5665
+size 1383

v3/DPO/DPO_10k/lora/checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c517d0407369461be1bd2161fd20105c2aa4886637aab49e0225f47034adb975
+size 1465

v3/DPO/DPO_10k/lora/checkpoint-1000/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

v3/DPO/DPO_10k/lora/checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

v3/DPO/DPO_10k/lora/checkpoint-1000/trainer_state.json

@@ -0,0 +1,1694 @@
+{
+  "best_global_step": 1000,
+  "best_metric": 0.6880542039871216,
+  "best_model_checkpoint": "output/lora/checkpoint-1000",
+  "epoch": 0.8,
+  "eval_steps": 100,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.008,
+      "grad_norm": 1.1179773807525635,
+      "learning_rate": 2.4e-08,
+      "logits/chosen": 1.4545083045959473,
+      "logits/rejected": 1.425490379333496,
+      "logps/chosen": -132.46189880371094,
+      "logps/rejected": -148.20260620117188,
+      "loss": 0.6930626392364502,
+      "rewards/accuracies": 0.21250000596046448,
+      "rewards/chosen": -6.165739614516497e-05,
+      "rewards/margins": 0.00017333509458694607,
+      "rewards/rejected": -0.00023499250528402627,
+      "step": 10
+    },
+    {
+      "epoch": 0.016,
+      "grad_norm": 1.3502839803695679,
+      "learning_rate": 5.0666666666666664e-08,
+      "logits/chosen": 1.5668801069259644,
+      "logits/rejected": 1.5191389322280884,
+      "logps/chosen": -157.63624572753906,
+      "logps/rejected": -144.2716827392578,
+      "loss": 0.6930380821228027,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0009244013344869018,
+      "rewards/margins": 0.00023505210992880166,
+      "rewards/rejected": 0.0006893490790389478,
+      "step": 20
+    },
+    {
+      "epoch": 0.024,
+      "grad_norm": 1.0219553709030151,
+      "learning_rate": 7.733333333333334e-08,
+      "logits/chosen": 1.8921797275543213,
+      "logits/rejected": 1.7342685461044312,
+      "logps/chosen": -164.55799865722656,
+      "logps/rejected": -158.92752075195312,
+      "loss": 0.69290189743042,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0009391927160322666,
+      "rewards/margins": 0.0005032324115745723,
+      "rewards/rejected": 0.0004359603044576943,
+      "step": 30
+    },
+    {
+      "epoch": 0.032,
+      "grad_norm": 1.2056113481521606,
+      "learning_rate": 1.0399999999999999e-07,
+      "logits/chosen": 1.4950047731399536,
+      "logits/rejected": 1.6163270473480225,
+      "logps/chosen": -145.54183959960938,
+      "logps/rejected": -134.19979858398438,
+      "loss": 0.6935500621795654,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": 0.0006421065190806985,
+      "rewards/margins": -0.0007943272357806563,
+      "rewards/rejected": 0.0014364338712766767,
+      "step": 40
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 1.1029725074768066,
+      "learning_rate": 1.3066666666666665e-07,
+      "logits/chosen": 1.4076533317565918,
+      "logits/rejected": 1.4791977405548096,
+      "logps/chosen": -133.23324584960938,
+      "logps/rejected": -138.3593292236328,
+      "loss": 0.6925833702087403,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0012372838100418448,
+      "rewards/margins": 0.001139522879384458,
+      "rewards/rejected": 9.776116348803043e-05,
+      "step": 50
+    },
+    {
+      "epoch": 0.048,
+      "grad_norm": 0.9960266351699829,
+      "learning_rate": 1.573333333333333e-07,
+      "logits/chosen": 1.407166600227356,
+      "logits/rejected": 1.376042127609253,
+      "logps/chosen": -137.8921661376953,
+      "logps/rejected": -127.97320556640625,
+      "loss": 0.693614149093628,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.00021316049969755113,
+      "rewards/margins": -0.0009249996510334313,
+      "rewards/rejected": 0.0007118391804397106,
+      "step": 60
+    },
+    {
+      "epoch": 0.056,
+      "grad_norm": 1.0946824550628662,
+      "learning_rate": 1.8399999999999998e-07,
+      "logits/chosen": 1.5944501161575317,
+      "logits/rejected": 1.7248146533966064,
+      "logps/chosen": -153.5526123046875,
+      "logps/rejected": -157.19735717773438,
+      "loss": 0.6928449153900147,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0026370862033218145,
+      "rewards/margins": 0.0006150150438770652,
+      "rewards/rejected": 0.0020220710430294275,
+      "step": 70
+    },
+    {
+      "epoch": 0.064,
+      "grad_norm": 1.0322489738464355,
+      "learning_rate": 2.1066666666666665e-07,
+      "logits/chosen": 1.4586353302001953,
+      "logits/rejected": 1.4842995405197144,
+      "logps/chosen": -149.03665161132812,
+      "logps/rejected": -136.24081420898438,
+      "loss": 0.6932015895843506,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.004061593674123287,
+      "rewards/margins": -9.99594121822156e-05,
+      "rewards/rejected": 0.004161553457379341,
+      "step": 80
+    },
+    {
+      "epoch": 0.072,
+      "grad_norm": 1.0012447834014893,
+      "learning_rate": 2.3733333333333334e-07,
+      "logits/chosen": 1.5075902938842773,
+      "logits/rejected": 1.5986944437026978,
+      "logps/chosen": -159.3433074951172,
+      "logps/rejected": -151.85072326660156,
+      "loss": 0.6928757190704345,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.006978762801736593,
+      "rewards/margins": 0.0005627512582577765,
+      "rewards/rejected": 0.006416010670363903,
+      "step": 90
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 1.280081033706665,
+      "learning_rate": 2.64e-07,
+      "logits/chosen": 1.448286533355713,
+      "logits/rejected": 1.5480695962905884,
+      "logps/chosen": -151.12625122070312,
+      "logps/rejected": -171.24545288085938,
+      "loss": 0.6931126117706299,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.00693341251462698,
+      "rewards/margins": 9.057526767719537e-05,
+      "rewards/rejected": 0.006842837668955326,
+      "step": 100
+    },
+    {
+      "epoch": 0.08,
+      "eval_logits/chosen": 1.518183946609497,
+      "eval_logits/rejected": 1.5525447130203247,
+      "eval_logps/chosen": -153.46630859375,
+      "eval_logps/rejected": -147.95005798339844,
+      "eval_loss": 0.6926330327987671,
+      "eval_rewards/accuracies": 0.5600000023841858,
+      "eval_rewards/chosen": 0.008253541775047779,
+      "eval_rewards/margins": 0.0010449704714119434,
+      "eval_rewards/rejected": 0.007208569906651974,
+      "eval_runtime": 91.0812,
+      "eval_samples_per_second": 5.49,
+      "eval_steps_per_second": 2.745,
+      "step": 100
+    },
+    {
+      "epoch": 0.088,
+      "grad_norm": 1.3912373781204224,
+      "learning_rate": 2.906666666666667e-07,
+      "logits/chosen": 1.6896222829818726,
+      "logits/rejected": 1.6516921520233154,
+      "logps/chosen": -173.70030212402344,
+      "logps/rejected": -155.5503692626953,
+      "loss": 0.6926521778106689,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.010179834440350533,
+      "rewards/margins": 0.0010137532372027636,
+      "rewards/rejected": 0.009166081435978413,
+      "step": 110
+    },
+    {
+      "epoch": 0.096,
+      "grad_norm": 1.1518888473510742,
+      "learning_rate": 3.173333333333333e-07,
+      "logits/chosen": 1.5166860818862915,
+      "logits/rejected": 1.5136979818344116,
+      "logps/chosen": -136.39906311035156,
+      "logps/rejected": -140.86561584472656,
+      "loss": 0.6932918548583984,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.00907944981008768,
+      "rewards/margins": -0.00026998287648893893,
+      "rewards/rejected": 0.009349432773888111,
+      "step": 120
+    },
+    {
+      "epoch": 0.104,
+      "grad_norm": 1.3213376998901367,
+      "learning_rate": 3.4399999999999996e-07,
+      "logits/chosen": 1.4598973989486694,
+      "logits/rejected": 1.6739647388458252,
+      "logps/chosen": -159.12461853027344,
+      "logps/rejected": -163.39346313476562,
+      "loss": 0.693086051940918,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.010477779433131218,
+      "rewards/margins": 0.00014539004769176245,
+      "rewards/rejected": 0.010332388803362846,
+      "step": 130
+    },
+    {
+      "epoch": 0.112,
+      "grad_norm": 0.8833863735198975,
+      "learning_rate": 3.7066666666666665e-07,
+      "logits/chosen": 1.3829412460327148,
+      "logits/rejected": 1.4100173711776733,
+      "logps/chosen": -132.7149658203125,
+      "logps/rejected": -126.85478210449219,
+      "loss": 0.6932382583618164,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.010262183845043182,
+      "rewards/margins": -0.0001672578218858689,
+      "rewards/rejected": 0.010429441928863525,
+      "step": 140
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 1.1472455263137817,
+      "learning_rate": 3.973333333333333e-07,
+      "logits/chosen": 1.4954869747161865,
+      "logits/rejected": 1.5857340097427368,
+      "logps/chosen": -165.10568237304688,
+      "logps/rejected": -158.97433471679688,
+      "loss": 0.6916103363037109,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.014177520759403706,
+      "rewards/margins": 0.003108317730948329,
+      "rewards/rejected": 0.011069201864302158,
+      "step": 150
+    },
+    {
+      "epoch": 0.128,
+      "grad_norm": 1.0628798007965088,
+      "learning_rate": 4.24e-07,
+      "logits/chosen": 1.6717243194580078,
+      "logits/rejected": 1.705092191696167,
+      "logps/chosen": -159.332275390625,
+      "logps/rejected": -147.21646118164062,
+      "loss": 0.6935629844665527,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.016994481906294823,
+      "rewards/margins": -0.0007874655420891941,
+      "rewards/rejected": 0.017781946808099747,
+      "step": 160
+    },
+    {
+      "epoch": 0.136,
+      "grad_norm": 1.390810251235962,
+      "learning_rate": 4.506666666666666e-07,
+      "logits/chosen": 1.4719234704971313,
+      "logits/rejected": 1.499526023864746,
+      "logps/chosen": -148.67251586914062,
+      "logps/rejected": -161.58021545410156,
+      "loss": 0.6926599979400635,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.021239640191197395,
+      "rewards/margins": 0.0010346340714022517,
+      "rewards/rejected": 0.020205006003379822,
+      "step": 170
+    },
+    {
+      "epoch": 0.144,
+      "grad_norm": 1.139707326889038,
+      "learning_rate": 4.773333333333333e-07,
+      "logits/chosen": 1.4833905696868896,
+      "logits/rejected": 1.637584924697876,
+      "logps/chosen": -158.96731567382812,
+      "logps/rejected": -161.9830322265625,
+      "loss": 0.6919547557830811,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.019416773691773415,
+      "rewards/margins": 0.0024296878837049007,
+      "rewards/rejected": 0.016987085342407227,
+      "step": 180
+    },
+    {
+      "epoch": 0.152,
+      "grad_norm": 1.2029796838760376,
+      "learning_rate": 5.04e-07,
+      "logits/chosen": 1.5398027896881104,
+      "logits/rejected": 1.6678705215454102,
+      "logps/chosen": -143.00283813476562,
+      "logps/rejected": -144.89378356933594,
+      "loss": 0.692262840270996,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.018960200250148773,
+      "rewards/margins": 0.0018175147706642747,
+      "rewards/rejected": 0.017142686992883682,
+      "step": 190
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 1.0527437925338745,
+      "learning_rate": 5.306666666666665e-07,
+      "logits/chosen": 1.6087623834609985,
+      "logits/rejected": 1.5181543827056885,
+      "logps/chosen": -159.35073852539062,
+      "logps/rejected": -133.59890747070312,
+      "loss": 0.6924069881439209,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.01926913857460022,
+      "rewards/margins": 0.0015339398523792624,
+      "rewards/rejected": 0.0177351962774992,
+      "step": 200
+    },
+    {
+      "epoch": 0.16,
+      "eval_logits/chosen": 1.5193672180175781,
+      "eval_logits/rejected": 1.5534639358520508,
+      "eval_logps/chosen": -153.34857177734375,
+      "eval_logps/rejected": -147.8425750732422,
+      "eval_loss": 0.6921458840370178,
+      "eval_rewards/accuracies": 0.5600000023841858,
+      "eval_rewards/chosen": 0.020027177408337593,
+      "eval_rewards/margins": 0.0020714527927339077,
+      "eval_rewards/rejected": 0.017955724149942398,
+      "eval_runtime": 90.9762,
+      "eval_samples_per_second": 5.496,
+      "eval_steps_per_second": 2.748,
+      "step": 200
+    },
+    {
+      "epoch": 0.168,
+      "grad_norm": 1.1897239685058594,
+      "learning_rate": 5.573333333333333e-07,
+      "logits/chosen": 1.6267732381820679,
+      "logits/rejected": 1.5775268077850342,
+      "logps/chosen": -159.51437377929688,
+      "logps/rejected": -170.17762756347656,
+      "loss": 0.6925770759582519,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.02078000269830227,
+      "rewards/margins": 0.0012053751852363348,
+      "rewards/rejected": 0.01957462914288044,
+      "step": 210
+    },
+    {
+      "epoch": 0.176,
+      "grad_norm": 0.9995236396789551,
+      "learning_rate": 5.839999999999999e-07,
+      "logits/chosen": 1.5781934261322021,
+      "logits/rejected": 1.642251968383789,
+      "logps/chosen": -159.95309448242188,
+      "logps/rejected": -144.36058044433594,
+      "loss": 0.6931475639343262,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.023322442546486855,
+      "rewards/margins": 6.392716750269756e-05,
+      "rewards/rejected": 0.02325851283967495,
+      "step": 220
+    },
+    {
+      "epoch": 0.184,
+      "grad_norm": 0.8819260597229004,
+      "learning_rate": 6.106666666666666e-07,
+      "logits/chosen": 1.459991693496704,
+      "logits/rejected": 1.6060739755630493,
+      "logps/chosen": -162.04745483398438,
+      "logps/rejected": -139.5411834716797,
+      "loss": 0.6919463634490967,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.026052182540297508,
+      "rewards/margins": 0.002481849165633321,
+      "rewards/rejected": 0.023570330813527107,
+      "step": 230
+    },
+    {
+      "epoch": 0.192,
+      "grad_norm": 1.1055208444595337,
+      "learning_rate": 6.373333333333333e-07,
+      "logits/chosen": 1.558739185333252,
+      "logits/rejected": 1.5965116024017334,
+      "logps/chosen": -171.9166259765625,
+      "logps/rejected": -145.66433715820312,
+      "loss": 0.6912176132202148,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.033868782222270966,
+      "rewards/margins": 0.004017929546535015,
+      "rewards/rejected": 0.029850851744413376,
+      "step": 240
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 1.1372733116149902,
+      "learning_rate": 6.64e-07,
+      "logits/chosen": 1.5322860479354858,
+      "logits/rejected": 1.4695795774459839,
+      "logps/chosen": -161.7803955078125,
+      "logps/rejected": -148.91949462890625,
+      "loss": 0.6940969944000244,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.034065864980220795,
+      "rewards/margins": -0.0016761379083618522,
+      "rewards/rejected": 0.035742007195949554,
+      "step": 250
+    },
+    {
+      "epoch": 0.208,
+      "grad_norm": 1.0917680263519287,
+      "learning_rate": 6.906666666666666e-07,
+      "logits/chosen": 1.4094547033309937,
+      "logits/rejected": 1.50186026096344,
+      "logps/chosen": -132.91455078125,
+      "logps/rejected": -146.79714965820312,
+      "loss": 0.6930203914642334,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": 0.03089061938226223,
+      "rewards/margins": 0.00040971505222842097,
+      "rewards/rejected": 0.030480902642011642,
+      "step": 260
+    },
+    {
+      "epoch": 0.216,
+      "grad_norm": 1.0280259847640991,
+      "learning_rate": 7.173333333333333e-07,
+      "logits/chosen": 1.531285285949707,
+      "logits/rejected": 1.6163132190704346,
+      "logps/chosen": -146.902587890625,
+      "logps/rejected": -147.8347625732422,
+      "loss": 0.6944749355316162,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.028514739125967026,
+      "rewards/margins": -0.0025452517438679934,
+      "rewards/rejected": 0.0310599897056818,
+      "step": 270
+    },
+    {
+      "epoch": 0.224,
+      "grad_norm": 1.398708701133728,
+      "learning_rate": 7.44e-07,
+      "logits/chosen": 1.563852071762085,
+      "logits/rejected": 1.396703839302063,
+      "logps/chosen": -172.68153381347656,
+      "logps/rejected": -155.80215454101562,
+      "loss": 0.689476490020752,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.03224216774106026,
+      "rewards/margins": 0.007473687641322613,
+      "rewards/rejected": 0.02476847730576992,
+      "step": 280
+    },
+    {
+      "epoch": 0.232,
+      "grad_norm": 1.0394141674041748,
+      "learning_rate": 7.706666666666667e-07,
+      "logits/chosen": 1.5581729412078857,
+      "logits/rejected": 1.5819021463394165,
+      "logps/chosen": -144.41465759277344,
+      "logps/rejected": -153.55406188964844,
+      "loss": 0.6927877902984619,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.03272664546966553,
+      "rewards/margins": 0.0008557910332456231,
+      "rewards/rejected": 0.031870849430561066,
+      "step": 290
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 1.2557185888290405,
+      "learning_rate": 7.973333333333333e-07,
+      "logits/chosen": 1.5062446594238281,
+      "logits/rejected": 1.5786281824111938,
+      "logps/chosen": -153.63558959960938,
+      "logps/rejected": -165.46617126464844,
+      "loss": 0.6897892951965332,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.03914078325033188,
+      "rewards/margins": 0.006873926613479853,
+      "rewards/rejected": 0.032266855239868164,
+      "step": 300
+    },
+    {
+      "epoch": 0.24,
+      "eval_logits/chosen": 1.5237834453582764,
+      "eval_logits/rejected": 1.557741641998291,
+      "eval_logps/chosen": -153.19883728027344,
+      "eval_logps/rejected": -147.7100830078125,
+      "eval_loss": 0.6913501024246216,
+      "eval_rewards/accuracies": 0.5339999794960022,
+      "eval_rewards/chosen": 0.03499976545572281,
+      "eval_rewards/margins": 0.0037925743963569403,
+      "eval_rewards/rejected": 0.031207194551825523,
+      "eval_runtime": 90.9133,
+      "eval_samples_per_second": 5.5,
+      "eval_steps_per_second": 2.75,
+      "step": 300
+    },
+    {
+      "epoch": 0.248,
+      "grad_norm": 1.1200144290924072,
+      "learning_rate": 8.24e-07,
+      "logits/chosen": 1.618949294090271,
+      "logits/rejected": 1.6625702381134033,
+      "logps/chosen": -141.8953857421875,
+      "logps/rejected": -156.6927032470703,
+      "loss": 0.694425106048584,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": 0.03162423521280289,
+      "rewards/margins": -0.0023833750747144222,
+      "rewards/rejected": 0.034007612615823746,
+      "step": 310
+    },
+    {
+      "epoch": 0.256,
+      "grad_norm": 1.0823743343353271,
+      "learning_rate": 8.506666666666667e-07,
+      "logits/chosen": 1.5974153280258179,
+      "logits/rejected": 1.6237560510635376,
+      "logps/chosen": -139.2996368408203,
+      "logps/rejected": -141.94407653808594,
+      "loss": 0.6914949893951416,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.029445698484778404,
+      "rewards/margins": 0.0034329916816204786,
+      "rewards/rejected": 0.026012707501649857,
+      "step": 320
+    },
+    {
+      "epoch": 0.264,
+      "grad_norm": 1.1765722036361694,
+      "learning_rate": 8.773333333333332e-07,
+      "logits/chosen": 1.465215802192688,
+      "logits/rejected": 1.4882014989852905,
+      "logps/chosen": -158.4329833984375,
+      "logps/rejected": -183.94259643554688,
+      "loss": 0.6925934791564942,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.026302698999643326,
+      "rewards/margins": 0.0012350418837741017,
+      "rewards/rejected": 0.025067657232284546,
+      "step": 330
+    },
+    {
+      "epoch": 0.272,
+      "grad_norm": 1.0667294263839722,
+      "learning_rate": 9.039999999999999e-07,
+      "logits/chosen": 1.7538875341415405,
+      "logits/rejected": 1.6171789169311523,
+      "logps/chosen": -159.63790893554688,
+      "logps/rejected": -151.1880340576172,
+      "loss": 0.6922041416168213,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.026619985699653625,
+      "rewards/margins": 0.002031774492934346,
+      "rewards/rejected": 0.024588212370872498,
+      "step": 340
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 1.2210006713867188,
+      "learning_rate": 9.306666666666666e-07,
+      "logits/chosen": 1.5815479755401611,
+      "logits/rejected": 1.710217833518982,
+      "logps/chosen": -167.21743774414062,
+      "logps/rejected": -196.77989196777344,
+      "loss": 0.6899692058563233,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.03636014088988304,
+      "rewards/margins": 0.006536121014505625,
+      "rewards/rejected": 0.029824022203683853,
+      "step": 350
+    },
+    {
+      "epoch": 0.288,
+      "grad_norm": 0.9623845219612122,
+      "learning_rate": 9.573333333333333e-07,
+      "logits/chosen": 1.4416790008544922,
+      "logits/rejected": 1.3618693351745605,
+      "logps/chosen": -163.27401733398438,
+      "logps/rejected": -169.3351287841797,
+      "loss": 0.6910821437835694,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.037200987339019775,
+      "rewards/margins": 0.004338678438216448,
+      "rewards/rejected": 0.032862309366464615,
+      "step": 360
+    },
+    {
+      "epoch": 0.296,
+      "grad_norm": 1.1005645990371704,
+      "learning_rate": 9.84e-07,
+      "logits/chosen": 1.6214616298675537,
+      "logits/rejected": 1.6572606563568115,
+      "logps/chosen": -162.95504760742188,
+      "logps/rejected": -154.91807556152344,
+      "loss": 0.6895030021667481,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.047198958694934845,
+      "rewards/margins": 0.0075958347879350185,
+      "rewards/rejected": 0.03960312157869339,
+      "step": 370
+    },
+    {
+      "epoch": 0.304,
+      "grad_norm": 1.1126338243484497,
+      "learning_rate": 9.988148148148148e-07,
+      "logits/chosen": 1.4842126369476318,
+      "logits/rejected": 1.410930871963501,
+      "logps/chosen": -143.09182739257812,
+      "logps/rejected": -153.48135375976562,
+      "loss": 0.6885370254516602,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.05975018069148064,
+      "rewards/margins": 0.00981281790882349,
+      "rewards/rejected": 0.04993735998868942,
+      "step": 380
+    },
+    {
+      "epoch": 0.312,
+      "grad_norm": 1.1146304607391357,
+      "learning_rate": 9.95851851851852e-07,
+      "logits/chosen": 1.608332633972168,
+      "logits/rejected": 1.5848472118377686,
+      "logps/chosen": -152.25250244140625,
+      "logps/rejected": -138.348876953125,
+      "loss": 0.6851807117462159,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.07630724459886551,
+      "rewards/margins": 0.016460279002785683,
+      "rewards/rejected": 0.059846967458724976,
+      "step": 390
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 1.2686601877212524,
+      "learning_rate": 9.928888888888889e-07,
+      "logits/chosen": 1.5840070247650146,
+      "logits/rejected": 1.5679785013198853,
+      "logps/chosen": -155.90219116210938,
+      "logps/rejected": -148.89549255371094,
+      "loss": 0.6891039371490478,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.08129279315471649,
+      "rewards/margins": 0.008997146971523762,
+      "rewards/rejected": 0.072295643389225,
+      "step": 400
+    },
+    {
+      "epoch": 0.32,
+      "eval_logits/chosen": 1.5358340740203857,
+      "eval_logits/rejected": 1.5689741373062134,
+      "eval_logps/chosen": -152.6895294189453,
+      "eval_logps/rejected": -147.23422241210938,
+      "eval_loss": 0.6900544166564941,
+      "eval_rewards/accuracies": 0.5419999957084656,
+      "eval_rewards/chosen": 0.08593228459358215,
+      "eval_rewards/margins": 0.0071397931315004826,
+      "eval_rewards/rejected": 0.0787924975156784,
+      "eval_runtime": 90.9072,
+      "eval_samples_per_second": 5.5,
+      "eval_steps_per_second": 2.75,
+      "step": 400
+    },
+    {
+      "epoch": 0.328,
+      "grad_norm": 1.1653108596801758,
+      "learning_rate": 9.899259259259258e-07,
+      "logits/chosen": 1.5450490713119507,
+      "logits/rejected": 1.4428151845932007,
+      "logps/chosen": -148.61581420898438,
+      "logps/rejected": -188.90919494628906,
+      "loss": 0.6918989181518554,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0946170911192894,
+      "rewards/margins": 0.003271549940109253,
+      "rewards/rejected": 0.09134554117918015,
+      "step": 410
+    },
+    {
+      "epoch": 0.336,
+      "grad_norm": 1.3114937543869019,
+      "learning_rate": 9.86962962962963e-07,
+      "logits/chosen": 1.5342715978622437,
+      "logits/rejected": 1.5572118759155273,
+      "logps/chosen": -152.65264892578125,
+      "logps/rejected": -168.03384399414062,
+      "loss": 0.6972955703735352,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.08869597315788269,
+      "rewards/margins": -0.007436127867549658,
+      "rewards/rejected": 0.09613210707902908,
+      "step": 420
+    },
+    {
+      "epoch": 0.344,
+      "grad_norm": 1.0040781497955322,
+      "learning_rate": 9.84e-07,
+      "logits/chosen": 1.5979712009429932,
+      "logits/rejected": 1.6713664531707764,
+      "logps/chosen": -164.2566375732422,
+      "logps/rejected": -152.1428680419922,
+      "loss": 0.6897010326385498,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.08101107180118561,
+      "rewards/margins": 0.007684691343456507,
+      "rewards/rejected": 0.07332637906074524,
+      "step": 430
+    },
+    {
+      "epoch": 0.352,
+      "grad_norm": 1.0801892280578613,
+      "learning_rate": 9.81037037037037e-07,
+      "logits/chosen": 1.439635992050171,
+      "logits/rejected": 1.3282365798950195,
+      "logps/chosen": -157.23403930664062,
+      "logps/rejected": -146.54806518554688,
+      "loss": 0.6881664752960205,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.08112544566392899,
+      "rewards/margins": 0.010971959680318832,
+      "rewards/rejected": 0.07015348225831985,
+      "step": 440
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 1.1139949560165405,
+      "learning_rate": 9.78074074074074e-07,
+      "logits/chosen": 1.6345123052597046,
+      "logits/rejected": 1.5876491069793701,
+      "logps/chosen": -157.6160125732422,
+      "logps/rejected": -160.02040100097656,
+      "loss": 0.6868602275848389,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.07638200372457504,
+      "rewards/margins": 0.013464884832501411,
+      "rewards/rejected": 0.06291711330413818,
+      "step": 450
+    },
+    {
+      "epoch": 0.368,
+      "grad_norm": 1.158035397529602,
+      "learning_rate": 9.751111111111112e-07,
+      "logits/chosen": 1.5430586338043213,
+      "logits/rejected": 1.6022913455963135,
+      "logps/chosen": -151.98919677734375,
+      "logps/rejected": -164.90565490722656,
+      "loss": 0.6891930580139161,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.06889279186725616,
+      "rewards/margins": 0.008791089057922363,
+      "rewards/rejected": 0.0601017065346241,
+      "step": 460
+    },
+    {
+      "epoch": 0.376,
+      "grad_norm": 1.0903704166412354,
+      "learning_rate": 9.721481481481481e-07,
+      "logits/chosen": 1.313881754875183,
+      "logits/rejected": 1.3182194232940674,
+      "logps/chosen": -131.5592041015625,
+      "logps/rejected": -123.84442138671875,
+      "loss": 0.6889429569244385,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.08150745928287506,
+      "rewards/margins": 0.009235726669430733,
+      "rewards/rejected": 0.07227173447608948,
+      "step": 470
+    },
+    {
+      "epoch": 0.384,
+      "grad_norm": 1.0766969919204712,
+      "learning_rate": 9.69185185185185e-07,
+      "logits/chosen": 1.425756812095642,
+      "logits/rejected": 1.476854920387268,
+      "logps/chosen": -148.72279357910156,
+      "logps/rejected": -164.40951538085938,
+      "loss": 0.6854116439819335,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.1009502187371254,
+      "rewards/margins": 0.0166789498180151,
+      "rewards/rejected": 0.08427127450704575,
+      "step": 480
+    },
+    {
+      "epoch": 0.392,
+      "grad_norm": 1.2078129053115845,
+      "learning_rate": 9.662222222222222e-07,
+      "logits/chosen": 1.5511282682418823,
+      "logits/rejected": 1.611383080482483,
+      "logps/chosen": -143.22006225585938,
+      "logps/rejected": -159.70176696777344,
+      "loss": 0.6968960762023926,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.08779363334178925,
+      "rewards/margins": -0.00598606513813138,
+      "rewards/rejected": 0.09377971291542053,
+      "step": 490
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 0.9106306433677673,
+      "learning_rate": 9.632592592592593e-07,
+      "logits/chosen": 1.4217239618301392,
+      "logits/rejected": 1.505446195602417,
+      "logps/chosen": -144.34109497070312,
+      "logps/rejected": -135.52430725097656,
+      "loss": 0.691196870803833,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.06902458518743515,
+      "rewards/margins": 0.005059319548308849,
+      "rewards/rejected": 0.06396526843309402,
+      "step": 500
+    },
+    {
+      "epoch": 0.4,
+      "eval_logits/chosen": 1.5185325145721436,
+      "eval_logits/rejected": 1.5510345697402954,
+      "eval_logps/chosen": -152.7921905517578,
+      "eval_logps/rejected": -147.3463134765625,
+      "eval_loss": 0.6898502111434937,
+      "eval_rewards/accuracies": 0.5099999904632568,
+      "eval_rewards/chosen": 0.07566512376070023,
+      "eval_rewards/margins": 0.008083080872893333,
+      "eval_rewards/rejected": 0.06758204847574234,
+      "eval_runtime": 90.8294,
+      "eval_samples_per_second": 5.505,
+      "eval_steps_per_second": 2.752,
+      "step": 500
+    },
+    {
+      "epoch": 0.408,
+      "grad_norm": 1.1637762784957886,
+      "learning_rate": 9.602962962962962e-07,
+      "logits/chosen": 1.5965330600738525,
+      "logits/rejected": 1.5338464975357056,
+      "logps/chosen": -131.49871826171875,
+      "logps/rejected": -140.05995178222656,
+      "loss": 0.6868171215057373,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.08385131508111954,
+      "rewards/margins": 0.013950209133327007,
+      "rewards/rejected": 0.06990110129117966,
+      "step": 510
+    },
+    {
+      "epoch": 0.416,
+      "grad_norm": 1.229719638824463,
+      "learning_rate": 9.573333333333333e-07,
+      "logits/chosen": 1.7057275772094727,
+      "logits/rejected": 1.5898948907852173,
+      "logps/chosen": -166.4267120361328,
+      "logps/rejected": -172.22337341308594,
+      "loss": 0.6912094116210937,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.07427279651165009,
+      "rewards/margins": 0.004661654122173786,
+      "rewards/rejected": 0.06961113959550858,
+      "step": 520
+    },
+    {
+      "epoch": 0.424,
+      "grad_norm": 1.136383295059204,
+      "learning_rate": 9.543703703703705e-07,
+      "logits/chosen": 1.6207077503204346,
+      "logits/rejected": 1.5417520999908447,
+      "logps/chosen": -166.81509399414062,
+      "logps/rejected": -168.5408172607422,
+      "loss": 0.6772748470306397,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.0953373983502388,
+      "rewards/margins": 0.033697597682476044,
+      "rewards/rejected": 0.06163979694247246,
+      "step": 530
+    },
+    {
+      "epoch": 0.432,
+      "grad_norm": 1.0640541315078735,
+      "learning_rate": 9.514074074074074e-07,
+      "logits/chosen": 1.6825854778289795,
+      "logits/rejected": 1.6327041387557983,
+      "logps/chosen": -181.39956665039062,
+      "logps/rejected": -158.49647521972656,
+      "loss": 0.6883615493774414,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.10187923908233643,
+      "rewards/margins": 0.010873330757021904,
+      "rewards/rejected": 0.09100590646266937,
+      "step": 540
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 1.477909803390503,
+      "learning_rate": 9.484444444444444e-07,
+      "logits/chosen": 1.5402686595916748,
+      "logits/rejected": 1.6487388610839844,
+      "logps/chosen": -151.53814697265625,
+      "logps/rejected": -167.8573455810547,
+      "loss": 0.6915213584899902,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.09472338110208511,
+      "rewards/margins": 0.005475065670907497,
+      "rewards/rejected": 0.08924831449985504,
+      "step": 550
+    },
+    {
+      "epoch": 0.448,
+      "grad_norm": 1.0138317346572876,
+      "learning_rate": 9.454814814814814e-07,
+      "logits/chosen": 1.6286503076553345,
+      "logits/rejected": 1.5223101377487183,
+      "logps/chosen": -154.72079467773438,
+      "logps/rejected": -139.2535858154297,
+      "loss": 0.6835216522216797,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.09194488823413849,
+      "rewards/margins": 0.02140027843415737,
+      "rewards/rejected": 0.07054460793733597,
+      "step": 560
+    },
+    {
+      "epoch": 0.456,
+      "grad_norm": 0.896689772605896,
+      "learning_rate": 9.425185185185184e-07,
+      "logits/chosen": 1.4857146739959717,
+      "logits/rejected": 1.4369819164276123,
+      "logps/chosen": -158.31809997558594,
+      "logps/rejected": -152.33676147460938,
+      "loss": 0.6863360404968262,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.07895292341709137,
+      "rewards/margins": 0.016260787844657898,
+      "rewards/rejected": 0.06269213557243347,
+      "step": 570
+    },
+    {
+      "epoch": 0.464,
+      "grad_norm": 1.4772330522537231,
+      "learning_rate": 9.395555555555556e-07,
+      "logits/chosen": 1.315298318862915,
+      "logits/rejected": 1.4757534265518188,
+      "logps/chosen": -136.4202117919922,
+      "logps/rejected": -154.99362182617188,
+      "loss": 0.6916167259216308,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.04677204787731171,
+      "rewards/margins": 0.005649724509567022,
+      "rewards/rejected": 0.041122324764728546,
+      "step": 580
+    },
+    {
+      "epoch": 0.472,
+      "grad_norm": 0.9806157350540161,
+      "learning_rate": 9.365925925925926e-07,
+      "logits/chosen": 1.465427279472351,
+      "logits/rejected": 1.459496259689331,
+      "logps/chosen": -145.07470703125,
+      "logps/rejected": -131.71974182128906,
+      "loss": 0.6857181549072265,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.07979898899793625,
+      "rewards/margins": 0.016965944319963455,
+      "rewards/rejected": 0.0628330409526825,
+      "step": 590
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 1.4313396215438843,
+      "learning_rate": 9.336296296296295e-07,
+      "logits/chosen": 1.5190367698669434,
+      "logits/rejected": 1.563504934310913,
+      "logps/chosen": -132.0201416015625,
+      "logps/rejected": -158.89865112304688,
+      "loss": 0.6964848041534424,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.06544725596904755,
+      "rewards/margins": -0.004280097782611847,
+      "rewards/rejected": 0.0697273463010788,
+      "step": 600
+    },
+    {
+      "epoch": 0.48,
+      "eval_logits/chosen": 1.5031698942184448,
+      "eval_logits/rejected": 1.5348584651947021,
+      "eval_logps/chosen": -152.76596069335938,
+      "eval_logps/rejected": -147.3241729736328,
+      "eval_loss": 0.6903045773506165,
+      "eval_rewards/accuracies": 0.527999997138977,
+      "eval_rewards/chosen": 0.07828804850578308,
+      "eval_rewards/margins": 0.008492168970406055,
+      "eval_rewards/rejected": 0.0697958841919899,
+      "eval_runtime": 90.9881,
+      "eval_samples_per_second": 5.495,
+      "eval_steps_per_second": 2.748,
+      "step": 600
+    },
+    {
+      "epoch": 0.488,
+      "grad_norm": 0.9467533230781555,
+      "learning_rate": 9.306666666666666e-07,
+      "logits/chosen": 1.5242283344268799,
+      "logits/rejected": 1.5798314809799194,
+      "logps/chosen": -131.90048217773438,
+      "logps/rejected": -149.93222045898438,
+      "loss": 0.6899854183197022,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.08419154584407806,
+      "rewards/margins": 0.00830511748790741,
+      "rewards/rejected": 0.07588644325733185,
+      "step": 610
+    },
+    {
+      "epoch": 0.496,
+      "grad_norm": 1.8463492393493652,
+      "learning_rate": 9.277037037037037e-07,
+      "logits/chosen": 1.4965155124664307,
+      "logits/rejected": 1.6336272954940796,
+      "logps/chosen": -156.1863555908203,
+      "logps/rejected": -167.74075317382812,
+      "loss": 0.6977802276611328,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.05518193915486336,
+      "rewards/margins": -0.006808985956013203,
+      "rewards/rejected": 0.061990927904844284,
+      "step": 620
+    },
+    {
+      "epoch": 0.504,
+      "grad_norm": 1.278316617012024,
+      "learning_rate": 9.247407407407407e-07,
+      "logits/chosen": 1.5074065923690796,
+      "logits/rejected": 1.535696029663086,
+      "logps/chosen": -167.56777954101562,
+      "logps/rejected": -147.04922485351562,
+      "loss": 0.6806219100952149,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0745314508676529,
+      "rewards/margins": 0.027104835957288742,
+      "rewards/rejected": 0.047426607459783554,
+      "step": 630
+    },
+    {
+      "epoch": 0.512,
+      "grad_norm": 1.0424679517745972,
+      "learning_rate": 9.217777777777778e-07,
+      "logits/chosen": 1.4299286603927612,
+      "logits/rejected": 1.4484076499938965,
+      "logps/chosen": -168.305908203125,
+      "logps/rejected": -152.61854553222656,
+      "loss": 0.6871783256530761,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.07751411944627762,
+      "rewards/margins": 0.017045259475708008,
+      "rewards/rejected": 0.06046885997056961,
+      "step": 640
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 1.1143999099731445,
+      "learning_rate": 9.188148148148148e-07,
+      "logits/chosen": 1.5520436763763428,
+      "logits/rejected": 1.531718134880066,
+      "logps/chosen": -154.92115783691406,
+      "logps/rejected": -126.78369140625,
+      "loss": 0.6889286518096924,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.0945608839392662,
+      "rewards/margins": 0.01080345083028078,
+      "rewards/rejected": 0.0837574377655983,
+      "step": 650
+    },
+    {
+      "epoch": 0.528,
+      "grad_norm": 1.2762326002120972,
+      "learning_rate": 9.158518518518517e-07,
+      "logits/chosen": 1.5747522115707397,
+      "logits/rejected": 1.4972833395004272,
+      "logps/chosen": -151.13914489746094,
+      "logps/rejected": -132.013916015625,
+      "loss": 0.6900454998016358,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.07303085923194885,
+      "rewards/margins": 0.008073865436017513,
+      "rewards/rejected": 0.06495700031518936,
+      "step": 660
+    },
+    {
+      "epoch": 0.536,
+      "grad_norm": 1.5207557678222656,
+      "learning_rate": 9.128888888888888e-07,
+      "logits/chosen": 1.6594665050506592,
+      "logits/rejected": 1.468492031097412,
+      "logps/chosen": -166.0108184814453,
+      "logps/rejected": -151.38555908203125,
+      "loss": 0.6934059143066407,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.09294265508651733,
+      "rewards/margins": 0.0021180796902626753,
+      "rewards/rejected": 0.09082455933094025,
+      "step": 670
+    },
+    {
+      "epoch": 0.544,
+      "grad_norm": 1.1073335409164429,
+      "learning_rate": 9.099259259259259e-07,
+      "logits/chosen": 1.5321934223175049,
+      "logits/rejected": 1.5313541889190674,
+      "logps/chosen": -144.5046844482422,
+      "logps/rejected": -133.05470275878906,
+      "loss": 0.6824594974517822,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.10782947391271591,
+      "rewards/margins": 0.0230915155261755,
+      "rewards/rejected": 0.08473796397447586,
+      "step": 680
+    },
+    {
+      "epoch": 0.552,
+      "grad_norm": 1.096970558166504,
+      "learning_rate": 9.069629629629629e-07,
+      "logits/chosen": 1.438305139541626,
+      "logits/rejected": 1.4338308572769165,
+      "logps/chosen": -173.1280517578125,
+      "logps/rejected": -160.20950317382812,
+      "loss": 0.6878445625305176,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.09615986794233322,
+      "rewards/margins": 0.012350928969681263,
+      "rewards/rejected": 0.08380892872810364,
+      "step": 690
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 1.3664878606796265,
+      "learning_rate": 9.039999999999999e-07,
+      "logits/chosen": 1.530369758605957,
+      "logits/rejected": 1.5520622730255127,
+      "logps/chosen": -159.60740661621094,
+      "logps/rejected": -137.77352905273438,
+      "loss": 0.6991429805755616,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.07249583303928375,
+      "rewards/margins": -0.009333941154181957,
+      "rewards/rejected": 0.08182977139949799,
+      "step": 700
+    },
+    {
+      "epoch": 0.56,
+      "eval_logits/chosen": 1.5136796236038208,
+      "eval_logits/rejected": 1.5456002950668335,
+      "eval_logps/chosen": -152.5748748779297,
+      "eval_logps/rejected": -147.17152404785156,
+      "eval_loss": 0.688511848449707,
+      "eval_rewards/accuracies": 0.5339999794960022,
+      "eval_rewards/chosen": 0.09739598631858826,
+      "eval_rewards/margins": 0.012335418723523617,
+      "eval_rewards/rejected": 0.08506056666374207,
+      "eval_runtime": 90.8734,
+      "eval_samples_per_second": 5.502,
+      "eval_steps_per_second": 2.751,
+      "step": 700
+    },
+    {
+      "epoch": 0.568,
+      "grad_norm": 1.224107265472412,
+      "learning_rate": 9.010370370370371e-07,
+      "logits/chosen": 1.6023967266082764,
+      "logits/rejected": 1.558224081993103,
+      "logps/chosen": -148.47120666503906,
+      "logps/rejected": -163.5648651123047,
+      "loss": 0.6865221500396729,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.12494877725839615,
+      "rewards/margins": 0.01545787788927555,
+      "rewards/rejected": 0.10949089378118515,
+      "step": 710
+    },
+    {
+      "epoch": 0.576,
+      "grad_norm": 1.2135164737701416,
+      "learning_rate": 8.98074074074074e-07,
+      "logits/chosen": 1.5902750492095947,
+      "logits/rejected": 1.7012746334075928,
+      "logps/chosen": -139.7012939453125,
+      "logps/rejected": -135.42225646972656,
+      "loss": 0.7014039039611817,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.09102506935596466,
+      "rewards/margins": -0.014695463702082634,
+      "rewards/rejected": 0.10572052001953125,
+      "step": 720
+    },
+    {
+      "epoch": 0.584,
+      "grad_norm": 1.1953649520874023,
+      "learning_rate": 8.95111111111111e-07,
+      "logits/chosen": 1.6540731191635132,
+      "logits/rejected": 1.6386349201202393,
+      "logps/chosen": -153.12847900390625,
+      "logps/rejected": -149.66319274902344,
+      "loss": 0.6952126502990723,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.1124584898352623,
+      "rewards/margins": -0.0012063594767823815,
+      "rewards/rejected": 0.11366486549377441,
+      "step": 730
+    },
+    {
+      "epoch": 0.592,
+      "grad_norm": 1.147875428199768,
+      "learning_rate": 8.921481481481481e-07,
+      "logits/chosen": 1.5578330755233765,
+      "logits/rejected": 1.4689290523529053,
+      "logps/chosen": -156.93788146972656,
+      "logps/rejected": -134.1832733154297,
+      "loss": 0.6919830799102783,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.09214536100625992,
+      "rewards/margins": 0.004291228950023651,
+      "rewards/rejected": 0.08785412460565567,
+      "step": 740
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 1.21685791015625,
+      "learning_rate": 8.891851851851852e-07,
+      "logits/chosen": 1.4241983890533447,
+      "logits/rejected": 1.4537718296051025,
+      "logps/chosen": -136.9163055419922,
+      "logps/rejected": -158.44491577148438,
+      "loss": 0.691422986984253,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.09173480421304703,
+      "rewards/margins": 0.00533579895272851,
+      "rewards/rejected": 0.08639900386333466,
+      "step": 750
+    },
+    {
+      "epoch": 0.608,
+      "grad_norm": 1.3629519939422607,
+      "learning_rate": 8.862222222222222e-07,
+      "logits/chosen": 1.565716028213501,
+      "logits/rejected": 1.3824570178985596,
+      "logps/chosen": -138.8521728515625,
+      "logps/rejected": -129.51028442382812,
+      "loss": 0.6931031227111817,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0763697624206543,
+      "rewards/margins": 0.0021251302678138018,
+      "rewards/rejected": 0.07424463331699371,
+      "step": 760
+    },
+    {
+      "epoch": 0.616,
+      "grad_norm": 1.1030747890472412,
+      "learning_rate": 8.832592592592593e-07,
+      "logits/chosen": 1.5781702995300293,
+      "logits/rejected": 1.5428775548934937,
+      "logps/chosen": -141.01480102539062,
+      "logps/rejected": -136.47189331054688,
+      "loss": 0.6777657032012939,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.10366280376911163,
+      "rewards/margins": 0.0333605632185936,
+      "rewards/rejected": 0.07030224055051804,
+      "step": 770
+    },
+    {
+      "epoch": 0.624,
+      "grad_norm": 1.3228758573532104,
+      "learning_rate": 8.802962962962962e-07,
+      "logits/chosen": 1.451596736907959,
+      "logits/rejected": 1.5489904880523682,
+      "logps/chosen": -153.74618530273438,
+      "logps/rejected": -159.26333618164062,
+      "loss": 0.6870471000671386,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.10403706133365631,
+      "rewards/margins": 0.014494165778160095,
+      "rewards/rejected": 0.08954289555549622,
+      "step": 780
+    },
+    {
+      "epoch": 0.632,
+      "grad_norm": 1.3267385959625244,
+      "learning_rate": 8.773333333333332e-07,
+      "logits/chosen": 1.4153831005096436,
+      "logits/rejected": 1.4497296810150146,
+      "logps/chosen": -148.05368041992188,
+      "logps/rejected": -151.3020477294922,
+      "loss": 0.6877872943878174,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.07300253212451935,
+      "rewards/margins": 0.013981563039124012,
+      "rewards/rejected": 0.05902096629142761,
+      "step": 790
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 1.2227483987808228,
+      "learning_rate": 8.743703703703703e-07,
+      "logits/chosen": 1.6773914098739624,
+      "logits/rejected": 1.6918518543243408,
+      "logps/chosen": -180.06863403320312,
+      "logps/rejected": -194.30752563476562,
+      "loss": 0.7015917778015137,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.07480724155902863,
+      "rewards/margins": -0.013628068380057812,
+      "rewards/rejected": 0.08843531459569931,
+      "step": 800
+    },
+    {
+      "epoch": 0.64,
+      "eval_logits/chosen": 1.5051140785217285,
+      "eval_logits/rejected": 1.5367870330810547,
+      "eval_logps/chosen": -152.8026885986328,
+      "eval_logps/rejected": -147.39437866210938,
+      "eval_loss": 0.688633918762207,
+      "eval_rewards/accuracies": 0.5220000147819519,
+      "eval_rewards/chosen": 0.07461711764335632,
+      "eval_rewards/margins": 0.011839868500828743,
+      "eval_rewards/rejected": 0.06277725100517273,
+      "eval_runtime": 90.9206,
+      "eval_samples_per_second": 5.499,
+      "eval_steps_per_second": 2.75,
+      "step": 800
+    },
+    {
+      "epoch": 0.648,
+      "grad_norm": 1.209770679473877,
+      "learning_rate": 8.714074074074074e-07,
+      "logits/chosen": 1.3785041570663452,
+      "logits/rejected": 1.60085928440094,
+      "logps/chosen": -114.5325927734375,
+      "logps/rejected": -151.88900756835938,
+      "loss": 0.6890413761138916,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0640551745891571,
+      "rewards/margins": 0.010126596316695213,
+      "rewards/rejected": 0.05392857640981674,
+      "step": 810
+    },
+    {
+      "epoch": 0.656,
+      "grad_norm": 1.0250163078308105,
+      "learning_rate": 8.684444444444444e-07,
+      "logits/chosen": 1.5827982425689697,
+      "logits/rejected": 1.6239618062973022,
+      "logps/chosen": -149.01904296875,
+      "logps/rejected": -141.9372100830078,
+      "loss": 0.6728014469146728,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0830039232969284,
+      "rewards/margins": 0.043090131133794785,
+      "rewards/rejected": 0.03991378843784332,
+      "step": 820
+    },
+    {
+      "epoch": 0.664,
+      "grad_norm": 1.0627655982971191,
+      "learning_rate": 8.654814814814814e-07,
+      "logits/chosen": 1.501873254776001,
+      "logits/rejected": 1.537386417388916,
+      "logps/chosen": -138.75169372558594,
+      "logps/rejected": -134.00970458984375,
+      "loss": 0.6801187992095947,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0640982910990715,
+      "rewards/margins": 0.02855527400970459,
+      "rewards/rejected": 0.03554301708936691,
+      "step": 830
+    },
+    {
+      "epoch": 0.672,
+      "grad_norm": 1.2453433275222778,
+      "learning_rate": 8.625185185185186e-07,
+      "logits/chosen": 1.4766005277633667,
+      "logits/rejected": 1.5185787677764893,
+      "logps/chosen": -148.41250610351562,
+      "logps/rejected": -157.73989868164062,
+      "loss": 0.6788079738616943,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.06551454961299896,
+      "rewards/margins": 0.03301671892404556,
+      "rewards/rejected": 0.0324978306889534,
+      "step": 840
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 1.191698670387268,
+      "learning_rate": 8.595555555555555e-07,
+      "logits/chosen": 1.555551528930664,
+      "logits/rejected": 1.5438499450683594,
+      "logps/chosen": -160.77488708496094,
+      "logps/rejected": -141.9876251220703,
+      "loss": 0.6801597595214843,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.053498052060604095,
+      "rewards/margins": 0.028470590710639954,
+      "rewards/rejected": 0.025027472525835037,
+      "step": 850
+    },
+    {
+      "epoch": 0.688,
+      "grad_norm": 0.9382092356681824,
+      "learning_rate": 8.565925925925925e-07,
+      "logits/chosen": 1.5482738018035889,
+      "logits/rejected": 1.5833700895309448,
+      "logps/chosen": -139.68118286132812,
+      "logps/rejected": -130.05947875976562,
+      "loss": 0.6825921535491943,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.06382572650909424,
+      "rewards/margins": 0.023111844435334206,
+      "rewards/rejected": 0.04071388021111488,
+      "step": 860
+    },
+    {
+      "epoch": 0.696,
+      "grad_norm": 1.2248382568359375,
+      "learning_rate": 8.536296296296296e-07,
+      "logits/chosen": 1.4529926776885986,
+      "logits/rejected": 1.4423902034759521,
+      "logps/chosen": -154.4588165283203,
+      "logps/rejected": -153.44949340820312,
+      "loss": 0.6915592193603516,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.03207828477025032,
+      "rewards/margins": 0.006785523146390915,
+      "rewards/rejected": 0.025292763486504555,
+      "step": 870
+    },
+    {
+      "epoch": 0.704,
+      "grad_norm": 1.2174893617630005,
+      "learning_rate": 8.506666666666667e-07,
+      "logits/chosen": 1.4073113203048706,
+      "logits/rejected": 1.668140172958374,
+      "logps/chosen": -145.18460083007812,
+      "logps/rejected": -157.0777587890625,
+      "loss": 0.6922356605529785,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.049450717866420746,
+      "rewards/margins": 0.004049716051667929,
+      "rewards/rejected": 0.04540099948644638,
+      "step": 880
+    },
+    {
+      "epoch": 0.712,
+      "grad_norm": 1.3186850547790527,
+      "learning_rate": 8.477037037037037e-07,
+      "logits/chosen": 1.2923892736434937,
+      "logits/rejected": 1.491051435470581,
+      "logps/chosen": -149.03018188476562,
+      "logps/rejected": -161.61715698242188,
+      "loss": 0.6841150760650635,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.038961172103881836,
+      "rewards/margins": 0.020759906619787216,
+      "rewards/rejected": 0.01820126175880432,
+      "step": 890
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 1.1123124361038208,
+      "learning_rate": 8.447407407407407e-07,
+      "logits/chosen": 1.546547293663025,
+      "logits/rejected": 1.682992696762085,
+      "logps/chosen": -166.2056884765625,
+      "logps/rejected": -180.73330688476562,
+      "loss": 0.6826215744018554,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.07279330492019653,
+      "rewards/margins": 0.025264525786042213,
+      "rewards/rejected": 0.04752878472208977,
+      "step": 900
+    },
+    {
+      "epoch": 0.72,
+      "eval_logits/chosen": 1.4819762706756592,
+      "eval_logits/rejected": 1.513027548789978,
+      "eval_logps/chosen": -153.1695556640625,
+      "eval_logps/rejected": -147.7560577392578,
+      "eval_loss": 0.689373254776001,
+      "eval_rewards/accuracies": 0.5299999713897705,
+      "eval_rewards/chosen": 0.03792775049805641,
+      "eval_rewards/margins": 0.011320074088871479,
+      "eval_rewards/rejected": 0.026607677340507507,
+      "eval_runtime": 90.9603,
+      "eval_samples_per_second": 5.497,
+      "eval_steps_per_second": 2.748,
+      "step": 900
+    },
+    {
+      "epoch": 0.728,
+      "grad_norm": 1.1507582664489746,
+      "learning_rate": 8.417777777777777e-07,
+      "logits/chosen": 1.5831308364868164,
+      "logits/rejected": 1.3737132549285889,
+      "logps/chosen": -140.00253295898438,
+      "logps/rejected": -138.45155334472656,
+      "loss": 0.6873391628265381,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.05066479369997978,
+      "rewards/margins": 0.015466553159058094,
+      "rewards/rejected": 0.03519824147224426,
+      "step": 910
+    },
+    {
+      "epoch": 0.736,
+      "grad_norm": 1.415456771850586,
+      "learning_rate": 8.388148148148147e-07,
+      "logits/chosen": 1.3000686168670654,
+      "logits/rejected": 1.2642180919647217,
+      "logps/chosen": -160.14027404785156,
+      "logps/rejected": -163.32827758789062,
+      "loss": 0.687142276763916,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.06618930399417877,
+      "rewards/margins": 0.014807088300585747,
+      "rewards/rejected": 0.05138222128152847,
+      "step": 920
+    },
+    {
+      "epoch": 0.744,
+      "grad_norm": 1.3228634595870972,
+      "learning_rate": 8.358518518518518e-07,
+      "logits/chosen": 1.5204170942306519,
+      "logits/rejected": 1.6129217147827148,
+      "logps/chosen": -157.99832153320312,
+      "logps/rejected": -162.09567260742188,
+      "loss": 0.6790886878967285,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.05940447002649307,
+      "rewards/margins": 0.03161252290010452,
+      "rewards/rejected": 0.027791941538453102,
+      "step": 930
+    },
+    {
+      "epoch": 0.752,
+      "grad_norm": 1.048064947128296,
+      "learning_rate": 8.328888888888889e-07,
+      "logits/chosen": 1.5515320301055908,
+      "logits/rejected": 1.676857352256775,
+      "logps/chosen": -148.4853515625,
+      "logps/rejected": -174.4320526123047,
+      "loss": 0.6935453414916992,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.010225490666925907,
+      "rewards/margins": 0.0021693313028663397,
+      "rewards/rejected": 0.008056161925196648,
+      "step": 940
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 1.247799038887024,
+      "learning_rate": 8.299259259259259e-07,
+      "logits/chosen": 1.384231686592102,
+      "logits/rejected": 1.4611561298370361,
+      "logps/chosen": -140.81443786621094,
+      "logps/rejected": -153.81326293945312,
+      "loss": 0.6805226325988769,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.053223587572574615,
+      "rewards/margins": 0.02933613583445549,
+      "rewards/rejected": 0.023887457326054573,
+      "step": 950
+    },
+    {
+      "epoch": 0.768,
+      "grad_norm": 1.1611264944076538,
+      "learning_rate": 8.269629629629629e-07,
+      "logits/chosen": 1.4987528324127197,
+      "logits/rejected": 1.3684321641921997,
+      "logps/chosen": -135.81056213378906,
+      "logps/rejected": -119.05133056640625,
+      "loss": 0.6954316616058349,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.0197703056037426,
+      "rewards/margins": -0.0012637012405321002,
+      "rewards/rejected": 0.021034007892012596,
+      "step": 960
+    },
+    {
+      "epoch": 0.776,
+      "grad_norm": 0.964948832988739,
+      "learning_rate": 8.24e-07,
+      "logits/chosen": 1.612053632736206,
+      "logits/rejected": 1.7114235162734985,
+      "logps/chosen": -153.56137084960938,
+      "logps/rejected": -149.17323303222656,
+      "loss": 0.6916662693023682,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.05082285404205322,
+      "rewards/margins": 0.0056606316938996315,
+      "rewards/rejected": 0.04516221210360527,
+      "step": 970
+    },
+    {
+      "epoch": 0.784,
+      "grad_norm": 1.409449815750122,
+      "learning_rate": 8.21037037037037e-07,
+      "logits/chosen": 1.384060025215149,
+      "logits/rejected": 1.316361904144287,
+      "logps/chosen": -168.75360107421875,
+      "logps/rejected": -141.79537963867188,
+      "loss": 0.6845529556274415,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0886571854352951,
+      "rewards/margins": 0.02084263041615486,
+      "rewards/rejected": 0.06781454384326935,
+      "step": 980
+    },
+    {
+      "epoch": 0.792,
+      "grad_norm": 1.0246562957763672,
+      "learning_rate": 8.18074074074074e-07,
+      "logits/chosen": 1.4986507892608643,
+      "logits/rejected": 1.571103811264038,
+      "logps/chosen": -165.22254943847656,
+      "logps/rejected": -168.63540649414062,
+      "loss": 0.6790967464447022,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.07517905533313751,
+      "rewards/margins": 0.03272219002246857,
+      "rewards/rejected": 0.04245685786008835,
+      "step": 990
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 1.014426827430725,
+      "learning_rate": 8.15111111111111e-07,
+      "logits/chosen": 1.4708755016326904,
+      "logits/rejected": 1.4550955295562744,
+      "logps/chosen": -149.49874877929688,
+      "logps/rejected": -125.0966567993164,
+      "loss": 0.6892098903656005,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.07825140655040741,
+      "rewards/margins": 0.01007872074842453,
+      "rewards/rejected": 0.06817268580198288,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8,
+      "eval_logits/chosen": 1.4972777366638184,
+      "eval_logits/rejected": 1.5283576250076294,
+      "eval_logps/chosen": -152.8240966796875,
+      "eval_logps/rejected": -147.44175720214844,
+      "eval_loss": 0.6880542039871216,
+      "eval_rewards/accuracies": 0.5419999957084656,
+      "eval_rewards/chosen": 0.07247376441955566,
+      "eval_rewards/margins": 0.01443479023873806,
+      "eval_rewards/rejected": 0.058038972318172455,
+      "eval_runtime": 90.827,
+      "eval_samples_per_second": 5.505,
+      "eval_steps_per_second": 2.752,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 3750,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

v3/DPO/DPO_10k/lora/checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4f90e4a7053b7fb440d68cb64a3d198396af09db942d08839c5f4f0f0b8a0c8
+size 6097

v3/DPO/DPO_10k/lora/checkpoint-1500/README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

v3/DPO/DPO_10k/lora/checkpoint-1500/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "gate_proj",
+    "down_proj",
+    "k_proj",
+    "o_proj",
+    "up_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

v3/DPO/DPO_10k/lora/checkpoint-1500/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7afd4ea14d3390375f16e1a10b8fcca1ede2eb7959bce12efb565ba4b2e00236
+size 180385008

v3/DPO/DPO_10k/lora/checkpoint-1500/chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

v3/DPO/DPO_10k/lora/checkpoint-1500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e76e51dcca53476f7d03879d08ea48c310a515073dcada2891911d9d2d8bca3
+size 360902475

v3/DPO/DPO_10k/lora/checkpoint-1500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b928d2d8033ac6bd87c58a39b741faace8bd1c6b0d070b7fad23c19520ff9f1a
+size 14645

v3/DPO/DPO_10k/lora/checkpoint-1500/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca372268f4fa9335030c0cb7aedb6cdba75f457da50e7a4034abb1a2d0843689
+size 1383

v3/DPO/DPO_10k/lora/checkpoint-1500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e10622725c891492a233a860ff051a91cd7997b7a70e593a4bc44e88f0ef3b5
+size 1465

v3/DPO/DPO_10k/lora/checkpoint-1500/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

v3/DPO/DPO_10k/lora/checkpoint-1500/tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

v3/DPO/DPO_10k/lora/checkpoint-1500/trainer_state.json

@@ -0,0 +1,2524 @@
+{
+  "best_global_step": 1400,
+  "best_metric": 0.6853657364845276,
+  "best_model_checkpoint": "output/lora/checkpoint-1000",
+  "epoch": 1.2,
+  "eval_steps": 100,
+  "global_step": 1500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.008,
+      "grad_norm": 1.1179773807525635,
+      "learning_rate": 2.4e-08,
+      "logits/chosen": 1.4545083045959473,
+      "logits/rejected": 1.425490379333496,
+      "logps/chosen": -132.46189880371094,
+      "logps/rejected": -148.20260620117188,
+      "loss": 0.6930626392364502,
+      "rewards/accuracies": 0.21250000596046448,
+      "rewards/chosen": -6.165739614516497e-05,
+      "rewards/margins": 0.00017333509458694607,
+      "rewards/rejected": -0.00023499250528402627,
+      "step": 10
+    },
+    {
+      "epoch": 0.016,
+      "grad_norm": 1.3502839803695679,
+      "learning_rate": 5.0666666666666664e-08,
+      "logits/chosen": 1.5668801069259644,
+      "logits/rejected": 1.5191389322280884,
+      "logps/chosen": -157.63624572753906,
+      "logps/rejected": -144.2716827392578,
+      "loss": 0.6930380821228027,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0009244013344869018,
+      "rewards/margins": 0.00023505210992880166,
+      "rewards/rejected": 0.0006893490790389478,
+      "step": 20
+    },
+    {
+      "epoch": 0.024,
+      "grad_norm": 1.0219553709030151,
+      "learning_rate": 7.733333333333334e-08,
+      "logits/chosen": 1.8921797275543213,
+      "logits/rejected": 1.7342685461044312,
+      "logps/chosen": -164.55799865722656,
+      "logps/rejected": -158.92752075195312,
+      "loss": 0.69290189743042,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0009391927160322666,
+      "rewards/margins": 0.0005032324115745723,
+      "rewards/rejected": 0.0004359603044576943,
+      "step": 30
+    },
+    {
+      "epoch": 0.032,
+      "grad_norm": 1.2056113481521606,
+      "learning_rate": 1.0399999999999999e-07,
+      "logits/chosen": 1.4950047731399536,
+      "logits/rejected": 1.6163270473480225,
+      "logps/chosen": -145.54183959960938,
+      "logps/rejected": -134.19979858398438,
+      "loss": 0.6935500621795654,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": 0.0006421065190806985,
+      "rewards/margins": -0.0007943272357806563,
+      "rewards/rejected": 0.0014364338712766767,
+      "step": 40
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 1.1029725074768066,
+      "learning_rate": 1.3066666666666665e-07,
+      "logits/chosen": 1.4076533317565918,
+      "logits/rejected": 1.4791977405548096,
+      "logps/chosen": -133.23324584960938,
+      "logps/rejected": -138.3593292236328,
+      "loss": 0.6925833702087403,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0012372838100418448,
+      "rewards/margins": 0.001139522879384458,
+      "rewards/rejected": 9.776116348803043e-05,
+      "step": 50
+    },
+    {
+      "epoch": 0.048,
+      "grad_norm": 0.9960266351699829,
+      "learning_rate": 1.573333333333333e-07,
+      "logits/chosen": 1.407166600227356,
+      "logits/rejected": 1.376042127609253,
+      "logps/chosen": -137.8921661376953,
+      "logps/rejected": -127.97320556640625,
+      "loss": 0.693614149093628,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.00021316049969755113,
+      "rewards/margins": -0.0009249996510334313,
+      "rewards/rejected": 0.0007118391804397106,
+      "step": 60
+    },
+    {
+      "epoch": 0.056,
+      "grad_norm": 1.0946824550628662,
+      "learning_rate": 1.8399999999999998e-07,
+      "logits/chosen": 1.5944501161575317,
+      "logits/rejected": 1.7248146533966064,
+      "logps/chosen": -153.5526123046875,
+      "logps/rejected": -157.19735717773438,
+      "loss": 0.6928449153900147,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0026370862033218145,
+      "rewards/margins": 0.0006150150438770652,
+      "rewards/rejected": 0.0020220710430294275,
+      "step": 70
+    },
+    {
+      "epoch": 0.064,
+      "grad_norm": 1.0322489738464355,
+      "learning_rate": 2.1066666666666665e-07,
+      "logits/chosen": 1.4586353302001953,
+      "logits/rejected": 1.4842995405197144,
+      "logps/chosen": -149.03665161132812,
+      "logps/rejected": -136.24081420898438,
+      "loss": 0.6932015895843506,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.004061593674123287,
+      "rewards/margins": -9.99594121822156e-05,
+      "rewards/rejected": 0.004161553457379341,
+      "step": 80
+    },
+    {
+      "epoch": 0.072,
+      "grad_norm": 1.0012447834014893,
+      "learning_rate": 2.3733333333333334e-07,
+      "logits/chosen": 1.5075902938842773,
+      "logits/rejected": 1.5986944437026978,
+      "logps/chosen": -159.3433074951172,
+      "logps/rejected": -151.85072326660156,
+      "loss": 0.6928757190704345,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.006978762801736593,
+      "rewards/margins": 0.0005627512582577765,
+      "rewards/rejected": 0.006416010670363903,
+      "step": 90
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 1.280081033706665,
+      "learning_rate": 2.64e-07,
+      "logits/chosen": 1.448286533355713,
+      "logits/rejected": 1.5480695962905884,
+      "logps/chosen": -151.12625122070312,
+      "logps/rejected": -171.24545288085938,
+      "loss": 0.6931126117706299,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.00693341251462698,
+      "rewards/margins": 9.057526767719537e-05,
+      "rewards/rejected": 0.006842837668955326,
+      "step": 100
+    },
+    {
+      "epoch": 0.08,
+      "eval_logits/chosen": 1.518183946609497,
+      "eval_logits/rejected": 1.5525447130203247,
+      "eval_logps/chosen": -153.46630859375,
+      "eval_logps/rejected": -147.95005798339844,
+      "eval_loss": 0.6926330327987671,
+      "eval_rewards/accuracies": 0.5600000023841858,
+      "eval_rewards/chosen": 0.008253541775047779,
+      "eval_rewards/margins": 0.0010449704714119434,
+      "eval_rewards/rejected": 0.007208569906651974,
+      "eval_runtime": 91.0812,
+      "eval_samples_per_second": 5.49,
+      "eval_steps_per_second": 2.745,
+      "step": 100
+    },
+    {
+      "epoch": 0.088,
+      "grad_norm": 1.3912373781204224,
+      "learning_rate": 2.906666666666667e-07,
+      "logits/chosen": 1.6896222829818726,
+      "logits/rejected": 1.6516921520233154,
+      "logps/chosen": -173.70030212402344,
+      "logps/rejected": -155.5503692626953,
+      "loss": 0.6926521778106689,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.010179834440350533,
+      "rewards/margins": 0.0010137532372027636,
+      "rewards/rejected": 0.009166081435978413,
+      "step": 110
+    },
+    {
+      "epoch": 0.096,
+      "grad_norm": 1.1518888473510742,
+      "learning_rate": 3.173333333333333e-07,
+      "logits/chosen": 1.5166860818862915,
+      "logits/rejected": 1.5136979818344116,
+      "logps/chosen": -136.39906311035156,
+      "logps/rejected": -140.86561584472656,
+      "loss": 0.6932918548583984,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.00907944981008768,
+      "rewards/margins": -0.00026998287648893893,
+      "rewards/rejected": 0.009349432773888111,
+      "step": 120
+    },
+    {
+      "epoch": 0.104,
+      "grad_norm": 1.3213376998901367,
+      "learning_rate": 3.4399999999999996e-07,
+      "logits/chosen": 1.4598973989486694,
+      "logits/rejected": 1.6739647388458252,
+      "logps/chosen": -159.12461853027344,
+      "logps/rejected": -163.39346313476562,
+      "loss": 0.693086051940918,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.010477779433131218,
+      "rewards/margins": 0.00014539004769176245,
+      "rewards/rejected": 0.010332388803362846,
+      "step": 130
+    },
+    {
+      "epoch": 0.112,
+      "grad_norm": 0.8833863735198975,
+      "learning_rate": 3.7066666666666665e-07,
+      "logits/chosen": 1.3829412460327148,
+      "logits/rejected": 1.4100173711776733,
+      "logps/chosen": -132.7149658203125,
+      "logps/rejected": -126.85478210449219,
+      "loss": 0.6932382583618164,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.010262183845043182,
+      "rewards/margins": -0.0001672578218858689,
+      "rewards/rejected": 0.010429441928863525,
+      "step": 140
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 1.1472455263137817,
+      "learning_rate": 3.973333333333333e-07,
+      "logits/chosen": 1.4954869747161865,
+      "logits/rejected": 1.5857340097427368,
+      "logps/chosen": -165.10568237304688,
+      "logps/rejected": -158.97433471679688,
+      "loss": 0.6916103363037109,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.014177520759403706,
+      "rewards/margins": 0.003108317730948329,
+      "rewards/rejected": 0.011069201864302158,
+      "step": 150
+    },
+    {
+      "epoch": 0.128,
+      "grad_norm": 1.0628798007965088,
+      "learning_rate": 4.24e-07,
+      "logits/chosen": 1.6717243194580078,
+      "logits/rejected": 1.705092191696167,
+      "logps/chosen": -159.332275390625,
+      "logps/rejected": -147.21646118164062,
+      "loss": 0.6935629844665527,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.016994481906294823,
+      "rewards/margins": -0.0007874655420891941,
+      "rewards/rejected": 0.017781946808099747,
+      "step": 160
+    },
+    {
+      "epoch": 0.136,
+      "grad_norm": 1.390810251235962,
+      "learning_rate": 4.506666666666666e-07,
+      "logits/chosen": 1.4719234704971313,
+      "logits/rejected": 1.499526023864746,
+      "logps/chosen": -148.67251586914062,
+      "logps/rejected": -161.58021545410156,
+      "loss": 0.6926599979400635,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.021239640191197395,
+      "rewards/margins": 0.0010346340714022517,
+      "rewards/rejected": 0.020205006003379822,
+      "step": 170
+    },
+    {
+      "epoch": 0.144,
+      "grad_norm": 1.139707326889038,
+      "learning_rate": 4.773333333333333e-07,
+      "logits/chosen": 1.4833905696868896,
+      "logits/rejected": 1.637584924697876,
+      "logps/chosen": -158.96731567382812,
+      "logps/rejected": -161.9830322265625,
+      "loss": 0.6919547557830811,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.019416773691773415,
+      "rewards/margins": 0.0024296878837049007,
+      "rewards/rejected": 0.016987085342407227,
+      "step": 180
+    },
+    {
+      "epoch": 0.152,
+      "grad_norm": 1.2029796838760376,
+      "learning_rate": 5.04e-07,
+      "logits/chosen": 1.5398027896881104,
+      "logits/rejected": 1.6678705215454102,
+      "logps/chosen": -143.00283813476562,
+      "logps/rejected": -144.89378356933594,
+      "loss": 0.692262840270996,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.018960200250148773,
+      "rewards/margins": 0.0018175147706642747,
+      "rewards/rejected": 0.017142686992883682,
+      "step": 190
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 1.0527437925338745,
+      "learning_rate": 5.306666666666665e-07,
+      "logits/chosen": 1.6087623834609985,
+      "logits/rejected": 1.5181543827056885,
+      "logps/chosen": -159.35073852539062,
+      "logps/rejected": -133.59890747070312,
+      "loss": 0.6924069881439209,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.01926913857460022,
+      "rewards/margins": 0.0015339398523792624,
+      "rewards/rejected": 0.0177351962774992,
+      "step": 200
+    },
+    {
+      "epoch": 0.16,
+      "eval_logits/chosen": 1.5193672180175781,
+      "eval_logits/rejected": 1.5534639358520508,
+      "eval_logps/chosen": -153.34857177734375,
+      "eval_logps/rejected": -147.8425750732422,
+      "eval_loss": 0.6921458840370178,
+      "eval_rewards/accuracies": 0.5600000023841858,
+      "eval_rewards/chosen": 0.020027177408337593,
+      "eval_rewards/margins": 0.0020714527927339077,
+      "eval_rewards/rejected": 0.017955724149942398,
+      "eval_runtime": 90.9762,
+      "eval_samples_per_second": 5.496,
+      "eval_steps_per_second": 2.748,
+      "step": 200
+    },
+    {
+      "epoch": 0.168,
+      "grad_norm": 1.1897239685058594,
+      "learning_rate": 5.573333333333333e-07,
+      "logits/chosen": 1.6267732381820679,
+      "logits/rejected": 1.5775268077850342,
+      "logps/chosen": -159.51437377929688,
+      "logps/rejected": -170.17762756347656,
+      "loss": 0.6925770759582519,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.02078000269830227,
+      "rewards/margins": 0.0012053751852363348,
+      "rewards/rejected": 0.01957462914288044,
+      "step": 210
+    },
+    {
+      "epoch": 0.176,
+      "grad_norm": 0.9995236396789551,
+      "learning_rate": 5.839999999999999e-07,
+      "logits/chosen": 1.5781934261322021,
+      "logits/rejected": 1.642251968383789,
+      "logps/chosen": -159.95309448242188,
+      "logps/rejected": -144.36058044433594,
+      "loss": 0.6931475639343262,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.023322442546486855,
+      "rewards/margins": 6.392716750269756e-05,
+      "rewards/rejected": 0.02325851283967495,
+      "step": 220
+    },
+    {
+      "epoch": 0.184,
+      "grad_norm": 0.8819260597229004,
+      "learning_rate": 6.106666666666666e-07,
+      "logits/chosen": 1.459991693496704,
+      "logits/rejected": 1.6060739755630493,
+      "logps/chosen": -162.04745483398438,
+      "logps/rejected": -139.5411834716797,
+      "loss": 0.6919463634490967,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.026052182540297508,
+      "rewards/margins": 0.002481849165633321,
+      "rewards/rejected": 0.023570330813527107,
+      "step": 230
+    },
+    {
+      "epoch": 0.192,
+      "grad_norm": 1.1055208444595337,
+      "learning_rate": 6.373333333333333e-07,
+      "logits/chosen": 1.558739185333252,
+      "logits/rejected": 1.5965116024017334,
+      "logps/chosen": -171.9166259765625,
+      "logps/rejected": -145.66433715820312,
+      "loss": 0.6912176132202148,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.033868782222270966,
+      "rewards/margins": 0.004017929546535015,
+      "rewards/rejected": 0.029850851744413376,
+      "step": 240
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 1.1372733116149902,
+      "learning_rate": 6.64e-07,
+      "logits/chosen": 1.5322860479354858,
+      "logits/rejected": 1.4695795774459839,
+      "logps/chosen": -161.7803955078125,
+      "logps/rejected": -148.91949462890625,
+      "loss": 0.6940969944000244,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.034065864980220795,
+      "rewards/margins": -0.0016761379083618522,
+      "rewards/rejected": 0.035742007195949554,
+      "step": 250
+    },
+    {
+      "epoch": 0.208,
+      "grad_norm": 1.0917680263519287,
+      "learning_rate": 6.906666666666666e-07,
+      "logits/chosen": 1.4094547033309937,
+      "logits/rejected": 1.50186026096344,
+      "logps/chosen": -132.91455078125,
+      "logps/rejected": -146.79714965820312,
+      "loss": 0.6930203914642334,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": 0.03089061938226223,
+      "rewards/margins": 0.00040971505222842097,
+      "rewards/rejected": 0.030480902642011642,
+      "step": 260
+    },
+    {
+      "epoch": 0.216,
+      "grad_norm": 1.0280259847640991,
+      "learning_rate": 7.173333333333333e-07,
+      "logits/chosen": 1.531285285949707,
+      "logits/rejected": 1.6163132190704346,
+      "logps/chosen": -146.902587890625,
+      "logps/rejected": -147.8347625732422,
+      "loss": 0.6944749355316162,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.028514739125967026,
+      "rewards/margins": -0.0025452517438679934,
+      "rewards/rejected": 0.0310599897056818,
+      "step": 270
+    },
+    {
+      "epoch": 0.224,
+      "grad_norm": 1.398708701133728,
+      "learning_rate": 7.44e-07,
+      "logits/chosen": 1.563852071762085,
+      "logits/rejected": 1.396703839302063,
+      "logps/chosen": -172.68153381347656,
+      "logps/rejected": -155.80215454101562,
+      "loss": 0.689476490020752,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.03224216774106026,
+      "rewards/margins": 0.007473687641322613,
+      "rewards/rejected": 0.02476847730576992,
+      "step": 280
+    },
+    {
+      "epoch": 0.232,
+      "grad_norm": 1.0394141674041748,
+      "learning_rate": 7.706666666666667e-07,
+      "logits/chosen": 1.5581729412078857,
+      "logits/rejected": 1.5819021463394165,
+      "logps/chosen": -144.41465759277344,
+      "logps/rejected": -153.55406188964844,
+      "loss": 0.6927877902984619,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.03272664546966553,
+      "rewards/margins": 0.0008557910332456231,
+      "rewards/rejected": 0.031870849430561066,
+      "step": 290
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 1.2557185888290405,
+      "learning_rate": 7.973333333333333e-07,
+      "logits/chosen": 1.5062446594238281,
+      "logits/rejected": 1.5786281824111938,
+      "logps/chosen": -153.63558959960938,
+      "logps/rejected": -165.46617126464844,
+      "loss": 0.6897892951965332,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.03914078325033188,
+      "rewards/margins": 0.006873926613479853,
+      "rewards/rejected": 0.032266855239868164,
+      "step": 300
+    },
+    {
+      "epoch": 0.24,
+      "eval_logits/chosen": 1.5237834453582764,
+      "eval_logits/rejected": 1.557741641998291,
+      "eval_logps/chosen": -153.19883728027344,
+      "eval_logps/rejected": -147.7100830078125,
+      "eval_loss": 0.6913501024246216,
+      "eval_rewards/accuracies": 0.5339999794960022,
+      "eval_rewards/chosen": 0.03499976545572281,
+      "eval_rewards/margins": 0.0037925743963569403,
+      "eval_rewards/rejected": 0.031207194551825523,
+      "eval_runtime": 90.9133,
+      "eval_samples_per_second": 5.5,
+      "eval_steps_per_second": 2.75,
+      "step": 300
+    },
+    {
+      "epoch": 0.248,
+      "grad_norm": 1.1200144290924072,
+      "learning_rate": 8.24e-07,
+      "logits/chosen": 1.618949294090271,
+      "logits/rejected": 1.6625702381134033,
+      "logps/chosen": -141.8953857421875,
+      "logps/rejected": -156.6927032470703,
+      "loss": 0.694425106048584,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": 0.03162423521280289,
+      "rewards/margins": -0.0023833750747144222,
+      "rewards/rejected": 0.034007612615823746,
+      "step": 310
+    },
+    {
+      "epoch": 0.256,
+      "grad_norm": 1.0823743343353271,
+      "learning_rate": 8.506666666666667e-07,
+      "logits/chosen": 1.5974153280258179,
+      "logits/rejected": 1.6237560510635376,
+      "logps/chosen": -139.2996368408203,
+      "logps/rejected": -141.94407653808594,
+      "loss": 0.6914949893951416,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.029445698484778404,
+      "rewards/margins": 0.0034329916816204786,
+      "rewards/rejected": 0.026012707501649857,
+      "step": 320
+    },
+    {
+      "epoch": 0.264,
+      "grad_norm": 1.1765722036361694,
+      "learning_rate": 8.773333333333332e-07,
+      "logits/chosen": 1.465215802192688,
+      "logits/rejected": 1.4882014989852905,
+      "logps/chosen": -158.4329833984375,
+      "logps/rejected": -183.94259643554688,
+      "loss": 0.6925934791564942,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.026302698999643326,
+      "rewards/margins": 0.0012350418837741017,
+      "rewards/rejected": 0.025067657232284546,
+      "step": 330
+    },
+    {
+      "epoch": 0.272,
+      "grad_norm": 1.0667294263839722,
+      "learning_rate": 9.039999999999999e-07,
+      "logits/chosen": 1.7538875341415405,
+      "logits/rejected": 1.6171789169311523,
+      "logps/chosen": -159.63790893554688,
+      "logps/rejected": -151.1880340576172,
+      "loss": 0.6922041416168213,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.026619985699653625,
+      "rewards/margins": 0.002031774492934346,
+      "rewards/rejected": 0.024588212370872498,
+      "step": 340
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 1.2210006713867188,
+      "learning_rate": 9.306666666666666e-07,
+      "logits/chosen": 1.5815479755401611,
+      "logits/rejected": 1.710217833518982,
+      "logps/chosen": -167.21743774414062,
+      "logps/rejected": -196.77989196777344,
+      "loss": 0.6899692058563233,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.03636014088988304,
+      "rewards/margins": 0.006536121014505625,
+      "rewards/rejected": 0.029824022203683853,
+      "step": 350
+    },
+    {
+      "epoch": 0.288,
+      "grad_norm": 0.9623845219612122,
+      "learning_rate": 9.573333333333333e-07,
+      "logits/chosen": 1.4416790008544922,
+      "logits/rejected": 1.3618693351745605,
+      "logps/chosen": -163.27401733398438,
+      "logps/rejected": -169.3351287841797,
+      "loss": 0.6910821437835694,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.037200987339019775,
+      "rewards/margins": 0.004338678438216448,
+      "rewards/rejected": 0.032862309366464615,
+      "step": 360
+    },
+    {
+      "epoch": 0.296,
+      "grad_norm": 1.1005645990371704,
+      "learning_rate": 9.84e-07,
+      "logits/chosen": 1.6214616298675537,
+      "logits/rejected": 1.6572606563568115,
+      "logps/chosen": -162.95504760742188,
+      "logps/rejected": -154.91807556152344,
+      "loss": 0.6895030021667481,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.047198958694934845,
+      "rewards/margins": 0.0075958347879350185,
+      "rewards/rejected": 0.03960312157869339,
+      "step": 370
+    },
+    {
+      "epoch": 0.304,
+      "grad_norm": 1.1126338243484497,
+      "learning_rate": 9.988148148148148e-07,
+      "logits/chosen": 1.4842126369476318,
+      "logits/rejected": 1.410930871963501,
+      "logps/chosen": -143.09182739257812,
+      "logps/rejected": -153.48135375976562,
+      "loss": 0.6885370254516602,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.05975018069148064,
+      "rewards/margins": 0.00981281790882349,
+      "rewards/rejected": 0.04993735998868942,
+      "step": 380
+    },
+    {
+      "epoch": 0.312,
+      "grad_norm": 1.1146304607391357,
+      "learning_rate": 9.95851851851852e-07,
+      "logits/chosen": 1.608332633972168,
+      "logits/rejected": 1.5848472118377686,
+      "logps/chosen": -152.25250244140625,
+      "logps/rejected": -138.348876953125,
+      "loss": 0.6851807117462159,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.07630724459886551,
+      "rewards/margins": 0.016460279002785683,
+      "rewards/rejected": 0.059846967458724976,
+      "step": 390
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 1.2686601877212524,
+      "learning_rate": 9.928888888888889e-07,
+      "logits/chosen": 1.5840070247650146,
+      "logits/rejected": 1.5679785013198853,
+      "logps/chosen": -155.90219116210938,
+      "logps/rejected": -148.89549255371094,
+      "loss": 0.6891039371490478,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.08129279315471649,
+      "rewards/margins": 0.008997146971523762,
+      "rewards/rejected": 0.072295643389225,
+      "step": 400
+    },
+    {
+      "epoch": 0.32,
+      "eval_logits/chosen": 1.5358340740203857,
+      "eval_logits/rejected": 1.5689741373062134,
+      "eval_logps/chosen": -152.6895294189453,
+      "eval_logps/rejected": -147.23422241210938,
+      "eval_loss": 0.6900544166564941,
+      "eval_rewards/accuracies": 0.5419999957084656,
+      "eval_rewards/chosen": 0.08593228459358215,
+      "eval_rewards/margins": 0.0071397931315004826,
+      "eval_rewards/rejected": 0.0787924975156784,
+      "eval_runtime": 90.9072,
+      "eval_samples_per_second": 5.5,
+      "eval_steps_per_second": 2.75,
+      "step": 400
+    },
+    {
+      "epoch": 0.328,
+      "grad_norm": 1.1653108596801758,
+      "learning_rate": 9.899259259259258e-07,
+      "logits/chosen": 1.5450490713119507,
+      "logits/rejected": 1.4428151845932007,
+      "logps/chosen": -148.61581420898438,
+      "logps/rejected": -188.90919494628906,
+      "loss": 0.6918989181518554,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0946170911192894,
+      "rewards/margins": 0.003271549940109253,
+      "rewards/rejected": 0.09134554117918015,
+      "step": 410
+    },
+    {
+      "epoch": 0.336,
+      "grad_norm": 1.3114937543869019,
+      "learning_rate": 9.86962962962963e-07,
+      "logits/chosen": 1.5342715978622437,
+      "logits/rejected": 1.5572118759155273,
+      "logps/chosen": -152.65264892578125,
+      "logps/rejected": -168.03384399414062,
+      "loss": 0.6972955703735352,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.08869597315788269,
+      "rewards/margins": -0.007436127867549658,
+      "rewards/rejected": 0.09613210707902908,
+      "step": 420
+    },
+    {
+      "epoch": 0.344,
+      "grad_norm": 1.0040781497955322,
+      "learning_rate": 9.84e-07,
+      "logits/chosen": 1.5979712009429932,
+      "logits/rejected": 1.6713664531707764,
+      "logps/chosen": -164.2566375732422,
+      "logps/rejected": -152.1428680419922,
+      "loss": 0.6897010326385498,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.08101107180118561,
+      "rewards/margins": 0.007684691343456507,
+      "rewards/rejected": 0.07332637906074524,
+      "step": 430
+    },
+    {
+      "epoch": 0.352,
+      "grad_norm": 1.0801892280578613,
+      "learning_rate": 9.81037037037037e-07,
+      "logits/chosen": 1.439635992050171,
+      "logits/rejected": 1.3282365798950195,
+      "logps/chosen": -157.23403930664062,
+      "logps/rejected": -146.54806518554688,
+      "loss": 0.6881664752960205,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.08112544566392899,
+      "rewards/margins": 0.010971959680318832,
+      "rewards/rejected": 0.07015348225831985,
+      "step": 440
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 1.1139949560165405,
+      "learning_rate": 9.78074074074074e-07,
+      "logits/chosen": 1.6345123052597046,
+      "logits/rejected": 1.5876491069793701,
+      "logps/chosen": -157.6160125732422,
+      "logps/rejected": -160.02040100097656,
+      "loss": 0.6868602275848389,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.07638200372457504,
+      "rewards/margins": 0.013464884832501411,
+      "rewards/rejected": 0.06291711330413818,
+      "step": 450
+    },
+    {
+      "epoch": 0.368,
+      "grad_norm": 1.158035397529602,
+      "learning_rate": 9.751111111111112e-07,
+      "logits/chosen": 1.5430586338043213,
+      "logits/rejected": 1.6022913455963135,
+      "logps/chosen": -151.98919677734375,
+      "logps/rejected": -164.90565490722656,
+      "loss": 0.6891930580139161,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.06889279186725616,
+      "rewards/margins": 0.008791089057922363,
+      "rewards/rejected": 0.0601017065346241,
+      "step": 460
+    },
+    {
+      "epoch": 0.376,
+      "grad_norm": 1.0903704166412354,
+      "learning_rate": 9.721481481481481e-07,
+      "logits/chosen": 1.313881754875183,
+      "logits/rejected": 1.3182194232940674,
+      "logps/chosen": -131.5592041015625,
+      "logps/rejected": -123.84442138671875,
+      "loss": 0.6889429569244385,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.08150745928287506,
+      "rewards/margins": 0.009235726669430733,
+      "rewards/rejected": 0.07227173447608948,
+      "step": 470
+    },
+    {
+      "epoch": 0.384,
+      "grad_norm": 1.0766969919204712,
+      "learning_rate": 9.69185185185185e-07,
+      "logits/chosen": 1.425756812095642,
+      "logits/rejected": 1.476854920387268,
+      "logps/chosen": -148.72279357910156,
+      "logps/rejected": -164.40951538085938,
+      "loss": 0.6854116439819335,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.1009502187371254,
+      "rewards/margins": 0.0166789498180151,
+      "rewards/rejected": 0.08427127450704575,
+      "step": 480
+    },
+    {
+      "epoch": 0.392,
+      "grad_norm": 1.2078129053115845,
+      "learning_rate": 9.662222222222222e-07,
+      "logits/chosen": 1.5511282682418823,
+      "logits/rejected": 1.611383080482483,
+      "logps/chosen": -143.22006225585938,
+      "logps/rejected": -159.70176696777344,
+      "loss": 0.6968960762023926,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.08779363334178925,
+      "rewards/margins": -0.00598606513813138,
+      "rewards/rejected": 0.09377971291542053,
+      "step": 490
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 0.9106306433677673,
+      "learning_rate": 9.632592592592593e-07,
+      "logits/chosen": 1.4217239618301392,
+      "logits/rejected": 1.505446195602417,
+      "logps/chosen": -144.34109497070312,
+      "logps/rejected": -135.52430725097656,
+      "loss": 0.691196870803833,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.06902458518743515,
+      "rewards/margins": 0.005059319548308849,
+      "rewards/rejected": 0.06396526843309402,
+      "step": 500
+    },
+    {
+      "epoch": 0.4,
+      "eval_logits/chosen": 1.5185325145721436,
+      "eval_logits/rejected": 1.5510345697402954,
+      "eval_logps/chosen": -152.7921905517578,
+      "eval_logps/rejected": -147.3463134765625,
+      "eval_loss": 0.6898502111434937,
+      "eval_rewards/accuracies": 0.5099999904632568,
+      "eval_rewards/chosen": 0.07566512376070023,
+      "eval_rewards/margins": 0.008083080872893333,
+      "eval_rewards/rejected": 0.06758204847574234,
+      "eval_runtime": 90.8294,
+      "eval_samples_per_second": 5.505,
+      "eval_steps_per_second": 2.752,
+      "step": 500
+    },
+    {
+      "epoch": 0.408,
+      "grad_norm": 1.1637762784957886,
+      "learning_rate": 9.602962962962962e-07,
+      "logits/chosen": 1.5965330600738525,
+      "logits/rejected": 1.5338464975357056,
+      "logps/chosen": -131.49871826171875,
+      "logps/rejected": -140.05995178222656,
+      "loss": 0.6868171215057373,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.08385131508111954,
+      "rewards/margins": 0.013950209133327007,
+      "rewards/rejected": 0.06990110129117966,
+      "step": 510
+    },
+    {
+      "epoch": 0.416,
+      "grad_norm": 1.229719638824463,
+      "learning_rate": 9.573333333333333e-07,
+      "logits/chosen": 1.7057275772094727,
+      "logits/rejected": 1.5898948907852173,
+      "logps/chosen": -166.4267120361328,
+      "logps/rejected": -172.22337341308594,
+      "loss": 0.6912094116210937,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.07427279651165009,
+      "rewards/margins": 0.004661654122173786,
+      "rewards/rejected": 0.06961113959550858,
+      "step": 520
+    },
+    {
+      "epoch": 0.424,
+      "grad_norm": 1.136383295059204,
+      "learning_rate": 9.543703703703705e-07,
+      "logits/chosen": 1.6207077503204346,
+      "logits/rejected": 1.5417520999908447,
+      "logps/chosen": -166.81509399414062,
+      "logps/rejected": -168.5408172607422,
+      "loss": 0.6772748470306397,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.0953373983502388,
+      "rewards/margins": 0.033697597682476044,
+      "rewards/rejected": 0.06163979694247246,
+      "step": 530
+    },
+    {
+      "epoch": 0.432,
+      "grad_norm": 1.0640541315078735,
+      "learning_rate": 9.514074074074074e-07,
+      "logits/chosen": 1.6825854778289795,
+      "logits/rejected": 1.6327041387557983,
+      "logps/chosen": -181.39956665039062,
+      "logps/rejected": -158.49647521972656,
+      "loss": 0.6883615493774414,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.10187923908233643,
+      "rewards/margins": 0.010873330757021904,
+      "rewards/rejected": 0.09100590646266937,
+      "step": 540
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 1.477909803390503,
+      "learning_rate": 9.484444444444444e-07,
+      "logits/chosen": 1.5402686595916748,
+      "logits/rejected": 1.6487388610839844,
+      "logps/chosen": -151.53814697265625,
+      "logps/rejected": -167.8573455810547,
+      "loss": 0.6915213584899902,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.09472338110208511,
+      "rewards/margins": 0.005475065670907497,
+      "rewards/rejected": 0.08924831449985504,
+      "step": 550
+    },
+    {
+      "epoch": 0.448,
+      "grad_norm": 1.0138317346572876,
+      "learning_rate": 9.454814814814814e-07,
+      "logits/chosen": 1.6286503076553345,
+      "logits/rejected": 1.5223101377487183,
+      "logps/chosen": -154.72079467773438,
+      "logps/rejected": -139.2535858154297,
+      "loss": 0.6835216522216797,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.09194488823413849,
+      "rewards/margins": 0.02140027843415737,
+      "rewards/rejected": 0.07054460793733597,
+      "step": 560
+    },
+    {
+      "epoch": 0.456,
+      "grad_norm": 0.896689772605896,
+      "learning_rate": 9.425185185185184e-07,
+      "logits/chosen": 1.4857146739959717,
+      "logits/rejected": 1.4369819164276123,
+      "logps/chosen": -158.31809997558594,
+      "logps/rejected": -152.33676147460938,
+      "loss": 0.6863360404968262,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.07895292341709137,
+      "rewards/margins": 0.016260787844657898,
+      "rewards/rejected": 0.06269213557243347,
+      "step": 570
+    },
+    {
+      "epoch": 0.464,
+      "grad_norm": 1.4772330522537231,
+      "learning_rate": 9.395555555555556e-07,
+      "logits/chosen": 1.315298318862915,
+      "logits/rejected": 1.4757534265518188,
+      "logps/chosen": -136.4202117919922,
+      "logps/rejected": -154.99362182617188,
+      "loss": 0.6916167259216308,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.04677204787731171,
+      "rewards/margins": 0.005649724509567022,
+      "rewards/rejected": 0.041122324764728546,
+      "step": 580
+    },
+    {
+      "epoch": 0.472,
+      "grad_norm": 0.9806157350540161,
+      "learning_rate": 9.365925925925926e-07,
+      "logits/chosen": 1.465427279472351,
+      "logits/rejected": 1.459496259689331,
+      "logps/chosen": -145.07470703125,
+      "logps/rejected": -131.71974182128906,
+      "loss": 0.6857181549072265,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.07979898899793625,
+      "rewards/margins": 0.016965944319963455,
+      "rewards/rejected": 0.0628330409526825,
+      "step": 590
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 1.4313396215438843,
+      "learning_rate": 9.336296296296295e-07,
+      "logits/chosen": 1.5190367698669434,
+      "logits/rejected": 1.563504934310913,
+      "logps/chosen": -132.0201416015625,
+      "logps/rejected": -158.89865112304688,
+      "loss": 0.6964848041534424,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.06544725596904755,
+      "rewards/margins": -0.004280097782611847,
+      "rewards/rejected": 0.0697273463010788,
+      "step": 600
+    },
+    {
+      "epoch": 0.48,
+      "eval_logits/chosen": 1.5031698942184448,
+      "eval_logits/rejected": 1.5348584651947021,
+      "eval_logps/chosen": -152.76596069335938,
+      "eval_logps/rejected": -147.3241729736328,
+      "eval_loss": 0.6903045773506165,
+      "eval_rewards/accuracies": 0.527999997138977,
+      "eval_rewards/chosen": 0.07828804850578308,
+      "eval_rewards/margins": 0.008492168970406055,
+      "eval_rewards/rejected": 0.0697958841919899,
+      "eval_runtime": 90.9881,
+      "eval_samples_per_second": 5.495,
+      "eval_steps_per_second": 2.748,
+      "step": 600
+    },
+    {
+      "epoch": 0.488,
+      "grad_norm": 0.9467533230781555,
+      "learning_rate": 9.306666666666666e-07,
+      "logits/chosen": 1.5242283344268799,
+      "logits/rejected": 1.5798314809799194,
+      "logps/chosen": -131.90048217773438,
+      "logps/rejected": -149.93222045898438,
+      "loss": 0.6899854183197022,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.08419154584407806,
+      "rewards/margins": 0.00830511748790741,
+      "rewards/rejected": 0.07588644325733185,
+      "step": 610
+    },
+    {
+      "epoch": 0.496,
+      "grad_norm": 1.8463492393493652,
+      "learning_rate": 9.277037037037037e-07,
+      "logits/chosen": 1.4965155124664307,
+      "logits/rejected": 1.6336272954940796,
+      "logps/chosen": -156.1863555908203,
+      "logps/rejected": -167.74075317382812,
+      "loss": 0.6977802276611328,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.05518193915486336,
+      "rewards/margins": -0.006808985956013203,
+      "rewards/rejected": 0.061990927904844284,
+      "step": 620
+    },
+    {
+      "epoch": 0.504,
+      "grad_norm": 1.278316617012024,
+      "learning_rate": 9.247407407407407e-07,
+      "logits/chosen": 1.5074065923690796,
+      "logits/rejected": 1.535696029663086,
+      "logps/chosen": -167.56777954101562,
+      "logps/rejected": -147.04922485351562,
+      "loss": 0.6806219100952149,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0745314508676529,
+      "rewards/margins": 0.027104835957288742,
+      "rewards/rejected": 0.047426607459783554,
+      "step": 630
+    },
+    {
+      "epoch": 0.512,
+      "grad_norm": 1.0424679517745972,
+      "learning_rate": 9.217777777777778e-07,
+      "logits/chosen": 1.4299286603927612,
+      "logits/rejected": 1.4484076499938965,
+      "logps/chosen": -168.305908203125,
+      "logps/rejected": -152.61854553222656,
+      "loss": 0.6871783256530761,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.07751411944627762,
+      "rewards/margins": 0.017045259475708008,
+      "rewards/rejected": 0.06046885997056961,
+      "step": 640
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 1.1143999099731445,
+      "learning_rate": 9.188148148148148e-07,
+      "logits/chosen": 1.5520436763763428,
+      "logits/rejected": 1.531718134880066,
+      "logps/chosen": -154.92115783691406,
+      "logps/rejected": -126.78369140625,
+      "loss": 0.6889286518096924,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.0945608839392662,
+      "rewards/margins": 0.01080345083028078,
+      "rewards/rejected": 0.0837574377655983,
+      "step": 650
+    },
+    {
+      "epoch": 0.528,
+      "grad_norm": 1.2762326002120972,
+      "learning_rate": 9.158518518518517e-07,
+      "logits/chosen": 1.5747522115707397,
+      "logits/rejected": 1.4972833395004272,
+      "logps/chosen": -151.13914489746094,
+      "logps/rejected": -132.013916015625,
+      "loss": 0.6900454998016358,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.07303085923194885,
+      "rewards/margins": 0.008073865436017513,
+      "rewards/rejected": 0.06495700031518936,
+      "step": 660
+    },
+    {
+      "epoch": 0.536,
+      "grad_norm": 1.5207557678222656,
+      "learning_rate": 9.128888888888888e-07,
+      "logits/chosen": 1.6594665050506592,
+      "logits/rejected": 1.468492031097412,
+      "logps/chosen": -166.0108184814453,
+      "logps/rejected": -151.38555908203125,
+      "loss": 0.6934059143066407,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.09294265508651733,
+      "rewards/margins": 0.0021180796902626753,
+      "rewards/rejected": 0.09082455933094025,
+      "step": 670
+    },
+    {
+      "epoch": 0.544,
+      "grad_norm": 1.1073335409164429,
+      "learning_rate": 9.099259259259259e-07,
+      "logits/chosen": 1.5321934223175049,
+      "logits/rejected": 1.5313541889190674,
+      "logps/chosen": -144.5046844482422,
+      "logps/rejected": -133.05470275878906,
+      "loss": 0.6824594974517822,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.10782947391271591,
+      "rewards/margins": 0.0230915155261755,
+      "rewards/rejected": 0.08473796397447586,
+      "step": 680
+    },
+    {
+      "epoch": 0.552,
+      "grad_norm": 1.096970558166504,
+      "learning_rate": 9.069629629629629e-07,
+      "logits/chosen": 1.438305139541626,
+      "logits/rejected": 1.4338308572769165,
+      "logps/chosen": -173.1280517578125,
+      "logps/rejected": -160.20950317382812,
+      "loss": 0.6878445625305176,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.09615986794233322,
+      "rewards/margins": 0.012350928969681263,
+      "rewards/rejected": 0.08380892872810364,
+      "step": 690
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 1.3664878606796265,
+      "learning_rate": 9.039999999999999e-07,
+      "logits/chosen": 1.530369758605957,
+      "logits/rejected": 1.5520622730255127,
+      "logps/chosen": -159.60740661621094,
+      "logps/rejected": -137.77352905273438,
+      "loss": 0.6991429805755616,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.07249583303928375,
+      "rewards/margins": -0.009333941154181957,
+      "rewards/rejected": 0.08182977139949799,
+      "step": 700
+    },
+    {
+      "epoch": 0.56,
+      "eval_logits/chosen": 1.5136796236038208,
+      "eval_logits/rejected": 1.5456002950668335,
+      "eval_logps/chosen": -152.5748748779297,
+      "eval_logps/rejected": -147.17152404785156,
+      "eval_loss": 0.688511848449707,
+      "eval_rewards/accuracies": 0.5339999794960022,
+      "eval_rewards/chosen": 0.09739598631858826,
+      "eval_rewards/margins": 0.012335418723523617,
+      "eval_rewards/rejected": 0.08506056666374207,
+      "eval_runtime": 90.8734,
+      "eval_samples_per_second": 5.502,
+      "eval_steps_per_second": 2.751,
+      "step": 700
+    },
+    {
+      "epoch": 0.568,
+      "grad_norm": 1.224107265472412,
+      "learning_rate": 9.010370370370371e-07,
+      "logits/chosen": 1.6023967266082764,
+      "logits/rejected": 1.558224081993103,
+      "logps/chosen": -148.47120666503906,
+      "logps/rejected": -163.5648651123047,
+      "loss": 0.6865221500396729,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.12494877725839615,
+      "rewards/margins": 0.01545787788927555,
+      "rewards/rejected": 0.10949089378118515,
+      "step": 710
+    },
+    {
+      "epoch": 0.576,
+      "grad_norm": 1.2135164737701416,
+      "learning_rate": 8.98074074074074e-07,
+      "logits/chosen": 1.5902750492095947,
+      "logits/rejected": 1.7012746334075928,
+      "logps/chosen": -139.7012939453125,
+      "logps/rejected": -135.42225646972656,
+      "loss": 0.7014039039611817,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.09102506935596466,
+      "rewards/margins": -0.014695463702082634,
+      "rewards/rejected": 0.10572052001953125,
+      "step": 720
+    },
+    {
+      "epoch": 0.584,
+      "grad_norm": 1.1953649520874023,
+      "learning_rate": 8.95111111111111e-07,
+      "logits/chosen": 1.6540731191635132,
+      "logits/rejected": 1.6386349201202393,
+      "logps/chosen": -153.12847900390625,
+      "logps/rejected": -149.66319274902344,
+      "loss": 0.6952126502990723,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.1124584898352623,
+      "rewards/margins": -0.0012063594767823815,
+      "rewards/rejected": 0.11366486549377441,
+      "step": 730
+    },
+    {
+      "epoch": 0.592,
+      "grad_norm": 1.147875428199768,
+      "learning_rate": 8.921481481481481e-07,
+      "logits/chosen": 1.5578330755233765,
+      "logits/rejected": 1.4689290523529053,
+      "logps/chosen": -156.93788146972656,
+      "logps/rejected": -134.1832733154297,
+      "loss": 0.6919830799102783,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.09214536100625992,
+      "rewards/margins": 0.004291228950023651,
+      "rewards/rejected": 0.08785412460565567,
+      "step": 740
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 1.21685791015625,
+      "learning_rate": 8.891851851851852e-07,
+      "logits/chosen": 1.4241983890533447,
+      "logits/rejected": 1.4537718296051025,
+      "logps/chosen": -136.9163055419922,
+      "logps/rejected": -158.44491577148438,
+      "loss": 0.691422986984253,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.09173480421304703,
+      "rewards/margins": 0.00533579895272851,
+      "rewards/rejected": 0.08639900386333466,
+      "step": 750
+    },
+    {
+      "epoch": 0.608,
+      "grad_norm": 1.3629519939422607,
+      "learning_rate": 8.862222222222222e-07,
+      "logits/chosen": 1.565716028213501,
+      "logits/rejected": 1.3824570178985596,
+      "logps/chosen": -138.8521728515625,
+      "logps/rejected": -129.51028442382812,
+      "loss": 0.6931031227111817,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0763697624206543,
+      "rewards/margins": 0.0021251302678138018,
+      "rewards/rejected": 0.07424463331699371,
+      "step": 760
+    },
+    {
+      "epoch": 0.616,
+      "grad_norm": 1.1030747890472412,
+      "learning_rate": 8.832592592592593e-07,
+      "logits/chosen": 1.5781702995300293,
+      "logits/rejected": 1.5428775548934937,
+      "logps/chosen": -141.01480102539062,
+      "logps/rejected": -136.47189331054688,
+      "loss": 0.6777657032012939,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.10366280376911163,
+      "rewards/margins": 0.0333605632185936,
+      "rewards/rejected": 0.07030224055051804,
+      "step": 770
+    },
+    {
+      "epoch": 0.624,
+      "grad_norm": 1.3228758573532104,
+      "learning_rate": 8.802962962962962e-07,
+      "logits/chosen": 1.451596736907959,
+      "logits/rejected": 1.5489904880523682,
+      "logps/chosen": -153.74618530273438,
+      "logps/rejected": -159.26333618164062,
+      "loss": 0.6870471000671386,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.10403706133365631,
+      "rewards/margins": 0.014494165778160095,
+      "rewards/rejected": 0.08954289555549622,
+      "step": 780
+    },
+    {
+      "epoch": 0.632,
+      "grad_norm": 1.3267385959625244,
+      "learning_rate": 8.773333333333332e-07,
+      "logits/chosen": 1.4153831005096436,
+      "logits/rejected": 1.4497296810150146,
+      "logps/chosen": -148.05368041992188,
+      "logps/rejected": -151.3020477294922,
+      "loss": 0.6877872943878174,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.07300253212451935,
+      "rewards/margins": 0.013981563039124012,
+      "rewards/rejected": 0.05902096629142761,
+      "step": 790
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 1.2227483987808228,
+      "learning_rate": 8.743703703703703e-07,
+      "logits/chosen": 1.6773914098739624,
+      "logits/rejected": 1.6918518543243408,
+      "logps/chosen": -180.06863403320312,
+      "logps/rejected": -194.30752563476562,
+      "loss": 0.7015917778015137,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.07480724155902863,
+      "rewards/margins": -0.013628068380057812,
+      "rewards/rejected": 0.08843531459569931,
+      "step": 800
+    },
+    {
+      "epoch": 0.64,
+      "eval_logits/chosen": 1.5051140785217285,
+      "eval_logits/rejected": 1.5367870330810547,
+      "eval_logps/chosen": -152.8026885986328,
+      "eval_logps/rejected": -147.39437866210938,
+      "eval_loss": 0.688633918762207,
+      "eval_rewards/accuracies": 0.5220000147819519,
+      "eval_rewards/chosen": 0.07461711764335632,
+      "eval_rewards/margins": 0.011839868500828743,
+      "eval_rewards/rejected": 0.06277725100517273,
+      "eval_runtime": 90.9206,
+      "eval_samples_per_second": 5.499,
+      "eval_steps_per_second": 2.75,
+      "step": 800
+    },
+    {
+      "epoch": 0.648,
+      "grad_norm": 1.209770679473877,
+      "learning_rate": 8.714074074074074e-07,
+      "logits/chosen": 1.3785041570663452,
+      "logits/rejected": 1.60085928440094,
+      "logps/chosen": -114.5325927734375,
+      "logps/rejected": -151.88900756835938,
+      "loss": 0.6890413761138916,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0640551745891571,
+      "rewards/margins": 0.010126596316695213,
+      "rewards/rejected": 0.05392857640981674,
+      "step": 810
+    },
+    {
+      "epoch": 0.656,
+      "grad_norm": 1.0250163078308105,
+      "learning_rate": 8.684444444444444e-07,
+      "logits/chosen": 1.5827982425689697,
+      "logits/rejected": 1.6239618062973022,
+      "logps/chosen": -149.01904296875,
+      "logps/rejected": -141.9372100830078,
+      "loss": 0.6728014469146728,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0830039232969284,
+      "rewards/margins": 0.043090131133794785,
+      "rewards/rejected": 0.03991378843784332,
+      "step": 820
+    },
+    {
+      "epoch": 0.664,
+      "grad_norm": 1.0627655982971191,
+      "learning_rate": 8.654814814814814e-07,
+      "logits/chosen": 1.501873254776001,
+      "logits/rejected": 1.537386417388916,
+      "logps/chosen": -138.75169372558594,
+      "logps/rejected": -134.00970458984375,
+      "loss": 0.6801187992095947,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0640982910990715,
+      "rewards/margins": 0.02855527400970459,
+      "rewards/rejected": 0.03554301708936691,
+      "step": 830
+    },
+    {
+      "epoch": 0.672,
+      "grad_norm": 1.2453433275222778,
+      "learning_rate": 8.625185185185186e-07,
+      "logits/chosen": 1.4766005277633667,
+      "logits/rejected": 1.5185787677764893,
+      "logps/chosen": -148.41250610351562,
+      "logps/rejected": -157.73989868164062,
+      "loss": 0.6788079738616943,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.06551454961299896,
+      "rewards/margins": 0.03301671892404556,
+      "rewards/rejected": 0.0324978306889534,
+      "step": 840
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 1.191698670387268,
+      "learning_rate": 8.595555555555555e-07,
+      "logits/chosen": 1.555551528930664,
+      "logits/rejected": 1.5438499450683594,
+      "logps/chosen": -160.77488708496094,
+      "logps/rejected": -141.9876251220703,
+      "loss": 0.6801597595214843,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.053498052060604095,
+      "rewards/margins": 0.028470590710639954,
+      "rewards/rejected": 0.025027472525835037,
+      "step": 850
+    },
+    {
+      "epoch": 0.688,
+      "grad_norm": 0.9382092356681824,
+      "learning_rate": 8.565925925925925e-07,
+      "logits/chosen": 1.5482738018035889,
+      "logits/rejected": 1.5833700895309448,
+      "logps/chosen": -139.68118286132812,
+      "logps/rejected": -130.05947875976562,
+      "loss": 0.6825921535491943,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.06382572650909424,
+      "rewards/margins": 0.023111844435334206,
+      "rewards/rejected": 0.04071388021111488,
+      "step": 860
+    },
+    {
+      "epoch": 0.696,
+      "grad_norm": 1.2248382568359375,
+      "learning_rate": 8.536296296296296e-07,
+      "logits/chosen": 1.4529926776885986,
+      "logits/rejected": 1.4423902034759521,
+      "logps/chosen": -154.4588165283203,
+      "logps/rejected": -153.44949340820312,
+      "loss": 0.6915592193603516,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.03207828477025032,
+      "rewards/margins": 0.006785523146390915,
+      "rewards/rejected": 0.025292763486504555,
+      "step": 870
+    },
+    {
+      "epoch": 0.704,
+      "grad_norm": 1.2174893617630005,
+      "learning_rate": 8.506666666666667e-07,
+      "logits/chosen": 1.4073113203048706,
+      "logits/rejected": 1.668140172958374,
+      "logps/chosen": -145.18460083007812,
+      "logps/rejected": -157.0777587890625,
+      "loss": 0.6922356605529785,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.049450717866420746,
+      "rewards/margins": 0.004049716051667929,
+      "rewards/rejected": 0.04540099948644638,
+      "step": 880
+    },
+    {
+      "epoch": 0.712,
+      "grad_norm": 1.3186850547790527,
+      "learning_rate": 8.477037037037037e-07,
+      "logits/chosen": 1.2923892736434937,
+      "logits/rejected": 1.491051435470581,
+      "logps/chosen": -149.03018188476562,
+      "logps/rejected": -161.61715698242188,
+      "loss": 0.6841150760650635,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.038961172103881836,
+      "rewards/margins": 0.020759906619787216,
+      "rewards/rejected": 0.01820126175880432,
+      "step": 890
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 1.1123124361038208,
+      "learning_rate": 8.447407407407407e-07,
+      "logits/chosen": 1.546547293663025,
+      "logits/rejected": 1.682992696762085,
+      "logps/chosen": -166.2056884765625,
+      "logps/rejected": -180.73330688476562,
+      "loss": 0.6826215744018554,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.07279330492019653,
+      "rewards/margins": 0.025264525786042213,
+      "rewards/rejected": 0.04752878472208977,
+      "step": 900
+    },
+    {
+      "epoch": 0.72,
+      "eval_logits/chosen": 1.4819762706756592,
+      "eval_logits/rejected": 1.513027548789978,
+      "eval_logps/chosen": -153.1695556640625,
+      "eval_logps/rejected": -147.7560577392578,
+      "eval_loss": 0.689373254776001,
+      "eval_rewards/accuracies": 0.5299999713897705,
+      "eval_rewards/chosen": 0.03792775049805641,
+      "eval_rewards/margins": 0.011320074088871479,
+      "eval_rewards/rejected": 0.026607677340507507,
+      "eval_runtime": 90.9603,
+      "eval_samples_per_second": 5.497,
+      "eval_steps_per_second": 2.748,
+      "step": 900
+    },
+    {
+      "epoch": 0.728,
+      "grad_norm": 1.1507582664489746,
+      "learning_rate": 8.417777777777777e-07,
+      "logits/chosen": 1.5831308364868164,
+      "logits/rejected": 1.3737132549285889,
+      "logps/chosen": -140.00253295898438,
+      "logps/rejected": -138.45155334472656,
+      "loss": 0.6873391628265381,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.05066479369997978,
+      "rewards/margins": 0.015466553159058094,
+      "rewards/rejected": 0.03519824147224426,
+      "step": 910
+    },
+    {
+      "epoch": 0.736,
+      "grad_norm": 1.415456771850586,
+      "learning_rate": 8.388148148148147e-07,
+      "logits/chosen": 1.3000686168670654,
+      "logits/rejected": 1.2642180919647217,
+      "logps/chosen": -160.14027404785156,
+      "logps/rejected": -163.32827758789062,
+      "loss": 0.687142276763916,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.06618930399417877,
+      "rewards/margins": 0.014807088300585747,
+      "rewards/rejected": 0.05138222128152847,
+      "step": 920
+    },
+    {
+      "epoch": 0.744,
+      "grad_norm": 1.3228634595870972,
+      "learning_rate": 8.358518518518518e-07,
+      "logits/chosen": 1.5204170942306519,
+      "logits/rejected": 1.6129217147827148,
+      "logps/chosen": -157.99832153320312,
+      "logps/rejected": -162.09567260742188,
+      "loss": 0.6790886878967285,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.05940447002649307,
+      "rewards/margins": 0.03161252290010452,
+      "rewards/rejected": 0.027791941538453102,
+      "step": 930
+    },
+    {
+      "epoch": 0.752,
+      "grad_norm": 1.048064947128296,
+      "learning_rate": 8.328888888888889e-07,
+      "logits/chosen": 1.5515320301055908,
+      "logits/rejected": 1.676857352256775,
+      "logps/chosen": -148.4853515625,
+      "logps/rejected": -174.4320526123047,
+      "loss": 0.6935453414916992,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.010225490666925907,
+      "rewards/margins": 0.0021693313028663397,
+      "rewards/rejected": 0.008056161925196648,
+      "step": 940
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 1.247799038887024,
+      "learning_rate": 8.299259259259259e-07,
+      "logits/chosen": 1.384231686592102,
+      "logits/rejected": 1.4611561298370361,
+      "logps/chosen": -140.81443786621094,
+      "logps/rejected": -153.81326293945312,
+      "loss": 0.6805226325988769,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.053223587572574615,
+      "rewards/margins": 0.02933613583445549,
+      "rewards/rejected": 0.023887457326054573,
+      "step": 950
+    },
+    {
+      "epoch": 0.768,
+      "grad_norm": 1.1611264944076538,
+      "learning_rate": 8.269629629629629e-07,
+      "logits/chosen": 1.4987528324127197,
+      "logits/rejected": 1.3684321641921997,
+      "logps/chosen": -135.81056213378906,
+      "logps/rejected": -119.05133056640625,
+      "loss": 0.6954316616058349,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.0197703056037426,
+      "rewards/margins": -0.0012637012405321002,
+      "rewards/rejected": 0.021034007892012596,
+      "step": 960
+    },
+    {
+      "epoch": 0.776,
+      "grad_norm": 0.964948832988739,
+      "learning_rate": 8.24e-07,
+      "logits/chosen": 1.612053632736206,
+      "logits/rejected": 1.7114235162734985,
+      "logps/chosen": -153.56137084960938,
+      "logps/rejected": -149.17323303222656,
+      "loss": 0.6916662693023682,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.05082285404205322,
+      "rewards/margins": 0.0056606316938996315,
+      "rewards/rejected": 0.04516221210360527,
+      "step": 970
+    },
+    {
+      "epoch": 0.784,
+      "grad_norm": 1.409449815750122,
+      "learning_rate": 8.21037037037037e-07,
+      "logits/chosen": 1.384060025215149,
+      "logits/rejected": 1.316361904144287,
+      "logps/chosen": -168.75360107421875,
+      "logps/rejected": -141.79537963867188,
+      "loss": 0.6845529556274415,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0886571854352951,
+      "rewards/margins": 0.02084263041615486,
+      "rewards/rejected": 0.06781454384326935,
+      "step": 980
+    },
+    {
+      "epoch": 0.792,
+      "grad_norm": 1.0246562957763672,
+      "learning_rate": 8.18074074074074e-07,
+      "logits/chosen": 1.4986507892608643,
+      "logits/rejected": 1.571103811264038,
+      "logps/chosen": -165.22254943847656,
+      "logps/rejected": -168.63540649414062,
+      "loss": 0.6790967464447022,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.07517905533313751,
+      "rewards/margins": 0.03272219002246857,
+      "rewards/rejected": 0.04245685786008835,
+      "step": 990
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 1.014426827430725,
+      "learning_rate": 8.15111111111111e-07,
+      "logits/chosen": 1.4708755016326904,
+      "logits/rejected": 1.4550955295562744,
+      "logps/chosen": -149.49874877929688,
+      "logps/rejected": -125.0966567993164,
+      "loss": 0.6892098903656005,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.07825140655040741,
+      "rewards/margins": 0.01007872074842453,
+      "rewards/rejected": 0.06817268580198288,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8,
+      "eval_logits/chosen": 1.4972777366638184,
+      "eval_logits/rejected": 1.5283576250076294,
+      "eval_logps/chosen": -152.8240966796875,
+      "eval_logps/rejected": -147.44175720214844,
+      "eval_loss": 0.6880542039871216,
+      "eval_rewards/accuracies": 0.5419999957084656,
+      "eval_rewards/chosen": 0.07247376441955566,
+      "eval_rewards/margins": 0.01443479023873806,
+      "eval_rewards/rejected": 0.058038972318172455,
+      "eval_runtime": 90.827,
+      "eval_samples_per_second": 5.505,
+      "eval_steps_per_second": 2.752,
+      "step": 1000
+    },
+    {
+      "epoch": 0.808,
+      "grad_norm": 1.1155726909637451,
+      "learning_rate": 8.121481481481482e-07,
+      "logits/chosen": 1.547481894493103,
+      "logits/rejected": 1.5461527109146118,
+      "logps/chosen": -154.89791870117188,
+      "logps/rejected": -142.05111694335938,
+      "loss": 0.6809016704559326,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.10093016922473907,
+      "rewards/margins": 0.027038773521780968,
+      "rewards/rejected": 0.07389138638973236,
+      "step": 1010
+    },
+    {
+      "epoch": 0.816,
+      "grad_norm": 1.6716786623001099,
+      "learning_rate": 8.091851851851852e-07,
+      "logits/chosen": 1.3954919576644897,
+      "logits/rejected": 1.5088036060333252,
+      "logps/chosen": -143.11846923828125,
+      "logps/rejected": -146.41482543945312,
+      "loss": 0.6849985122680664,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.05970728397369385,
+      "rewards/margins": 0.021999523043632507,
+      "rewards/rejected": 0.03770776465535164,
+      "step": 1020
+    },
+    {
+      "epoch": 0.824,
+      "grad_norm": 1.308182716369629,
+      "learning_rate": 8.062222222222221e-07,
+      "logits/chosen": 1.532488465309143,
+      "logits/rejected": 1.3316718339920044,
+      "logps/chosen": -163.06858825683594,
+      "logps/rejected": -136.57989501953125,
+      "loss": 0.6789252281188964,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.11086118221282959,
+      "rewards/margins": 0.033117204904556274,
+      "rewards/rejected": 0.07774396985769272,
+      "step": 1030
+    },
+    {
+      "epoch": 0.832,
+      "grad_norm": 1.1019353866577148,
+      "learning_rate": 8.032592592592592e-07,
+      "logits/chosen": 1.4798121452331543,
+      "logits/rejected": 1.5203083753585815,
+      "logps/chosen": -140.3281707763672,
+      "logps/rejected": -141.06216430664062,
+      "loss": 0.6724937915802002,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.1202714815735817,
+      "rewards/margins": 0.046712182462215424,
+      "rewards/rejected": 0.07355931401252747,
+      "step": 1040
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 1.2966418266296387,
+      "learning_rate": 8.002962962962963e-07,
+      "logits/chosen": 1.3034439086914062,
+      "logits/rejected": 1.4363505840301514,
+      "logps/chosen": -142.1992645263672,
+      "logps/rejected": -132.0046844482422,
+      "loss": 0.6848338603973388,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.11126589775085449,
+      "rewards/margins": 0.019870441406965256,
+      "rewards/rejected": 0.09139545261859894,
+      "step": 1050
+    },
+    {
+      "epoch": 0.848,
+      "grad_norm": 1.0321130752563477,
+      "learning_rate": 7.973333333333333e-07,
+      "logits/chosen": 1.3257265090942383,
+      "logits/rejected": 1.4649587869644165,
+      "logps/chosen": -112.98075103759766,
+      "logps/rejected": -127.4059829711914,
+      "loss": 0.6679783344268799,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.17311830818653107,
+      "rewards/margins": 0.05631983280181885,
+      "rewards/rejected": 0.11679847538471222,
+      "step": 1060
+    },
+    {
+      "epoch": 0.856,
+      "grad_norm": 1.2037222385406494,
+      "learning_rate": 7.943703703703704e-07,
+      "logits/chosen": 1.3766987323760986,
+      "logits/rejected": 1.2802103757858276,
+      "logps/chosen": -154.8142547607422,
+      "logps/rejected": -148.680908203125,
+      "loss": 0.668931531906128,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.1449051946401596,
+      "rewards/margins": 0.05721765011548996,
+      "rewards/rejected": 0.08768755197525024,
+      "step": 1070
+    },
+    {
+      "epoch": 0.864,
+      "grad_norm": 0.9230766296386719,
+      "learning_rate": 7.914074074074074e-07,
+      "logits/chosen": 1.447328805923462,
+      "logits/rejected": 1.5235106945037842,
+      "logps/chosen": -150.21812438964844,
+      "logps/rejected": -123.29923248291016,
+      "loss": 0.6716424465179444,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.16066041588783264,
+      "rewards/margins": 0.04824484512209892,
+      "rewards/rejected": 0.11241557449102402,
+      "step": 1080
+    },
+    {
+      "epoch": 0.872,
+      "grad_norm": 1.0423877239227295,
+      "learning_rate": 7.884444444444443e-07,
+      "logits/chosen": 1.6920299530029297,
+      "logits/rejected": 1.6123542785644531,
+      "logps/chosen": -175.1676025390625,
+      "logps/rejected": -185.07223510742188,
+      "loss": 0.6723968505859375,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.16490009427070618,
+      "rewards/margins": 0.046577490866184235,
+      "rewards/rejected": 0.11832261085510254,
+      "step": 1090
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 1.4142779111862183,
+      "learning_rate": 7.854814814814814e-07,
+      "logits/chosen": 1.453111171722412,
+      "logits/rejected": 1.4981731176376343,
+      "logps/chosen": -139.9765167236328,
+      "logps/rejected": -153.63058471679688,
+      "loss": 0.6872457027435303,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.13096000254154205,
+      "rewards/margins": 0.01702108606696129,
+      "rewards/rejected": 0.11393891274929047,
+      "step": 1100
+    },
+    {
+      "epoch": 0.88,
+      "eval_logits/chosen": 1.5076195001602173,
+      "eval_logits/rejected": 1.5373249053955078,
+      "eval_logps/chosen": -152.1697998046875,
+      "eval_logps/rejected": -146.83502197265625,
+      "eval_loss": 0.6873304843902588,
+      "eval_rewards/accuracies": 0.5360000133514404,
+      "eval_rewards/chosen": 0.13790392875671387,
+      "eval_rewards/margins": 0.019191108644008636,
+      "eval_rewards/rejected": 0.11871281266212463,
+      "eval_runtime": 91.0157,
+      "eval_samples_per_second": 5.494,
+      "eval_steps_per_second": 2.747,
+      "step": 1100
+    },
+    {
+      "epoch": 0.888,
+      "grad_norm": 1.413996696472168,
+      "learning_rate": 7.825185185185185e-07,
+      "logits/chosen": 1.5183905363082886,
+      "logits/rejected": 1.4942539930343628,
+      "logps/chosen": -156.383544921875,
+      "logps/rejected": -145.61398315429688,
+      "loss": 0.6810199737548828,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.14243564009666443,
+      "rewards/margins": 0.03136260434985161,
+      "rewards/rejected": 0.11107305437326431,
+      "step": 1110
+    },
+    {
+      "epoch": 0.896,
+      "grad_norm": 1.0046707391738892,
+      "learning_rate": 7.795555555555555e-07,
+      "logits/chosen": 1.4982213973999023,
+      "logits/rejected": 1.5465660095214844,
+      "logps/chosen": -156.79293823242188,
+      "logps/rejected": -150.77853393554688,
+      "loss": 0.6808318614959716,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.1551930010318756,
+      "rewards/margins": 0.028187204152345657,
+      "rewards/rejected": 0.12700578570365906,
+      "step": 1120
+    },
+    {
+      "epoch": 0.904,
+      "grad_norm": 1.125625729560852,
+      "learning_rate": 7.765925925925925e-07,
+      "logits/chosen": 1.6053575277328491,
+      "logits/rejected": 1.47335946559906,
+      "logps/chosen": -148.05690002441406,
+      "logps/rejected": -135.36598205566406,
+      "loss": 0.6889908790588379,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.15128108859062195,
+      "rewards/margins": 0.015811312943696976,
+      "rewards/rejected": 0.13546979427337646,
+      "step": 1130
+    },
+    {
+      "epoch": 0.912,
+      "grad_norm": 1.0933659076690674,
+      "learning_rate": 7.736296296296297e-07,
+      "logits/chosen": 1.5862042903900146,
+      "logits/rejected": 1.5827430486679077,
+      "logps/chosen": -156.10145568847656,
+      "logps/rejected": -142.94851684570312,
+      "loss": 0.6753206253051758,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.1547432243824005,
+      "rewards/margins": 0.04229574277997017,
+      "rewards/rejected": 0.11244747787714005,
+      "step": 1140
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 1.0774686336517334,
+      "learning_rate": 7.706666666666667e-07,
+      "logits/chosen": 1.2369152307510376,
+      "logits/rejected": 1.2727023363113403,
+      "logps/chosen": -139.23147583007812,
+      "logps/rejected": -130.53038024902344,
+      "loss": 0.6967329025268555,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.10801626741886139,
+      "rewards/margins": 0.004670143127441406,
+      "rewards/rejected": 0.10334610939025879,
+      "step": 1150
+    },
+    {
+      "epoch": 0.928,
+      "grad_norm": 1.1111780405044556,
+      "learning_rate": 7.677037037037036e-07,
+      "logits/chosen": 1.501250982284546,
+      "logits/rejected": 1.4272868633270264,
+      "logps/chosen": -127.09858703613281,
+      "logps/rejected": -114.9041748046875,
+      "loss": 0.6853152275085449,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.1366150826215744,
+      "rewards/margins": 0.021945688873529434,
+      "rewards/rejected": 0.11466939747333527,
+      "step": 1160
+    },
+    {
+      "epoch": 0.936,
+      "grad_norm": 1.5593854188919067,
+      "learning_rate": 7.647407407407407e-07,
+      "logits/chosen": 1.727927803993225,
+      "logits/rejected": 1.6510727405548096,
+      "logps/chosen": -156.70993041992188,
+      "logps/rejected": -167.07266235351562,
+      "loss": 0.6884279251098633,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.14902853965759277,
+      "rewards/margins": 0.01685408689081669,
+      "rewards/rejected": 0.13217447698116302,
+      "step": 1170
+    },
+    {
+      "epoch": 0.944,
+      "grad_norm": 2.0064918994903564,
+      "learning_rate": 7.617777777777778e-07,
+      "logits/chosen": 1.6262975931167603,
+      "logits/rejected": 1.58951997756958,
+      "logps/chosen": -186.59210205078125,
+      "logps/rejected": -151.2965850830078,
+      "loss": 0.6906405448913574,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.14898104965686798,
+      "rewards/margins": 0.013303476385772228,
+      "rewards/rejected": 0.13567757606506348,
+      "step": 1180
+    },
+    {
+      "epoch": 0.952,
+      "grad_norm": 1.209291696548462,
+      "learning_rate": 7.588148148148148e-07,
+      "logits/chosen": 1.639369249343872,
+      "logits/rejected": 1.6400636434555054,
+      "logps/chosen": -166.24684143066406,
+      "logps/rejected": -196.35305786132812,
+      "loss": 0.6789956092834473,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.1348457634449005,
+      "rewards/margins": 0.03475186973810196,
+      "rewards/rejected": 0.10009388625621796,
+      "step": 1190
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 1.209230899810791,
+      "learning_rate": 7.558518518518519e-07,
+      "logits/chosen": 1.542942762374878,
+      "logits/rejected": 1.5128916501998901,
+      "logps/chosen": -138.994384765625,
+      "logps/rejected": -145.42880249023438,
+      "loss": 0.6936234951019287,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.13895569741725922,
+      "rewards/margins": 0.005582497920840979,
+      "rewards/rejected": 0.1333732008934021,
+      "step": 1200
+    },
+    {
+      "epoch": 0.96,
+      "eval_logits/chosen": 1.5055651664733887,
+      "eval_logits/rejected": 1.5351229906082153,
+      "eval_logps/chosen": -152.2256622314453,
+      "eval_logps/rejected": -146.909912109375,
+      "eval_loss": 0.6866763830184937,
+      "eval_rewards/accuracies": 0.5360000133514404,
+      "eval_rewards/chosen": 0.13231882452964783,
+      "eval_rewards/margins": 0.02109396643936634,
+      "eval_rewards/rejected": 0.11122485250234604,
+      "eval_runtime": 90.8141,
+      "eval_samples_per_second": 5.506,
+      "eval_steps_per_second": 2.753,
+      "step": 1200
+    },
+    {
+      "epoch": 0.968,
+      "grad_norm": 1.2058950662612915,
+      "learning_rate": 7.528888888888889e-07,
+      "logits/chosen": 1.3591210842132568,
+      "logits/rejected": 1.3811699151992798,
+      "logps/chosen": -155.1759033203125,
+      "logps/rejected": -141.97772216796875,
+      "loss": 0.6902606964111329,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.153532013297081,
+      "rewards/margins": 0.011359233409166336,
+      "rewards/rejected": 0.14217278361320496,
+      "step": 1210
+    },
+    {
+      "epoch": 0.976,
+      "grad_norm": 1.366702914237976,
+      "learning_rate": 7.499259259259258e-07,
+      "logits/chosen": 1.3841370344161987,
+      "logits/rejected": 1.3791372776031494,
+      "logps/chosen": -145.47613525390625,
+      "logps/rejected": -123.9173583984375,
+      "loss": 0.6971709251403808,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.10521620512008667,
+      "rewards/margins": -0.0030749079305678606,
+      "rewards/rejected": 0.1082911267876625,
+      "step": 1220
+    },
+    {
+      "epoch": 0.984,
+      "grad_norm": 1.477455496788025,
+      "learning_rate": 7.469629629629629e-07,
+      "logits/chosen": 1.4447228908538818,
+      "logits/rejected": 1.6022417545318604,
+      "logps/chosen": -137.9462127685547,
+      "logps/rejected": -150.25559997558594,
+      "loss": 0.6915127277374268,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.15108394622802734,
+      "rewards/margins": 0.009388584643602371,
+      "rewards/rejected": 0.14169538021087646,
+      "step": 1230
+    },
+    {
+      "epoch": 0.992,
+      "grad_norm": 1.4407854080200195,
+      "learning_rate": 7.44e-07,
+      "logits/chosen": 1.3998825550079346,
+      "logits/rejected": 1.5924925804138184,
+      "logps/chosen": -144.93775939941406,
+      "logps/rejected": -148.02542114257812,
+      "loss": 0.6699621677398682,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.146137997508049,
+      "rewards/margins": 0.05326627567410469,
+      "rewards/rejected": 0.09287171810865402,
+      "step": 1240
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 1.183438777923584,
+      "learning_rate": 7.41037037037037e-07,
+      "logits/chosen": 1.5393705368041992,
+      "logits/rejected": 1.6312555074691772,
+      "logps/chosen": -145.61672973632812,
+      "logps/rejected": -158.9256591796875,
+      "loss": 0.6811168670654297,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.1304798424243927,
+      "rewards/margins": 0.03360626846551895,
+      "rewards/rejected": 0.09687358886003494,
+      "step": 1250
+    },
+    {
+      "epoch": 1.008,
+      "grad_norm": 1.2436553239822388,
+      "learning_rate": 7.38074074074074e-07,
+      "logits/chosen": 1.4758602380752563,
+      "logits/rejected": 1.534609079360962,
+      "logps/chosen": -124.26841735839844,
+      "logps/rejected": -132.41867065429688,
+      "loss": 0.688177490234375,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.12570872902870178,
+      "rewards/margins": 0.014875045046210289,
+      "rewards/rejected": 0.11083368957042694,
+      "step": 1260
+    },
+    {
+      "epoch": 1.016,
+      "grad_norm": 1.3853868246078491,
+      "learning_rate": 7.351111111111112e-07,
+      "logits/chosen": 1.7195091247558594,
+      "logits/rejected": 1.632851004600525,
+      "logps/chosen": -160.8590850830078,
+      "logps/rejected": -153.91671752929688,
+      "loss": 0.6669504165649414,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.181864932179451,
+      "rewards/margins": 0.06175640970468521,
+      "rewards/rejected": 0.12010850757360458,
+      "step": 1270
+    },
+    {
+      "epoch": 1.024,
+      "grad_norm": 1.2765302658081055,
+      "learning_rate": 7.321481481481481e-07,
+      "logits/chosen": 1.5782036781311035,
+      "logits/rejected": 1.5713088512420654,
+      "logps/chosen": -154.3319549560547,
+      "logps/rejected": -141.43345642089844,
+      "loss": 0.6734190464019776,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.12361389398574829,
+      "rewards/margins": 0.04626934230327606,
+      "rewards/rejected": 0.07734455168247223,
+      "step": 1280
+    },
+    {
+      "epoch": 1.032,
+      "grad_norm": 1.0421397686004639,
+      "learning_rate": 7.291851851851851e-07,
+      "logits/chosen": 1.5199967622756958,
+      "logits/rejected": 1.5965670347213745,
+      "logps/chosen": -145.28416442871094,
+      "logps/rejected": -133.72256469726562,
+      "loss": 0.6804514408111573,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.11794980615377426,
+      "rewards/margins": 0.030916064977645874,
+      "rewards/rejected": 0.08703374862670898,
+      "step": 1290
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 1.032158613204956,
+      "learning_rate": 7.262222222222222e-07,
+      "logits/chosen": 1.6042639017105103,
+      "logits/rejected": 1.4255346059799194,
+      "logps/chosen": -150.4803009033203,
+      "logps/rejected": -130.2232208251953,
+      "loss": 0.6676507472991944,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.15466037392616272,
+      "rewards/margins": 0.05808692425489426,
+      "rewards/rejected": 0.09657344967126846,
+      "step": 1300
+    },
+    {
+      "epoch": 1.04,
+      "eval_logits/chosen": 1.5118356943130493,
+      "eval_logits/rejected": 1.5418950319290161,
+      "eval_logps/chosen": -152.23300170898438,
+      "eval_logps/rejected": -146.9257049560547,
+      "eval_loss": 0.6859617233276367,
+      "eval_rewards/accuracies": 0.5339999794960022,
+      "eval_rewards/chosen": 0.13158410787582397,
+      "eval_rewards/margins": 0.021939631551504135,
+      "eval_rewards/rejected": 0.10964448750019073,
+      "eval_runtime": 90.9624,
+      "eval_samples_per_second": 5.497,
+      "eval_steps_per_second": 2.748,
+      "step": 1300
+    },
+    {
+      "epoch": 1.048,
+      "grad_norm": 1.4942574501037598,
+      "learning_rate": 7.232592592592593e-07,
+      "logits/chosen": 1.689648985862732,
+      "logits/rejected": 1.691888451576233,
+      "logps/chosen": -166.02809143066406,
+      "logps/rejected": -160.30601501464844,
+      "loss": 0.6770285606384278,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.14377735555171967,
+      "rewards/margins": 0.03825496509671211,
+      "rewards/rejected": 0.10552239418029785,
+      "step": 1310
+    },
+    {
+      "epoch": 1.056,
+      "grad_norm": 1.3656011819839478,
+      "learning_rate": 7.202962962962963e-07,
+      "logits/chosen": 1.3065154552459717,
+      "logits/rejected": 1.5681203603744507,
+      "logps/chosen": -129.1146240234375,
+      "logps/rejected": -151.2584991455078,
+      "loss": 0.6844655036926269,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.12841743230819702,
+      "rewards/margins": 0.02178655005991459,
+      "rewards/rejected": 0.10663086175918579,
+      "step": 1320
+    },
+    {
+      "epoch": 1.064,
+      "grad_norm": 1.1723405122756958,
+      "learning_rate": 7.173333333333333e-07,
+      "logits/chosen": 1.6219158172607422,
+      "logits/rejected": 1.6032555103302002,
+      "logps/chosen": -156.4185028076172,
+      "logps/rejected": -146.28067016601562,
+      "loss": 0.6650479316711426,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.1538492739200592,
+      "rewards/margins": 0.06132146716117859,
+      "rewards/rejected": 0.09252782166004181,
+      "step": 1330
+    },
+    {
+      "epoch": 1.072,
+      "grad_norm": 1.2311245203018188,
+      "learning_rate": 7.143703703703703e-07,
+      "logits/chosen": 1.4625260829925537,
+      "logits/rejected": 1.392937421798706,
+      "logps/chosen": -153.05941772460938,
+      "logps/rejected": -145.37841796875,
+      "loss": 0.6570021152496338,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.16768500208854675,
+      "rewards/margins": 0.0787494108080864,
+      "rewards/rejected": 0.08893559873104095,
+      "step": 1340
+    },
+    {
+      "epoch": 1.08,
+      "grad_norm": 1.314666748046875,
+      "learning_rate": 7.114074074074073e-07,
+      "logits/chosen": 1.4812809228897095,
+      "logits/rejected": 1.3796050548553467,
+      "logps/chosen": -165.70205688476562,
+      "logps/rejected": -140.13150024414062,
+      "loss": 0.6535200595855712,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.20691323280334473,
+      "rewards/margins": 0.0911526307463646,
+      "rewards/rejected": 0.11576057970523834,
+      "step": 1350
+    },
+    {
+      "epoch": 1.088,
+      "grad_norm": 0.9556720852851868,
+      "learning_rate": 7.084444444444444e-07,
+      "logits/chosen": 1.6368863582611084,
+      "logits/rejected": 1.678134560585022,
+      "logps/chosen": -166.56253051757812,
+      "logps/rejected": -164.36416625976562,
+      "loss": 0.6742628574371338,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.15145304799079895,
+      "rewards/margins": 0.04319921135902405,
+      "rewards/rejected": 0.1082538366317749,
+      "step": 1360
+    },
+    {
+      "epoch": 1.096,
+      "grad_norm": 0.945633053779602,
+      "learning_rate": 7.054814814814815e-07,
+      "logits/chosen": 1.4285938739776611,
+      "logits/rejected": 1.5576260089874268,
+      "logps/chosen": -131.5234832763672,
+      "logps/rejected": -128.02340698242188,
+      "loss": 0.6670140743255615,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.16399362683296204,
+      "rewards/margins": 0.05922812223434448,
+      "rewards/rejected": 0.10476551204919815,
+      "step": 1370
+    },
+    {
+      "epoch": 1.104,
+      "grad_norm": 1.1189475059509277,
+      "learning_rate": 7.025185185185185e-07,
+      "logits/chosen": 1.5019981861114502,
+      "logits/rejected": 1.667614221572876,
+      "logps/chosen": -161.61854553222656,
+      "logps/rejected": -166.39996337890625,
+      "loss": 0.6742690086364747,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.15786050260066986,
+      "rewards/margins": 0.044500257819890976,
+      "rewards/rejected": 0.11336024850606918,
+      "step": 1380
+    },
+    {
+      "epoch": 1.112,
+      "grad_norm": 1.3382912874221802,
+      "learning_rate": 6.995555555555555e-07,
+      "logits/chosen": 1.4972480535507202,
+      "logits/rejected": 1.4946129322052002,
+      "logps/chosen": -166.43682861328125,
+      "logps/rejected": -162.3004913330078,
+      "loss": 0.6557814121246338,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.18964914977550507,
+      "rewards/margins": 0.08500295877456665,
+      "rewards/rejected": 0.10464620590209961,
+      "step": 1390
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 1.2123515605926514,
+      "learning_rate": 6.965925925925926e-07,
+      "logits/chosen": 1.456635594367981,
+      "logits/rejected": 1.5964475870132446,
+      "logps/chosen": -152.08261108398438,
+      "logps/rejected": -161.51023864746094,
+      "loss": 0.6708066940307618,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.216823011636734,
+      "rewards/margins": 0.050363689661026,
+      "rewards/rejected": 0.166459321975708,
+      "step": 1400
+    },
+    {
+      "epoch": 1.12,
+      "eval_logits/chosen": 1.521518588066101,
+      "eval_logits/rejected": 1.5510718822479248,
+      "eval_logps/chosen": -151.858154296875,
+      "eval_logps/rejected": -146.5819549560547,
+      "eval_loss": 0.6853657364845276,
+      "eval_rewards/accuracies": 0.5440000295639038,
+      "eval_rewards/chosen": 0.16906805336475372,
+      "eval_rewards/margins": 0.025049500167369843,
+      "eval_rewards/rejected": 0.14401854574680328,
+      "eval_runtime": 90.8779,
+      "eval_samples_per_second": 5.502,
+      "eval_steps_per_second": 2.751,
+      "step": 1400
+    },
+    {
+      "epoch": 1.1280000000000001,
+      "grad_norm": 1.3488291501998901,
+      "learning_rate": 6.936296296296296e-07,
+      "logits/chosen": 1.6009830236434937,
+      "logits/rejected": 1.6035544872283936,
+      "logps/chosen": -145.44662475585938,
+      "logps/rejected": -149.9284210205078,
+      "loss": 0.679435396194458,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.1647307425737381,
+      "rewards/margins": 0.03348377346992493,
+      "rewards/rejected": 0.13124695420265198,
+      "step": 1410
+    },
+    {
+      "epoch": 1.1360000000000001,
+      "grad_norm": 1.6453970670700073,
+      "learning_rate": 6.906666666666666e-07,
+      "logits/chosen": 1.593658447265625,
+      "logits/rejected": 1.648276925086975,
+      "logps/chosen": -135.2926788330078,
+      "logps/rejected": -150.9585723876953,
+      "loss": 0.6875734329223633,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.15669891238212585,
+      "rewards/margins": 0.01690087839961052,
+      "rewards/rejected": 0.13979804515838623,
+      "step": 1420
+    },
+    {
+      "epoch": 1.144,
+      "grad_norm": 1.6117807626724243,
+      "learning_rate": 6.877037037037036e-07,
+      "logits/chosen": 1.4510356187820435,
+      "logits/rejected": 1.2334567308425903,
+      "logps/chosen": -153.94854736328125,
+      "logps/rejected": -147.78079223632812,
+      "loss": 0.6689413070678711,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.1819056123495102,
+      "rewards/margins": 0.058564942330121994,
+      "rewards/rejected": 0.1233406662940979,
+      "step": 1430
+    },
+    {
+      "epoch": 1.152,
+      "grad_norm": 1.5533809661865234,
+      "learning_rate": 6.847407407407408e-07,
+      "logits/chosen": 1.4535889625549316,
+      "logits/rejected": 1.4273865222930908,
+      "logps/chosen": -170.42904663085938,
+      "logps/rejected": -151.9979705810547,
+      "loss": 0.6845246315002441,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.14223966002464294,
+      "rewards/margins": 0.022389881312847137,
+      "rewards/rejected": 0.1198497861623764,
+      "step": 1440
+    },
+    {
+      "epoch": 1.16,
+      "grad_norm": 1.5696005821228027,
+      "learning_rate": 6.817777777777778e-07,
+      "logits/chosen": 1.4418725967407227,
+      "logits/rejected": 1.4354021549224854,
+      "logps/chosen": -153.962158203125,
+      "logps/rejected": -148.40512084960938,
+      "loss": 0.6761108875274658,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.15529735386371613,
+      "rewards/margins": 0.040999218821525574,
+      "rewards/rejected": 0.11429812759160995,
+      "step": 1450
+    },
+    {
+      "epoch": 1.168,
+      "grad_norm": 1.0711781978607178,
+      "learning_rate": 6.788148148148147e-07,
+      "logits/chosen": 1.5659481287002563,
+      "logits/rejected": 1.5640677213668823,
+      "logps/chosen": -149.816162109375,
+      "logps/rejected": -174.68643188476562,
+      "loss": 0.677489185333252,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.1510745733976364,
+      "rewards/margins": 0.03914733976125717,
+      "rewards/rejected": 0.11192724853754044,
+      "step": 1460
+    },
+    {
+      "epoch": 1.176,
+      "grad_norm": 1.2336673736572266,
+      "learning_rate": 6.758518518518518e-07,
+      "logits/chosen": 1.4275394678115845,
+      "logits/rejected": 1.6394407749176025,
+      "logps/chosen": -137.93788146972656,
+      "logps/rejected": -154.97195434570312,
+      "loss": 0.6785802364349365,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.10557998716831207,
+      "rewards/margins": 0.0352485254406929,
+      "rewards/rejected": 0.07033145427703857,
+      "step": 1470
+    },
+    {
+      "epoch": 1.184,
+      "grad_norm": 1.0030723810195923,
+      "learning_rate": 6.728888888888888e-07,
+      "logits/chosen": 1.4652725458145142,
+      "logits/rejected": 1.4992272853851318,
+      "logps/chosen": -143.87374877929688,
+      "logps/rejected": -139.5693817138672,
+      "loss": 0.6700259208679199,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.13436684012413025,
+      "rewards/margins": 0.05286343768239021,
+      "rewards/rejected": 0.08150340616703033,
+      "step": 1480
+    },
+    {
+      "epoch": 1.192,
+      "grad_norm": 1.0897104740142822,
+      "learning_rate": 6.699259259259259e-07,
+      "logits/chosen": 1.7033601999282837,
+      "logits/rejected": 1.617489218711853,
+      "logps/chosen": -140.43667602539062,
+      "logps/rejected": -146.59683227539062,
+      "loss": 0.6685672283172608,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.11724893748760223,
+      "rewards/margins": 0.055138181895017624,
+      "rewards/rejected": 0.06211075931787491,
+      "step": 1490
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 0.8573256731033325,
+      "learning_rate": 6.66962962962963e-07,
+      "logits/chosen": 1.5181013345718384,
+      "logits/rejected": 1.5965152978897095,
+      "logps/chosen": -147.78506469726562,
+      "logps/rejected": -142.18930053710938,
+      "loss": 0.6564288139343262,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.14890792965888977,
+      "rewards/margins": 0.08058460801839828,
+      "rewards/rejected": 0.06832331418991089,
+      "step": 1500
+    },
+    {
+      "epoch": 1.2,
+      "eval_logits/chosen": 1.488415002822876,
+      "eval_logits/rejected": 1.5179662704467773,
+      "eval_logps/chosen": -152.60232543945312,
+      "eval_logps/rejected": -147.2954864501953,
+      "eval_loss": 0.6861968040466309,
+      "eval_rewards/accuracies": 0.5400000214576721,
+      "eval_rewards/chosen": 0.09465231001377106,
+      "eval_rewards/margins": 0.021986663341522217,
+      "eval_rewards/rejected": 0.07266565412282944,
+      "eval_runtime": 90.8951,
+      "eval_samples_per_second": 5.501,
+      "eval_steps_per_second": 2.75,
+      "step": 1500
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 3750,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

v3/DPO/DPO_10k/lora/checkpoint-1500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4f90e4a7053b7fb440d68cb64a3d198396af09db942d08839c5f4f0f0b8a0c8
+size 6097

v3/DPO/DPO_10k/lora/checkpoint-2000/README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

v3/DPO/DPO_10k/lora/checkpoint-2000/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "gate_proj",
+    "down_proj",
+    "k_proj",
+    "o_proj",
+    "up_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

v3/DPO/DPO_10k/lora/checkpoint-2000/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cd69c523e1e782aa020a6dd4857a965f4cb7c32d5baa557697e95c17d5c604e
+size 180385008

v3/DPO/DPO_10k/lora/checkpoint-2000/chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

v3/DPO/DPO_10k/lora/checkpoint-2000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9fab96ca3915ebdde3a55a6d95b9e0ffb2c8eb2ed59436fcdae3be4c642b46f
+size 360902475

v3/DPO/DPO_10k/lora/checkpoint-2000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f27403649e37262609f871dba600d7da4f0bfc387b6571584c6bc75289b7c853
+size 14645

v3/DPO/DPO_10k/lora/checkpoint-2000/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4aa03f6e0cd07cf67ce1fbe3101d545f5771ef9148b9debf02b11cf6948da5c
+size 1383

v3/DPO/DPO_10k/lora/checkpoint-2000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:318db60b99958c78b78a2ad7d7ff224500c5019593c5cf0c4944785079b20a61
+size 1465

v3/DPO/DPO_10k/lora/checkpoint-2000/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

v3/DPO/DPO_10k/lora/checkpoint-2000/tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

v3/DPO/DPO_10k/lora/checkpoint-2000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4f90e4a7053b7fb440d68cb64a3d198396af09db942d08839c5f4f0f0b8a0c8
+size 6097