diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..52ff3839701acc195c05c413809b35c3ec5f70af 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,26 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_10k/MDPO_10k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_10k/lora/checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_10k/lora/checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_10k/lora/checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_10k/lora/checkpoint-2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_10k/lora/checkpoint-3000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_10k/lora/checkpoint-3500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_10k/lora/checkpoint-3750/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_10k/lora/checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_1k/MDPO_1k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_1k/lora/checkpoint-375/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_5k/MDPO_5k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_5k/lora/checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_5k/lora/checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_5k/lora/checkpoint-1875/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/DPO/DPO_5k/lora/checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/ORPO/ORPO_1k/MORPO_1k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/ORPO/ORPO_1k/lora/checkpoint-375/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/ORPO/ORPO_5k/MORPO_5k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/ORPO/ORPO_5k/lora/checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/ORPO/ORPO_5k/lora/checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/ORPO/ORPO_5k/lora/checkpoint-1875/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v3/ORPO/ORPO_5k/lora/checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/v3/Base/gen-output/data-00000-of-00001.arrow b/v3/Base/gen-output/data-00000-of-00001.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..336120ed99e8e51ecdffba3c2e4cdf8a5f2e14ac
--- /dev/null
+++ b/v3/Base/gen-output/data-00000-of-00001.arrow
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efabd6f3e982da43a3b3d18d95b36f512aee97801c2e980bce071c90f75c41d3
+size 982376
diff --git a/v3/Base/gen-output/dataset_info.json b/v3/Base/gen-output/dataset_info.json
new file mode 100644
index 0000000000000000000000000000000000000000..523b881cd388b7ceb2401ad4759de06c0653cbef
--- /dev/null
+++ b/v3/Base/gen-output/dataset_info.json
@@ -0,0 +1,34 @@
+{
+ "citation": "",
+ "description": "",
+ "features": {
+ "prompt": {
+ "feature": {
+ "content": {
+ "dtype": "string",
+ "_type": "Value"
+ },
+ "role": {
+ "dtype": "string",
+ "_type": "Value"
+ }
+ },
+ "_type": "List"
+ },
+ "generated_text": {
+ "feature": {
+ "content": {
+ "dtype": "string",
+ "_type": "Value"
+ },
+ "role": {
+ "dtype": "string",
+ "_type": "Value"
+ }
+ },
+ "_type": "List"
+ }
+ },
+ "homepage": "",
+ "license": ""
+}
\ No newline at end of file
diff --git a/v3/Base/gen-output/state.json b/v3/Base/gen-output/state.json
new file mode 100644
index 0000000000000000000000000000000000000000..c2203042fa006b7f86899673cc5704d31c323d83
--- /dev/null
+++ b/v3/Base/gen-output/state.json
@@ -0,0 +1,13 @@
+{
+ "_data_files": [
+ {
+ "filename": "data-00000-of-00001.arrow"
+ }
+ ],
+ "_fingerprint": "f319c5a85dc7aa54",
+ "_format_columns": null,
+ "_format_kwargs": {},
+ "_format_type": null,
+ "_output_all_columns": false,
+ "_split": null
+}
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/DPO_10k/README.md b/v3/DPO/DPO_10k/DPO_10k/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..250500f798b4682b2cd2c35cd1fc366677c215dd
--- /dev/null
+++ b/v3/DPO/DPO_10k/DPO_10k/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/DPO_10k/adapter_config.json b/v3/DPO/DPO_10k/DPO_10k/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..039869f8d79ca6c48a793a29a79267da6d8f81c2
--- /dev/null
+++ b/v3/DPO/DPO_10k/DPO_10k/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "gate_proj",
+ "down_proj",
+ "k_proj",
+ "o_proj",
+ "up_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/DPO_10k/adapter_model.safetensors b/v3/DPO/DPO_10k/DPO_10k/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ea99abfc2b2f7e8033d92153d8b0e0a30ae9423b
--- /dev/null
+++ b/v3/DPO/DPO_10k/DPO_10k/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27a5e5a70889c639646183be14875b89ae81e677265281b25eadec5735c154e0
+size 180385008
diff --git a/v3/DPO/DPO_10k/MDPO_10k/chat_template.jinja b/v3/DPO/DPO_10k/MDPO_10k/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/DPO/DPO_10k/MDPO_10k/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/DPO/DPO_10k/MDPO_10k/config.json b/v3/DPO/DPO_10k/MDPO_10k/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e9ec18bc1ae18922052a57480e28401f3b9c6b84
--- /dev/null
+++ b/v3/DPO/DPO_10k/MDPO_10k/config.json
@@ -0,0 +1,40 @@
+{
+ "architectures": [
+ "LlamaForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 128000,
+ "dtype": "float16",
+ "eos_token_id": [
+ 128001,
+ 128008,
+ 128009
+ ],
+ "head_dim": 64,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 8192,
+ "max_position_embeddings": 131072,
+ "mlp_bias": false,
+ "model_type": "llama",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 16,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "pretraining_tp": 1,
+ "rms_norm_eps": 1e-05,
+ "rope_parameters": {
+ "factor": 32.0,
+ "high_freq_factor": 4.0,
+ "low_freq_factor": 1.0,
+ "original_max_position_embeddings": 8192,
+ "rope_theta": 500000.0,
+ "rope_type": "llama3"
+ },
+ "tie_word_embeddings": true,
+ "transformers_version": "5.2.0",
+ "use_cache": true,
+ "vocab_size": 128256
+}
diff --git a/v3/DPO/DPO_10k/MDPO_10k/generation_config.json b/v3/DPO/DPO_10k/MDPO_10k/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..898a2e748a41a4b7a931b754f8abbdac02039fae
--- /dev/null
+++ b/v3/DPO/DPO_10k/MDPO_10k/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "bos_token_id": 128000,
+ "do_sample": true,
+ "eos_token_id": [
+ 128001,
+ 128008,
+ 128009
+ ],
+ "temperature": 0.6,
+ "top_p": 0.9,
+ "transformers_version": "5.2.0"
+}
diff --git a/v3/DPO/DPO_10k/MDPO_10k/model.safetensors b/v3/DPO/DPO_10k/MDPO_10k/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..15722101795872d7511fbb8e3fdb5ec4c64e5653
--- /dev/null
+++ b/v3/DPO/DPO_10k/MDPO_10k/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85c9d82fdf7dbcecd629e2cc9efd75e800f619818033fb81d1f7884887a67cc7
+size 2471645464
diff --git a/v3/DPO/DPO_10k/MDPO_10k/tokenizer.json b/v3/DPO/DPO_10k/MDPO_10k/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/DPO/DPO_10k/MDPO_10k/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/DPO/DPO_10k/MDPO_10k/tokenizer_config.json b/v3/DPO/DPO_10k/MDPO_10k/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/DPO/DPO_10k/MDPO_10k/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/DPO/DPO_10k/lora/README.md b/v3/DPO/DPO_10k/lora/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..6c810ce1b1fdcd23f3c05186b6ceac5cd5c41721
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/README.md
@@ -0,0 +1,68 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: transformers
+model_name: lora
+tags:
+- generated_from_trainer
+- trl
+- dpo
+licence: license
+---
+
+# Model Card for lora
+
+This model is a fine-tuned version of [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[
](https://wandb.ai/sea-rod/huggingface/runs/9nfqa09t)
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- TRL: 0.28.0
+- Transformers: 5.2.0
+- Pytorch: 2.10.0
+- Datasets: 4.5.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+ title = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+ author = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+ year = 2023,
+ booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+ url = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+ editor = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+ title = {{TRL: Transformers Reinforcement Learning}},
+ author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+ license = {Apache-2.0},
+ url = {https://github.com/huggingface/trl},
+ year = {2020}
+}
+```
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1000/README.md b/v3/DPO/DPO_10k/lora/checkpoint-1000/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..250500f798b4682b2cd2c35cd1fc366677c215dd
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1000/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1000/adapter_config.json b/v3/DPO/DPO_10k/lora/checkpoint-1000/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..039869f8d79ca6c48a793a29a79267da6d8f81c2
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1000/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "gate_proj",
+ "down_proj",
+ "k_proj",
+ "o_proj",
+ "up_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1000/adapter_model.safetensors b/v3/DPO/DPO_10k/lora/checkpoint-1000/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ea99abfc2b2f7e8033d92153d8b0e0a30ae9423b
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1000/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27a5e5a70889c639646183be14875b89ae81e677265281b25eadec5735c154e0
+size 180385008
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1000/chat_template.jinja b/v3/DPO/DPO_10k/lora/checkpoint-1000/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1000/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1000/optimizer.pt b/v3/DPO/DPO_10k/lora/checkpoint-1000/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8fe30d91ba021202226c0c2472788542c8517d2c
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1000/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d87beff974577f720d7e8125b66164907ad5b681a0dedce34f2ee0ab3d344976
+size 360902475
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1000/rng_state.pth b/v3/DPO/DPO_10k/lora/checkpoint-1000/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..d5e20c9b1725549f13f103bd76fa989eb82478a1
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1000/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea11996454b5587fcf33ae0ab5cf14b2031bf5f53f8c2ed5a48e87de31e29c84
+size 14645
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1000/scaler.pt b/v3/DPO/DPO_10k/lora/checkpoint-1000/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0405b11307032f43fcf2212da19c52823cc01ac4
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1000/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14ae2a2128444abab378aa06c09a61a84665f758fcc19fc46f5789b0bc1b5665
+size 1383
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1000/scheduler.pt b/v3/DPO/DPO_10k/lora/checkpoint-1000/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f12764eabb1bdbbc95e74d89f292100162b8f27a
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1000/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c517d0407369461be1bd2161fd20105c2aa4886637aab49e0225f47034adb975
+size 1465
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1000/tokenizer.json b/v3/DPO/DPO_10k/lora/checkpoint-1000/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1000/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1000/tokenizer_config.json b/v3/DPO/DPO_10k/lora/checkpoint-1000/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1000/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1000/trainer_state.json b/v3/DPO/DPO_10k/lora/checkpoint-1000/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..5cb1a3488a3a36ec65d2b25c0d3d5b4d92178428
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1000/trainer_state.json
@@ -0,0 +1,1694 @@
+{
+ "best_global_step": 1000,
+ "best_metric": 0.6880542039871216,
+ "best_model_checkpoint": "output/lora/checkpoint-1000",
+ "epoch": 0.8,
+ "eval_steps": 100,
+ "global_step": 1000,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.008,
+ "grad_norm": 1.1179773807525635,
+ "learning_rate": 2.4e-08,
+ "logits/chosen": 1.4545083045959473,
+ "logits/rejected": 1.425490379333496,
+ "logps/chosen": -132.46189880371094,
+ "logps/rejected": -148.20260620117188,
+ "loss": 0.6930626392364502,
+ "rewards/accuracies": 0.21250000596046448,
+ "rewards/chosen": -6.165739614516497e-05,
+ "rewards/margins": 0.00017333509458694607,
+ "rewards/rejected": -0.00023499250528402627,
+ "step": 10
+ },
+ {
+ "epoch": 0.016,
+ "grad_norm": 1.3502839803695679,
+ "learning_rate": 5.0666666666666664e-08,
+ "logits/chosen": 1.5668801069259644,
+ "logits/rejected": 1.5191389322280884,
+ "logps/chosen": -157.63624572753906,
+ "logps/rejected": -144.2716827392578,
+ "loss": 0.6930380821228027,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.0009244013344869018,
+ "rewards/margins": 0.00023505210992880166,
+ "rewards/rejected": 0.0006893490790389478,
+ "step": 20
+ },
+ {
+ "epoch": 0.024,
+ "grad_norm": 1.0219553709030151,
+ "learning_rate": 7.733333333333334e-08,
+ "logits/chosen": 1.8921797275543213,
+ "logits/rejected": 1.7342685461044312,
+ "logps/chosen": -164.55799865722656,
+ "logps/rejected": -158.92752075195312,
+ "loss": 0.69290189743042,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0009391927160322666,
+ "rewards/margins": 0.0005032324115745723,
+ "rewards/rejected": 0.0004359603044576943,
+ "step": 30
+ },
+ {
+ "epoch": 0.032,
+ "grad_norm": 1.2056113481521606,
+ "learning_rate": 1.0399999999999999e-07,
+ "logits/chosen": 1.4950047731399536,
+ "logits/rejected": 1.6163270473480225,
+ "logps/chosen": -145.54183959960938,
+ "logps/rejected": -134.19979858398438,
+ "loss": 0.6935500621795654,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.0006421065190806985,
+ "rewards/margins": -0.0007943272357806563,
+ "rewards/rejected": 0.0014364338712766767,
+ "step": 40
+ },
+ {
+ "epoch": 0.04,
+ "grad_norm": 1.1029725074768066,
+ "learning_rate": 1.3066666666666665e-07,
+ "logits/chosen": 1.4076533317565918,
+ "logits/rejected": 1.4791977405548096,
+ "logps/chosen": -133.23324584960938,
+ "logps/rejected": -138.3593292236328,
+ "loss": 0.6925833702087403,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.0012372838100418448,
+ "rewards/margins": 0.001139522879384458,
+ "rewards/rejected": 9.776116348803043e-05,
+ "step": 50
+ },
+ {
+ "epoch": 0.048,
+ "grad_norm": 0.9960266351699829,
+ "learning_rate": 1.573333333333333e-07,
+ "logits/chosen": 1.407166600227356,
+ "logits/rejected": 1.376042127609253,
+ "logps/chosen": -137.8921661376953,
+ "logps/rejected": -127.97320556640625,
+ "loss": 0.693614149093628,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.00021316049969755113,
+ "rewards/margins": -0.0009249996510334313,
+ "rewards/rejected": 0.0007118391804397106,
+ "step": 60
+ },
+ {
+ "epoch": 0.056,
+ "grad_norm": 1.0946824550628662,
+ "learning_rate": 1.8399999999999998e-07,
+ "logits/chosen": 1.5944501161575317,
+ "logits/rejected": 1.7248146533966064,
+ "logps/chosen": -153.5526123046875,
+ "logps/rejected": -157.19735717773438,
+ "loss": 0.6928449153900147,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0026370862033218145,
+ "rewards/margins": 0.0006150150438770652,
+ "rewards/rejected": 0.0020220710430294275,
+ "step": 70
+ },
+ {
+ "epoch": 0.064,
+ "grad_norm": 1.0322489738464355,
+ "learning_rate": 2.1066666666666665e-07,
+ "logits/chosen": 1.4586353302001953,
+ "logits/rejected": 1.4842995405197144,
+ "logps/chosen": -149.03665161132812,
+ "logps/rejected": -136.24081420898438,
+ "loss": 0.6932015895843506,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.004061593674123287,
+ "rewards/margins": -9.99594121822156e-05,
+ "rewards/rejected": 0.004161553457379341,
+ "step": 80
+ },
+ {
+ "epoch": 0.072,
+ "grad_norm": 1.0012447834014893,
+ "learning_rate": 2.3733333333333334e-07,
+ "logits/chosen": 1.5075902938842773,
+ "logits/rejected": 1.5986944437026978,
+ "logps/chosen": -159.3433074951172,
+ "logps/rejected": -151.85072326660156,
+ "loss": 0.6928757190704345,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.006978762801736593,
+ "rewards/margins": 0.0005627512582577765,
+ "rewards/rejected": 0.006416010670363903,
+ "step": 90
+ },
+ {
+ "epoch": 0.08,
+ "grad_norm": 1.280081033706665,
+ "learning_rate": 2.64e-07,
+ "logits/chosen": 1.448286533355713,
+ "logits/rejected": 1.5480695962905884,
+ "logps/chosen": -151.12625122070312,
+ "logps/rejected": -171.24545288085938,
+ "loss": 0.6931126117706299,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.00693341251462698,
+ "rewards/margins": 9.057526767719537e-05,
+ "rewards/rejected": 0.006842837668955326,
+ "step": 100
+ },
+ {
+ "epoch": 0.08,
+ "eval_logits/chosen": 1.518183946609497,
+ "eval_logits/rejected": 1.5525447130203247,
+ "eval_logps/chosen": -153.46630859375,
+ "eval_logps/rejected": -147.95005798339844,
+ "eval_loss": 0.6926330327987671,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.008253541775047779,
+ "eval_rewards/margins": 0.0010449704714119434,
+ "eval_rewards/rejected": 0.007208569906651974,
+ "eval_runtime": 91.0812,
+ "eval_samples_per_second": 5.49,
+ "eval_steps_per_second": 2.745,
+ "step": 100
+ },
+ {
+ "epoch": 0.088,
+ "grad_norm": 1.3912373781204224,
+ "learning_rate": 2.906666666666667e-07,
+ "logits/chosen": 1.6896222829818726,
+ "logits/rejected": 1.6516921520233154,
+ "logps/chosen": -173.70030212402344,
+ "logps/rejected": -155.5503692626953,
+ "loss": 0.6926521778106689,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.010179834440350533,
+ "rewards/margins": 0.0010137532372027636,
+ "rewards/rejected": 0.009166081435978413,
+ "step": 110
+ },
+ {
+ "epoch": 0.096,
+ "grad_norm": 1.1518888473510742,
+ "learning_rate": 3.173333333333333e-07,
+ "logits/chosen": 1.5166860818862915,
+ "logits/rejected": 1.5136979818344116,
+ "logps/chosen": -136.39906311035156,
+ "logps/rejected": -140.86561584472656,
+ "loss": 0.6932918548583984,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": 0.00907944981008768,
+ "rewards/margins": -0.00026998287648893893,
+ "rewards/rejected": 0.009349432773888111,
+ "step": 120
+ },
+ {
+ "epoch": 0.104,
+ "grad_norm": 1.3213376998901367,
+ "learning_rate": 3.4399999999999996e-07,
+ "logits/chosen": 1.4598973989486694,
+ "logits/rejected": 1.6739647388458252,
+ "logps/chosen": -159.12461853027344,
+ "logps/rejected": -163.39346313476562,
+ "loss": 0.693086051940918,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.010477779433131218,
+ "rewards/margins": 0.00014539004769176245,
+ "rewards/rejected": 0.010332388803362846,
+ "step": 130
+ },
+ {
+ "epoch": 0.112,
+ "grad_norm": 0.8833863735198975,
+ "learning_rate": 3.7066666666666665e-07,
+ "logits/chosen": 1.3829412460327148,
+ "logits/rejected": 1.4100173711776733,
+ "logps/chosen": -132.7149658203125,
+ "logps/rejected": -126.85478210449219,
+ "loss": 0.6932382583618164,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": 0.010262183845043182,
+ "rewards/margins": -0.0001672578218858689,
+ "rewards/rejected": 0.010429441928863525,
+ "step": 140
+ },
+ {
+ "epoch": 0.12,
+ "grad_norm": 1.1472455263137817,
+ "learning_rate": 3.973333333333333e-07,
+ "logits/chosen": 1.4954869747161865,
+ "logits/rejected": 1.5857340097427368,
+ "logps/chosen": -165.10568237304688,
+ "logps/rejected": -158.97433471679688,
+ "loss": 0.6916103363037109,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.014177520759403706,
+ "rewards/margins": 0.003108317730948329,
+ "rewards/rejected": 0.011069201864302158,
+ "step": 150
+ },
+ {
+ "epoch": 0.128,
+ "grad_norm": 1.0628798007965088,
+ "learning_rate": 4.24e-07,
+ "logits/chosen": 1.6717243194580078,
+ "logits/rejected": 1.705092191696167,
+ "logps/chosen": -159.332275390625,
+ "logps/rejected": -147.21646118164062,
+ "loss": 0.6935629844665527,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": 0.016994481906294823,
+ "rewards/margins": -0.0007874655420891941,
+ "rewards/rejected": 0.017781946808099747,
+ "step": 160
+ },
+ {
+ "epoch": 0.136,
+ "grad_norm": 1.390810251235962,
+ "learning_rate": 4.506666666666666e-07,
+ "logits/chosen": 1.4719234704971313,
+ "logits/rejected": 1.499526023864746,
+ "logps/chosen": -148.67251586914062,
+ "logps/rejected": -161.58021545410156,
+ "loss": 0.6926599979400635,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.021239640191197395,
+ "rewards/margins": 0.0010346340714022517,
+ "rewards/rejected": 0.020205006003379822,
+ "step": 170
+ },
+ {
+ "epoch": 0.144,
+ "grad_norm": 1.139707326889038,
+ "learning_rate": 4.773333333333333e-07,
+ "logits/chosen": 1.4833905696868896,
+ "logits/rejected": 1.637584924697876,
+ "logps/chosen": -158.96731567382812,
+ "logps/rejected": -161.9830322265625,
+ "loss": 0.6919547557830811,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.019416773691773415,
+ "rewards/margins": 0.0024296878837049007,
+ "rewards/rejected": 0.016987085342407227,
+ "step": 180
+ },
+ {
+ "epoch": 0.152,
+ "grad_norm": 1.2029796838760376,
+ "learning_rate": 5.04e-07,
+ "logits/chosen": 1.5398027896881104,
+ "logits/rejected": 1.6678705215454102,
+ "logps/chosen": -143.00283813476562,
+ "logps/rejected": -144.89378356933594,
+ "loss": 0.692262840270996,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.018960200250148773,
+ "rewards/margins": 0.0018175147706642747,
+ "rewards/rejected": 0.017142686992883682,
+ "step": 190
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 1.0527437925338745,
+ "learning_rate": 5.306666666666665e-07,
+ "logits/chosen": 1.6087623834609985,
+ "logits/rejected": 1.5181543827056885,
+ "logps/chosen": -159.35073852539062,
+ "logps/rejected": -133.59890747070312,
+ "loss": 0.6924069881439209,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.01926913857460022,
+ "rewards/margins": 0.0015339398523792624,
+ "rewards/rejected": 0.0177351962774992,
+ "step": 200
+ },
+ {
+ "epoch": 0.16,
+ "eval_logits/chosen": 1.5193672180175781,
+ "eval_logits/rejected": 1.5534639358520508,
+ "eval_logps/chosen": -153.34857177734375,
+ "eval_logps/rejected": -147.8425750732422,
+ "eval_loss": 0.6921458840370178,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.020027177408337593,
+ "eval_rewards/margins": 0.0020714527927339077,
+ "eval_rewards/rejected": 0.017955724149942398,
+ "eval_runtime": 90.9762,
+ "eval_samples_per_second": 5.496,
+ "eval_steps_per_second": 2.748,
+ "step": 200
+ },
+ {
+ "epoch": 0.168,
+ "grad_norm": 1.1897239685058594,
+ "learning_rate": 5.573333333333333e-07,
+ "logits/chosen": 1.6267732381820679,
+ "logits/rejected": 1.5775268077850342,
+ "logps/chosen": -159.51437377929688,
+ "logps/rejected": -170.17762756347656,
+ "loss": 0.6925770759582519,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.02078000269830227,
+ "rewards/margins": 0.0012053751852363348,
+ "rewards/rejected": 0.01957462914288044,
+ "step": 210
+ },
+ {
+ "epoch": 0.176,
+ "grad_norm": 0.9995236396789551,
+ "learning_rate": 5.839999999999999e-07,
+ "logits/chosen": 1.5781934261322021,
+ "logits/rejected": 1.642251968383789,
+ "logps/chosen": -159.95309448242188,
+ "logps/rejected": -144.36058044433594,
+ "loss": 0.6931475639343262,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.023322442546486855,
+ "rewards/margins": 6.392716750269756e-05,
+ "rewards/rejected": 0.02325851283967495,
+ "step": 220
+ },
+ {
+ "epoch": 0.184,
+ "grad_norm": 0.8819260597229004,
+ "learning_rate": 6.106666666666666e-07,
+ "logits/chosen": 1.459991693496704,
+ "logits/rejected": 1.6060739755630493,
+ "logps/chosen": -162.04745483398438,
+ "logps/rejected": -139.5411834716797,
+ "loss": 0.6919463634490967,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.026052182540297508,
+ "rewards/margins": 0.002481849165633321,
+ "rewards/rejected": 0.023570330813527107,
+ "step": 230
+ },
+ {
+ "epoch": 0.192,
+ "grad_norm": 1.1055208444595337,
+ "learning_rate": 6.373333333333333e-07,
+ "logits/chosen": 1.558739185333252,
+ "logits/rejected": 1.5965116024017334,
+ "logps/chosen": -171.9166259765625,
+ "logps/rejected": -145.66433715820312,
+ "loss": 0.6912176132202148,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.033868782222270966,
+ "rewards/margins": 0.004017929546535015,
+ "rewards/rejected": 0.029850851744413376,
+ "step": 240
+ },
+ {
+ "epoch": 0.2,
+ "grad_norm": 1.1372733116149902,
+ "learning_rate": 6.64e-07,
+ "logits/chosen": 1.5322860479354858,
+ "logits/rejected": 1.4695795774459839,
+ "logps/chosen": -161.7803955078125,
+ "logps/rejected": -148.91949462890625,
+ "loss": 0.6940969944000244,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.034065864980220795,
+ "rewards/margins": -0.0016761379083618522,
+ "rewards/rejected": 0.035742007195949554,
+ "step": 250
+ },
+ {
+ "epoch": 0.208,
+ "grad_norm": 1.0917680263519287,
+ "learning_rate": 6.906666666666666e-07,
+ "logits/chosen": 1.4094547033309937,
+ "logits/rejected": 1.50186026096344,
+ "logps/chosen": -132.91455078125,
+ "logps/rejected": -146.79714965820312,
+ "loss": 0.6930203914642334,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.03089061938226223,
+ "rewards/margins": 0.00040971505222842097,
+ "rewards/rejected": 0.030480902642011642,
+ "step": 260
+ },
+ {
+ "epoch": 0.216,
+ "grad_norm": 1.0280259847640991,
+ "learning_rate": 7.173333333333333e-07,
+ "logits/chosen": 1.531285285949707,
+ "logits/rejected": 1.6163132190704346,
+ "logps/chosen": -146.902587890625,
+ "logps/rejected": -147.8347625732422,
+ "loss": 0.6944749355316162,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.028514739125967026,
+ "rewards/margins": -0.0025452517438679934,
+ "rewards/rejected": 0.0310599897056818,
+ "step": 270
+ },
+ {
+ "epoch": 0.224,
+ "grad_norm": 1.398708701133728,
+ "learning_rate": 7.44e-07,
+ "logits/chosen": 1.563852071762085,
+ "logits/rejected": 1.396703839302063,
+ "logps/chosen": -172.68153381347656,
+ "logps/rejected": -155.80215454101562,
+ "loss": 0.689476490020752,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03224216774106026,
+ "rewards/margins": 0.007473687641322613,
+ "rewards/rejected": 0.02476847730576992,
+ "step": 280
+ },
+ {
+ "epoch": 0.232,
+ "grad_norm": 1.0394141674041748,
+ "learning_rate": 7.706666666666667e-07,
+ "logits/chosen": 1.5581729412078857,
+ "logits/rejected": 1.5819021463394165,
+ "logps/chosen": -144.41465759277344,
+ "logps/rejected": -153.55406188964844,
+ "loss": 0.6927877902984619,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.03272664546966553,
+ "rewards/margins": 0.0008557910332456231,
+ "rewards/rejected": 0.031870849430561066,
+ "step": 290
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 1.2557185888290405,
+ "learning_rate": 7.973333333333333e-07,
+ "logits/chosen": 1.5062446594238281,
+ "logits/rejected": 1.5786281824111938,
+ "logps/chosen": -153.63558959960938,
+ "logps/rejected": -165.46617126464844,
+ "loss": 0.6897892951965332,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03914078325033188,
+ "rewards/margins": 0.006873926613479853,
+ "rewards/rejected": 0.032266855239868164,
+ "step": 300
+ },
+ {
+ "epoch": 0.24,
+ "eval_logits/chosen": 1.5237834453582764,
+ "eval_logits/rejected": 1.557741641998291,
+ "eval_logps/chosen": -153.19883728027344,
+ "eval_logps/rejected": -147.7100830078125,
+ "eval_loss": 0.6913501024246216,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.03499976545572281,
+ "eval_rewards/margins": 0.0037925743963569403,
+ "eval_rewards/rejected": 0.031207194551825523,
+ "eval_runtime": 90.9133,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 300
+ },
+ {
+ "epoch": 0.248,
+ "grad_norm": 1.1200144290924072,
+ "learning_rate": 8.24e-07,
+ "logits/chosen": 1.618949294090271,
+ "logits/rejected": 1.6625702381134033,
+ "logps/chosen": -141.8953857421875,
+ "logps/rejected": -156.6927032470703,
+ "loss": 0.694425106048584,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.03162423521280289,
+ "rewards/margins": -0.0023833750747144222,
+ "rewards/rejected": 0.034007612615823746,
+ "step": 310
+ },
+ {
+ "epoch": 0.256,
+ "grad_norm": 1.0823743343353271,
+ "learning_rate": 8.506666666666667e-07,
+ "logits/chosen": 1.5974153280258179,
+ "logits/rejected": 1.6237560510635376,
+ "logps/chosen": -139.2996368408203,
+ "logps/rejected": -141.94407653808594,
+ "loss": 0.6914949893951416,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.029445698484778404,
+ "rewards/margins": 0.0034329916816204786,
+ "rewards/rejected": 0.026012707501649857,
+ "step": 320
+ },
+ {
+ "epoch": 0.264,
+ "grad_norm": 1.1765722036361694,
+ "learning_rate": 8.773333333333332e-07,
+ "logits/chosen": 1.465215802192688,
+ "logits/rejected": 1.4882014989852905,
+ "logps/chosen": -158.4329833984375,
+ "logps/rejected": -183.94259643554688,
+ "loss": 0.6925934791564942,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.026302698999643326,
+ "rewards/margins": 0.0012350418837741017,
+ "rewards/rejected": 0.025067657232284546,
+ "step": 330
+ },
+ {
+ "epoch": 0.272,
+ "grad_norm": 1.0667294263839722,
+ "learning_rate": 9.039999999999999e-07,
+ "logits/chosen": 1.7538875341415405,
+ "logits/rejected": 1.6171789169311523,
+ "logps/chosen": -159.63790893554688,
+ "logps/rejected": -151.1880340576172,
+ "loss": 0.6922041416168213,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.026619985699653625,
+ "rewards/margins": 0.002031774492934346,
+ "rewards/rejected": 0.024588212370872498,
+ "step": 340
+ },
+ {
+ "epoch": 0.28,
+ "grad_norm": 1.2210006713867188,
+ "learning_rate": 9.306666666666666e-07,
+ "logits/chosen": 1.5815479755401611,
+ "logits/rejected": 1.710217833518982,
+ "logps/chosen": -167.21743774414062,
+ "logps/rejected": -196.77989196777344,
+ "loss": 0.6899692058563233,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.03636014088988304,
+ "rewards/margins": 0.006536121014505625,
+ "rewards/rejected": 0.029824022203683853,
+ "step": 350
+ },
+ {
+ "epoch": 0.288,
+ "grad_norm": 0.9623845219612122,
+ "learning_rate": 9.573333333333333e-07,
+ "logits/chosen": 1.4416790008544922,
+ "logits/rejected": 1.3618693351745605,
+ "logps/chosen": -163.27401733398438,
+ "logps/rejected": -169.3351287841797,
+ "loss": 0.6910821437835694,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.037200987339019775,
+ "rewards/margins": 0.004338678438216448,
+ "rewards/rejected": 0.032862309366464615,
+ "step": 360
+ },
+ {
+ "epoch": 0.296,
+ "grad_norm": 1.1005645990371704,
+ "learning_rate": 9.84e-07,
+ "logits/chosen": 1.6214616298675537,
+ "logits/rejected": 1.6572606563568115,
+ "logps/chosen": -162.95504760742188,
+ "logps/rejected": -154.91807556152344,
+ "loss": 0.6895030021667481,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.047198958694934845,
+ "rewards/margins": 0.0075958347879350185,
+ "rewards/rejected": 0.03960312157869339,
+ "step": 370
+ },
+ {
+ "epoch": 0.304,
+ "grad_norm": 1.1126338243484497,
+ "learning_rate": 9.988148148148148e-07,
+ "logits/chosen": 1.4842126369476318,
+ "logits/rejected": 1.410930871963501,
+ "logps/chosen": -143.09182739257812,
+ "logps/rejected": -153.48135375976562,
+ "loss": 0.6885370254516602,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.05975018069148064,
+ "rewards/margins": 0.00981281790882349,
+ "rewards/rejected": 0.04993735998868942,
+ "step": 380
+ },
+ {
+ "epoch": 0.312,
+ "grad_norm": 1.1146304607391357,
+ "learning_rate": 9.95851851851852e-07,
+ "logits/chosen": 1.608332633972168,
+ "logits/rejected": 1.5848472118377686,
+ "logps/chosen": -152.25250244140625,
+ "logps/rejected": -138.348876953125,
+ "loss": 0.6851807117462159,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.07630724459886551,
+ "rewards/margins": 0.016460279002785683,
+ "rewards/rejected": 0.059846967458724976,
+ "step": 390
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 1.2686601877212524,
+ "learning_rate": 9.928888888888889e-07,
+ "logits/chosen": 1.5840070247650146,
+ "logits/rejected": 1.5679785013198853,
+ "logps/chosen": -155.90219116210938,
+ "logps/rejected": -148.89549255371094,
+ "loss": 0.6891039371490478,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.08129279315471649,
+ "rewards/margins": 0.008997146971523762,
+ "rewards/rejected": 0.072295643389225,
+ "step": 400
+ },
+ {
+ "epoch": 0.32,
+ "eval_logits/chosen": 1.5358340740203857,
+ "eval_logits/rejected": 1.5689741373062134,
+ "eval_logps/chosen": -152.6895294189453,
+ "eval_logps/rejected": -147.23422241210938,
+ "eval_loss": 0.6900544166564941,
+ "eval_rewards/accuracies": 0.5419999957084656,
+ "eval_rewards/chosen": 0.08593228459358215,
+ "eval_rewards/margins": 0.0071397931315004826,
+ "eval_rewards/rejected": 0.0787924975156784,
+ "eval_runtime": 90.9072,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 400
+ },
+ {
+ "epoch": 0.328,
+ "grad_norm": 1.1653108596801758,
+ "learning_rate": 9.899259259259258e-07,
+ "logits/chosen": 1.5450490713119507,
+ "logits/rejected": 1.4428151845932007,
+ "logps/chosen": -148.61581420898438,
+ "logps/rejected": -188.90919494628906,
+ "loss": 0.6918989181518554,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0946170911192894,
+ "rewards/margins": 0.003271549940109253,
+ "rewards/rejected": 0.09134554117918015,
+ "step": 410
+ },
+ {
+ "epoch": 0.336,
+ "grad_norm": 1.3114937543869019,
+ "learning_rate": 9.86962962962963e-07,
+ "logits/chosen": 1.5342715978622437,
+ "logits/rejected": 1.5572118759155273,
+ "logps/chosen": -152.65264892578125,
+ "logps/rejected": -168.03384399414062,
+ "loss": 0.6972955703735352,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.08869597315788269,
+ "rewards/margins": -0.007436127867549658,
+ "rewards/rejected": 0.09613210707902908,
+ "step": 420
+ },
+ {
+ "epoch": 0.344,
+ "grad_norm": 1.0040781497955322,
+ "learning_rate": 9.84e-07,
+ "logits/chosen": 1.5979712009429932,
+ "logits/rejected": 1.6713664531707764,
+ "logps/chosen": -164.2566375732422,
+ "logps/rejected": -152.1428680419922,
+ "loss": 0.6897010326385498,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.08101107180118561,
+ "rewards/margins": 0.007684691343456507,
+ "rewards/rejected": 0.07332637906074524,
+ "step": 430
+ },
+ {
+ "epoch": 0.352,
+ "grad_norm": 1.0801892280578613,
+ "learning_rate": 9.81037037037037e-07,
+ "logits/chosen": 1.439635992050171,
+ "logits/rejected": 1.3282365798950195,
+ "logps/chosen": -157.23403930664062,
+ "logps/rejected": -146.54806518554688,
+ "loss": 0.6881664752960205,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.08112544566392899,
+ "rewards/margins": 0.010971959680318832,
+ "rewards/rejected": 0.07015348225831985,
+ "step": 440
+ },
+ {
+ "epoch": 0.36,
+ "grad_norm": 1.1139949560165405,
+ "learning_rate": 9.78074074074074e-07,
+ "logits/chosen": 1.6345123052597046,
+ "logits/rejected": 1.5876491069793701,
+ "logps/chosen": -157.6160125732422,
+ "logps/rejected": -160.02040100097656,
+ "loss": 0.6868602275848389,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.07638200372457504,
+ "rewards/margins": 0.013464884832501411,
+ "rewards/rejected": 0.06291711330413818,
+ "step": 450
+ },
+ {
+ "epoch": 0.368,
+ "grad_norm": 1.158035397529602,
+ "learning_rate": 9.751111111111112e-07,
+ "logits/chosen": 1.5430586338043213,
+ "logits/rejected": 1.6022913455963135,
+ "logps/chosen": -151.98919677734375,
+ "logps/rejected": -164.90565490722656,
+ "loss": 0.6891930580139161,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06889279186725616,
+ "rewards/margins": 0.008791089057922363,
+ "rewards/rejected": 0.0601017065346241,
+ "step": 460
+ },
+ {
+ "epoch": 0.376,
+ "grad_norm": 1.0903704166412354,
+ "learning_rate": 9.721481481481481e-07,
+ "logits/chosen": 1.313881754875183,
+ "logits/rejected": 1.3182194232940674,
+ "logps/chosen": -131.5592041015625,
+ "logps/rejected": -123.84442138671875,
+ "loss": 0.6889429569244385,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.08150745928287506,
+ "rewards/margins": 0.009235726669430733,
+ "rewards/rejected": 0.07227173447608948,
+ "step": 470
+ },
+ {
+ "epoch": 0.384,
+ "grad_norm": 1.0766969919204712,
+ "learning_rate": 9.69185185185185e-07,
+ "logits/chosen": 1.425756812095642,
+ "logits/rejected": 1.476854920387268,
+ "logps/chosen": -148.72279357910156,
+ "logps/rejected": -164.40951538085938,
+ "loss": 0.6854116439819335,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1009502187371254,
+ "rewards/margins": 0.0166789498180151,
+ "rewards/rejected": 0.08427127450704575,
+ "step": 480
+ },
+ {
+ "epoch": 0.392,
+ "grad_norm": 1.2078129053115845,
+ "learning_rate": 9.662222222222222e-07,
+ "logits/chosen": 1.5511282682418823,
+ "logits/rejected": 1.611383080482483,
+ "logps/chosen": -143.22006225585938,
+ "logps/rejected": -159.70176696777344,
+ "loss": 0.6968960762023926,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.08779363334178925,
+ "rewards/margins": -0.00598606513813138,
+ "rewards/rejected": 0.09377971291542053,
+ "step": 490
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.9106306433677673,
+ "learning_rate": 9.632592592592593e-07,
+ "logits/chosen": 1.4217239618301392,
+ "logits/rejected": 1.505446195602417,
+ "logps/chosen": -144.34109497070312,
+ "logps/rejected": -135.52430725097656,
+ "loss": 0.691196870803833,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06902458518743515,
+ "rewards/margins": 0.005059319548308849,
+ "rewards/rejected": 0.06396526843309402,
+ "step": 500
+ },
+ {
+ "epoch": 0.4,
+ "eval_logits/chosen": 1.5185325145721436,
+ "eval_logits/rejected": 1.5510345697402954,
+ "eval_logps/chosen": -152.7921905517578,
+ "eval_logps/rejected": -147.3463134765625,
+ "eval_loss": 0.6898502111434937,
+ "eval_rewards/accuracies": 0.5099999904632568,
+ "eval_rewards/chosen": 0.07566512376070023,
+ "eval_rewards/margins": 0.008083080872893333,
+ "eval_rewards/rejected": 0.06758204847574234,
+ "eval_runtime": 90.8294,
+ "eval_samples_per_second": 5.505,
+ "eval_steps_per_second": 2.752,
+ "step": 500
+ },
+ {
+ "epoch": 0.408,
+ "grad_norm": 1.1637762784957886,
+ "learning_rate": 9.602962962962962e-07,
+ "logits/chosen": 1.5965330600738525,
+ "logits/rejected": 1.5338464975357056,
+ "logps/chosen": -131.49871826171875,
+ "logps/rejected": -140.05995178222656,
+ "loss": 0.6868171215057373,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.08385131508111954,
+ "rewards/margins": 0.013950209133327007,
+ "rewards/rejected": 0.06990110129117966,
+ "step": 510
+ },
+ {
+ "epoch": 0.416,
+ "grad_norm": 1.229719638824463,
+ "learning_rate": 9.573333333333333e-07,
+ "logits/chosen": 1.7057275772094727,
+ "logits/rejected": 1.5898948907852173,
+ "logps/chosen": -166.4267120361328,
+ "logps/rejected": -172.22337341308594,
+ "loss": 0.6912094116210937,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.07427279651165009,
+ "rewards/margins": 0.004661654122173786,
+ "rewards/rejected": 0.06961113959550858,
+ "step": 520
+ },
+ {
+ "epoch": 0.424,
+ "grad_norm": 1.136383295059204,
+ "learning_rate": 9.543703703703705e-07,
+ "logits/chosen": 1.6207077503204346,
+ "logits/rejected": 1.5417520999908447,
+ "logps/chosen": -166.81509399414062,
+ "logps/rejected": -168.5408172607422,
+ "loss": 0.6772748470306397,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.0953373983502388,
+ "rewards/margins": 0.033697597682476044,
+ "rewards/rejected": 0.06163979694247246,
+ "step": 530
+ },
+ {
+ "epoch": 0.432,
+ "grad_norm": 1.0640541315078735,
+ "learning_rate": 9.514074074074074e-07,
+ "logits/chosen": 1.6825854778289795,
+ "logits/rejected": 1.6327041387557983,
+ "logps/chosen": -181.39956665039062,
+ "logps/rejected": -158.49647521972656,
+ "loss": 0.6883615493774414,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.10187923908233643,
+ "rewards/margins": 0.010873330757021904,
+ "rewards/rejected": 0.09100590646266937,
+ "step": 540
+ },
+ {
+ "epoch": 0.44,
+ "grad_norm": 1.477909803390503,
+ "learning_rate": 9.484444444444444e-07,
+ "logits/chosen": 1.5402686595916748,
+ "logits/rejected": 1.6487388610839844,
+ "logps/chosen": -151.53814697265625,
+ "logps/rejected": -167.8573455810547,
+ "loss": 0.6915213584899902,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09472338110208511,
+ "rewards/margins": 0.005475065670907497,
+ "rewards/rejected": 0.08924831449985504,
+ "step": 550
+ },
+ {
+ "epoch": 0.448,
+ "grad_norm": 1.0138317346572876,
+ "learning_rate": 9.454814814814814e-07,
+ "logits/chosen": 1.6286503076553345,
+ "logits/rejected": 1.5223101377487183,
+ "logps/chosen": -154.72079467773438,
+ "logps/rejected": -139.2535858154297,
+ "loss": 0.6835216522216797,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.09194488823413849,
+ "rewards/margins": 0.02140027843415737,
+ "rewards/rejected": 0.07054460793733597,
+ "step": 560
+ },
+ {
+ "epoch": 0.456,
+ "grad_norm": 0.896689772605896,
+ "learning_rate": 9.425185185185184e-07,
+ "logits/chosen": 1.4857146739959717,
+ "logits/rejected": 1.4369819164276123,
+ "logps/chosen": -158.31809997558594,
+ "logps/rejected": -152.33676147460938,
+ "loss": 0.6863360404968262,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.07895292341709137,
+ "rewards/margins": 0.016260787844657898,
+ "rewards/rejected": 0.06269213557243347,
+ "step": 570
+ },
+ {
+ "epoch": 0.464,
+ "grad_norm": 1.4772330522537231,
+ "learning_rate": 9.395555555555556e-07,
+ "logits/chosen": 1.315298318862915,
+ "logits/rejected": 1.4757534265518188,
+ "logps/chosen": -136.4202117919922,
+ "logps/rejected": -154.99362182617188,
+ "loss": 0.6916167259216308,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.04677204787731171,
+ "rewards/margins": 0.005649724509567022,
+ "rewards/rejected": 0.041122324764728546,
+ "step": 580
+ },
+ {
+ "epoch": 0.472,
+ "grad_norm": 0.9806157350540161,
+ "learning_rate": 9.365925925925926e-07,
+ "logits/chosen": 1.465427279472351,
+ "logits/rejected": 1.459496259689331,
+ "logps/chosen": -145.07470703125,
+ "logps/rejected": -131.71974182128906,
+ "loss": 0.6857181549072265,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.07979898899793625,
+ "rewards/margins": 0.016965944319963455,
+ "rewards/rejected": 0.0628330409526825,
+ "step": 590
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 1.4313396215438843,
+ "learning_rate": 9.336296296296295e-07,
+ "logits/chosen": 1.5190367698669434,
+ "logits/rejected": 1.563504934310913,
+ "logps/chosen": -132.0201416015625,
+ "logps/rejected": -158.89865112304688,
+ "loss": 0.6964848041534424,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.06544725596904755,
+ "rewards/margins": -0.004280097782611847,
+ "rewards/rejected": 0.0697273463010788,
+ "step": 600
+ },
+ {
+ "epoch": 0.48,
+ "eval_logits/chosen": 1.5031698942184448,
+ "eval_logits/rejected": 1.5348584651947021,
+ "eval_logps/chosen": -152.76596069335938,
+ "eval_logps/rejected": -147.3241729736328,
+ "eval_loss": 0.6903045773506165,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": 0.07828804850578308,
+ "eval_rewards/margins": 0.008492168970406055,
+ "eval_rewards/rejected": 0.0697958841919899,
+ "eval_runtime": 90.9881,
+ "eval_samples_per_second": 5.495,
+ "eval_steps_per_second": 2.748,
+ "step": 600
+ },
+ {
+ "epoch": 0.488,
+ "grad_norm": 0.9467533230781555,
+ "learning_rate": 9.306666666666666e-07,
+ "logits/chosen": 1.5242283344268799,
+ "logits/rejected": 1.5798314809799194,
+ "logps/chosen": -131.90048217773438,
+ "logps/rejected": -149.93222045898438,
+ "loss": 0.6899854183197022,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.08419154584407806,
+ "rewards/margins": 0.00830511748790741,
+ "rewards/rejected": 0.07588644325733185,
+ "step": 610
+ },
+ {
+ "epoch": 0.496,
+ "grad_norm": 1.8463492393493652,
+ "learning_rate": 9.277037037037037e-07,
+ "logits/chosen": 1.4965155124664307,
+ "logits/rejected": 1.6336272954940796,
+ "logps/chosen": -156.1863555908203,
+ "logps/rejected": -167.74075317382812,
+ "loss": 0.6977802276611328,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.05518193915486336,
+ "rewards/margins": -0.006808985956013203,
+ "rewards/rejected": 0.061990927904844284,
+ "step": 620
+ },
+ {
+ "epoch": 0.504,
+ "grad_norm": 1.278316617012024,
+ "learning_rate": 9.247407407407407e-07,
+ "logits/chosen": 1.5074065923690796,
+ "logits/rejected": 1.535696029663086,
+ "logps/chosen": -167.56777954101562,
+ "logps/rejected": -147.04922485351562,
+ "loss": 0.6806219100952149,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.0745314508676529,
+ "rewards/margins": 0.027104835957288742,
+ "rewards/rejected": 0.047426607459783554,
+ "step": 630
+ },
+ {
+ "epoch": 0.512,
+ "grad_norm": 1.0424679517745972,
+ "learning_rate": 9.217777777777778e-07,
+ "logits/chosen": 1.4299286603927612,
+ "logits/rejected": 1.4484076499938965,
+ "logps/chosen": -168.305908203125,
+ "logps/rejected": -152.61854553222656,
+ "loss": 0.6871783256530761,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07751411944627762,
+ "rewards/margins": 0.017045259475708008,
+ "rewards/rejected": 0.06046885997056961,
+ "step": 640
+ },
+ {
+ "epoch": 0.52,
+ "grad_norm": 1.1143999099731445,
+ "learning_rate": 9.188148148148148e-07,
+ "logits/chosen": 1.5520436763763428,
+ "logits/rejected": 1.531718134880066,
+ "logps/chosen": -154.92115783691406,
+ "logps/rejected": -126.78369140625,
+ "loss": 0.6889286518096924,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.0945608839392662,
+ "rewards/margins": 0.01080345083028078,
+ "rewards/rejected": 0.0837574377655983,
+ "step": 650
+ },
+ {
+ "epoch": 0.528,
+ "grad_norm": 1.2762326002120972,
+ "learning_rate": 9.158518518518517e-07,
+ "logits/chosen": 1.5747522115707397,
+ "logits/rejected": 1.4972833395004272,
+ "logps/chosen": -151.13914489746094,
+ "logps/rejected": -132.013916015625,
+ "loss": 0.6900454998016358,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.07303085923194885,
+ "rewards/margins": 0.008073865436017513,
+ "rewards/rejected": 0.06495700031518936,
+ "step": 660
+ },
+ {
+ "epoch": 0.536,
+ "grad_norm": 1.5207557678222656,
+ "learning_rate": 9.128888888888888e-07,
+ "logits/chosen": 1.6594665050506592,
+ "logits/rejected": 1.468492031097412,
+ "logps/chosen": -166.0108184814453,
+ "logps/rejected": -151.38555908203125,
+ "loss": 0.6934059143066407,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.09294265508651733,
+ "rewards/margins": 0.0021180796902626753,
+ "rewards/rejected": 0.09082455933094025,
+ "step": 670
+ },
+ {
+ "epoch": 0.544,
+ "grad_norm": 1.1073335409164429,
+ "learning_rate": 9.099259259259259e-07,
+ "logits/chosen": 1.5321934223175049,
+ "logits/rejected": 1.5313541889190674,
+ "logps/chosen": -144.5046844482422,
+ "logps/rejected": -133.05470275878906,
+ "loss": 0.6824594974517822,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.10782947391271591,
+ "rewards/margins": 0.0230915155261755,
+ "rewards/rejected": 0.08473796397447586,
+ "step": 680
+ },
+ {
+ "epoch": 0.552,
+ "grad_norm": 1.096970558166504,
+ "learning_rate": 9.069629629629629e-07,
+ "logits/chosen": 1.438305139541626,
+ "logits/rejected": 1.4338308572769165,
+ "logps/chosen": -173.1280517578125,
+ "logps/rejected": -160.20950317382812,
+ "loss": 0.6878445625305176,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09615986794233322,
+ "rewards/margins": 0.012350928969681263,
+ "rewards/rejected": 0.08380892872810364,
+ "step": 690
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 1.3664878606796265,
+ "learning_rate": 9.039999999999999e-07,
+ "logits/chosen": 1.530369758605957,
+ "logits/rejected": 1.5520622730255127,
+ "logps/chosen": -159.60740661621094,
+ "logps/rejected": -137.77352905273438,
+ "loss": 0.6991429805755616,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07249583303928375,
+ "rewards/margins": -0.009333941154181957,
+ "rewards/rejected": 0.08182977139949799,
+ "step": 700
+ },
+ {
+ "epoch": 0.56,
+ "eval_logits/chosen": 1.5136796236038208,
+ "eval_logits/rejected": 1.5456002950668335,
+ "eval_logps/chosen": -152.5748748779297,
+ "eval_logps/rejected": -147.17152404785156,
+ "eval_loss": 0.688511848449707,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.09739598631858826,
+ "eval_rewards/margins": 0.012335418723523617,
+ "eval_rewards/rejected": 0.08506056666374207,
+ "eval_runtime": 90.8734,
+ "eval_samples_per_second": 5.502,
+ "eval_steps_per_second": 2.751,
+ "step": 700
+ },
+ {
+ "epoch": 0.568,
+ "grad_norm": 1.224107265472412,
+ "learning_rate": 9.010370370370371e-07,
+ "logits/chosen": 1.6023967266082764,
+ "logits/rejected": 1.558224081993103,
+ "logps/chosen": -148.47120666503906,
+ "logps/rejected": -163.5648651123047,
+ "loss": 0.6865221500396729,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.12494877725839615,
+ "rewards/margins": 0.01545787788927555,
+ "rewards/rejected": 0.10949089378118515,
+ "step": 710
+ },
+ {
+ "epoch": 0.576,
+ "grad_norm": 1.2135164737701416,
+ "learning_rate": 8.98074074074074e-07,
+ "logits/chosen": 1.5902750492095947,
+ "logits/rejected": 1.7012746334075928,
+ "logps/chosen": -139.7012939453125,
+ "logps/rejected": -135.42225646972656,
+ "loss": 0.7014039039611817,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.09102506935596466,
+ "rewards/margins": -0.014695463702082634,
+ "rewards/rejected": 0.10572052001953125,
+ "step": 720
+ },
+ {
+ "epoch": 0.584,
+ "grad_norm": 1.1953649520874023,
+ "learning_rate": 8.95111111111111e-07,
+ "logits/chosen": 1.6540731191635132,
+ "logits/rejected": 1.6386349201202393,
+ "logps/chosen": -153.12847900390625,
+ "logps/rejected": -149.66319274902344,
+ "loss": 0.6952126502990723,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.1124584898352623,
+ "rewards/margins": -0.0012063594767823815,
+ "rewards/rejected": 0.11366486549377441,
+ "step": 730
+ },
+ {
+ "epoch": 0.592,
+ "grad_norm": 1.147875428199768,
+ "learning_rate": 8.921481481481481e-07,
+ "logits/chosen": 1.5578330755233765,
+ "logits/rejected": 1.4689290523529053,
+ "logps/chosen": -156.93788146972656,
+ "logps/rejected": -134.1832733154297,
+ "loss": 0.6919830799102783,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09214536100625992,
+ "rewards/margins": 0.004291228950023651,
+ "rewards/rejected": 0.08785412460565567,
+ "step": 740
+ },
+ {
+ "epoch": 0.6,
+ "grad_norm": 1.21685791015625,
+ "learning_rate": 8.891851851851852e-07,
+ "logits/chosen": 1.4241983890533447,
+ "logits/rejected": 1.4537718296051025,
+ "logps/chosen": -136.9163055419922,
+ "logps/rejected": -158.44491577148438,
+ "loss": 0.691422986984253,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.09173480421304703,
+ "rewards/margins": 0.00533579895272851,
+ "rewards/rejected": 0.08639900386333466,
+ "step": 750
+ },
+ {
+ "epoch": 0.608,
+ "grad_norm": 1.3629519939422607,
+ "learning_rate": 8.862222222222222e-07,
+ "logits/chosen": 1.565716028213501,
+ "logits/rejected": 1.3824570178985596,
+ "logps/chosen": -138.8521728515625,
+ "logps/rejected": -129.51028442382812,
+ "loss": 0.6931031227111817,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0763697624206543,
+ "rewards/margins": 0.0021251302678138018,
+ "rewards/rejected": 0.07424463331699371,
+ "step": 760
+ },
+ {
+ "epoch": 0.616,
+ "grad_norm": 1.1030747890472412,
+ "learning_rate": 8.832592592592593e-07,
+ "logits/chosen": 1.5781702995300293,
+ "logits/rejected": 1.5428775548934937,
+ "logps/chosen": -141.01480102539062,
+ "logps/rejected": -136.47189331054688,
+ "loss": 0.6777657032012939,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.10366280376911163,
+ "rewards/margins": 0.0333605632185936,
+ "rewards/rejected": 0.07030224055051804,
+ "step": 770
+ },
+ {
+ "epoch": 0.624,
+ "grad_norm": 1.3228758573532104,
+ "learning_rate": 8.802962962962962e-07,
+ "logits/chosen": 1.451596736907959,
+ "logits/rejected": 1.5489904880523682,
+ "logps/chosen": -153.74618530273438,
+ "logps/rejected": -159.26333618164062,
+ "loss": 0.6870471000671386,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.10403706133365631,
+ "rewards/margins": 0.014494165778160095,
+ "rewards/rejected": 0.08954289555549622,
+ "step": 780
+ },
+ {
+ "epoch": 0.632,
+ "grad_norm": 1.3267385959625244,
+ "learning_rate": 8.773333333333332e-07,
+ "logits/chosen": 1.4153831005096436,
+ "logits/rejected": 1.4497296810150146,
+ "logps/chosen": -148.05368041992188,
+ "logps/rejected": -151.3020477294922,
+ "loss": 0.6877872943878174,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.07300253212451935,
+ "rewards/margins": 0.013981563039124012,
+ "rewards/rejected": 0.05902096629142761,
+ "step": 790
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 1.2227483987808228,
+ "learning_rate": 8.743703703703703e-07,
+ "logits/chosen": 1.6773914098739624,
+ "logits/rejected": 1.6918518543243408,
+ "logps/chosen": -180.06863403320312,
+ "logps/rejected": -194.30752563476562,
+ "loss": 0.7015917778015137,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07480724155902863,
+ "rewards/margins": -0.013628068380057812,
+ "rewards/rejected": 0.08843531459569931,
+ "step": 800
+ },
+ {
+ "epoch": 0.64,
+ "eval_logits/chosen": 1.5051140785217285,
+ "eval_logits/rejected": 1.5367870330810547,
+ "eval_logps/chosen": -152.8026885986328,
+ "eval_logps/rejected": -147.39437866210938,
+ "eval_loss": 0.688633918762207,
+ "eval_rewards/accuracies": 0.5220000147819519,
+ "eval_rewards/chosen": 0.07461711764335632,
+ "eval_rewards/margins": 0.011839868500828743,
+ "eval_rewards/rejected": 0.06277725100517273,
+ "eval_runtime": 90.9206,
+ "eval_samples_per_second": 5.499,
+ "eval_steps_per_second": 2.75,
+ "step": 800
+ },
+ {
+ "epoch": 0.648,
+ "grad_norm": 1.209770679473877,
+ "learning_rate": 8.714074074074074e-07,
+ "logits/chosen": 1.3785041570663452,
+ "logits/rejected": 1.60085928440094,
+ "logps/chosen": -114.5325927734375,
+ "logps/rejected": -151.88900756835938,
+ "loss": 0.6890413761138916,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0640551745891571,
+ "rewards/margins": 0.010126596316695213,
+ "rewards/rejected": 0.05392857640981674,
+ "step": 810
+ },
+ {
+ "epoch": 0.656,
+ "grad_norm": 1.0250163078308105,
+ "learning_rate": 8.684444444444444e-07,
+ "logits/chosen": 1.5827982425689697,
+ "logits/rejected": 1.6239618062973022,
+ "logps/chosen": -149.01904296875,
+ "logps/rejected": -141.9372100830078,
+ "loss": 0.6728014469146728,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.0830039232969284,
+ "rewards/margins": 0.043090131133794785,
+ "rewards/rejected": 0.03991378843784332,
+ "step": 820
+ },
+ {
+ "epoch": 0.664,
+ "grad_norm": 1.0627655982971191,
+ "learning_rate": 8.654814814814814e-07,
+ "logits/chosen": 1.501873254776001,
+ "logits/rejected": 1.537386417388916,
+ "logps/chosen": -138.75169372558594,
+ "logps/rejected": -134.00970458984375,
+ "loss": 0.6801187992095947,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0640982910990715,
+ "rewards/margins": 0.02855527400970459,
+ "rewards/rejected": 0.03554301708936691,
+ "step": 830
+ },
+ {
+ "epoch": 0.672,
+ "grad_norm": 1.2453433275222778,
+ "learning_rate": 8.625185185185186e-07,
+ "logits/chosen": 1.4766005277633667,
+ "logits/rejected": 1.5185787677764893,
+ "logps/chosen": -148.41250610351562,
+ "logps/rejected": -157.73989868164062,
+ "loss": 0.6788079738616943,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.06551454961299896,
+ "rewards/margins": 0.03301671892404556,
+ "rewards/rejected": 0.0324978306889534,
+ "step": 840
+ },
+ {
+ "epoch": 0.68,
+ "grad_norm": 1.191698670387268,
+ "learning_rate": 8.595555555555555e-07,
+ "logits/chosen": 1.555551528930664,
+ "logits/rejected": 1.5438499450683594,
+ "logps/chosen": -160.77488708496094,
+ "logps/rejected": -141.9876251220703,
+ "loss": 0.6801597595214843,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.053498052060604095,
+ "rewards/margins": 0.028470590710639954,
+ "rewards/rejected": 0.025027472525835037,
+ "step": 850
+ },
+ {
+ "epoch": 0.688,
+ "grad_norm": 0.9382092356681824,
+ "learning_rate": 8.565925925925925e-07,
+ "logits/chosen": 1.5482738018035889,
+ "logits/rejected": 1.5833700895309448,
+ "logps/chosen": -139.68118286132812,
+ "logps/rejected": -130.05947875976562,
+ "loss": 0.6825921535491943,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.06382572650909424,
+ "rewards/margins": 0.023111844435334206,
+ "rewards/rejected": 0.04071388021111488,
+ "step": 860
+ },
+ {
+ "epoch": 0.696,
+ "grad_norm": 1.2248382568359375,
+ "learning_rate": 8.536296296296296e-07,
+ "logits/chosen": 1.4529926776885986,
+ "logits/rejected": 1.4423902034759521,
+ "logps/chosen": -154.4588165283203,
+ "logps/rejected": -153.44949340820312,
+ "loss": 0.6915592193603516,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.03207828477025032,
+ "rewards/margins": 0.006785523146390915,
+ "rewards/rejected": 0.025292763486504555,
+ "step": 870
+ },
+ {
+ "epoch": 0.704,
+ "grad_norm": 1.2174893617630005,
+ "learning_rate": 8.506666666666667e-07,
+ "logits/chosen": 1.4073113203048706,
+ "logits/rejected": 1.668140172958374,
+ "logps/chosen": -145.18460083007812,
+ "logps/rejected": -157.0777587890625,
+ "loss": 0.6922356605529785,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.049450717866420746,
+ "rewards/margins": 0.004049716051667929,
+ "rewards/rejected": 0.04540099948644638,
+ "step": 880
+ },
+ {
+ "epoch": 0.712,
+ "grad_norm": 1.3186850547790527,
+ "learning_rate": 8.477037037037037e-07,
+ "logits/chosen": 1.2923892736434937,
+ "logits/rejected": 1.491051435470581,
+ "logps/chosen": -149.03018188476562,
+ "logps/rejected": -161.61715698242188,
+ "loss": 0.6841150760650635,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.038961172103881836,
+ "rewards/margins": 0.020759906619787216,
+ "rewards/rejected": 0.01820126175880432,
+ "step": 890
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 1.1123124361038208,
+ "learning_rate": 8.447407407407407e-07,
+ "logits/chosen": 1.546547293663025,
+ "logits/rejected": 1.682992696762085,
+ "logps/chosen": -166.2056884765625,
+ "logps/rejected": -180.73330688476562,
+ "loss": 0.6826215744018554,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.07279330492019653,
+ "rewards/margins": 0.025264525786042213,
+ "rewards/rejected": 0.04752878472208977,
+ "step": 900
+ },
+ {
+ "epoch": 0.72,
+ "eval_logits/chosen": 1.4819762706756592,
+ "eval_logits/rejected": 1.513027548789978,
+ "eval_logps/chosen": -153.1695556640625,
+ "eval_logps/rejected": -147.7560577392578,
+ "eval_loss": 0.689373254776001,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": 0.03792775049805641,
+ "eval_rewards/margins": 0.011320074088871479,
+ "eval_rewards/rejected": 0.026607677340507507,
+ "eval_runtime": 90.9603,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 900
+ },
+ {
+ "epoch": 0.728,
+ "grad_norm": 1.1507582664489746,
+ "learning_rate": 8.417777777777777e-07,
+ "logits/chosen": 1.5831308364868164,
+ "logits/rejected": 1.3737132549285889,
+ "logps/chosen": -140.00253295898438,
+ "logps/rejected": -138.45155334472656,
+ "loss": 0.6873391628265381,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.05066479369997978,
+ "rewards/margins": 0.015466553159058094,
+ "rewards/rejected": 0.03519824147224426,
+ "step": 910
+ },
+ {
+ "epoch": 0.736,
+ "grad_norm": 1.415456771850586,
+ "learning_rate": 8.388148148148147e-07,
+ "logits/chosen": 1.3000686168670654,
+ "logits/rejected": 1.2642180919647217,
+ "logps/chosen": -160.14027404785156,
+ "logps/rejected": -163.32827758789062,
+ "loss": 0.687142276763916,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.06618930399417877,
+ "rewards/margins": 0.014807088300585747,
+ "rewards/rejected": 0.05138222128152847,
+ "step": 920
+ },
+ {
+ "epoch": 0.744,
+ "grad_norm": 1.3228634595870972,
+ "learning_rate": 8.358518518518518e-07,
+ "logits/chosen": 1.5204170942306519,
+ "logits/rejected": 1.6129217147827148,
+ "logps/chosen": -157.99832153320312,
+ "logps/rejected": -162.09567260742188,
+ "loss": 0.6790886878967285,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.05940447002649307,
+ "rewards/margins": 0.03161252290010452,
+ "rewards/rejected": 0.027791941538453102,
+ "step": 930
+ },
+ {
+ "epoch": 0.752,
+ "grad_norm": 1.048064947128296,
+ "learning_rate": 8.328888888888889e-07,
+ "logits/chosen": 1.5515320301055908,
+ "logits/rejected": 1.676857352256775,
+ "logps/chosen": -148.4853515625,
+ "logps/rejected": -174.4320526123047,
+ "loss": 0.6935453414916992,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.010225490666925907,
+ "rewards/margins": 0.0021693313028663397,
+ "rewards/rejected": 0.008056161925196648,
+ "step": 940
+ },
+ {
+ "epoch": 0.76,
+ "grad_norm": 1.247799038887024,
+ "learning_rate": 8.299259259259259e-07,
+ "logits/chosen": 1.384231686592102,
+ "logits/rejected": 1.4611561298370361,
+ "logps/chosen": -140.81443786621094,
+ "logps/rejected": -153.81326293945312,
+ "loss": 0.6805226325988769,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.053223587572574615,
+ "rewards/margins": 0.02933613583445549,
+ "rewards/rejected": 0.023887457326054573,
+ "step": 950
+ },
+ {
+ "epoch": 0.768,
+ "grad_norm": 1.1611264944076538,
+ "learning_rate": 8.269629629629629e-07,
+ "logits/chosen": 1.4987528324127197,
+ "logits/rejected": 1.3684321641921997,
+ "logps/chosen": -135.81056213378906,
+ "logps/rejected": -119.05133056640625,
+ "loss": 0.6954316616058349,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.0197703056037426,
+ "rewards/margins": -0.0012637012405321002,
+ "rewards/rejected": 0.021034007892012596,
+ "step": 960
+ },
+ {
+ "epoch": 0.776,
+ "grad_norm": 0.964948832988739,
+ "learning_rate": 8.24e-07,
+ "logits/chosen": 1.612053632736206,
+ "logits/rejected": 1.7114235162734985,
+ "logps/chosen": -153.56137084960938,
+ "logps/rejected": -149.17323303222656,
+ "loss": 0.6916662693023682,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.05082285404205322,
+ "rewards/margins": 0.0056606316938996315,
+ "rewards/rejected": 0.04516221210360527,
+ "step": 970
+ },
+ {
+ "epoch": 0.784,
+ "grad_norm": 1.409449815750122,
+ "learning_rate": 8.21037037037037e-07,
+ "logits/chosen": 1.384060025215149,
+ "logits/rejected": 1.316361904144287,
+ "logps/chosen": -168.75360107421875,
+ "logps/rejected": -141.79537963867188,
+ "loss": 0.6845529556274415,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.0886571854352951,
+ "rewards/margins": 0.02084263041615486,
+ "rewards/rejected": 0.06781454384326935,
+ "step": 980
+ },
+ {
+ "epoch": 0.792,
+ "grad_norm": 1.0246562957763672,
+ "learning_rate": 8.18074074074074e-07,
+ "logits/chosen": 1.4986507892608643,
+ "logits/rejected": 1.571103811264038,
+ "logps/chosen": -165.22254943847656,
+ "logps/rejected": -168.63540649414062,
+ "loss": 0.6790967464447022,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.07517905533313751,
+ "rewards/margins": 0.03272219002246857,
+ "rewards/rejected": 0.04245685786008835,
+ "step": 990
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 1.014426827430725,
+ "learning_rate": 8.15111111111111e-07,
+ "logits/chosen": 1.4708755016326904,
+ "logits/rejected": 1.4550955295562744,
+ "logps/chosen": -149.49874877929688,
+ "logps/rejected": -125.0966567993164,
+ "loss": 0.6892098903656005,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.07825140655040741,
+ "rewards/margins": 0.01007872074842453,
+ "rewards/rejected": 0.06817268580198288,
+ "step": 1000
+ },
+ {
+ "epoch": 0.8,
+ "eval_logits/chosen": 1.4972777366638184,
+ "eval_logits/rejected": 1.5283576250076294,
+ "eval_logps/chosen": -152.8240966796875,
+ "eval_logps/rejected": -147.44175720214844,
+ "eval_loss": 0.6880542039871216,
+ "eval_rewards/accuracies": 0.5419999957084656,
+ "eval_rewards/chosen": 0.07247376441955566,
+ "eval_rewards/margins": 0.01443479023873806,
+ "eval_rewards/rejected": 0.058038972318172455,
+ "eval_runtime": 90.827,
+ "eval_samples_per_second": 5.505,
+ "eval_steps_per_second": 2.752,
+ "step": 1000
+ }
+ ],
+ "logging_steps": 10,
+ "max_steps": 3750,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1000/training_args.bin b/v3/DPO/DPO_10k/lora/checkpoint-1000/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..b3fc3d76093802087869ab1fee5e37b9c830fe77
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4f90e4a7053b7fb440d68cb64a3d198396af09db942d08839c5f4f0f0b8a0c8
+size 6097
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1500/README.md b/v3/DPO/DPO_10k/lora/checkpoint-1500/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..250500f798b4682b2cd2c35cd1fc366677c215dd
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1500/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1500/adapter_config.json b/v3/DPO/DPO_10k/lora/checkpoint-1500/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..039869f8d79ca6c48a793a29a79267da6d8f81c2
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1500/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "gate_proj",
+ "down_proj",
+ "k_proj",
+ "o_proj",
+ "up_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1500/adapter_model.safetensors b/v3/DPO/DPO_10k/lora/checkpoint-1500/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1b406b77bafe0414dc9cfbf2103be8732b5d8bcb
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1500/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7afd4ea14d3390375f16e1a10b8fcca1ede2eb7959bce12efb565ba4b2e00236
+size 180385008
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1500/chat_template.jinja b/v3/DPO/DPO_10k/lora/checkpoint-1500/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1500/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1500/optimizer.pt b/v3/DPO/DPO_10k/lora/checkpoint-1500/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..64d6283d76cb927ba75f705069d68b56ebce5b0f
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1500/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e76e51dcca53476f7d03879d08ea48c310a515073dcada2891911d9d2d8bca3
+size 360902475
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1500/rng_state.pth b/v3/DPO/DPO_10k/lora/checkpoint-1500/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..7cd34506332c2c2b22e1158344d58b303b6ffc2a
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1500/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b928d2d8033ac6bd87c58a39b741faace8bd1c6b0d070b7fad23c19520ff9f1a
+size 14645
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1500/scaler.pt b/v3/DPO/DPO_10k/lora/checkpoint-1500/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3664bb212147e2acc5cde00bad4860996ad25fe2
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1500/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca372268f4fa9335030c0cb7aedb6cdba75f457da50e7a4034abb1a2d0843689
+size 1383
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1500/scheduler.pt b/v3/DPO/DPO_10k/lora/checkpoint-1500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..82bc213b41ea37ad8da03fba6756d8c1661dd57b
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e10622725c891492a233a860ff051a91cd7997b7a70e593a4bc44e88f0ef3b5
+size 1465
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1500/tokenizer.json b/v3/DPO/DPO_10k/lora/checkpoint-1500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1500/tokenizer_config.json b/v3/DPO/DPO_10k/lora/checkpoint-1500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1500/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1500/trainer_state.json b/v3/DPO/DPO_10k/lora/checkpoint-1500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..91b7f55ced9152eaf74e347c036a1acae3166511
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1500/trainer_state.json
@@ -0,0 +1,2524 @@
+{
+ "best_global_step": 1400,
+ "best_metric": 0.6853657364845276,
+ "best_model_checkpoint": "output/lora/checkpoint-1000",
+ "epoch": 1.2,
+ "eval_steps": 100,
+ "global_step": 1500,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.008,
+ "grad_norm": 1.1179773807525635,
+ "learning_rate": 2.4e-08,
+ "logits/chosen": 1.4545083045959473,
+ "logits/rejected": 1.425490379333496,
+ "logps/chosen": -132.46189880371094,
+ "logps/rejected": -148.20260620117188,
+ "loss": 0.6930626392364502,
+ "rewards/accuracies": 0.21250000596046448,
+ "rewards/chosen": -6.165739614516497e-05,
+ "rewards/margins": 0.00017333509458694607,
+ "rewards/rejected": -0.00023499250528402627,
+ "step": 10
+ },
+ {
+ "epoch": 0.016,
+ "grad_norm": 1.3502839803695679,
+ "learning_rate": 5.0666666666666664e-08,
+ "logits/chosen": 1.5668801069259644,
+ "logits/rejected": 1.5191389322280884,
+ "logps/chosen": -157.63624572753906,
+ "logps/rejected": -144.2716827392578,
+ "loss": 0.6930380821228027,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.0009244013344869018,
+ "rewards/margins": 0.00023505210992880166,
+ "rewards/rejected": 0.0006893490790389478,
+ "step": 20
+ },
+ {
+ "epoch": 0.024,
+ "grad_norm": 1.0219553709030151,
+ "learning_rate": 7.733333333333334e-08,
+ "logits/chosen": 1.8921797275543213,
+ "logits/rejected": 1.7342685461044312,
+ "logps/chosen": -164.55799865722656,
+ "logps/rejected": -158.92752075195312,
+ "loss": 0.69290189743042,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0009391927160322666,
+ "rewards/margins": 0.0005032324115745723,
+ "rewards/rejected": 0.0004359603044576943,
+ "step": 30
+ },
+ {
+ "epoch": 0.032,
+ "grad_norm": 1.2056113481521606,
+ "learning_rate": 1.0399999999999999e-07,
+ "logits/chosen": 1.4950047731399536,
+ "logits/rejected": 1.6163270473480225,
+ "logps/chosen": -145.54183959960938,
+ "logps/rejected": -134.19979858398438,
+ "loss": 0.6935500621795654,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.0006421065190806985,
+ "rewards/margins": -0.0007943272357806563,
+ "rewards/rejected": 0.0014364338712766767,
+ "step": 40
+ },
+ {
+ "epoch": 0.04,
+ "grad_norm": 1.1029725074768066,
+ "learning_rate": 1.3066666666666665e-07,
+ "logits/chosen": 1.4076533317565918,
+ "logits/rejected": 1.4791977405548096,
+ "logps/chosen": -133.23324584960938,
+ "logps/rejected": -138.3593292236328,
+ "loss": 0.6925833702087403,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.0012372838100418448,
+ "rewards/margins": 0.001139522879384458,
+ "rewards/rejected": 9.776116348803043e-05,
+ "step": 50
+ },
+ {
+ "epoch": 0.048,
+ "grad_norm": 0.9960266351699829,
+ "learning_rate": 1.573333333333333e-07,
+ "logits/chosen": 1.407166600227356,
+ "logits/rejected": 1.376042127609253,
+ "logps/chosen": -137.8921661376953,
+ "logps/rejected": -127.97320556640625,
+ "loss": 0.693614149093628,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.00021316049969755113,
+ "rewards/margins": -0.0009249996510334313,
+ "rewards/rejected": 0.0007118391804397106,
+ "step": 60
+ },
+ {
+ "epoch": 0.056,
+ "grad_norm": 1.0946824550628662,
+ "learning_rate": 1.8399999999999998e-07,
+ "logits/chosen": 1.5944501161575317,
+ "logits/rejected": 1.7248146533966064,
+ "logps/chosen": -153.5526123046875,
+ "logps/rejected": -157.19735717773438,
+ "loss": 0.6928449153900147,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0026370862033218145,
+ "rewards/margins": 0.0006150150438770652,
+ "rewards/rejected": 0.0020220710430294275,
+ "step": 70
+ },
+ {
+ "epoch": 0.064,
+ "grad_norm": 1.0322489738464355,
+ "learning_rate": 2.1066666666666665e-07,
+ "logits/chosen": 1.4586353302001953,
+ "logits/rejected": 1.4842995405197144,
+ "logps/chosen": -149.03665161132812,
+ "logps/rejected": -136.24081420898438,
+ "loss": 0.6932015895843506,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.004061593674123287,
+ "rewards/margins": -9.99594121822156e-05,
+ "rewards/rejected": 0.004161553457379341,
+ "step": 80
+ },
+ {
+ "epoch": 0.072,
+ "grad_norm": 1.0012447834014893,
+ "learning_rate": 2.3733333333333334e-07,
+ "logits/chosen": 1.5075902938842773,
+ "logits/rejected": 1.5986944437026978,
+ "logps/chosen": -159.3433074951172,
+ "logps/rejected": -151.85072326660156,
+ "loss": 0.6928757190704345,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.006978762801736593,
+ "rewards/margins": 0.0005627512582577765,
+ "rewards/rejected": 0.006416010670363903,
+ "step": 90
+ },
+ {
+ "epoch": 0.08,
+ "grad_norm": 1.280081033706665,
+ "learning_rate": 2.64e-07,
+ "logits/chosen": 1.448286533355713,
+ "logits/rejected": 1.5480695962905884,
+ "logps/chosen": -151.12625122070312,
+ "logps/rejected": -171.24545288085938,
+ "loss": 0.6931126117706299,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.00693341251462698,
+ "rewards/margins": 9.057526767719537e-05,
+ "rewards/rejected": 0.006842837668955326,
+ "step": 100
+ },
+ {
+ "epoch": 0.08,
+ "eval_logits/chosen": 1.518183946609497,
+ "eval_logits/rejected": 1.5525447130203247,
+ "eval_logps/chosen": -153.46630859375,
+ "eval_logps/rejected": -147.95005798339844,
+ "eval_loss": 0.6926330327987671,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.008253541775047779,
+ "eval_rewards/margins": 0.0010449704714119434,
+ "eval_rewards/rejected": 0.007208569906651974,
+ "eval_runtime": 91.0812,
+ "eval_samples_per_second": 5.49,
+ "eval_steps_per_second": 2.745,
+ "step": 100
+ },
+ {
+ "epoch": 0.088,
+ "grad_norm": 1.3912373781204224,
+ "learning_rate": 2.906666666666667e-07,
+ "logits/chosen": 1.6896222829818726,
+ "logits/rejected": 1.6516921520233154,
+ "logps/chosen": -173.70030212402344,
+ "logps/rejected": -155.5503692626953,
+ "loss": 0.6926521778106689,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.010179834440350533,
+ "rewards/margins": 0.0010137532372027636,
+ "rewards/rejected": 0.009166081435978413,
+ "step": 110
+ },
+ {
+ "epoch": 0.096,
+ "grad_norm": 1.1518888473510742,
+ "learning_rate": 3.173333333333333e-07,
+ "logits/chosen": 1.5166860818862915,
+ "logits/rejected": 1.5136979818344116,
+ "logps/chosen": -136.39906311035156,
+ "logps/rejected": -140.86561584472656,
+ "loss": 0.6932918548583984,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": 0.00907944981008768,
+ "rewards/margins": -0.00026998287648893893,
+ "rewards/rejected": 0.009349432773888111,
+ "step": 120
+ },
+ {
+ "epoch": 0.104,
+ "grad_norm": 1.3213376998901367,
+ "learning_rate": 3.4399999999999996e-07,
+ "logits/chosen": 1.4598973989486694,
+ "logits/rejected": 1.6739647388458252,
+ "logps/chosen": -159.12461853027344,
+ "logps/rejected": -163.39346313476562,
+ "loss": 0.693086051940918,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.010477779433131218,
+ "rewards/margins": 0.00014539004769176245,
+ "rewards/rejected": 0.010332388803362846,
+ "step": 130
+ },
+ {
+ "epoch": 0.112,
+ "grad_norm": 0.8833863735198975,
+ "learning_rate": 3.7066666666666665e-07,
+ "logits/chosen": 1.3829412460327148,
+ "logits/rejected": 1.4100173711776733,
+ "logps/chosen": -132.7149658203125,
+ "logps/rejected": -126.85478210449219,
+ "loss": 0.6932382583618164,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": 0.010262183845043182,
+ "rewards/margins": -0.0001672578218858689,
+ "rewards/rejected": 0.010429441928863525,
+ "step": 140
+ },
+ {
+ "epoch": 0.12,
+ "grad_norm": 1.1472455263137817,
+ "learning_rate": 3.973333333333333e-07,
+ "logits/chosen": 1.4954869747161865,
+ "logits/rejected": 1.5857340097427368,
+ "logps/chosen": -165.10568237304688,
+ "logps/rejected": -158.97433471679688,
+ "loss": 0.6916103363037109,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.014177520759403706,
+ "rewards/margins": 0.003108317730948329,
+ "rewards/rejected": 0.011069201864302158,
+ "step": 150
+ },
+ {
+ "epoch": 0.128,
+ "grad_norm": 1.0628798007965088,
+ "learning_rate": 4.24e-07,
+ "logits/chosen": 1.6717243194580078,
+ "logits/rejected": 1.705092191696167,
+ "logps/chosen": -159.332275390625,
+ "logps/rejected": -147.21646118164062,
+ "loss": 0.6935629844665527,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": 0.016994481906294823,
+ "rewards/margins": -0.0007874655420891941,
+ "rewards/rejected": 0.017781946808099747,
+ "step": 160
+ },
+ {
+ "epoch": 0.136,
+ "grad_norm": 1.390810251235962,
+ "learning_rate": 4.506666666666666e-07,
+ "logits/chosen": 1.4719234704971313,
+ "logits/rejected": 1.499526023864746,
+ "logps/chosen": -148.67251586914062,
+ "logps/rejected": -161.58021545410156,
+ "loss": 0.6926599979400635,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.021239640191197395,
+ "rewards/margins": 0.0010346340714022517,
+ "rewards/rejected": 0.020205006003379822,
+ "step": 170
+ },
+ {
+ "epoch": 0.144,
+ "grad_norm": 1.139707326889038,
+ "learning_rate": 4.773333333333333e-07,
+ "logits/chosen": 1.4833905696868896,
+ "logits/rejected": 1.637584924697876,
+ "logps/chosen": -158.96731567382812,
+ "logps/rejected": -161.9830322265625,
+ "loss": 0.6919547557830811,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.019416773691773415,
+ "rewards/margins": 0.0024296878837049007,
+ "rewards/rejected": 0.016987085342407227,
+ "step": 180
+ },
+ {
+ "epoch": 0.152,
+ "grad_norm": 1.2029796838760376,
+ "learning_rate": 5.04e-07,
+ "logits/chosen": 1.5398027896881104,
+ "logits/rejected": 1.6678705215454102,
+ "logps/chosen": -143.00283813476562,
+ "logps/rejected": -144.89378356933594,
+ "loss": 0.692262840270996,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.018960200250148773,
+ "rewards/margins": 0.0018175147706642747,
+ "rewards/rejected": 0.017142686992883682,
+ "step": 190
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 1.0527437925338745,
+ "learning_rate": 5.306666666666665e-07,
+ "logits/chosen": 1.6087623834609985,
+ "logits/rejected": 1.5181543827056885,
+ "logps/chosen": -159.35073852539062,
+ "logps/rejected": -133.59890747070312,
+ "loss": 0.6924069881439209,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.01926913857460022,
+ "rewards/margins": 0.0015339398523792624,
+ "rewards/rejected": 0.0177351962774992,
+ "step": 200
+ },
+ {
+ "epoch": 0.16,
+ "eval_logits/chosen": 1.5193672180175781,
+ "eval_logits/rejected": 1.5534639358520508,
+ "eval_logps/chosen": -153.34857177734375,
+ "eval_logps/rejected": -147.8425750732422,
+ "eval_loss": 0.6921458840370178,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.020027177408337593,
+ "eval_rewards/margins": 0.0020714527927339077,
+ "eval_rewards/rejected": 0.017955724149942398,
+ "eval_runtime": 90.9762,
+ "eval_samples_per_second": 5.496,
+ "eval_steps_per_second": 2.748,
+ "step": 200
+ },
+ {
+ "epoch": 0.168,
+ "grad_norm": 1.1897239685058594,
+ "learning_rate": 5.573333333333333e-07,
+ "logits/chosen": 1.6267732381820679,
+ "logits/rejected": 1.5775268077850342,
+ "logps/chosen": -159.51437377929688,
+ "logps/rejected": -170.17762756347656,
+ "loss": 0.6925770759582519,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.02078000269830227,
+ "rewards/margins": 0.0012053751852363348,
+ "rewards/rejected": 0.01957462914288044,
+ "step": 210
+ },
+ {
+ "epoch": 0.176,
+ "grad_norm": 0.9995236396789551,
+ "learning_rate": 5.839999999999999e-07,
+ "logits/chosen": 1.5781934261322021,
+ "logits/rejected": 1.642251968383789,
+ "logps/chosen": -159.95309448242188,
+ "logps/rejected": -144.36058044433594,
+ "loss": 0.6931475639343262,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.023322442546486855,
+ "rewards/margins": 6.392716750269756e-05,
+ "rewards/rejected": 0.02325851283967495,
+ "step": 220
+ },
+ {
+ "epoch": 0.184,
+ "grad_norm": 0.8819260597229004,
+ "learning_rate": 6.106666666666666e-07,
+ "logits/chosen": 1.459991693496704,
+ "logits/rejected": 1.6060739755630493,
+ "logps/chosen": -162.04745483398438,
+ "logps/rejected": -139.5411834716797,
+ "loss": 0.6919463634490967,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.026052182540297508,
+ "rewards/margins": 0.002481849165633321,
+ "rewards/rejected": 0.023570330813527107,
+ "step": 230
+ },
+ {
+ "epoch": 0.192,
+ "grad_norm": 1.1055208444595337,
+ "learning_rate": 6.373333333333333e-07,
+ "logits/chosen": 1.558739185333252,
+ "logits/rejected": 1.5965116024017334,
+ "logps/chosen": -171.9166259765625,
+ "logps/rejected": -145.66433715820312,
+ "loss": 0.6912176132202148,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.033868782222270966,
+ "rewards/margins": 0.004017929546535015,
+ "rewards/rejected": 0.029850851744413376,
+ "step": 240
+ },
+ {
+ "epoch": 0.2,
+ "grad_norm": 1.1372733116149902,
+ "learning_rate": 6.64e-07,
+ "logits/chosen": 1.5322860479354858,
+ "logits/rejected": 1.4695795774459839,
+ "logps/chosen": -161.7803955078125,
+ "logps/rejected": -148.91949462890625,
+ "loss": 0.6940969944000244,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.034065864980220795,
+ "rewards/margins": -0.0016761379083618522,
+ "rewards/rejected": 0.035742007195949554,
+ "step": 250
+ },
+ {
+ "epoch": 0.208,
+ "grad_norm": 1.0917680263519287,
+ "learning_rate": 6.906666666666666e-07,
+ "logits/chosen": 1.4094547033309937,
+ "logits/rejected": 1.50186026096344,
+ "logps/chosen": -132.91455078125,
+ "logps/rejected": -146.79714965820312,
+ "loss": 0.6930203914642334,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.03089061938226223,
+ "rewards/margins": 0.00040971505222842097,
+ "rewards/rejected": 0.030480902642011642,
+ "step": 260
+ },
+ {
+ "epoch": 0.216,
+ "grad_norm": 1.0280259847640991,
+ "learning_rate": 7.173333333333333e-07,
+ "logits/chosen": 1.531285285949707,
+ "logits/rejected": 1.6163132190704346,
+ "logps/chosen": -146.902587890625,
+ "logps/rejected": -147.8347625732422,
+ "loss": 0.6944749355316162,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.028514739125967026,
+ "rewards/margins": -0.0025452517438679934,
+ "rewards/rejected": 0.0310599897056818,
+ "step": 270
+ },
+ {
+ "epoch": 0.224,
+ "grad_norm": 1.398708701133728,
+ "learning_rate": 7.44e-07,
+ "logits/chosen": 1.563852071762085,
+ "logits/rejected": 1.396703839302063,
+ "logps/chosen": -172.68153381347656,
+ "logps/rejected": -155.80215454101562,
+ "loss": 0.689476490020752,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03224216774106026,
+ "rewards/margins": 0.007473687641322613,
+ "rewards/rejected": 0.02476847730576992,
+ "step": 280
+ },
+ {
+ "epoch": 0.232,
+ "grad_norm": 1.0394141674041748,
+ "learning_rate": 7.706666666666667e-07,
+ "logits/chosen": 1.5581729412078857,
+ "logits/rejected": 1.5819021463394165,
+ "logps/chosen": -144.41465759277344,
+ "logps/rejected": -153.55406188964844,
+ "loss": 0.6927877902984619,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.03272664546966553,
+ "rewards/margins": 0.0008557910332456231,
+ "rewards/rejected": 0.031870849430561066,
+ "step": 290
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 1.2557185888290405,
+ "learning_rate": 7.973333333333333e-07,
+ "logits/chosen": 1.5062446594238281,
+ "logits/rejected": 1.5786281824111938,
+ "logps/chosen": -153.63558959960938,
+ "logps/rejected": -165.46617126464844,
+ "loss": 0.6897892951965332,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03914078325033188,
+ "rewards/margins": 0.006873926613479853,
+ "rewards/rejected": 0.032266855239868164,
+ "step": 300
+ },
+ {
+ "epoch": 0.24,
+ "eval_logits/chosen": 1.5237834453582764,
+ "eval_logits/rejected": 1.557741641998291,
+ "eval_logps/chosen": -153.19883728027344,
+ "eval_logps/rejected": -147.7100830078125,
+ "eval_loss": 0.6913501024246216,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.03499976545572281,
+ "eval_rewards/margins": 0.0037925743963569403,
+ "eval_rewards/rejected": 0.031207194551825523,
+ "eval_runtime": 90.9133,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 300
+ },
+ {
+ "epoch": 0.248,
+ "grad_norm": 1.1200144290924072,
+ "learning_rate": 8.24e-07,
+ "logits/chosen": 1.618949294090271,
+ "logits/rejected": 1.6625702381134033,
+ "logps/chosen": -141.8953857421875,
+ "logps/rejected": -156.6927032470703,
+ "loss": 0.694425106048584,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.03162423521280289,
+ "rewards/margins": -0.0023833750747144222,
+ "rewards/rejected": 0.034007612615823746,
+ "step": 310
+ },
+ {
+ "epoch": 0.256,
+ "grad_norm": 1.0823743343353271,
+ "learning_rate": 8.506666666666667e-07,
+ "logits/chosen": 1.5974153280258179,
+ "logits/rejected": 1.6237560510635376,
+ "logps/chosen": -139.2996368408203,
+ "logps/rejected": -141.94407653808594,
+ "loss": 0.6914949893951416,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.029445698484778404,
+ "rewards/margins": 0.0034329916816204786,
+ "rewards/rejected": 0.026012707501649857,
+ "step": 320
+ },
+ {
+ "epoch": 0.264,
+ "grad_norm": 1.1765722036361694,
+ "learning_rate": 8.773333333333332e-07,
+ "logits/chosen": 1.465215802192688,
+ "logits/rejected": 1.4882014989852905,
+ "logps/chosen": -158.4329833984375,
+ "logps/rejected": -183.94259643554688,
+ "loss": 0.6925934791564942,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.026302698999643326,
+ "rewards/margins": 0.0012350418837741017,
+ "rewards/rejected": 0.025067657232284546,
+ "step": 330
+ },
+ {
+ "epoch": 0.272,
+ "grad_norm": 1.0667294263839722,
+ "learning_rate": 9.039999999999999e-07,
+ "logits/chosen": 1.7538875341415405,
+ "logits/rejected": 1.6171789169311523,
+ "logps/chosen": -159.63790893554688,
+ "logps/rejected": -151.1880340576172,
+ "loss": 0.6922041416168213,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.026619985699653625,
+ "rewards/margins": 0.002031774492934346,
+ "rewards/rejected": 0.024588212370872498,
+ "step": 340
+ },
+ {
+ "epoch": 0.28,
+ "grad_norm": 1.2210006713867188,
+ "learning_rate": 9.306666666666666e-07,
+ "logits/chosen": 1.5815479755401611,
+ "logits/rejected": 1.710217833518982,
+ "logps/chosen": -167.21743774414062,
+ "logps/rejected": -196.77989196777344,
+ "loss": 0.6899692058563233,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.03636014088988304,
+ "rewards/margins": 0.006536121014505625,
+ "rewards/rejected": 0.029824022203683853,
+ "step": 350
+ },
+ {
+ "epoch": 0.288,
+ "grad_norm": 0.9623845219612122,
+ "learning_rate": 9.573333333333333e-07,
+ "logits/chosen": 1.4416790008544922,
+ "logits/rejected": 1.3618693351745605,
+ "logps/chosen": -163.27401733398438,
+ "logps/rejected": -169.3351287841797,
+ "loss": 0.6910821437835694,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.037200987339019775,
+ "rewards/margins": 0.004338678438216448,
+ "rewards/rejected": 0.032862309366464615,
+ "step": 360
+ },
+ {
+ "epoch": 0.296,
+ "grad_norm": 1.1005645990371704,
+ "learning_rate": 9.84e-07,
+ "logits/chosen": 1.6214616298675537,
+ "logits/rejected": 1.6572606563568115,
+ "logps/chosen": -162.95504760742188,
+ "logps/rejected": -154.91807556152344,
+ "loss": 0.6895030021667481,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.047198958694934845,
+ "rewards/margins": 0.0075958347879350185,
+ "rewards/rejected": 0.03960312157869339,
+ "step": 370
+ },
+ {
+ "epoch": 0.304,
+ "grad_norm": 1.1126338243484497,
+ "learning_rate": 9.988148148148148e-07,
+ "logits/chosen": 1.4842126369476318,
+ "logits/rejected": 1.410930871963501,
+ "logps/chosen": -143.09182739257812,
+ "logps/rejected": -153.48135375976562,
+ "loss": 0.6885370254516602,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.05975018069148064,
+ "rewards/margins": 0.00981281790882349,
+ "rewards/rejected": 0.04993735998868942,
+ "step": 380
+ },
+ {
+ "epoch": 0.312,
+ "grad_norm": 1.1146304607391357,
+ "learning_rate": 9.95851851851852e-07,
+ "logits/chosen": 1.608332633972168,
+ "logits/rejected": 1.5848472118377686,
+ "logps/chosen": -152.25250244140625,
+ "logps/rejected": -138.348876953125,
+ "loss": 0.6851807117462159,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.07630724459886551,
+ "rewards/margins": 0.016460279002785683,
+ "rewards/rejected": 0.059846967458724976,
+ "step": 390
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 1.2686601877212524,
+ "learning_rate": 9.928888888888889e-07,
+ "logits/chosen": 1.5840070247650146,
+ "logits/rejected": 1.5679785013198853,
+ "logps/chosen": -155.90219116210938,
+ "logps/rejected": -148.89549255371094,
+ "loss": 0.6891039371490478,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.08129279315471649,
+ "rewards/margins": 0.008997146971523762,
+ "rewards/rejected": 0.072295643389225,
+ "step": 400
+ },
+ {
+ "epoch": 0.32,
+ "eval_logits/chosen": 1.5358340740203857,
+ "eval_logits/rejected": 1.5689741373062134,
+ "eval_logps/chosen": -152.6895294189453,
+ "eval_logps/rejected": -147.23422241210938,
+ "eval_loss": 0.6900544166564941,
+ "eval_rewards/accuracies": 0.5419999957084656,
+ "eval_rewards/chosen": 0.08593228459358215,
+ "eval_rewards/margins": 0.0071397931315004826,
+ "eval_rewards/rejected": 0.0787924975156784,
+ "eval_runtime": 90.9072,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 400
+ },
+ {
+ "epoch": 0.328,
+ "grad_norm": 1.1653108596801758,
+ "learning_rate": 9.899259259259258e-07,
+ "logits/chosen": 1.5450490713119507,
+ "logits/rejected": 1.4428151845932007,
+ "logps/chosen": -148.61581420898438,
+ "logps/rejected": -188.90919494628906,
+ "loss": 0.6918989181518554,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0946170911192894,
+ "rewards/margins": 0.003271549940109253,
+ "rewards/rejected": 0.09134554117918015,
+ "step": 410
+ },
+ {
+ "epoch": 0.336,
+ "grad_norm": 1.3114937543869019,
+ "learning_rate": 9.86962962962963e-07,
+ "logits/chosen": 1.5342715978622437,
+ "logits/rejected": 1.5572118759155273,
+ "logps/chosen": -152.65264892578125,
+ "logps/rejected": -168.03384399414062,
+ "loss": 0.6972955703735352,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.08869597315788269,
+ "rewards/margins": -0.007436127867549658,
+ "rewards/rejected": 0.09613210707902908,
+ "step": 420
+ },
+ {
+ "epoch": 0.344,
+ "grad_norm": 1.0040781497955322,
+ "learning_rate": 9.84e-07,
+ "logits/chosen": 1.5979712009429932,
+ "logits/rejected": 1.6713664531707764,
+ "logps/chosen": -164.2566375732422,
+ "logps/rejected": -152.1428680419922,
+ "loss": 0.6897010326385498,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.08101107180118561,
+ "rewards/margins": 0.007684691343456507,
+ "rewards/rejected": 0.07332637906074524,
+ "step": 430
+ },
+ {
+ "epoch": 0.352,
+ "grad_norm": 1.0801892280578613,
+ "learning_rate": 9.81037037037037e-07,
+ "logits/chosen": 1.439635992050171,
+ "logits/rejected": 1.3282365798950195,
+ "logps/chosen": -157.23403930664062,
+ "logps/rejected": -146.54806518554688,
+ "loss": 0.6881664752960205,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.08112544566392899,
+ "rewards/margins": 0.010971959680318832,
+ "rewards/rejected": 0.07015348225831985,
+ "step": 440
+ },
+ {
+ "epoch": 0.36,
+ "grad_norm": 1.1139949560165405,
+ "learning_rate": 9.78074074074074e-07,
+ "logits/chosen": 1.6345123052597046,
+ "logits/rejected": 1.5876491069793701,
+ "logps/chosen": -157.6160125732422,
+ "logps/rejected": -160.02040100097656,
+ "loss": 0.6868602275848389,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.07638200372457504,
+ "rewards/margins": 0.013464884832501411,
+ "rewards/rejected": 0.06291711330413818,
+ "step": 450
+ },
+ {
+ "epoch": 0.368,
+ "grad_norm": 1.158035397529602,
+ "learning_rate": 9.751111111111112e-07,
+ "logits/chosen": 1.5430586338043213,
+ "logits/rejected": 1.6022913455963135,
+ "logps/chosen": -151.98919677734375,
+ "logps/rejected": -164.90565490722656,
+ "loss": 0.6891930580139161,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06889279186725616,
+ "rewards/margins": 0.008791089057922363,
+ "rewards/rejected": 0.0601017065346241,
+ "step": 460
+ },
+ {
+ "epoch": 0.376,
+ "grad_norm": 1.0903704166412354,
+ "learning_rate": 9.721481481481481e-07,
+ "logits/chosen": 1.313881754875183,
+ "logits/rejected": 1.3182194232940674,
+ "logps/chosen": -131.5592041015625,
+ "logps/rejected": -123.84442138671875,
+ "loss": 0.6889429569244385,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.08150745928287506,
+ "rewards/margins": 0.009235726669430733,
+ "rewards/rejected": 0.07227173447608948,
+ "step": 470
+ },
+ {
+ "epoch": 0.384,
+ "grad_norm": 1.0766969919204712,
+ "learning_rate": 9.69185185185185e-07,
+ "logits/chosen": 1.425756812095642,
+ "logits/rejected": 1.476854920387268,
+ "logps/chosen": -148.72279357910156,
+ "logps/rejected": -164.40951538085938,
+ "loss": 0.6854116439819335,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1009502187371254,
+ "rewards/margins": 0.0166789498180151,
+ "rewards/rejected": 0.08427127450704575,
+ "step": 480
+ },
+ {
+ "epoch": 0.392,
+ "grad_norm": 1.2078129053115845,
+ "learning_rate": 9.662222222222222e-07,
+ "logits/chosen": 1.5511282682418823,
+ "logits/rejected": 1.611383080482483,
+ "logps/chosen": -143.22006225585938,
+ "logps/rejected": -159.70176696777344,
+ "loss": 0.6968960762023926,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.08779363334178925,
+ "rewards/margins": -0.00598606513813138,
+ "rewards/rejected": 0.09377971291542053,
+ "step": 490
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.9106306433677673,
+ "learning_rate": 9.632592592592593e-07,
+ "logits/chosen": 1.4217239618301392,
+ "logits/rejected": 1.505446195602417,
+ "logps/chosen": -144.34109497070312,
+ "logps/rejected": -135.52430725097656,
+ "loss": 0.691196870803833,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06902458518743515,
+ "rewards/margins": 0.005059319548308849,
+ "rewards/rejected": 0.06396526843309402,
+ "step": 500
+ },
+ {
+ "epoch": 0.4,
+ "eval_logits/chosen": 1.5185325145721436,
+ "eval_logits/rejected": 1.5510345697402954,
+ "eval_logps/chosen": -152.7921905517578,
+ "eval_logps/rejected": -147.3463134765625,
+ "eval_loss": 0.6898502111434937,
+ "eval_rewards/accuracies": 0.5099999904632568,
+ "eval_rewards/chosen": 0.07566512376070023,
+ "eval_rewards/margins": 0.008083080872893333,
+ "eval_rewards/rejected": 0.06758204847574234,
+ "eval_runtime": 90.8294,
+ "eval_samples_per_second": 5.505,
+ "eval_steps_per_second": 2.752,
+ "step": 500
+ },
+ {
+ "epoch": 0.408,
+ "grad_norm": 1.1637762784957886,
+ "learning_rate": 9.602962962962962e-07,
+ "logits/chosen": 1.5965330600738525,
+ "logits/rejected": 1.5338464975357056,
+ "logps/chosen": -131.49871826171875,
+ "logps/rejected": -140.05995178222656,
+ "loss": 0.6868171215057373,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.08385131508111954,
+ "rewards/margins": 0.013950209133327007,
+ "rewards/rejected": 0.06990110129117966,
+ "step": 510
+ },
+ {
+ "epoch": 0.416,
+ "grad_norm": 1.229719638824463,
+ "learning_rate": 9.573333333333333e-07,
+ "logits/chosen": 1.7057275772094727,
+ "logits/rejected": 1.5898948907852173,
+ "logps/chosen": -166.4267120361328,
+ "logps/rejected": -172.22337341308594,
+ "loss": 0.6912094116210937,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.07427279651165009,
+ "rewards/margins": 0.004661654122173786,
+ "rewards/rejected": 0.06961113959550858,
+ "step": 520
+ },
+ {
+ "epoch": 0.424,
+ "grad_norm": 1.136383295059204,
+ "learning_rate": 9.543703703703705e-07,
+ "logits/chosen": 1.6207077503204346,
+ "logits/rejected": 1.5417520999908447,
+ "logps/chosen": -166.81509399414062,
+ "logps/rejected": -168.5408172607422,
+ "loss": 0.6772748470306397,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.0953373983502388,
+ "rewards/margins": 0.033697597682476044,
+ "rewards/rejected": 0.06163979694247246,
+ "step": 530
+ },
+ {
+ "epoch": 0.432,
+ "grad_norm": 1.0640541315078735,
+ "learning_rate": 9.514074074074074e-07,
+ "logits/chosen": 1.6825854778289795,
+ "logits/rejected": 1.6327041387557983,
+ "logps/chosen": -181.39956665039062,
+ "logps/rejected": -158.49647521972656,
+ "loss": 0.6883615493774414,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.10187923908233643,
+ "rewards/margins": 0.010873330757021904,
+ "rewards/rejected": 0.09100590646266937,
+ "step": 540
+ },
+ {
+ "epoch": 0.44,
+ "grad_norm": 1.477909803390503,
+ "learning_rate": 9.484444444444444e-07,
+ "logits/chosen": 1.5402686595916748,
+ "logits/rejected": 1.6487388610839844,
+ "logps/chosen": -151.53814697265625,
+ "logps/rejected": -167.8573455810547,
+ "loss": 0.6915213584899902,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09472338110208511,
+ "rewards/margins": 0.005475065670907497,
+ "rewards/rejected": 0.08924831449985504,
+ "step": 550
+ },
+ {
+ "epoch": 0.448,
+ "grad_norm": 1.0138317346572876,
+ "learning_rate": 9.454814814814814e-07,
+ "logits/chosen": 1.6286503076553345,
+ "logits/rejected": 1.5223101377487183,
+ "logps/chosen": -154.72079467773438,
+ "logps/rejected": -139.2535858154297,
+ "loss": 0.6835216522216797,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.09194488823413849,
+ "rewards/margins": 0.02140027843415737,
+ "rewards/rejected": 0.07054460793733597,
+ "step": 560
+ },
+ {
+ "epoch": 0.456,
+ "grad_norm": 0.896689772605896,
+ "learning_rate": 9.425185185185184e-07,
+ "logits/chosen": 1.4857146739959717,
+ "logits/rejected": 1.4369819164276123,
+ "logps/chosen": -158.31809997558594,
+ "logps/rejected": -152.33676147460938,
+ "loss": 0.6863360404968262,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.07895292341709137,
+ "rewards/margins": 0.016260787844657898,
+ "rewards/rejected": 0.06269213557243347,
+ "step": 570
+ },
+ {
+ "epoch": 0.464,
+ "grad_norm": 1.4772330522537231,
+ "learning_rate": 9.395555555555556e-07,
+ "logits/chosen": 1.315298318862915,
+ "logits/rejected": 1.4757534265518188,
+ "logps/chosen": -136.4202117919922,
+ "logps/rejected": -154.99362182617188,
+ "loss": 0.6916167259216308,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.04677204787731171,
+ "rewards/margins": 0.005649724509567022,
+ "rewards/rejected": 0.041122324764728546,
+ "step": 580
+ },
+ {
+ "epoch": 0.472,
+ "grad_norm": 0.9806157350540161,
+ "learning_rate": 9.365925925925926e-07,
+ "logits/chosen": 1.465427279472351,
+ "logits/rejected": 1.459496259689331,
+ "logps/chosen": -145.07470703125,
+ "logps/rejected": -131.71974182128906,
+ "loss": 0.6857181549072265,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.07979898899793625,
+ "rewards/margins": 0.016965944319963455,
+ "rewards/rejected": 0.0628330409526825,
+ "step": 590
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 1.4313396215438843,
+ "learning_rate": 9.336296296296295e-07,
+ "logits/chosen": 1.5190367698669434,
+ "logits/rejected": 1.563504934310913,
+ "logps/chosen": -132.0201416015625,
+ "logps/rejected": -158.89865112304688,
+ "loss": 0.6964848041534424,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.06544725596904755,
+ "rewards/margins": -0.004280097782611847,
+ "rewards/rejected": 0.0697273463010788,
+ "step": 600
+ },
+ {
+ "epoch": 0.48,
+ "eval_logits/chosen": 1.5031698942184448,
+ "eval_logits/rejected": 1.5348584651947021,
+ "eval_logps/chosen": -152.76596069335938,
+ "eval_logps/rejected": -147.3241729736328,
+ "eval_loss": 0.6903045773506165,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": 0.07828804850578308,
+ "eval_rewards/margins": 0.008492168970406055,
+ "eval_rewards/rejected": 0.0697958841919899,
+ "eval_runtime": 90.9881,
+ "eval_samples_per_second": 5.495,
+ "eval_steps_per_second": 2.748,
+ "step": 600
+ },
+ {
+ "epoch": 0.488,
+ "grad_norm": 0.9467533230781555,
+ "learning_rate": 9.306666666666666e-07,
+ "logits/chosen": 1.5242283344268799,
+ "logits/rejected": 1.5798314809799194,
+ "logps/chosen": -131.90048217773438,
+ "logps/rejected": -149.93222045898438,
+ "loss": 0.6899854183197022,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.08419154584407806,
+ "rewards/margins": 0.00830511748790741,
+ "rewards/rejected": 0.07588644325733185,
+ "step": 610
+ },
+ {
+ "epoch": 0.496,
+ "grad_norm": 1.8463492393493652,
+ "learning_rate": 9.277037037037037e-07,
+ "logits/chosen": 1.4965155124664307,
+ "logits/rejected": 1.6336272954940796,
+ "logps/chosen": -156.1863555908203,
+ "logps/rejected": -167.74075317382812,
+ "loss": 0.6977802276611328,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.05518193915486336,
+ "rewards/margins": -0.006808985956013203,
+ "rewards/rejected": 0.061990927904844284,
+ "step": 620
+ },
+ {
+ "epoch": 0.504,
+ "grad_norm": 1.278316617012024,
+ "learning_rate": 9.247407407407407e-07,
+ "logits/chosen": 1.5074065923690796,
+ "logits/rejected": 1.535696029663086,
+ "logps/chosen": -167.56777954101562,
+ "logps/rejected": -147.04922485351562,
+ "loss": 0.6806219100952149,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.0745314508676529,
+ "rewards/margins": 0.027104835957288742,
+ "rewards/rejected": 0.047426607459783554,
+ "step": 630
+ },
+ {
+ "epoch": 0.512,
+ "grad_norm": 1.0424679517745972,
+ "learning_rate": 9.217777777777778e-07,
+ "logits/chosen": 1.4299286603927612,
+ "logits/rejected": 1.4484076499938965,
+ "logps/chosen": -168.305908203125,
+ "logps/rejected": -152.61854553222656,
+ "loss": 0.6871783256530761,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07751411944627762,
+ "rewards/margins": 0.017045259475708008,
+ "rewards/rejected": 0.06046885997056961,
+ "step": 640
+ },
+ {
+ "epoch": 0.52,
+ "grad_norm": 1.1143999099731445,
+ "learning_rate": 9.188148148148148e-07,
+ "logits/chosen": 1.5520436763763428,
+ "logits/rejected": 1.531718134880066,
+ "logps/chosen": -154.92115783691406,
+ "logps/rejected": -126.78369140625,
+ "loss": 0.6889286518096924,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.0945608839392662,
+ "rewards/margins": 0.01080345083028078,
+ "rewards/rejected": 0.0837574377655983,
+ "step": 650
+ },
+ {
+ "epoch": 0.528,
+ "grad_norm": 1.2762326002120972,
+ "learning_rate": 9.158518518518517e-07,
+ "logits/chosen": 1.5747522115707397,
+ "logits/rejected": 1.4972833395004272,
+ "logps/chosen": -151.13914489746094,
+ "logps/rejected": -132.013916015625,
+ "loss": 0.6900454998016358,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.07303085923194885,
+ "rewards/margins": 0.008073865436017513,
+ "rewards/rejected": 0.06495700031518936,
+ "step": 660
+ },
+ {
+ "epoch": 0.536,
+ "grad_norm": 1.5207557678222656,
+ "learning_rate": 9.128888888888888e-07,
+ "logits/chosen": 1.6594665050506592,
+ "logits/rejected": 1.468492031097412,
+ "logps/chosen": -166.0108184814453,
+ "logps/rejected": -151.38555908203125,
+ "loss": 0.6934059143066407,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.09294265508651733,
+ "rewards/margins": 0.0021180796902626753,
+ "rewards/rejected": 0.09082455933094025,
+ "step": 670
+ },
+ {
+ "epoch": 0.544,
+ "grad_norm": 1.1073335409164429,
+ "learning_rate": 9.099259259259259e-07,
+ "logits/chosen": 1.5321934223175049,
+ "logits/rejected": 1.5313541889190674,
+ "logps/chosen": -144.5046844482422,
+ "logps/rejected": -133.05470275878906,
+ "loss": 0.6824594974517822,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.10782947391271591,
+ "rewards/margins": 0.0230915155261755,
+ "rewards/rejected": 0.08473796397447586,
+ "step": 680
+ },
+ {
+ "epoch": 0.552,
+ "grad_norm": 1.096970558166504,
+ "learning_rate": 9.069629629629629e-07,
+ "logits/chosen": 1.438305139541626,
+ "logits/rejected": 1.4338308572769165,
+ "logps/chosen": -173.1280517578125,
+ "logps/rejected": -160.20950317382812,
+ "loss": 0.6878445625305176,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09615986794233322,
+ "rewards/margins": 0.012350928969681263,
+ "rewards/rejected": 0.08380892872810364,
+ "step": 690
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 1.3664878606796265,
+ "learning_rate": 9.039999999999999e-07,
+ "logits/chosen": 1.530369758605957,
+ "logits/rejected": 1.5520622730255127,
+ "logps/chosen": -159.60740661621094,
+ "logps/rejected": -137.77352905273438,
+ "loss": 0.6991429805755616,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07249583303928375,
+ "rewards/margins": -0.009333941154181957,
+ "rewards/rejected": 0.08182977139949799,
+ "step": 700
+ },
+ {
+ "epoch": 0.56,
+ "eval_logits/chosen": 1.5136796236038208,
+ "eval_logits/rejected": 1.5456002950668335,
+ "eval_logps/chosen": -152.5748748779297,
+ "eval_logps/rejected": -147.17152404785156,
+ "eval_loss": 0.688511848449707,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.09739598631858826,
+ "eval_rewards/margins": 0.012335418723523617,
+ "eval_rewards/rejected": 0.08506056666374207,
+ "eval_runtime": 90.8734,
+ "eval_samples_per_second": 5.502,
+ "eval_steps_per_second": 2.751,
+ "step": 700
+ },
+ {
+ "epoch": 0.568,
+ "grad_norm": 1.224107265472412,
+ "learning_rate": 9.010370370370371e-07,
+ "logits/chosen": 1.6023967266082764,
+ "logits/rejected": 1.558224081993103,
+ "logps/chosen": -148.47120666503906,
+ "logps/rejected": -163.5648651123047,
+ "loss": 0.6865221500396729,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.12494877725839615,
+ "rewards/margins": 0.01545787788927555,
+ "rewards/rejected": 0.10949089378118515,
+ "step": 710
+ },
+ {
+ "epoch": 0.576,
+ "grad_norm": 1.2135164737701416,
+ "learning_rate": 8.98074074074074e-07,
+ "logits/chosen": 1.5902750492095947,
+ "logits/rejected": 1.7012746334075928,
+ "logps/chosen": -139.7012939453125,
+ "logps/rejected": -135.42225646972656,
+ "loss": 0.7014039039611817,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.09102506935596466,
+ "rewards/margins": -0.014695463702082634,
+ "rewards/rejected": 0.10572052001953125,
+ "step": 720
+ },
+ {
+ "epoch": 0.584,
+ "grad_norm": 1.1953649520874023,
+ "learning_rate": 8.95111111111111e-07,
+ "logits/chosen": 1.6540731191635132,
+ "logits/rejected": 1.6386349201202393,
+ "logps/chosen": -153.12847900390625,
+ "logps/rejected": -149.66319274902344,
+ "loss": 0.6952126502990723,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.1124584898352623,
+ "rewards/margins": -0.0012063594767823815,
+ "rewards/rejected": 0.11366486549377441,
+ "step": 730
+ },
+ {
+ "epoch": 0.592,
+ "grad_norm": 1.147875428199768,
+ "learning_rate": 8.921481481481481e-07,
+ "logits/chosen": 1.5578330755233765,
+ "logits/rejected": 1.4689290523529053,
+ "logps/chosen": -156.93788146972656,
+ "logps/rejected": -134.1832733154297,
+ "loss": 0.6919830799102783,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09214536100625992,
+ "rewards/margins": 0.004291228950023651,
+ "rewards/rejected": 0.08785412460565567,
+ "step": 740
+ },
+ {
+ "epoch": 0.6,
+ "grad_norm": 1.21685791015625,
+ "learning_rate": 8.891851851851852e-07,
+ "logits/chosen": 1.4241983890533447,
+ "logits/rejected": 1.4537718296051025,
+ "logps/chosen": -136.9163055419922,
+ "logps/rejected": -158.44491577148438,
+ "loss": 0.691422986984253,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.09173480421304703,
+ "rewards/margins": 0.00533579895272851,
+ "rewards/rejected": 0.08639900386333466,
+ "step": 750
+ },
+ {
+ "epoch": 0.608,
+ "grad_norm": 1.3629519939422607,
+ "learning_rate": 8.862222222222222e-07,
+ "logits/chosen": 1.565716028213501,
+ "logits/rejected": 1.3824570178985596,
+ "logps/chosen": -138.8521728515625,
+ "logps/rejected": -129.51028442382812,
+ "loss": 0.6931031227111817,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0763697624206543,
+ "rewards/margins": 0.0021251302678138018,
+ "rewards/rejected": 0.07424463331699371,
+ "step": 760
+ },
+ {
+ "epoch": 0.616,
+ "grad_norm": 1.1030747890472412,
+ "learning_rate": 8.832592592592593e-07,
+ "logits/chosen": 1.5781702995300293,
+ "logits/rejected": 1.5428775548934937,
+ "logps/chosen": -141.01480102539062,
+ "logps/rejected": -136.47189331054688,
+ "loss": 0.6777657032012939,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.10366280376911163,
+ "rewards/margins": 0.0333605632185936,
+ "rewards/rejected": 0.07030224055051804,
+ "step": 770
+ },
+ {
+ "epoch": 0.624,
+ "grad_norm": 1.3228758573532104,
+ "learning_rate": 8.802962962962962e-07,
+ "logits/chosen": 1.451596736907959,
+ "logits/rejected": 1.5489904880523682,
+ "logps/chosen": -153.74618530273438,
+ "logps/rejected": -159.26333618164062,
+ "loss": 0.6870471000671386,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.10403706133365631,
+ "rewards/margins": 0.014494165778160095,
+ "rewards/rejected": 0.08954289555549622,
+ "step": 780
+ },
+ {
+ "epoch": 0.632,
+ "grad_norm": 1.3267385959625244,
+ "learning_rate": 8.773333333333332e-07,
+ "logits/chosen": 1.4153831005096436,
+ "logits/rejected": 1.4497296810150146,
+ "logps/chosen": -148.05368041992188,
+ "logps/rejected": -151.3020477294922,
+ "loss": 0.6877872943878174,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.07300253212451935,
+ "rewards/margins": 0.013981563039124012,
+ "rewards/rejected": 0.05902096629142761,
+ "step": 790
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 1.2227483987808228,
+ "learning_rate": 8.743703703703703e-07,
+ "logits/chosen": 1.6773914098739624,
+ "logits/rejected": 1.6918518543243408,
+ "logps/chosen": -180.06863403320312,
+ "logps/rejected": -194.30752563476562,
+ "loss": 0.7015917778015137,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07480724155902863,
+ "rewards/margins": -0.013628068380057812,
+ "rewards/rejected": 0.08843531459569931,
+ "step": 800
+ },
+ {
+ "epoch": 0.64,
+ "eval_logits/chosen": 1.5051140785217285,
+ "eval_logits/rejected": 1.5367870330810547,
+ "eval_logps/chosen": -152.8026885986328,
+ "eval_logps/rejected": -147.39437866210938,
+ "eval_loss": 0.688633918762207,
+ "eval_rewards/accuracies": 0.5220000147819519,
+ "eval_rewards/chosen": 0.07461711764335632,
+ "eval_rewards/margins": 0.011839868500828743,
+ "eval_rewards/rejected": 0.06277725100517273,
+ "eval_runtime": 90.9206,
+ "eval_samples_per_second": 5.499,
+ "eval_steps_per_second": 2.75,
+ "step": 800
+ },
+ {
+ "epoch": 0.648,
+ "grad_norm": 1.209770679473877,
+ "learning_rate": 8.714074074074074e-07,
+ "logits/chosen": 1.3785041570663452,
+ "logits/rejected": 1.60085928440094,
+ "logps/chosen": -114.5325927734375,
+ "logps/rejected": -151.88900756835938,
+ "loss": 0.6890413761138916,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0640551745891571,
+ "rewards/margins": 0.010126596316695213,
+ "rewards/rejected": 0.05392857640981674,
+ "step": 810
+ },
+ {
+ "epoch": 0.656,
+ "grad_norm": 1.0250163078308105,
+ "learning_rate": 8.684444444444444e-07,
+ "logits/chosen": 1.5827982425689697,
+ "logits/rejected": 1.6239618062973022,
+ "logps/chosen": -149.01904296875,
+ "logps/rejected": -141.9372100830078,
+ "loss": 0.6728014469146728,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.0830039232969284,
+ "rewards/margins": 0.043090131133794785,
+ "rewards/rejected": 0.03991378843784332,
+ "step": 820
+ },
+ {
+ "epoch": 0.664,
+ "grad_norm": 1.0627655982971191,
+ "learning_rate": 8.654814814814814e-07,
+ "logits/chosen": 1.501873254776001,
+ "logits/rejected": 1.537386417388916,
+ "logps/chosen": -138.75169372558594,
+ "logps/rejected": -134.00970458984375,
+ "loss": 0.6801187992095947,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0640982910990715,
+ "rewards/margins": 0.02855527400970459,
+ "rewards/rejected": 0.03554301708936691,
+ "step": 830
+ },
+ {
+ "epoch": 0.672,
+ "grad_norm": 1.2453433275222778,
+ "learning_rate": 8.625185185185186e-07,
+ "logits/chosen": 1.4766005277633667,
+ "logits/rejected": 1.5185787677764893,
+ "logps/chosen": -148.41250610351562,
+ "logps/rejected": -157.73989868164062,
+ "loss": 0.6788079738616943,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.06551454961299896,
+ "rewards/margins": 0.03301671892404556,
+ "rewards/rejected": 0.0324978306889534,
+ "step": 840
+ },
+ {
+ "epoch": 0.68,
+ "grad_norm": 1.191698670387268,
+ "learning_rate": 8.595555555555555e-07,
+ "logits/chosen": 1.555551528930664,
+ "logits/rejected": 1.5438499450683594,
+ "logps/chosen": -160.77488708496094,
+ "logps/rejected": -141.9876251220703,
+ "loss": 0.6801597595214843,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.053498052060604095,
+ "rewards/margins": 0.028470590710639954,
+ "rewards/rejected": 0.025027472525835037,
+ "step": 850
+ },
+ {
+ "epoch": 0.688,
+ "grad_norm": 0.9382092356681824,
+ "learning_rate": 8.565925925925925e-07,
+ "logits/chosen": 1.5482738018035889,
+ "logits/rejected": 1.5833700895309448,
+ "logps/chosen": -139.68118286132812,
+ "logps/rejected": -130.05947875976562,
+ "loss": 0.6825921535491943,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.06382572650909424,
+ "rewards/margins": 0.023111844435334206,
+ "rewards/rejected": 0.04071388021111488,
+ "step": 860
+ },
+ {
+ "epoch": 0.696,
+ "grad_norm": 1.2248382568359375,
+ "learning_rate": 8.536296296296296e-07,
+ "logits/chosen": 1.4529926776885986,
+ "logits/rejected": 1.4423902034759521,
+ "logps/chosen": -154.4588165283203,
+ "logps/rejected": -153.44949340820312,
+ "loss": 0.6915592193603516,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.03207828477025032,
+ "rewards/margins": 0.006785523146390915,
+ "rewards/rejected": 0.025292763486504555,
+ "step": 870
+ },
+ {
+ "epoch": 0.704,
+ "grad_norm": 1.2174893617630005,
+ "learning_rate": 8.506666666666667e-07,
+ "logits/chosen": 1.4073113203048706,
+ "logits/rejected": 1.668140172958374,
+ "logps/chosen": -145.18460083007812,
+ "logps/rejected": -157.0777587890625,
+ "loss": 0.6922356605529785,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.049450717866420746,
+ "rewards/margins": 0.004049716051667929,
+ "rewards/rejected": 0.04540099948644638,
+ "step": 880
+ },
+ {
+ "epoch": 0.712,
+ "grad_norm": 1.3186850547790527,
+ "learning_rate": 8.477037037037037e-07,
+ "logits/chosen": 1.2923892736434937,
+ "logits/rejected": 1.491051435470581,
+ "logps/chosen": -149.03018188476562,
+ "logps/rejected": -161.61715698242188,
+ "loss": 0.6841150760650635,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.038961172103881836,
+ "rewards/margins": 0.020759906619787216,
+ "rewards/rejected": 0.01820126175880432,
+ "step": 890
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 1.1123124361038208,
+ "learning_rate": 8.447407407407407e-07,
+ "logits/chosen": 1.546547293663025,
+ "logits/rejected": 1.682992696762085,
+ "logps/chosen": -166.2056884765625,
+ "logps/rejected": -180.73330688476562,
+ "loss": 0.6826215744018554,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.07279330492019653,
+ "rewards/margins": 0.025264525786042213,
+ "rewards/rejected": 0.04752878472208977,
+ "step": 900
+ },
+ {
+ "epoch": 0.72,
+ "eval_logits/chosen": 1.4819762706756592,
+ "eval_logits/rejected": 1.513027548789978,
+ "eval_logps/chosen": -153.1695556640625,
+ "eval_logps/rejected": -147.7560577392578,
+ "eval_loss": 0.689373254776001,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": 0.03792775049805641,
+ "eval_rewards/margins": 0.011320074088871479,
+ "eval_rewards/rejected": 0.026607677340507507,
+ "eval_runtime": 90.9603,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 900
+ },
+ {
+ "epoch": 0.728,
+ "grad_norm": 1.1507582664489746,
+ "learning_rate": 8.417777777777777e-07,
+ "logits/chosen": 1.5831308364868164,
+ "logits/rejected": 1.3737132549285889,
+ "logps/chosen": -140.00253295898438,
+ "logps/rejected": -138.45155334472656,
+ "loss": 0.6873391628265381,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.05066479369997978,
+ "rewards/margins": 0.015466553159058094,
+ "rewards/rejected": 0.03519824147224426,
+ "step": 910
+ },
+ {
+ "epoch": 0.736,
+ "grad_norm": 1.415456771850586,
+ "learning_rate": 8.388148148148147e-07,
+ "logits/chosen": 1.3000686168670654,
+ "logits/rejected": 1.2642180919647217,
+ "logps/chosen": -160.14027404785156,
+ "logps/rejected": -163.32827758789062,
+ "loss": 0.687142276763916,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.06618930399417877,
+ "rewards/margins": 0.014807088300585747,
+ "rewards/rejected": 0.05138222128152847,
+ "step": 920
+ },
+ {
+ "epoch": 0.744,
+ "grad_norm": 1.3228634595870972,
+ "learning_rate": 8.358518518518518e-07,
+ "logits/chosen": 1.5204170942306519,
+ "logits/rejected": 1.6129217147827148,
+ "logps/chosen": -157.99832153320312,
+ "logps/rejected": -162.09567260742188,
+ "loss": 0.6790886878967285,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.05940447002649307,
+ "rewards/margins": 0.03161252290010452,
+ "rewards/rejected": 0.027791941538453102,
+ "step": 930
+ },
+ {
+ "epoch": 0.752,
+ "grad_norm": 1.048064947128296,
+ "learning_rate": 8.328888888888889e-07,
+ "logits/chosen": 1.5515320301055908,
+ "logits/rejected": 1.676857352256775,
+ "logps/chosen": -148.4853515625,
+ "logps/rejected": -174.4320526123047,
+ "loss": 0.6935453414916992,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.010225490666925907,
+ "rewards/margins": 0.0021693313028663397,
+ "rewards/rejected": 0.008056161925196648,
+ "step": 940
+ },
+ {
+ "epoch": 0.76,
+ "grad_norm": 1.247799038887024,
+ "learning_rate": 8.299259259259259e-07,
+ "logits/chosen": 1.384231686592102,
+ "logits/rejected": 1.4611561298370361,
+ "logps/chosen": -140.81443786621094,
+ "logps/rejected": -153.81326293945312,
+ "loss": 0.6805226325988769,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.053223587572574615,
+ "rewards/margins": 0.02933613583445549,
+ "rewards/rejected": 0.023887457326054573,
+ "step": 950
+ },
+ {
+ "epoch": 0.768,
+ "grad_norm": 1.1611264944076538,
+ "learning_rate": 8.269629629629629e-07,
+ "logits/chosen": 1.4987528324127197,
+ "logits/rejected": 1.3684321641921997,
+ "logps/chosen": -135.81056213378906,
+ "logps/rejected": -119.05133056640625,
+ "loss": 0.6954316616058349,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.0197703056037426,
+ "rewards/margins": -0.0012637012405321002,
+ "rewards/rejected": 0.021034007892012596,
+ "step": 960
+ },
+ {
+ "epoch": 0.776,
+ "grad_norm": 0.964948832988739,
+ "learning_rate": 8.24e-07,
+ "logits/chosen": 1.612053632736206,
+ "logits/rejected": 1.7114235162734985,
+ "logps/chosen": -153.56137084960938,
+ "logps/rejected": -149.17323303222656,
+ "loss": 0.6916662693023682,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.05082285404205322,
+ "rewards/margins": 0.0056606316938996315,
+ "rewards/rejected": 0.04516221210360527,
+ "step": 970
+ },
+ {
+ "epoch": 0.784,
+ "grad_norm": 1.409449815750122,
+ "learning_rate": 8.21037037037037e-07,
+ "logits/chosen": 1.384060025215149,
+ "logits/rejected": 1.316361904144287,
+ "logps/chosen": -168.75360107421875,
+ "logps/rejected": -141.79537963867188,
+ "loss": 0.6845529556274415,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.0886571854352951,
+ "rewards/margins": 0.02084263041615486,
+ "rewards/rejected": 0.06781454384326935,
+ "step": 980
+ },
+ {
+ "epoch": 0.792,
+ "grad_norm": 1.0246562957763672,
+ "learning_rate": 8.18074074074074e-07,
+ "logits/chosen": 1.4986507892608643,
+ "logits/rejected": 1.571103811264038,
+ "logps/chosen": -165.22254943847656,
+ "logps/rejected": -168.63540649414062,
+ "loss": 0.6790967464447022,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.07517905533313751,
+ "rewards/margins": 0.03272219002246857,
+ "rewards/rejected": 0.04245685786008835,
+ "step": 990
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 1.014426827430725,
+ "learning_rate": 8.15111111111111e-07,
+ "logits/chosen": 1.4708755016326904,
+ "logits/rejected": 1.4550955295562744,
+ "logps/chosen": -149.49874877929688,
+ "logps/rejected": -125.0966567993164,
+ "loss": 0.6892098903656005,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.07825140655040741,
+ "rewards/margins": 0.01007872074842453,
+ "rewards/rejected": 0.06817268580198288,
+ "step": 1000
+ },
+ {
+ "epoch": 0.8,
+ "eval_logits/chosen": 1.4972777366638184,
+ "eval_logits/rejected": 1.5283576250076294,
+ "eval_logps/chosen": -152.8240966796875,
+ "eval_logps/rejected": -147.44175720214844,
+ "eval_loss": 0.6880542039871216,
+ "eval_rewards/accuracies": 0.5419999957084656,
+ "eval_rewards/chosen": 0.07247376441955566,
+ "eval_rewards/margins": 0.01443479023873806,
+ "eval_rewards/rejected": 0.058038972318172455,
+ "eval_runtime": 90.827,
+ "eval_samples_per_second": 5.505,
+ "eval_steps_per_second": 2.752,
+ "step": 1000
+ },
+ {
+ "epoch": 0.808,
+ "grad_norm": 1.1155726909637451,
+ "learning_rate": 8.121481481481482e-07,
+ "logits/chosen": 1.547481894493103,
+ "logits/rejected": 1.5461527109146118,
+ "logps/chosen": -154.89791870117188,
+ "logps/rejected": -142.05111694335938,
+ "loss": 0.6809016704559326,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.10093016922473907,
+ "rewards/margins": 0.027038773521780968,
+ "rewards/rejected": 0.07389138638973236,
+ "step": 1010
+ },
+ {
+ "epoch": 0.816,
+ "grad_norm": 1.6716786623001099,
+ "learning_rate": 8.091851851851852e-07,
+ "logits/chosen": 1.3954919576644897,
+ "logits/rejected": 1.5088036060333252,
+ "logps/chosen": -143.11846923828125,
+ "logps/rejected": -146.41482543945312,
+ "loss": 0.6849985122680664,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.05970728397369385,
+ "rewards/margins": 0.021999523043632507,
+ "rewards/rejected": 0.03770776465535164,
+ "step": 1020
+ },
+ {
+ "epoch": 0.824,
+ "grad_norm": 1.308182716369629,
+ "learning_rate": 8.062222222222221e-07,
+ "logits/chosen": 1.532488465309143,
+ "logits/rejected": 1.3316718339920044,
+ "logps/chosen": -163.06858825683594,
+ "logps/rejected": -136.57989501953125,
+ "loss": 0.6789252281188964,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.11086118221282959,
+ "rewards/margins": 0.033117204904556274,
+ "rewards/rejected": 0.07774396985769272,
+ "step": 1030
+ },
+ {
+ "epoch": 0.832,
+ "grad_norm": 1.1019353866577148,
+ "learning_rate": 8.032592592592592e-07,
+ "logits/chosen": 1.4798121452331543,
+ "logits/rejected": 1.5203083753585815,
+ "logps/chosen": -140.3281707763672,
+ "logps/rejected": -141.06216430664062,
+ "loss": 0.6724937915802002,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.1202714815735817,
+ "rewards/margins": 0.046712182462215424,
+ "rewards/rejected": 0.07355931401252747,
+ "step": 1040
+ },
+ {
+ "epoch": 0.84,
+ "grad_norm": 1.2966418266296387,
+ "learning_rate": 8.002962962962963e-07,
+ "logits/chosen": 1.3034439086914062,
+ "logits/rejected": 1.4363505840301514,
+ "logps/chosen": -142.1992645263672,
+ "logps/rejected": -132.0046844482422,
+ "loss": 0.6848338603973388,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.11126589775085449,
+ "rewards/margins": 0.019870441406965256,
+ "rewards/rejected": 0.09139545261859894,
+ "step": 1050
+ },
+ {
+ "epoch": 0.848,
+ "grad_norm": 1.0321130752563477,
+ "learning_rate": 7.973333333333333e-07,
+ "logits/chosen": 1.3257265090942383,
+ "logits/rejected": 1.4649587869644165,
+ "logps/chosen": -112.98075103759766,
+ "logps/rejected": -127.4059829711914,
+ "loss": 0.6679783344268799,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.17311830818653107,
+ "rewards/margins": 0.05631983280181885,
+ "rewards/rejected": 0.11679847538471222,
+ "step": 1060
+ },
+ {
+ "epoch": 0.856,
+ "grad_norm": 1.2037222385406494,
+ "learning_rate": 7.943703703703704e-07,
+ "logits/chosen": 1.3766987323760986,
+ "logits/rejected": 1.2802103757858276,
+ "logps/chosen": -154.8142547607422,
+ "logps/rejected": -148.680908203125,
+ "loss": 0.668931531906128,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1449051946401596,
+ "rewards/margins": 0.05721765011548996,
+ "rewards/rejected": 0.08768755197525024,
+ "step": 1070
+ },
+ {
+ "epoch": 0.864,
+ "grad_norm": 0.9230766296386719,
+ "learning_rate": 7.914074074074074e-07,
+ "logits/chosen": 1.447328805923462,
+ "logits/rejected": 1.5235106945037842,
+ "logps/chosen": -150.21812438964844,
+ "logps/rejected": -123.29923248291016,
+ "loss": 0.6716424465179444,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.16066041588783264,
+ "rewards/margins": 0.04824484512209892,
+ "rewards/rejected": 0.11241557449102402,
+ "step": 1080
+ },
+ {
+ "epoch": 0.872,
+ "grad_norm": 1.0423877239227295,
+ "learning_rate": 7.884444444444443e-07,
+ "logits/chosen": 1.6920299530029297,
+ "logits/rejected": 1.6123542785644531,
+ "logps/chosen": -175.1676025390625,
+ "logps/rejected": -185.07223510742188,
+ "loss": 0.6723968505859375,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.16490009427070618,
+ "rewards/margins": 0.046577490866184235,
+ "rewards/rejected": 0.11832261085510254,
+ "step": 1090
+ },
+ {
+ "epoch": 0.88,
+ "grad_norm": 1.4142779111862183,
+ "learning_rate": 7.854814814814814e-07,
+ "logits/chosen": 1.453111171722412,
+ "logits/rejected": 1.4981731176376343,
+ "logps/chosen": -139.9765167236328,
+ "logps/rejected": -153.63058471679688,
+ "loss": 0.6872457027435303,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.13096000254154205,
+ "rewards/margins": 0.01702108606696129,
+ "rewards/rejected": 0.11393891274929047,
+ "step": 1100
+ },
+ {
+ "epoch": 0.88,
+ "eval_logits/chosen": 1.5076195001602173,
+ "eval_logits/rejected": 1.5373249053955078,
+ "eval_logps/chosen": -152.1697998046875,
+ "eval_logps/rejected": -146.83502197265625,
+ "eval_loss": 0.6873304843902588,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.13790392875671387,
+ "eval_rewards/margins": 0.019191108644008636,
+ "eval_rewards/rejected": 0.11871281266212463,
+ "eval_runtime": 91.0157,
+ "eval_samples_per_second": 5.494,
+ "eval_steps_per_second": 2.747,
+ "step": 1100
+ },
+ {
+ "epoch": 0.888,
+ "grad_norm": 1.413996696472168,
+ "learning_rate": 7.825185185185185e-07,
+ "logits/chosen": 1.5183905363082886,
+ "logits/rejected": 1.4942539930343628,
+ "logps/chosen": -156.383544921875,
+ "logps/rejected": -145.61398315429688,
+ "loss": 0.6810199737548828,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.14243564009666443,
+ "rewards/margins": 0.03136260434985161,
+ "rewards/rejected": 0.11107305437326431,
+ "step": 1110
+ },
+ {
+ "epoch": 0.896,
+ "grad_norm": 1.0046707391738892,
+ "learning_rate": 7.795555555555555e-07,
+ "logits/chosen": 1.4982213973999023,
+ "logits/rejected": 1.5465660095214844,
+ "logps/chosen": -156.79293823242188,
+ "logps/rejected": -150.77853393554688,
+ "loss": 0.6808318614959716,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1551930010318756,
+ "rewards/margins": 0.028187204152345657,
+ "rewards/rejected": 0.12700578570365906,
+ "step": 1120
+ },
+ {
+ "epoch": 0.904,
+ "grad_norm": 1.125625729560852,
+ "learning_rate": 7.765925925925925e-07,
+ "logits/chosen": 1.6053575277328491,
+ "logits/rejected": 1.47335946559906,
+ "logps/chosen": -148.05690002441406,
+ "logps/rejected": -135.36598205566406,
+ "loss": 0.6889908790588379,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.15128108859062195,
+ "rewards/margins": 0.015811312943696976,
+ "rewards/rejected": 0.13546979427337646,
+ "step": 1130
+ },
+ {
+ "epoch": 0.912,
+ "grad_norm": 1.0933659076690674,
+ "learning_rate": 7.736296296296297e-07,
+ "logits/chosen": 1.5862042903900146,
+ "logits/rejected": 1.5827430486679077,
+ "logps/chosen": -156.10145568847656,
+ "logps/rejected": -142.94851684570312,
+ "loss": 0.6753206253051758,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1547432243824005,
+ "rewards/margins": 0.04229574277997017,
+ "rewards/rejected": 0.11244747787714005,
+ "step": 1140
+ },
+ {
+ "epoch": 0.92,
+ "grad_norm": 1.0774686336517334,
+ "learning_rate": 7.706666666666667e-07,
+ "logits/chosen": 1.2369152307510376,
+ "logits/rejected": 1.2727023363113403,
+ "logps/chosen": -139.23147583007812,
+ "logps/rejected": -130.53038024902344,
+ "loss": 0.6967329025268555,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": 0.10801626741886139,
+ "rewards/margins": 0.004670143127441406,
+ "rewards/rejected": 0.10334610939025879,
+ "step": 1150
+ },
+ {
+ "epoch": 0.928,
+ "grad_norm": 1.1111780405044556,
+ "learning_rate": 7.677037037037036e-07,
+ "logits/chosen": 1.501250982284546,
+ "logits/rejected": 1.4272868633270264,
+ "logps/chosen": -127.09858703613281,
+ "logps/rejected": -114.9041748046875,
+ "loss": 0.6853152275085449,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.1366150826215744,
+ "rewards/margins": 0.021945688873529434,
+ "rewards/rejected": 0.11466939747333527,
+ "step": 1160
+ },
+ {
+ "epoch": 0.936,
+ "grad_norm": 1.5593854188919067,
+ "learning_rate": 7.647407407407407e-07,
+ "logits/chosen": 1.727927803993225,
+ "logits/rejected": 1.6510727405548096,
+ "logps/chosen": -156.70993041992188,
+ "logps/rejected": -167.07266235351562,
+ "loss": 0.6884279251098633,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.14902853965759277,
+ "rewards/margins": 0.01685408689081669,
+ "rewards/rejected": 0.13217447698116302,
+ "step": 1170
+ },
+ {
+ "epoch": 0.944,
+ "grad_norm": 2.0064918994903564,
+ "learning_rate": 7.617777777777778e-07,
+ "logits/chosen": 1.6262975931167603,
+ "logits/rejected": 1.58951997756958,
+ "logps/chosen": -186.59210205078125,
+ "logps/rejected": -151.2965850830078,
+ "loss": 0.6906405448913574,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.14898104965686798,
+ "rewards/margins": 0.013303476385772228,
+ "rewards/rejected": 0.13567757606506348,
+ "step": 1180
+ },
+ {
+ "epoch": 0.952,
+ "grad_norm": 1.209291696548462,
+ "learning_rate": 7.588148148148148e-07,
+ "logits/chosen": 1.639369249343872,
+ "logits/rejected": 1.6400636434555054,
+ "logps/chosen": -166.24684143066406,
+ "logps/rejected": -196.35305786132812,
+ "loss": 0.6789956092834473,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1348457634449005,
+ "rewards/margins": 0.03475186973810196,
+ "rewards/rejected": 0.10009388625621796,
+ "step": 1190
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 1.209230899810791,
+ "learning_rate": 7.558518518518519e-07,
+ "logits/chosen": 1.542942762374878,
+ "logits/rejected": 1.5128916501998901,
+ "logps/chosen": -138.994384765625,
+ "logps/rejected": -145.42880249023438,
+ "loss": 0.6936234951019287,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.13895569741725922,
+ "rewards/margins": 0.005582497920840979,
+ "rewards/rejected": 0.1333732008934021,
+ "step": 1200
+ },
+ {
+ "epoch": 0.96,
+ "eval_logits/chosen": 1.5055651664733887,
+ "eval_logits/rejected": 1.5351229906082153,
+ "eval_logps/chosen": -152.2256622314453,
+ "eval_logps/rejected": -146.909912109375,
+ "eval_loss": 0.6866763830184937,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.13231882452964783,
+ "eval_rewards/margins": 0.02109396643936634,
+ "eval_rewards/rejected": 0.11122485250234604,
+ "eval_runtime": 90.8141,
+ "eval_samples_per_second": 5.506,
+ "eval_steps_per_second": 2.753,
+ "step": 1200
+ },
+ {
+ "epoch": 0.968,
+ "grad_norm": 1.2058950662612915,
+ "learning_rate": 7.528888888888889e-07,
+ "logits/chosen": 1.3591210842132568,
+ "logits/rejected": 1.3811699151992798,
+ "logps/chosen": -155.1759033203125,
+ "logps/rejected": -141.97772216796875,
+ "loss": 0.6902606964111329,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.153532013297081,
+ "rewards/margins": 0.011359233409166336,
+ "rewards/rejected": 0.14217278361320496,
+ "step": 1210
+ },
+ {
+ "epoch": 0.976,
+ "grad_norm": 1.366702914237976,
+ "learning_rate": 7.499259259259258e-07,
+ "logits/chosen": 1.3841370344161987,
+ "logits/rejected": 1.3791372776031494,
+ "logps/chosen": -145.47613525390625,
+ "logps/rejected": -123.9173583984375,
+ "loss": 0.6971709251403808,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.10521620512008667,
+ "rewards/margins": -0.0030749079305678606,
+ "rewards/rejected": 0.1082911267876625,
+ "step": 1220
+ },
+ {
+ "epoch": 0.984,
+ "grad_norm": 1.477455496788025,
+ "learning_rate": 7.469629629629629e-07,
+ "logits/chosen": 1.4447228908538818,
+ "logits/rejected": 1.6022417545318604,
+ "logps/chosen": -137.9462127685547,
+ "logps/rejected": -150.25559997558594,
+ "loss": 0.6915127277374268,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.15108394622802734,
+ "rewards/margins": 0.009388584643602371,
+ "rewards/rejected": 0.14169538021087646,
+ "step": 1230
+ },
+ {
+ "epoch": 0.992,
+ "grad_norm": 1.4407854080200195,
+ "learning_rate": 7.44e-07,
+ "logits/chosen": 1.3998825550079346,
+ "logits/rejected": 1.5924925804138184,
+ "logps/chosen": -144.93775939941406,
+ "logps/rejected": -148.02542114257812,
+ "loss": 0.6699621677398682,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.146137997508049,
+ "rewards/margins": 0.05326627567410469,
+ "rewards/rejected": 0.09287171810865402,
+ "step": 1240
+ },
+ {
+ "epoch": 1.0,
+ "grad_norm": 1.183438777923584,
+ "learning_rate": 7.41037037037037e-07,
+ "logits/chosen": 1.5393705368041992,
+ "logits/rejected": 1.6312555074691772,
+ "logps/chosen": -145.61672973632812,
+ "logps/rejected": -158.9256591796875,
+ "loss": 0.6811168670654297,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.1304798424243927,
+ "rewards/margins": 0.03360626846551895,
+ "rewards/rejected": 0.09687358886003494,
+ "step": 1250
+ },
+ {
+ "epoch": 1.008,
+ "grad_norm": 1.2436553239822388,
+ "learning_rate": 7.38074074074074e-07,
+ "logits/chosen": 1.4758602380752563,
+ "logits/rejected": 1.534609079360962,
+ "logps/chosen": -124.26841735839844,
+ "logps/rejected": -132.41867065429688,
+ "loss": 0.688177490234375,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.12570872902870178,
+ "rewards/margins": 0.014875045046210289,
+ "rewards/rejected": 0.11083368957042694,
+ "step": 1260
+ },
+ {
+ "epoch": 1.016,
+ "grad_norm": 1.3853868246078491,
+ "learning_rate": 7.351111111111112e-07,
+ "logits/chosen": 1.7195091247558594,
+ "logits/rejected": 1.632851004600525,
+ "logps/chosen": -160.8590850830078,
+ "logps/rejected": -153.91671752929688,
+ "loss": 0.6669504165649414,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.181864932179451,
+ "rewards/margins": 0.06175640970468521,
+ "rewards/rejected": 0.12010850757360458,
+ "step": 1270
+ },
+ {
+ "epoch": 1.024,
+ "grad_norm": 1.2765302658081055,
+ "learning_rate": 7.321481481481481e-07,
+ "logits/chosen": 1.5782036781311035,
+ "logits/rejected": 1.5713088512420654,
+ "logps/chosen": -154.3319549560547,
+ "logps/rejected": -141.43345642089844,
+ "loss": 0.6734190464019776,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.12361389398574829,
+ "rewards/margins": 0.04626934230327606,
+ "rewards/rejected": 0.07734455168247223,
+ "step": 1280
+ },
+ {
+ "epoch": 1.032,
+ "grad_norm": 1.0421397686004639,
+ "learning_rate": 7.291851851851851e-07,
+ "logits/chosen": 1.5199967622756958,
+ "logits/rejected": 1.5965670347213745,
+ "logps/chosen": -145.28416442871094,
+ "logps/rejected": -133.72256469726562,
+ "loss": 0.6804514408111573,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.11794980615377426,
+ "rewards/margins": 0.030916064977645874,
+ "rewards/rejected": 0.08703374862670898,
+ "step": 1290
+ },
+ {
+ "epoch": 1.04,
+ "grad_norm": 1.032158613204956,
+ "learning_rate": 7.262222222222222e-07,
+ "logits/chosen": 1.6042639017105103,
+ "logits/rejected": 1.4255346059799194,
+ "logps/chosen": -150.4803009033203,
+ "logps/rejected": -130.2232208251953,
+ "loss": 0.6676507472991944,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.15466037392616272,
+ "rewards/margins": 0.05808692425489426,
+ "rewards/rejected": 0.09657344967126846,
+ "step": 1300
+ },
+ {
+ "epoch": 1.04,
+ "eval_logits/chosen": 1.5118356943130493,
+ "eval_logits/rejected": 1.5418950319290161,
+ "eval_logps/chosen": -152.23300170898438,
+ "eval_logps/rejected": -146.9257049560547,
+ "eval_loss": 0.6859617233276367,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.13158410787582397,
+ "eval_rewards/margins": 0.021939631551504135,
+ "eval_rewards/rejected": 0.10964448750019073,
+ "eval_runtime": 90.9624,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 1300
+ },
+ {
+ "epoch": 1.048,
+ "grad_norm": 1.4942574501037598,
+ "learning_rate": 7.232592592592593e-07,
+ "logits/chosen": 1.689648985862732,
+ "logits/rejected": 1.691888451576233,
+ "logps/chosen": -166.02809143066406,
+ "logps/rejected": -160.30601501464844,
+ "loss": 0.6770285606384278,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.14377735555171967,
+ "rewards/margins": 0.03825496509671211,
+ "rewards/rejected": 0.10552239418029785,
+ "step": 1310
+ },
+ {
+ "epoch": 1.056,
+ "grad_norm": 1.3656011819839478,
+ "learning_rate": 7.202962962962963e-07,
+ "logits/chosen": 1.3065154552459717,
+ "logits/rejected": 1.5681203603744507,
+ "logps/chosen": -129.1146240234375,
+ "logps/rejected": -151.2584991455078,
+ "loss": 0.6844655036926269,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.12841743230819702,
+ "rewards/margins": 0.02178655005991459,
+ "rewards/rejected": 0.10663086175918579,
+ "step": 1320
+ },
+ {
+ "epoch": 1.064,
+ "grad_norm": 1.1723405122756958,
+ "learning_rate": 7.173333333333333e-07,
+ "logits/chosen": 1.6219158172607422,
+ "logits/rejected": 1.6032555103302002,
+ "logps/chosen": -156.4185028076172,
+ "logps/rejected": -146.28067016601562,
+ "loss": 0.6650479316711426,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1538492739200592,
+ "rewards/margins": 0.06132146716117859,
+ "rewards/rejected": 0.09252782166004181,
+ "step": 1330
+ },
+ {
+ "epoch": 1.072,
+ "grad_norm": 1.2311245203018188,
+ "learning_rate": 7.143703703703703e-07,
+ "logits/chosen": 1.4625260829925537,
+ "logits/rejected": 1.392937421798706,
+ "logps/chosen": -153.05941772460938,
+ "logps/rejected": -145.37841796875,
+ "loss": 0.6570021152496338,
+ "rewards/accuracies": 0.762499988079071,
+ "rewards/chosen": 0.16768500208854675,
+ "rewards/margins": 0.0787494108080864,
+ "rewards/rejected": 0.08893559873104095,
+ "step": 1340
+ },
+ {
+ "epoch": 1.08,
+ "grad_norm": 1.314666748046875,
+ "learning_rate": 7.114074074074073e-07,
+ "logits/chosen": 1.4812809228897095,
+ "logits/rejected": 1.3796050548553467,
+ "logps/chosen": -165.70205688476562,
+ "logps/rejected": -140.13150024414062,
+ "loss": 0.6535200595855712,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.20691323280334473,
+ "rewards/margins": 0.0911526307463646,
+ "rewards/rejected": 0.11576057970523834,
+ "step": 1350
+ },
+ {
+ "epoch": 1.088,
+ "grad_norm": 0.9556720852851868,
+ "learning_rate": 7.084444444444444e-07,
+ "logits/chosen": 1.6368863582611084,
+ "logits/rejected": 1.678134560585022,
+ "logps/chosen": -166.56253051757812,
+ "logps/rejected": -164.36416625976562,
+ "loss": 0.6742628574371338,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.15145304799079895,
+ "rewards/margins": 0.04319921135902405,
+ "rewards/rejected": 0.1082538366317749,
+ "step": 1360
+ },
+ {
+ "epoch": 1.096,
+ "grad_norm": 0.945633053779602,
+ "learning_rate": 7.054814814814815e-07,
+ "logits/chosen": 1.4285938739776611,
+ "logits/rejected": 1.5576260089874268,
+ "logps/chosen": -131.5234832763672,
+ "logps/rejected": -128.02340698242188,
+ "loss": 0.6670140743255615,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.16399362683296204,
+ "rewards/margins": 0.05922812223434448,
+ "rewards/rejected": 0.10476551204919815,
+ "step": 1370
+ },
+ {
+ "epoch": 1.104,
+ "grad_norm": 1.1189475059509277,
+ "learning_rate": 7.025185185185185e-07,
+ "logits/chosen": 1.5019981861114502,
+ "logits/rejected": 1.667614221572876,
+ "logps/chosen": -161.61854553222656,
+ "logps/rejected": -166.39996337890625,
+ "loss": 0.6742690086364747,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.15786050260066986,
+ "rewards/margins": 0.044500257819890976,
+ "rewards/rejected": 0.11336024850606918,
+ "step": 1380
+ },
+ {
+ "epoch": 1.112,
+ "grad_norm": 1.3382912874221802,
+ "learning_rate": 6.995555555555555e-07,
+ "logits/chosen": 1.4972480535507202,
+ "logits/rejected": 1.4946129322052002,
+ "logps/chosen": -166.43682861328125,
+ "logps/rejected": -162.3004913330078,
+ "loss": 0.6557814121246338,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.18964914977550507,
+ "rewards/margins": 0.08500295877456665,
+ "rewards/rejected": 0.10464620590209961,
+ "step": 1390
+ },
+ {
+ "epoch": 1.12,
+ "grad_norm": 1.2123515605926514,
+ "learning_rate": 6.965925925925926e-07,
+ "logits/chosen": 1.456635594367981,
+ "logits/rejected": 1.5964475870132446,
+ "logps/chosen": -152.08261108398438,
+ "logps/rejected": -161.51023864746094,
+ "loss": 0.6708066940307618,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.216823011636734,
+ "rewards/margins": 0.050363689661026,
+ "rewards/rejected": 0.166459321975708,
+ "step": 1400
+ },
+ {
+ "epoch": 1.12,
+ "eval_logits/chosen": 1.521518588066101,
+ "eval_logits/rejected": 1.5510718822479248,
+ "eval_logps/chosen": -151.858154296875,
+ "eval_logps/rejected": -146.5819549560547,
+ "eval_loss": 0.6853657364845276,
+ "eval_rewards/accuracies": 0.5440000295639038,
+ "eval_rewards/chosen": 0.16906805336475372,
+ "eval_rewards/margins": 0.025049500167369843,
+ "eval_rewards/rejected": 0.14401854574680328,
+ "eval_runtime": 90.8779,
+ "eval_samples_per_second": 5.502,
+ "eval_steps_per_second": 2.751,
+ "step": 1400
+ },
+ {
+ "epoch": 1.1280000000000001,
+ "grad_norm": 1.3488291501998901,
+ "learning_rate": 6.936296296296296e-07,
+ "logits/chosen": 1.6009830236434937,
+ "logits/rejected": 1.6035544872283936,
+ "logps/chosen": -145.44662475585938,
+ "logps/rejected": -149.9284210205078,
+ "loss": 0.679435396194458,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.1647307425737381,
+ "rewards/margins": 0.03348377346992493,
+ "rewards/rejected": 0.13124695420265198,
+ "step": 1410
+ },
+ {
+ "epoch": 1.1360000000000001,
+ "grad_norm": 1.6453970670700073,
+ "learning_rate": 6.906666666666666e-07,
+ "logits/chosen": 1.593658447265625,
+ "logits/rejected": 1.648276925086975,
+ "logps/chosen": -135.2926788330078,
+ "logps/rejected": -150.9585723876953,
+ "loss": 0.6875734329223633,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.15669891238212585,
+ "rewards/margins": 0.01690087839961052,
+ "rewards/rejected": 0.13979804515838623,
+ "step": 1420
+ },
+ {
+ "epoch": 1.144,
+ "grad_norm": 1.6117807626724243,
+ "learning_rate": 6.877037037037036e-07,
+ "logits/chosen": 1.4510356187820435,
+ "logits/rejected": 1.2334567308425903,
+ "logps/chosen": -153.94854736328125,
+ "logps/rejected": -147.78079223632812,
+ "loss": 0.6689413070678711,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1819056123495102,
+ "rewards/margins": 0.058564942330121994,
+ "rewards/rejected": 0.1233406662940979,
+ "step": 1430
+ },
+ {
+ "epoch": 1.152,
+ "grad_norm": 1.5533809661865234,
+ "learning_rate": 6.847407407407408e-07,
+ "logits/chosen": 1.4535889625549316,
+ "logits/rejected": 1.4273865222930908,
+ "logps/chosen": -170.42904663085938,
+ "logps/rejected": -151.9979705810547,
+ "loss": 0.6845246315002441,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.14223966002464294,
+ "rewards/margins": 0.022389881312847137,
+ "rewards/rejected": 0.1198497861623764,
+ "step": 1440
+ },
+ {
+ "epoch": 1.16,
+ "grad_norm": 1.5696005821228027,
+ "learning_rate": 6.817777777777778e-07,
+ "logits/chosen": 1.4418725967407227,
+ "logits/rejected": 1.4354021549224854,
+ "logps/chosen": -153.962158203125,
+ "logps/rejected": -148.40512084960938,
+ "loss": 0.6761108875274658,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.15529735386371613,
+ "rewards/margins": 0.040999218821525574,
+ "rewards/rejected": 0.11429812759160995,
+ "step": 1450
+ },
+ {
+ "epoch": 1.168,
+ "grad_norm": 1.0711781978607178,
+ "learning_rate": 6.788148148148147e-07,
+ "logits/chosen": 1.5659481287002563,
+ "logits/rejected": 1.5640677213668823,
+ "logps/chosen": -149.816162109375,
+ "logps/rejected": -174.68643188476562,
+ "loss": 0.677489185333252,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1510745733976364,
+ "rewards/margins": 0.03914733976125717,
+ "rewards/rejected": 0.11192724853754044,
+ "step": 1460
+ },
+ {
+ "epoch": 1.176,
+ "grad_norm": 1.2336673736572266,
+ "learning_rate": 6.758518518518518e-07,
+ "logits/chosen": 1.4275394678115845,
+ "logits/rejected": 1.6394407749176025,
+ "logps/chosen": -137.93788146972656,
+ "logps/rejected": -154.97195434570312,
+ "loss": 0.6785802364349365,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10557998716831207,
+ "rewards/margins": 0.0352485254406929,
+ "rewards/rejected": 0.07033145427703857,
+ "step": 1470
+ },
+ {
+ "epoch": 1.184,
+ "grad_norm": 1.0030723810195923,
+ "learning_rate": 6.728888888888888e-07,
+ "logits/chosen": 1.4652725458145142,
+ "logits/rejected": 1.4992272853851318,
+ "logps/chosen": -143.87374877929688,
+ "logps/rejected": -139.5693817138672,
+ "loss": 0.6700259208679199,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.13436684012413025,
+ "rewards/margins": 0.05286343768239021,
+ "rewards/rejected": 0.08150340616703033,
+ "step": 1480
+ },
+ {
+ "epoch": 1.192,
+ "grad_norm": 1.0897104740142822,
+ "learning_rate": 6.699259259259259e-07,
+ "logits/chosen": 1.7033601999282837,
+ "logits/rejected": 1.617489218711853,
+ "logps/chosen": -140.43667602539062,
+ "logps/rejected": -146.59683227539062,
+ "loss": 0.6685672283172608,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.11724893748760223,
+ "rewards/margins": 0.055138181895017624,
+ "rewards/rejected": 0.06211075931787491,
+ "step": 1490
+ },
+ {
+ "epoch": 1.2,
+ "grad_norm": 0.8573256731033325,
+ "learning_rate": 6.66962962962963e-07,
+ "logits/chosen": 1.5181013345718384,
+ "logits/rejected": 1.5965152978897095,
+ "logps/chosen": -147.78506469726562,
+ "logps/rejected": -142.18930053710938,
+ "loss": 0.6564288139343262,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.14890792965888977,
+ "rewards/margins": 0.08058460801839828,
+ "rewards/rejected": 0.06832331418991089,
+ "step": 1500
+ },
+ {
+ "epoch": 1.2,
+ "eval_logits/chosen": 1.488415002822876,
+ "eval_logits/rejected": 1.5179662704467773,
+ "eval_logps/chosen": -152.60232543945312,
+ "eval_logps/rejected": -147.2954864501953,
+ "eval_loss": 0.6861968040466309,
+ "eval_rewards/accuracies": 0.5400000214576721,
+ "eval_rewards/chosen": 0.09465231001377106,
+ "eval_rewards/margins": 0.021986663341522217,
+ "eval_rewards/rejected": 0.07266565412282944,
+ "eval_runtime": 90.8951,
+ "eval_samples_per_second": 5.501,
+ "eval_steps_per_second": 2.75,
+ "step": 1500
+ }
+ ],
+ "logging_steps": 10,
+ "max_steps": 3750,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-1500/training_args.bin b/v3/DPO/DPO_10k/lora/checkpoint-1500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..b3fc3d76093802087869ab1fee5e37b9c830fe77
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-1500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4f90e4a7053b7fb440d68cb64a3d198396af09db942d08839c5f4f0f0b8a0c8
+size 6097
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2000/README.md b/v3/DPO/DPO_10k/lora/checkpoint-2000/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..250500f798b4682b2cd2c35cd1fc366677c215dd
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2000/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2000/adapter_config.json b/v3/DPO/DPO_10k/lora/checkpoint-2000/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..039869f8d79ca6c48a793a29a79267da6d8f81c2
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2000/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "gate_proj",
+ "down_proj",
+ "k_proj",
+ "o_proj",
+ "up_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2000/adapter_model.safetensors b/v3/DPO/DPO_10k/lora/checkpoint-2000/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1d149e4057820747d0b93f59b09781e87e8b4734
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2000/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cd69c523e1e782aa020a6dd4857a965f4cb7c32d5baa557697e95c17d5c604e
+size 180385008
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2000/chat_template.jinja b/v3/DPO/DPO_10k/lora/checkpoint-2000/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2000/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2000/optimizer.pt b/v3/DPO/DPO_10k/lora/checkpoint-2000/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..af41051fddbe89e607c70f44442b8e9ab8a95744
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2000/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9fab96ca3915ebdde3a55a6d95b9e0ffb2c8eb2ed59436fcdae3be4c642b46f
+size 360902475
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2000/rng_state.pth b/v3/DPO/DPO_10k/lora/checkpoint-2000/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..45da84cb9f96139d23fe2df13e9f26310fc87471
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2000/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f27403649e37262609f871dba600d7da4f0bfc387b6571584c6bc75289b7c853
+size 14645
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2000/scaler.pt b/v3/DPO/DPO_10k/lora/checkpoint-2000/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..05c642874425671923b2af80a1aa6317e758b492
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2000/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4aa03f6e0cd07cf67ce1fbe3101d545f5771ef9148b9debf02b11cf6948da5c
+size 1383
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2000/scheduler.pt b/v3/DPO/DPO_10k/lora/checkpoint-2000/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..461b87c5a3c410d70833878cff07b7db25b011b8
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2000/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:318db60b99958c78b78a2ad7d7ff224500c5019593c5cf0c4944785079b20a61
+size 1465
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2000/tokenizer.json b/v3/DPO/DPO_10k/lora/checkpoint-2000/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2000/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2000/tokenizer_config.json b/v3/DPO/DPO_10k/lora/checkpoint-2000/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2000/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2000/trainer_state.json b/v3/DPO/DPO_10k/lora/checkpoint-2000/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..a97c5b7b1d1637ba33a8925e1160a55c23eaf9bd
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2000/trainer_state.json
@@ -0,0 +1,3354 @@
+{
+ "best_global_step": 1400,
+ "best_metric": 0.6853657364845276,
+ "best_model_checkpoint": "output/lora/checkpoint-1000",
+ "epoch": 1.6,
+ "eval_steps": 100,
+ "global_step": 2000,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.008,
+ "grad_norm": 1.1179773807525635,
+ "learning_rate": 2.4e-08,
+ "logits/chosen": 1.4545083045959473,
+ "logits/rejected": 1.425490379333496,
+ "logps/chosen": -132.46189880371094,
+ "logps/rejected": -148.20260620117188,
+ "loss": 0.6930626392364502,
+ "rewards/accuracies": 0.21250000596046448,
+ "rewards/chosen": -6.165739614516497e-05,
+ "rewards/margins": 0.00017333509458694607,
+ "rewards/rejected": -0.00023499250528402627,
+ "step": 10
+ },
+ {
+ "epoch": 0.016,
+ "grad_norm": 1.3502839803695679,
+ "learning_rate": 5.0666666666666664e-08,
+ "logits/chosen": 1.5668801069259644,
+ "logits/rejected": 1.5191389322280884,
+ "logps/chosen": -157.63624572753906,
+ "logps/rejected": -144.2716827392578,
+ "loss": 0.6930380821228027,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.0009244013344869018,
+ "rewards/margins": 0.00023505210992880166,
+ "rewards/rejected": 0.0006893490790389478,
+ "step": 20
+ },
+ {
+ "epoch": 0.024,
+ "grad_norm": 1.0219553709030151,
+ "learning_rate": 7.733333333333334e-08,
+ "logits/chosen": 1.8921797275543213,
+ "logits/rejected": 1.7342685461044312,
+ "logps/chosen": -164.55799865722656,
+ "logps/rejected": -158.92752075195312,
+ "loss": 0.69290189743042,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0009391927160322666,
+ "rewards/margins": 0.0005032324115745723,
+ "rewards/rejected": 0.0004359603044576943,
+ "step": 30
+ },
+ {
+ "epoch": 0.032,
+ "grad_norm": 1.2056113481521606,
+ "learning_rate": 1.0399999999999999e-07,
+ "logits/chosen": 1.4950047731399536,
+ "logits/rejected": 1.6163270473480225,
+ "logps/chosen": -145.54183959960938,
+ "logps/rejected": -134.19979858398438,
+ "loss": 0.6935500621795654,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.0006421065190806985,
+ "rewards/margins": -0.0007943272357806563,
+ "rewards/rejected": 0.0014364338712766767,
+ "step": 40
+ },
+ {
+ "epoch": 0.04,
+ "grad_norm": 1.1029725074768066,
+ "learning_rate": 1.3066666666666665e-07,
+ "logits/chosen": 1.4076533317565918,
+ "logits/rejected": 1.4791977405548096,
+ "logps/chosen": -133.23324584960938,
+ "logps/rejected": -138.3593292236328,
+ "loss": 0.6925833702087403,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.0012372838100418448,
+ "rewards/margins": 0.001139522879384458,
+ "rewards/rejected": 9.776116348803043e-05,
+ "step": 50
+ },
+ {
+ "epoch": 0.048,
+ "grad_norm": 0.9960266351699829,
+ "learning_rate": 1.573333333333333e-07,
+ "logits/chosen": 1.407166600227356,
+ "logits/rejected": 1.376042127609253,
+ "logps/chosen": -137.8921661376953,
+ "logps/rejected": -127.97320556640625,
+ "loss": 0.693614149093628,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.00021316049969755113,
+ "rewards/margins": -0.0009249996510334313,
+ "rewards/rejected": 0.0007118391804397106,
+ "step": 60
+ },
+ {
+ "epoch": 0.056,
+ "grad_norm": 1.0946824550628662,
+ "learning_rate": 1.8399999999999998e-07,
+ "logits/chosen": 1.5944501161575317,
+ "logits/rejected": 1.7248146533966064,
+ "logps/chosen": -153.5526123046875,
+ "logps/rejected": -157.19735717773438,
+ "loss": 0.6928449153900147,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0026370862033218145,
+ "rewards/margins": 0.0006150150438770652,
+ "rewards/rejected": 0.0020220710430294275,
+ "step": 70
+ },
+ {
+ "epoch": 0.064,
+ "grad_norm": 1.0322489738464355,
+ "learning_rate": 2.1066666666666665e-07,
+ "logits/chosen": 1.4586353302001953,
+ "logits/rejected": 1.4842995405197144,
+ "logps/chosen": -149.03665161132812,
+ "logps/rejected": -136.24081420898438,
+ "loss": 0.6932015895843506,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.004061593674123287,
+ "rewards/margins": -9.99594121822156e-05,
+ "rewards/rejected": 0.004161553457379341,
+ "step": 80
+ },
+ {
+ "epoch": 0.072,
+ "grad_norm": 1.0012447834014893,
+ "learning_rate": 2.3733333333333334e-07,
+ "logits/chosen": 1.5075902938842773,
+ "logits/rejected": 1.5986944437026978,
+ "logps/chosen": -159.3433074951172,
+ "logps/rejected": -151.85072326660156,
+ "loss": 0.6928757190704345,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.006978762801736593,
+ "rewards/margins": 0.0005627512582577765,
+ "rewards/rejected": 0.006416010670363903,
+ "step": 90
+ },
+ {
+ "epoch": 0.08,
+ "grad_norm": 1.280081033706665,
+ "learning_rate": 2.64e-07,
+ "logits/chosen": 1.448286533355713,
+ "logits/rejected": 1.5480695962905884,
+ "logps/chosen": -151.12625122070312,
+ "logps/rejected": -171.24545288085938,
+ "loss": 0.6931126117706299,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.00693341251462698,
+ "rewards/margins": 9.057526767719537e-05,
+ "rewards/rejected": 0.006842837668955326,
+ "step": 100
+ },
+ {
+ "epoch": 0.08,
+ "eval_logits/chosen": 1.518183946609497,
+ "eval_logits/rejected": 1.5525447130203247,
+ "eval_logps/chosen": -153.46630859375,
+ "eval_logps/rejected": -147.95005798339844,
+ "eval_loss": 0.6926330327987671,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.008253541775047779,
+ "eval_rewards/margins": 0.0010449704714119434,
+ "eval_rewards/rejected": 0.007208569906651974,
+ "eval_runtime": 91.0812,
+ "eval_samples_per_second": 5.49,
+ "eval_steps_per_second": 2.745,
+ "step": 100
+ },
+ {
+ "epoch": 0.088,
+ "grad_norm": 1.3912373781204224,
+ "learning_rate": 2.906666666666667e-07,
+ "logits/chosen": 1.6896222829818726,
+ "logits/rejected": 1.6516921520233154,
+ "logps/chosen": -173.70030212402344,
+ "logps/rejected": -155.5503692626953,
+ "loss": 0.6926521778106689,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.010179834440350533,
+ "rewards/margins": 0.0010137532372027636,
+ "rewards/rejected": 0.009166081435978413,
+ "step": 110
+ },
+ {
+ "epoch": 0.096,
+ "grad_norm": 1.1518888473510742,
+ "learning_rate": 3.173333333333333e-07,
+ "logits/chosen": 1.5166860818862915,
+ "logits/rejected": 1.5136979818344116,
+ "logps/chosen": -136.39906311035156,
+ "logps/rejected": -140.86561584472656,
+ "loss": 0.6932918548583984,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": 0.00907944981008768,
+ "rewards/margins": -0.00026998287648893893,
+ "rewards/rejected": 0.009349432773888111,
+ "step": 120
+ },
+ {
+ "epoch": 0.104,
+ "grad_norm": 1.3213376998901367,
+ "learning_rate": 3.4399999999999996e-07,
+ "logits/chosen": 1.4598973989486694,
+ "logits/rejected": 1.6739647388458252,
+ "logps/chosen": -159.12461853027344,
+ "logps/rejected": -163.39346313476562,
+ "loss": 0.693086051940918,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.010477779433131218,
+ "rewards/margins": 0.00014539004769176245,
+ "rewards/rejected": 0.010332388803362846,
+ "step": 130
+ },
+ {
+ "epoch": 0.112,
+ "grad_norm": 0.8833863735198975,
+ "learning_rate": 3.7066666666666665e-07,
+ "logits/chosen": 1.3829412460327148,
+ "logits/rejected": 1.4100173711776733,
+ "logps/chosen": -132.7149658203125,
+ "logps/rejected": -126.85478210449219,
+ "loss": 0.6932382583618164,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": 0.010262183845043182,
+ "rewards/margins": -0.0001672578218858689,
+ "rewards/rejected": 0.010429441928863525,
+ "step": 140
+ },
+ {
+ "epoch": 0.12,
+ "grad_norm": 1.1472455263137817,
+ "learning_rate": 3.973333333333333e-07,
+ "logits/chosen": 1.4954869747161865,
+ "logits/rejected": 1.5857340097427368,
+ "logps/chosen": -165.10568237304688,
+ "logps/rejected": -158.97433471679688,
+ "loss": 0.6916103363037109,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.014177520759403706,
+ "rewards/margins": 0.003108317730948329,
+ "rewards/rejected": 0.011069201864302158,
+ "step": 150
+ },
+ {
+ "epoch": 0.128,
+ "grad_norm": 1.0628798007965088,
+ "learning_rate": 4.24e-07,
+ "logits/chosen": 1.6717243194580078,
+ "logits/rejected": 1.705092191696167,
+ "logps/chosen": -159.332275390625,
+ "logps/rejected": -147.21646118164062,
+ "loss": 0.6935629844665527,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": 0.016994481906294823,
+ "rewards/margins": -0.0007874655420891941,
+ "rewards/rejected": 0.017781946808099747,
+ "step": 160
+ },
+ {
+ "epoch": 0.136,
+ "grad_norm": 1.390810251235962,
+ "learning_rate": 4.506666666666666e-07,
+ "logits/chosen": 1.4719234704971313,
+ "logits/rejected": 1.499526023864746,
+ "logps/chosen": -148.67251586914062,
+ "logps/rejected": -161.58021545410156,
+ "loss": 0.6926599979400635,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.021239640191197395,
+ "rewards/margins": 0.0010346340714022517,
+ "rewards/rejected": 0.020205006003379822,
+ "step": 170
+ },
+ {
+ "epoch": 0.144,
+ "grad_norm": 1.139707326889038,
+ "learning_rate": 4.773333333333333e-07,
+ "logits/chosen": 1.4833905696868896,
+ "logits/rejected": 1.637584924697876,
+ "logps/chosen": -158.96731567382812,
+ "logps/rejected": -161.9830322265625,
+ "loss": 0.6919547557830811,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.019416773691773415,
+ "rewards/margins": 0.0024296878837049007,
+ "rewards/rejected": 0.016987085342407227,
+ "step": 180
+ },
+ {
+ "epoch": 0.152,
+ "grad_norm": 1.2029796838760376,
+ "learning_rate": 5.04e-07,
+ "logits/chosen": 1.5398027896881104,
+ "logits/rejected": 1.6678705215454102,
+ "logps/chosen": -143.00283813476562,
+ "logps/rejected": -144.89378356933594,
+ "loss": 0.692262840270996,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.018960200250148773,
+ "rewards/margins": 0.0018175147706642747,
+ "rewards/rejected": 0.017142686992883682,
+ "step": 190
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 1.0527437925338745,
+ "learning_rate": 5.306666666666665e-07,
+ "logits/chosen": 1.6087623834609985,
+ "logits/rejected": 1.5181543827056885,
+ "logps/chosen": -159.35073852539062,
+ "logps/rejected": -133.59890747070312,
+ "loss": 0.6924069881439209,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.01926913857460022,
+ "rewards/margins": 0.0015339398523792624,
+ "rewards/rejected": 0.0177351962774992,
+ "step": 200
+ },
+ {
+ "epoch": 0.16,
+ "eval_logits/chosen": 1.5193672180175781,
+ "eval_logits/rejected": 1.5534639358520508,
+ "eval_logps/chosen": -153.34857177734375,
+ "eval_logps/rejected": -147.8425750732422,
+ "eval_loss": 0.6921458840370178,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.020027177408337593,
+ "eval_rewards/margins": 0.0020714527927339077,
+ "eval_rewards/rejected": 0.017955724149942398,
+ "eval_runtime": 90.9762,
+ "eval_samples_per_second": 5.496,
+ "eval_steps_per_second": 2.748,
+ "step": 200
+ },
+ {
+ "epoch": 0.168,
+ "grad_norm": 1.1897239685058594,
+ "learning_rate": 5.573333333333333e-07,
+ "logits/chosen": 1.6267732381820679,
+ "logits/rejected": 1.5775268077850342,
+ "logps/chosen": -159.51437377929688,
+ "logps/rejected": -170.17762756347656,
+ "loss": 0.6925770759582519,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.02078000269830227,
+ "rewards/margins": 0.0012053751852363348,
+ "rewards/rejected": 0.01957462914288044,
+ "step": 210
+ },
+ {
+ "epoch": 0.176,
+ "grad_norm": 0.9995236396789551,
+ "learning_rate": 5.839999999999999e-07,
+ "logits/chosen": 1.5781934261322021,
+ "logits/rejected": 1.642251968383789,
+ "logps/chosen": -159.95309448242188,
+ "logps/rejected": -144.36058044433594,
+ "loss": 0.6931475639343262,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.023322442546486855,
+ "rewards/margins": 6.392716750269756e-05,
+ "rewards/rejected": 0.02325851283967495,
+ "step": 220
+ },
+ {
+ "epoch": 0.184,
+ "grad_norm": 0.8819260597229004,
+ "learning_rate": 6.106666666666666e-07,
+ "logits/chosen": 1.459991693496704,
+ "logits/rejected": 1.6060739755630493,
+ "logps/chosen": -162.04745483398438,
+ "logps/rejected": -139.5411834716797,
+ "loss": 0.6919463634490967,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.026052182540297508,
+ "rewards/margins": 0.002481849165633321,
+ "rewards/rejected": 0.023570330813527107,
+ "step": 230
+ },
+ {
+ "epoch": 0.192,
+ "grad_norm": 1.1055208444595337,
+ "learning_rate": 6.373333333333333e-07,
+ "logits/chosen": 1.558739185333252,
+ "logits/rejected": 1.5965116024017334,
+ "logps/chosen": -171.9166259765625,
+ "logps/rejected": -145.66433715820312,
+ "loss": 0.6912176132202148,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.033868782222270966,
+ "rewards/margins": 0.004017929546535015,
+ "rewards/rejected": 0.029850851744413376,
+ "step": 240
+ },
+ {
+ "epoch": 0.2,
+ "grad_norm": 1.1372733116149902,
+ "learning_rate": 6.64e-07,
+ "logits/chosen": 1.5322860479354858,
+ "logits/rejected": 1.4695795774459839,
+ "logps/chosen": -161.7803955078125,
+ "logps/rejected": -148.91949462890625,
+ "loss": 0.6940969944000244,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.034065864980220795,
+ "rewards/margins": -0.0016761379083618522,
+ "rewards/rejected": 0.035742007195949554,
+ "step": 250
+ },
+ {
+ "epoch": 0.208,
+ "grad_norm": 1.0917680263519287,
+ "learning_rate": 6.906666666666666e-07,
+ "logits/chosen": 1.4094547033309937,
+ "logits/rejected": 1.50186026096344,
+ "logps/chosen": -132.91455078125,
+ "logps/rejected": -146.79714965820312,
+ "loss": 0.6930203914642334,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.03089061938226223,
+ "rewards/margins": 0.00040971505222842097,
+ "rewards/rejected": 0.030480902642011642,
+ "step": 260
+ },
+ {
+ "epoch": 0.216,
+ "grad_norm": 1.0280259847640991,
+ "learning_rate": 7.173333333333333e-07,
+ "logits/chosen": 1.531285285949707,
+ "logits/rejected": 1.6163132190704346,
+ "logps/chosen": -146.902587890625,
+ "logps/rejected": -147.8347625732422,
+ "loss": 0.6944749355316162,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.028514739125967026,
+ "rewards/margins": -0.0025452517438679934,
+ "rewards/rejected": 0.0310599897056818,
+ "step": 270
+ },
+ {
+ "epoch": 0.224,
+ "grad_norm": 1.398708701133728,
+ "learning_rate": 7.44e-07,
+ "logits/chosen": 1.563852071762085,
+ "logits/rejected": 1.396703839302063,
+ "logps/chosen": -172.68153381347656,
+ "logps/rejected": -155.80215454101562,
+ "loss": 0.689476490020752,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03224216774106026,
+ "rewards/margins": 0.007473687641322613,
+ "rewards/rejected": 0.02476847730576992,
+ "step": 280
+ },
+ {
+ "epoch": 0.232,
+ "grad_norm": 1.0394141674041748,
+ "learning_rate": 7.706666666666667e-07,
+ "logits/chosen": 1.5581729412078857,
+ "logits/rejected": 1.5819021463394165,
+ "logps/chosen": -144.41465759277344,
+ "logps/rejected": -153.55406188964844,
+ "loss": 0.6927877902984619,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.03272664546966553,
+ "rewards/margins": 0.0008557910332456231,
+ "rewards/rejected": 0.031870849430561066,
+ "step": 290
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 1.2557185888290405,
+ "learning_rate": 7.973333333333333e-07,
+ "logits/chosen": 1.5062446594238281,
+ "logits/rejected": 1.5786281824111938,
+ "logps/chosen": -153.63558959960938,
+ "logps/rejected": -165.46617126464844,
+ "loss": 0.6897892951965332,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03914078325033188,
+ "rewards/margins": 0.006873926613479853,
+ "rewards/rejected": 0.032266855239868164,
+ "step": 300
+ },
+ {
+ "epoch": 0.24,
+ "eval_logits/chosen": 1.5237834453582764,
+ "eval_logits/rejected": 1.557741641998291,
+ "eval_logps/chosen": -153.19883728027344,
+ "eval_logps/rejected": -147.7100830078125,
+ "eval_loss": 0.6913501024246216,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.03499976545572281,
+ "eval_rewards/margins": 0.0037925743963569403,
+ "eval_rewards/rejected": 0.031207194551825523,
+ "eval_runtime": 90.9133,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 300
+ },
+ {
+ "epoch": 0.248,
+ "grad_norm": 1.1200144290924072,
+ "learning_rate": 8.24e-07,
+ "logits/chosen": 1.618949294090271,
+ "logits/rejected": 1.6625702381134033,
+ "logps/chosen": -141.8953857421875,
+ "logps/rejected": -156.6927032470703,
+ "loss": 0.694425106048584,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.03162423521280289,
+ "rewards/margins": -0.0023833750747144222,
+ "rewards/rejected": 0.034007612615823746,
+ "step": 310
+ },
+ {
+ "epoch": 0.256,
+ "grad_norm": 1.0823743343353271,
+ "learning_rate": 8.506666666666667e-07,
+ "logits/chosen": 1.5974153280258179,
+ "logits/rejected": 1.6237560510635376,
+ "logps/chosen": -139.2996368408203,
+ "logps/rejected": -141.94407653808594,
+ "loss": 0.6914949893951416,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.029445698484778404,
+ "rewards/margins": 0.0034329916816204786,
+ "rewards/rejected": 0.026012707501649857,
+ "step": 320
+ },
+ {
+ "epoch": 0.264,
+ "grad_norm": 1.1765722036361694,
+ "learning_rate": 8.773333333333332e-07,
+ "logits/chosen": 1.465215802192688,
+ "logits/rejected": 1.4882014989852905,
+ "logps/chosen": -158.4329833984375,
+ "logps/rejected": -183.94259643554688,
+ "loss": 0.6925934791564942,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.026302698999643326,
+ "rewards/margins": 0.0012350418837741017,
+ "rewards/rejected": 0.025067657232284546,
+ "step": 330
+ },
+ {
+ "epoch": 0.272,
+ "grad_norm": 1.0667294263839722,
+ "learning_rate": 9.039999999999999e-07,
+ "logits/chosen": 1.7538875341415405,
+ "logits/rejected": 1.6171789169311523,
+ "logps/chosen": -159.63790893554688,
+ "logps/rejected": -151.1880340576172,
+ "loss": 0.6922041416168213,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.026619985699653625,
+ "rewards/margins": 0.002031774492934346,
+ "rewards/rejected": 0.024588212370872498,
+ "step": 340
+ },
+ {
+ "epoch": 0.28,
+ "grad_norm": 1.2210006713867188,
+ "learning_rate": 9.306666666666666e-07,
+ "logits/chosen": 1.5815479755401611,
+ "logits/rejected": 1.710217833518982,
+ "logps/chosen": -167.21743774414062,
+ "logps/rejected": -196.77989196777344,
+ "loss": 0.6899692058563233,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.03636014088988304,
+ "rewards/margins": 0.006536121014505625,
+ "rewards/rejected": 0.029824022203683853,
+ "step": 350
+ },
+ {
+ "epoch": 0.288,
+ "grad_norm": 0.9623845219612122,
+ "learning_rate": 9.573333333333333e-07,
+ "logits/chosen": 1.4416790008544922,
+ "logits/rejected": 1.3618693351745605,
+ "logps/chosen": -163.27401733398438,
+ "logps/rejected": -169.3351287841797,
+ "loss": 0.6910821437835694,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.037200987339019775,
+ "rewards/margins": 0.004338678438216448,
+ "rewards/rejected": 0.032862309366464615,
+ "step": 360
+ },
+ {
+ "epoch": 0.296,
+ "grad_norm": 1.1005645990371704,
+ "learning_rate": 9.84e-07,
+ "logits/chosen": 1.6214616298675537,
+ "logits/rejected": 1.6572606563568115,
+ "logps/chosen": -162.95504760742188,
+ "logps/rejected": -154.91807556152344,
+ "loss": 0.6895030021667481,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.047198958694934845,
+ "rewards/margins": 0.0075958347879350185,
+ "rewards/rejected": 0.03960312157869339,
+ "step": 370
+ },
+ {
+ "epoch": 0.304,
+ "grad_norm": 1.1126338243484497,
+ "learning_rate": 9.988148148148148e-07,
+ "logits/chosen": 1.4842126369476318,
+ "logits/rejected": 1.410930871963501,
+ "logps/chosen": -143.09182739257812,
+ "logps/rejected": -153.48135375976562,
+ "loss": 0.6885370254516602,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.05975018069148064,
+ "rewards/margins": 0.00981281790882349,
+ "rewards/rejected": 0.04993735998868942,
+ "step": 380
+ },
+ {
+ "epoch": 0.312,
+ "grad_norm": 1.1146304607391357,
+ "learning_rate": 9.95851851851852e-07,
+ "logits/chosen": 1.608332633972168,
+ "logits/rejected": 1.5848472118377686,
+ "logps/chosen": -152.25250244140625,
+ "logps/rejected": -138.348876953125,
+ "loss": 0.6851807117462159,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.07630724459886551,
+ "rewards/margins": 0.016460279002785683,
+ "rewards/rejected": 0.059846967458724976,
+ "step": 390
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 1.2686601877212524,
+ "learning_rate": 9.928888888888889e-07,
+ "logits/chosen": 1.5840070247650146,
+ "logits/rejected": 1.5679785013198853,
+ "logps/chosen": -155.90219116210938,
+ "logps/rejected": -148.89549255371094,
+ "loss": 0.6891039371490478,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.08129279315471649,
+ "rewards/margins": 0.008997146971523762,
+ "rewards/rejected": 0.072295643389225,
+ "step": 400
+ },
+ {
+ "epoch": 0.32,
+ "eval_logits/chosen": 1.5358340740203857,
+ "eval_logits/rejected": 1.5689741373062134,
+ "eval_logps/chosen": -152.6895294189453,
+ "eval_logps/rejected": -147.23422241210938,
+ "eval_loss": 0.6900544166564941,
+ "eval_rewards/accuracies": 0.5419999957084656,
+ "eval_rewards/chosen": 0.08593228459358215,
+ "eval_rewards/margins": 0.0071397931315004826,
+ "eval_rewards/rejected": 0.0787924975156784,
+ "eval_runtime": 90.9072,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 400
+ },
+ {
+ "epoch": 0.328,
+ "grad_norm": 1.1653108596801758,
+ "learning_rate": 9.899259259259258e-07,
+ "logits/chosen": 1.5450490713119507,
+ "logits/rejected": 1.4428151845932007,
+ "logps/chosen": -148.61581420898438,
+ "logps/rejected": -188.90919494628906,
+ "loss": 0.6918989181518554,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0946170911192894,
+ "rewards/margins": 0.003271549940109253,
+ "rewards/rejected": 0.09134554117918015,
+ "step": 410
+ },
+ {
+ "epoch": 0.336,
+ "grad_norm": 1.3114937543869019,
+ "learning_rate": 9.86962962962963e-07,
+ "logits/chosen": 1.5342715978622437,
+ "logits/rejected": 1.5572118759155273,
+ "logps/chosen": -152.65264892578125,
+ "logps/rejected": -168.03384399414062,
+ "loss": 0.6972955703735352,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.08869597315788269,
+ "rewards/margins": -0.007436127867549658,
+ "rewards/rejected": 0.09613210707902908,
+ "step": 420
+ },
+ {
+ "epoch": 0.344,
+ "grad_norm": 1.0040781497955322,
+ "learning_rate": 9.84e-07,
+ "logits/chosen": 1.5979712009429932,
+ "logits/rejected": 1.6713664531707764,
+ "logps/chosen": -164.2566375732422,
+ "logps/rejected": -152.1428680419922,
+ "loss": 0.6897010326385498,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.08101107180118561,
+ "rewards/margins": 0.007684691343456507,
+ "rewards/rejected": 0.07332637906074524,
+ "step": 430
+ },
+ {
+ "epoch": 0.352,
+ "grad_norm": 1.0801892280578613,
+ "learning_rate": 9.81037037037037e-07,
+ "logits/chosen": 1.439635992050171,
+ "logits/rejected": 1.3282365798950195,
+ "logps/chosen": -157.23403930664062,
+ "logps/rejected": -146.54806518554688,
+ "loss": 0.6881664752960205,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.08112544566392899,
+ "rewards/margins": 0.010971959680318832,
+ "rewards/rejected": 0.07015348225831985,
+ "step": 440
+ },
+ {
+ "epoch": 0.36,
+ "grad_norm": 1.1139949560165405,
+ "learning_rate": 9.78074074074074e-07,
+ "logits/chosen": 1.6345123052597046,
+ "logits/rejected": 1.5876491069793701,
+ "logps/chosen": -157.6160125732422,
+ "logps/rejected": -160.02040100097656,
+ "loss": 0.6868602275848389,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.07638200372457504,
+ "rewards/margins": 0.013464884832501411,
+ "rewards/rejected": 0.06291711330413818,
+ "step": 450
+ },
+ {
+ "epoch": 0.368,
+ "grad_norm": 1.158035397529602,
+ "learning_rate": 9.751111111111112e-07,
+ "logits/chosen": 1.5430586338043213,
+ "logits/rejected": 1.6022913455963135,
+ "logps/chosen": -151.98919677734375,
+ "logps/rejected": -164.90565490722656,
+ "loss": 0.6891930580139161,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06889279186725616,
+ "rewards/margins": 0.008791089057922363,
+ "rewards/rejected": 0.0601017065346241,
+ "step": 460
+ },
+ {
+ "epoch": 0.376,
+ "grad_norm": 1.0903704166412354,
+ "learning_rate": 9.721481481481481e-07,
+ "logits/chosen": 1.313881754875183,
+ "logits/rejected": 1.3182194232940674,
+ "logps/chosen": -131.5592041015625,
+ "logps/rejected": -123.84442138671875,
+ "loss": 0.6889429569244385,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.08150745928287506,
+ "rewards/margins": 0.009235726669430733,
+ "rewards/rejected": 0.07227173447608948,
+ "step": 470
+ },
+ {
+ "epoch": 0.384,
+ "grad_norm": 1.0766969919204712,
+ "learning_rate": 9.69185185185185e-07,
+ "logits/chosen": 1.425756812095642,
+ "logits/rejected": 1.476854920387268,
+ "logps/chosen": -148.72279357910156,
+ "logps/rejected": -164.40951538085938,
+ "loss": 0.6854116439819335,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1009502187371254,
+ "rewards/margins": 0.0166789498180151,
+ "rewards/rejected": 0.08427127450704575,
+ "step": 480
+ },
+ {
+ "epoch": 0.392,
+ "grad_norm": 1.2078129053115845,
+ "learning_rate": 9.662222222222222e-07,
+ "logits/chosen": 1.5511282682418823,
+ "logits/rejected": 1.611383080482483,
+ "logps/chosen": -143.22006225585938,
+ "logps/rejected": -159.70176696777344,
+ "loss": 0.6968960762023926,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.08779363334178925,
+ "rewards/margins": -0.00598606513813138,
+ "rewards/rejected": 0.09377971291542053,
+ "step": 490
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.9106306433677673,
+ "learning_rate": 9.632592592592593e-07,
+ "logits/chosen": 1.4217239618301392,
+ "logits/rejected": 1.505446195602417,
+ "logps/chosen": -144.34109497070312,
+ "logps/rejected": -135.52430725097656,
+ "loss": 0.691196870803833,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06902458518743515,
+ "rewards/margins": 0.005059319548308849,
+ "rewards/rejected": 0.06396526843309402,
+ "step": 500
+ },
+ {
+ "epoch": 0.4,
+ "eval_logits/chosen": 1.5185325145721436,
+ "eval_logits/rejected": 1.5510345697402954,
+ "eval_logps/chosen": -152.7921905517578,
+ "eval_logps/rejected": -147.3463134765625,
+ "eval_loss": 0.6898502111434937,
+ "eval_rewards/accuracies": 0.5099999904632568,
+ "eval_rewards/chosen": 0.07566512376070023,
+ "eval_rewards/margins": 0.008083080872893333,
+ "eval_rewards/rejected": 0.06758204847574234,
+ "eval_runtime": 90.8294,
+ "eval_samples_per_second": 5.505,
+ "eval_steps_per_second": 2.752,
+ "step": 500
+ },
+ {
+ "epoch": 0.408,
+ "grad_norm": 1.1637762784957886,
+ "learning_rate": 9.602962962962962e-07,
+ "logits/chosen": 1.5965330600738525,
+ "logits/rejected": 1.5338464975357056,
+ "logps/chosen": -131.49871826171875,
+ "logps/rejected": -140.05995178222656,
+ "loss": 0.6868171215057373,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.08385131508111954,
+ "rewards/margins": 0.013950209133327007,
+ "rewards/rejected": 0.06990110129117966,
+ "step": 510
+ },
+ {
+ "epoch": 0.416,
+ "grad_norm": 1.229719638824463,
+ "learning_rate": 9.573333333333333e-07,
+ "logits/chosen": 1.7057275772094727,
+ "logits/rejected": 1.5898948907852173,
+ "logps/chosen": -166.4267120361328,
+ "logps/rejected": -172.22337341308594,
+ "loss": 0.6912094116210937,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.07427279651165009,
+ "rewards/margins": 0.004661654122173786,
+ "rewards/rejected": 0.06961113959550858,
+ "step": 520
+ },
+ {
+ "epoch": 0.424,
+ "grad_norm": 1.136383295059204,
+ "learning_rate": 9.543703703703705e-07,
+ "logits/chosen": 1.6207077503204346,
+ "logits/rejected": 1.5417520999908447,
+ "logps/chosen": -166.81509399414062,
+ "logps/rejected": -168.5408172607422,
+ "loss": 0.6772748470306397,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.0953373983502388,
+ "rewards/margins": 0.033697597682476044,
+ "rewards/rejected": 0.06163979694247246,
+ "step": 530
+ },
+ {
+ "epoch": 0.432,
+ "grad_norm": 1.0640541315078735,
+ "learning_rate": 9.514074074074074e-07,
+ "logits/chosen": 1.6825854778289795,
+ "logits/rejected": 1.6327041387557983,
+ "logps/chosen": -181.39956665039062,
+ "logps/rejected": -158.49647521972656,
+ "loss": 0.6883615493774414,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.10187923908233643,
+ "rewards/margins": 0.010873330757021904,
+ "rewards/rejected": 0.09100590646266937,
+ "step": 540
+ },
+ {
+ "epoch": 0.44,
+ "grad_norm": 1.477909803390503,
+ "learning_rate": 9.484444444444444e-07,
+ "logits/chosen": 1.5402686595916748,
+ "logits/rejected": 1.6487388610839844,
+ "logps/chosen": -151.53814697265625,
+ "logps/rejected": -167.8573455810547,
+ "loss": 0.6915213584899902,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09472338110208511,
+ "rewards/margins": 0.005475065670907497,
+ "rewards/rejected": 0.08924831449985504,
+ "step": 550
+ },
+ {
+ "epoch": 0.448,
+ "grad_norm": 1.0138317346572876,
+ "learning_rate": 9.454814814814814e-07,
+ "logits/chosen": 1.6286503076553345,
+ "logits/rejected": 1.5223101377487183,
+ "logps/chosen": -154.72079467773438,
+ "logps/rejected": -139.2535858154297,
+ "loss": 0.6835216522216797,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.09194488823413849,
+ "rewards/margins": 0.02140027843415737,
+ "rewards/rejected": 0.07054460793733597,
+ "step": 560
+ },
+ {
+ "epoch": 0.456,
+ "grad_norm": 0.896689772605896,
+ "learning_rate": 9.425185185185184e-07,
+ "logits/chosen": 1.4857146739959717,
+ "logits/rejected": 1.4369819164276123,
+ "logps/chosen": -158.31809997558594,
+ "logps/rejected": -152.33676147460938,
+ "loss": 0.6863360404968262,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.07895292341709137,
+ "rewards/margins": 0.016260787844657898,
+ "rewards/rejected": 0.06269213557243347,
+ "step": 570
+ },
+ {
+ "epoch": 0.464,
+ "grad_norm": 1.4772330522537231,
+ "learning_rate": 9.395555555555556e-07,
+ "logits/chosen": 1.315298318862915,
+ "logits/rejected": 1.4757534265518188,
+ "logps/chosen": -136.4202117919922,
+ "logps/rejected": -154.99362182617188,
+ "loss": 0.6916167259216308,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.04677204787731171,
+ "rewards/margins": 0.005649724509567022,
+ "rewards/rejected": 0.041122324764728546,
+ "step": 580
+ },
+ {
+ "epoch": 0.472,
+ "grad_norm": 0.9806157350540161,
+ "learning_rate": 9.365925925925926e-07,
+ "logits/chosen": 1.465427279472351,
+ "logits/rejected": 1.459496259689331,
+ "logps/chosen": -145.07470703125,
+ "logps/rejected": -131.71974182128906,
+ "loss": 0.6857181549072265,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.07979898899793625,
+ "rewards/margins": 0.016965944319963455,
+ "rewards/rejected": 0.0628330409526825,
+ "step": 590
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 1.4313396215438843,
+ "learning_rate": 9.336296296296295e-07,
+ "logits/chosen": 1.5190367698669434,
+ "logits/rejected": 1.563504934310913,
+ "logps/chosen": -132.0201416015625,
+ "logps/rejected": -158.89865112304688,
+ "loss": 0.6964848041534424,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.06544725596904755,
+ "rewards/margins": -0.004280097782611847,
+ "rewards/rejected": 0.0697273463010788,
+ "step": 600
+ },
+ {
+ "epoch": 0.48,
+ "eval_logits/chosen": 1.5031698942184448,
+ "eval_logits/rejected": 1.5348584651947021,
+ "eval_logps/chosen": -152.76596069335938,
+ "eval_logps/rejected": -147.3241729736328,
+ "eval_loss": 0.6903045773506165,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": 0.07828804850578308,
+ "eval_rewards/margins": 0.008492168970406055,
+ "eval_rewards/rejected": 0.0697958841919899,
+ "eval_runtime": 90.9881,
+ "eval_samples_per_second": 5.495,
+ "eval_steps_per_second": 2.748,
+ "step": 600
+ },
+ {
+ "epoch": 0.488,
+ "grad_norm": 0.9467533230781555,
+ "learning_rate": 9.306666666666666e-07,
+ "logits/chosen": 1.5242283344268799,
+ "logits/rejected": 1.5798314809799194,
+ "logps/chosen": -131.90048217773438,
+ "logps/rejected": -149.93222045898438,
+ "loss": 0.6899854183197022,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.08419154584407806,
+ "rewards/margins": 0.00830511748790741,
+ "rewards/rejected": 0.07588644325733185,
+ "step": 610
+ },
+ {
+ "epoch": 0.496,
+ "grad_norm": 1.8463492393493652,
+ "learning_rate": 9.277037037037037e-07,
+ "logits/chosen": 1.4965155124664307,
+ "logits/rejected": 1.6336272954940796,
+ "logps/chosen": -156.1863555908203,
+ "logps/rejected": -167.74075317382812,
+ "loss": 0.6977802276611328,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.05518193915486336,
+ "rewards/margins": -0.006808985956013203,
+ "rewards/rejected": 0.061990927904844284,
+ "step": 620
+ },
+ {
+ "epoch": 0.504,
+ "grad_norm": 1.278316617012024,
+ "learning_rate": 9.247407407407407e-07,
+ "logits/chosen": 1.5074065923690796,
+ "logits/rejected": 1.535696029663086,
+ "logps/chosen": -167.56777954101562,
+ "logps/rejected": -147.04922485351562,
+ "loss": 0.6806219100952149,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.0745314508676529,
+ "rewards/margins": 0.027104835957288742,
+ "rewards/rejected": 0.047426607459783554,
+ "step": 630
+ },
+ {
+ "epoch": 0.512,
+ "grad_norm": 1.0424679517745972,
+ "learning_rate": 9.217777777777778e-07,
+ "logits/chosen": 1.4299286603927612,
+ "logits/rejected": 1.4484076499938965,
+ "logps/chosen": -168.305908203125,
+ "logps/rejected": -152.61854553222656,
+ "loss": 0.6871783256530761,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07751411944627762,
+ "rewards/margins": 0.017045259475708008,
+ "rewards/rejected": 0.06046885997056961,
+ "step": 640
+ },
+ {
+ "epoch": 0.52,
+ "grad_norm": 1.1143999099731445,
+ "learning_rate": 9.188148148148148e-07,
+ "logits/chosen": 1.5520436763763428,
+ "logits/rejected": 1.531718134880066,
+ "logps/chosen": -154.92115783691406,
+ "logps/rejected": -126.78369140625,
+ "loss": 0.6889286518096924,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.0945608839392662,
+ "rewards/margins": 0.01080345083028078,
+ "rewards/rejected": 0.0837574377655983,
+ "step": 650
+ },
+ {
+ "epoch": 0.528,
+ "grad_norm": 1.2762326002120972,
+ "learning_rate": 9.158518518518517e-07,
+ "logits/chosen": 1.5747522115707397,
+ "logits/rejected": 1.4972833395004272,
+ "logps/chosen": -151.13914489746094,
+ "logps/rejected": -132.013916015625,
+ "loss": 0.6900454998016358,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.07303085923194885,
+ "rewards/margins": 0.008073865436017513,
+ "rewards/rejected": 0.06495700031518936,
+ "step": 660
+ },
+ {
+ "epoch": 0.536,
+ "grad_norm": 1.5207557678222656,
+ "learning_rate": 9.128888888888888e-07,
+ "logits/chosen": 1.6594665050506592,
+ "logits/rejected": 1.468492031097412,
+ "logps/chosen": -166.0108184814453,
+ "logps/rejected": -151.38555908203125,
+ "loss": 0.6934059143066407,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.09294265508651733,
+ "rewards/margins": 0.0021180796902626753,
+ "rewards/rejected": 0.09082455933094025,
+ "step": 670
+ },
+ {
+ "epoch": 0.544,
+ "grad_norm": 1.1073335409164429,
+ "learning_rate": 9.099259259259259e-07,
+ "logits/chosen": 1.5321934223175049,
+ "logits/rejected": 1.5313541889190674,
+ "logps/chosen": -144.5046844482422,
+ "logps/rejected": -133.05470275878906,
+ "loss": 0.6824594974517822,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.10782947391271591,
+ "rewards/margins": 0.0230915155261755,
+ "rewards/rejected": 0.08473796397447586,
+ "step": 680
+ },
+ {
+ "epoch": 0.552,
+ "grad_norm": 1.096970558166504,
+ "learning_rate": 9.069629629629629e-07,
+ "logits/chosen": 1.438305139541626,
+ "logits/rejected": 1.4338308572769165,
+ "logps/chosen": -173.1280517578125,
+ "logps/rejected": -160.20950317382812,
+ "loss": 0.6878445625305176,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09615986794233322,
+ "rewards/margins": 0.012350928969681263,
+ "rewards/rejected": 0.08380892872810364,
+ "step": 690
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 1.3664878606796265,
+ "learning_rate": 9.039999999999999e-07,
+ "logits/chosen": 1.530369758605957,
+ "logits/rejected": 1.5520622730255127,
+ "logps/chosen": -159.60740661621094,
+ "logps/rejected": -137.77352905273438,
+ "loss": 0.6991429805755616,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07249583303928375,
+ "rewards/margins": -0.009333941154181957,
+ "rewards/rejected": 0.08182977139949799,
+ "step": 700
+ },
+ {
+ "epoch": 0.56,
+ "eval_logits/chosen": 1.5136796236038208,
+ "eval_logits/rejected": 1.5456002950668335,
+ "eval_logps/chosen": -152.5748748779297,
+ "eval_logps/rejected": -147.17152404785156,
+ "eval_loss": 0.688511848449707,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.09739598631858826,
+ "eval_rewards/margins": 0.012335418723523617,
+ "eval_rewards/rejected": 0.08506056666374207,
+ "eval_runtime": 90.8734,
+ "eval_samples_per_second": 5.502,
+ "eval_steps_per_second": 2.751,
+ "step": 700
+ },
+ {
+ "epoch": 0.568,
+ "grad_norm": 1.224107265472412,
+ "learning_rate": 9.010370370370371e-07,
+ "logits/chosen": 1.6023967266082764,
+ "logits/rejected": 1.558224081993103,
+ "logps/chosen": -148.47120666503906,
+ "logps/rejected": -163.5648651123047,
+ "loss": 0.6865221500396729,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.12494877725839615,
+ "rewards/margins": 0.01545787788927555,
+ "rewards/rejected": 0.10949089378118515,
+ "step": 710
+ },
+ {
+ "epoch": 0.576,
+ "grad_norm": 1.2135164737701416,
+ "learning_rate": 8.98074074074074e-07,
+ "logits/chosen": 1.5902750492095947,
+ "logits/rejected": 1.7012746334075928,
+ "logps/chosen": -139.7012939453125,
+ "logps/rejected": -135.42225646972656,
+ "loss": 0.7014039039611817,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.09102506935596466,
+ "rewards/margins": -0.014695463702082634,
+ "rewards/rejected": 0.10572052001953125,
+ "step": 720
+ },
+ {
+ "epoch": 0.584,
+ "grad_norm": 1.1953649520874023,
+ "learning_rate": 8.95111111111111e-07,
+ "logits/chosen": 1.6540731191635132,
+ "logits/rejected": 1.6386349201202393,
+ "logps/chosen": -153.12847900390625,
+ "logps/rejected": -149.66319274902344,
+ "loss": 0.6952126502990723,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.1124584898352623,
+ "rewards/margins": -0.0012063594767823815,
+ "rewards/rejected": 0.11366486549377441,
+ "step": 730
+ },
+ {
+ "epoch": 0.592,
+ "grad_norm": 1.147875428199768,
+ "learning_rate": 8.921481481481481e-07,
+ "logits/chosen": 1.5578330755233765,
+ "logits/rejected": 1.4689290523529053,
+ "logps/chosen": -156.93788146972656,
+ "logps/rejected": -134.1832733154297,
+ "loss": 0.6919830799102783,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09214536100625992,
+ "rewards/margins": 0.004291228950023651,
+ "rewards/rejected": 0.08785412460565567,
+ "step": 740
+ },
+ {
+ "epoch": 0.6,
+ "grad_norm": 1.21685791015625,
+ "learning_rate": 8.891851851851852e-07,
+ "logits/chosen": 1.4241983890533447,
+ "logits/rejected": 1.4537718296051025,
+ "logps/chosen": -136.9163055419922,
+ "logps/rejected": -158.44491577148438,
+ "loss": 0.691422986984253,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.09173480421304703,
+ "rewards/margins": 0.00533579895272851,
+ "rewards/rejected": 0.08639900386333466,
+ "step": 750
+ },
+ {
+ "epoch": 0.608,
+ "grad_norm": 1.3629519939422607,
+ "learning_rate": 8.862222222222222e-07,
+ "logits/chosen": 1.565716028213501,
+ "logits/rejected": 1.3824570178985596,
+ "logps/chosen": -138.8521728515625,
+ "logps/rejected": -129.51028442382812,
+ "loss": 0.6931031227111817,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0763697624206543,
+ "rewards/margins": 0.0021251302678138018,
+ "rewards/rejected": 0.07424463331699371,
+ "step": 760
+ },
+ {
+ "epoch": 0.616,
+ "grad_norm": 1.1030747890472412,
+ "learning_rate": 8.832592592592593e-07,
+ "logits/chosen": 1.5781702995300293,
+ "logits/rejected": 1.5428775548934937,
+ "logps/chosen": -141.01480102539062,
+ "logps/rejected": -136.47189331054688,
+ "loss": 0.6777657032012939,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.10366280376911163,
+ "rewards/margins": 0.0333605632185936,
+ "rewards/rejected": 0.07030224055051804,
+ "step": 770
+ },
+ {
+ "epoch": 0.624,
+ "grad_norm": 1.3228758573532104,
+ "learning_rate": 8.802962962962962e-07,
+ "logits/chosen": 1.451596736907959,
+ "logits/rejected": 1.5489904880523682,
+ "logps/chosen": -153.74618530273438,
+ "logps/rejected": -159.26333618164062,
+ "loss": 0.6870471000671386,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.10403706133365631,
+ "rewards/margins": 0.014494165778160095,
+ "rewards/rejected": 0.08954289555549622,
+ "step": 780
+ },
+ {
+ "epoch": 0.632,
+ "grad_norm": 1.3267385959625244,
+ "learning_rate": 8.773333333333332e-07,
+ "logits/chosen": 1.4153831005096436,
+ "logits/rejected": 1.4497296810150146,
+ "logps/chosen": -148.05368041992188,
+ "logps/rejected": -151.3020477294922,
+ "loss": 0.6877872943878174,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.07300253212451935,
+ "rewards/margins": 0.013981563039124012,
+ "rewards/rejected": 0.05902096629142761,
+ "step": 790
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 1.2227483987808228,
+ "learning_rate": 8.743703703703703e-07,
+ "logits/chosen": 1.6773914098739624,
+ "logits/rejected": 1.6918518543243408,
+ "logps/chosen": -180.06863403320312,
+ "logps/rejected": -194.30752563476562,
+ "loss": 0.7015917778015137,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07480724155902863,
+ "rewards/margins": -0.013628068380057812,
+ "rewards/rejected": 0.08843531459569931,
+ "step": 800
+ },
+ {
+ "epoch": 0.64,
+ "eval_logits/chosen": 1.5051140785217285,
+ "eval_logits/rejected": 1.5367870330810547,
+ "eval_logps/chosen": -152.8026885986328,
+ "eval_logps/rejected": -147.39437866210938,
+ "eval_loss": 0.688633918762207,
+ "eval_rewards/accuracies": 0.5220000147819519,
+ "eval_rewards/chosen": 0.07461711764335632,
+ "eval_rewards/margins": 0.011839868500828743,
+ "eval_rewards/rejected": 0.06277725100517273,
+ "eval_runtime": 90.9206,
+ "eval_samples_per_second": 5.499,
+ "eval_steps_per_second": 2.75,
+ "step": 800
+ },
+ {
+ "epoch": 0.648,
+ "grad_norm": 1.209770679473877,
+ "learning_rate": 8.714074074074074e-07,
+ "logits/chosen": 1.3785041570663452,
+ "logits/rejected": 1.60085928440094,
+ "logps/chosen": -114.5325927734375,
+ "logps/rejected": -151.88900756835938,
+ "loss": 0.6890413761138916,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0640551745891571,
+ "rewards/margins": 0.010126596316695213,
+ "rewards/rejected": 0.05392857640981674,
+ "step": 810
+ },
+ {
+ "epoch": 0.656,
+ "grad_norm": 1.0250163078308105,
+ "learning_rate": 8.684444444444444e-07,
+ "logits/chosen": 1.5827982425689697,
+ "logits/rejected": 1.6239618062973022,
+ "logps/chosen": -149.01904296875,
+ "logps/rejected": -141.9372100830078,
+ "loss": 0.6728014469146728,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.0830039232969284,
+ "rewards/margins": 0.043090131133794785,
+ "rewards/rejected": 0.03991378843784332,
+ "step": 820
+ },
+ {
+ "epoch": 0.664,
+ "grad_norm": 1.0627655982971191,
+ "learning_rate": 8.654814814814814e-07,
+ "logits/chosen": 1.501873254776001,
+ "logits/rejected": 1.537386417388916,
+ "logps/chosen": -138.75169372558594,
+ "logps/rejected": -134.00970458984375,
+ "loss": 0.6801187992095947,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0640982910990715,
+ "rewards/margins": 0.02855527400970459,
+ "rewards/rejected": 0.03554301708936691,
+ "step": 830
+ },
+ {
+ "epoch": 0.672,
+ "grad_norm": 1.2453433275222778,
+ "learning_rate": 8.625185185185186e-07,
+ "logits/chosen": 1.4766005277633667,
+ "logits/rejected": 1.5185787677764893,
+ "logps/chosen": -148.41250610351562,
+ "logps/rejected": -157.73989868164062,
+ "loss": 0.6788079738616943,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.06551454961299896,
+ "rewards/margins": 0.03301671892404556,
+ "rewards/rejected": 0.0324978306889534,
+ "step": 840
+ },
+ {
+ "epoch": 0.68,
+ "grad_norm": 1.191698670387268,
+ "learning_rate": 8.595555555555555e-07,
+ "logits/chosen": 1.555551528930664,
+ "logits/rejected": 1.5438499450683594,
+ "logps/chosen": -160.77488708496094,
+ "logps/rejected": -141.9876251220703,
+ "loss": 0.6801597595214843,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.053498052060604095,
+ "rewards/margins": 0.028470590710639954,
+ "rewards/rejected": 0.025027472525835037,
+ "step": 850
+ },
+ {
+ "epoch": 0.688,
+ "grad_norm": 0.9382092356681824,
+ "learning_rate": 8.565925925925925e-07,
+ "logits/chosen": 1.5482738018035889,
+ "logits/rejected": 1.5833700895309448,
+ "logps/chosen": -139.68118286132812,
+ "logps/rejected": -130.05947875976562,
+ "loss": 0.6825921535491943,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.06382572650909424,
+ "rewards/margins": 0.023111844435334206,
+ "rewards/rejected": 0.04071388021111488,
+ "step": 860
+ },
+ {
+ "epoch": 0.696,
+ "grad_norm": 1.2248382568359375,
+ "learning_rate": 8.536296296296296e-07,
+ "logits/chosen": 1.4529926776885986,
+ "logits/rejected": 1.4423902034759521,
+ "logps/chosen": -154.4588165283203,
+ "logps/rejected": -153.44949340820312,
+ "loss": 0.6915592193603516,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.03207828477025032,
+ "rewards/margins": 0.006785523146390915,
+ "rewards/rejected": 0.025292763486504555,
+ "step": 870
+ },
+ {
+ "epoch": 0.704,
+ "grad_norm": 1.2174893617630005,
+ "learning_rate": 8.506666666666667e-07,
+ "logits/chosen": 1.4073113203048706,
+ "logits/rejected": 1.668140172958374,
+ "logps/chosen": -145.18460083007812,
+ "logps/rejected": -157.0777587890625,
+ "loss": 0.6922356605529785,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.049450717866420746,
+ "rewards/margins": 0.004049716051667929,
+ "rewards/rejected": 0.04540099948644638,
+ "step": 880
+ },
+ {
+ "epoch": 0.712,
+ "grad_norm": 1.3186850547790527,
+ "learning_rate": 8.477037037037037e-07,
+ "logits/chosen": 1.2923892736434937,
+ "logits/rejected": 1.491051435470581,
+ "logps/chosen": -149.03018188476562,
+ "logps/rejected": -161.61715698242188,
+ "loss": 0.6841150760650635,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.038961172103881836,
+ "rewards/margins": 0.020759906619787216,
+ "rewards/rejected": 0.01820126175880432,
+ "step": 890
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 1.1123124361038208,
+ "learning_rate": 8.447407407407407e-07,
+ "logits/chosen": 1.546547293663025,
+ "logits/rejected": 1.682992696762085,
+ "logps/chosen": -166.2056884765625,
+ "logps/rejected": -180.73330688476562,
+ "loss": 0.6826215744018554,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.07279330492019653,
+ "rewards/margins": 0.025264525786042213,
+ "rewards/rejected": 0.04752878472208977,
+ "step": 900
+ },
+ {
+ "epoch": 0.72,
+ "eval_logits/chosen": 1.4819762706756592,
+ "eval_logits/rejected": 1.513027548789978,
+ "eval_logps/chosen": -153.1695556640625,
+ "eval_logps/rejected": -147.7560577392578,
+ "eval_loss": 0.689373254776001,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": 0.03792775049805641,
+ "eval_rewards/margins": 0.011320074088871479,
+ "eval_rewards/rejected": 0.026607677340507507,
+ "eval_runtime": 90.9603,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 900
+ },
+ {
+ "epoch": 0.728,
+ "grad_norm": 1.1507582664489746,
+ "learning_rate": 8.417777777777777e-07,
+ "logits/chosen": 1.5831308364868164,
+ "logits/rejected": 1.3737132549285889,
+ "logps/chosen": -140.00253295898438,
+ "logps/rejected": -138.45155334472656,
+ "loss": 0.6873391628265381,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.05066479369997978,
+ "rewards/margins": 0.015466553159058094,
+ "rewards/rejected": 0.03519824147224426,
+ "step": 910
+ },
+ {
+ "epoch": 0.736,
+ "grad_norm": 1.415456771850586,
+ "learning_rate": 8.388148148148147e-07,
+ "logits/chosen": 1.3000686168670654,
+ "logits/rejected": 1.2642180919647217,
+ "logps/chosen": -160.14027404785156,
+ "logps/rejected": -163.32827758789062,
+ "loss": 0.687142276763916,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.06618930399417877,
+ "rewards/margins": 0.014807088300585747,
+ "rewards/rejected": 0.05138222128152847,
+ "step": 920
+ },
+ {
+ "epoch": 0.744,
+ "grad_norm": 1.3228634595870972,
+ "learning_rate": 8.358518518518518e-07,
+ "logits/chosen": 1.5204170942306519,
+ "logits/rejected": 1.6129217147827148,
+ "logps/chosen": -157.99832153320312,
+ "logps/rejected": -162.09567260742188,
+ "loss": 0.6790886878967285,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.05940447002649307,
+ "rewards/margins": 0.03161252290010452,
+ "rewards/rejected": 0.027791941538453102,
+ "step": 930
+ },
+ {
+ "epoch": 0.752,
+ "grad_norm": 1.048064947128296,
+ "learning_rate": 8.328888888888889e-07,
+ "logits/chosen": 1.5515320301055908,
+ "logits/rejected": 1.676857352256775,
+ "logps/chosen": -148.4853515625,
+ "logps/rejected": -174.4320526123047,
+ "loss": 0.6935453414916992,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.010225490666925907,
+ "rewards/margins": 0.0021693313028663397,
+ "rewards/rejected": 0.008056161925196648,
+ "step": 940
+ },
+ {
+ "epoch": 0.76,
+ "grad_norm": 1.247799038887024,
+ "learning_rate": 8.299259259259259e-07,
+ "logits/chosen": 1.384231686592102,
+ "logits/rejected": 1.4611561298370361,
+ "logps/chosen": -140.81443786621094,
+ "logps/rejected": -153.81326293945312,
+ "loss": 0.6805226325988769,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.053223587572574615,
+ "rewards/margins": 0.02933613583445549,
+ "rewards/rejected": 0.023887457326054573,
+ "step": 950
+ },
+ {
+ "epoch": 0.768,
+ "grad_norm": 1.1611264944076538,
+ "learning_rate": 8.269629629629629e-07,
+ "logits/chosen": 1.4987528324127197,
+ "logits/rejected": 1.3684321641921997,
+ "logps/chosen": -135.81056213378906,
+ "logps/rejected": -119.05133056640625,
+ "loss": 0.6954316616058349,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.0197703056037426,
+ "rewards/margins": -0.0012637012405321002,
+ "rewards/rejected": 0.021034007892012596,
+ "step": 960
+ },
+ {
+ "epoch": 0.776,
+ "grad_norm": 0.964948832988739,
+ "learning_rate": 8.24e-07,
+ "logits/chosen": 1.612053632736206,
+ "logits/rejected": 1.7114235162734985,
+ "logps/chosen": -153.56137084960938,
+ "logps/rejected": -149.17323303222656,
+ "loss": 0.6916662693023682,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.05082285404205322,
+ "rewards/margins": 0.0056606316938996315,
+ "rewards/rejected": 0.04516221210360527,
+ "step": 970
+ },
+ {
+ "epoch": 0.784,
+ "grad_norm": 1.409449815750122,
+ "learning_rate": 8.21037037037037e-07,
+ "logits/chosen": 1.384060025215149,
+ "logits/rejected": 1.316361904144287,
+ "logps/chosen": -168.75360107421875,
+ "logps/rejected": -141.79537963867188,
+ "loss": 0.6845529556274415,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.0886571854352951,
+ "rewards/margins": 0.02084263041615486,
+ "rewards/rejected": 0.06781454384326935,
+ "step": 980
+ },
+ {
+ "epoch": 0.792,
+ "grad_norm": 1.0246562957763672,
+ "learning_rate": 8.18074074074074e-07,
+ "logits/chosen": 1.4986507892608643,
+ "logits/rejected": 1.571103811264038,
+ "logps/chosen": -165.22254943847656,
+ "logps/rejected": -168.63540649414062,
+ "loss": 0.6790967464447022,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.07517905533313751,
+ "rewards/margins": 0.03272219002246857,
+ "rewards/rejected": 0.04245685786008835,
+ "step": 990
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 1.014426827430725,
+ "learning_rate": 8.15111111111111e-07,
+ "logits/chosen": 1.4708755016326904,
+ "logits/rejected": 1.4550955295562744,
+ "logps/chosen": -149.49874877929688,
+ "logps/rejected": -125.0966567993164,
+ "loss": 0.6892098903656005,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.07825140655040741,
+ "rewards/margins": 0.01007872074842453,
+ "rewards/rejected": 0.06817268580198288,
+ "step": 1000
+ },
+ {
+ "epoch": 0.8,
+ "eval_logits/chosen": 1.4972777366638184,
+ "eval_logits/rejected": 1.5283576250076294,
+ "eval_logps/chosen": -152.8240966796875,
+ "eval_logps/rejected": -147.44175720214844,
+ "eval_loss": 0.6880542039871216,
+ "eval_rewards/accuracies": 0.5419999957084656,
+ "eval_rewards/chosen": 0.07247376441955566,
+ "eval_rewards/margins": 0.01443479023873806,
+ "eval_rewards/rejected": 0.058038972318172455,
+ "eval_runtime": 90.827,
+ "eval_samples_per_second": 5.505,
+ "eval_steps_per_second": 2.752,
+ "step": 1000
+ },
+ {
+ "epoch": 0.808,
+ "grad_norm": 1.1155726909637451,
+ "learning_rate": 8.121481481481482e-07,
+ "logits/chosen": 1.547481894493103,
+ "logits/rejected": 1.5461527109146118,
+ "logps/chosen": -154.89791870117188,
+ "logps/rejected": -142.05111694335938,
+ "loss": 0.6809016704559326,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.10093016922473907,
+ "rewards/margins": 0.027038773521780968,
+ "rewards/rejected": 0.07389138638973236,
+ "step": 1010
+ },
+ {
+ "epoch": 0.816,
+ "grad_norm": 1.6716786623001099,
+ "learning_rate": 8.091851851851852e-07,
+ "logits/chosen": 1.3954919576644897,
+ "logits/rejected": 1.5088036060333252,
+ "logps/chosen": -143.11846923828125,
+ "logps/rejected": -146.41482543945312,
+ "loss": 0.6849985122680664,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.05970728397369385,
+ "rewards/margins": 0.021999523043632507,
+ "rewards/rejected": 0.03770776465535164,
+ "step": 1020
+ },
+ {
+ "epoch": 0.824,
+ "grad_norm": 1.308182716369629,
+ "learning_rate": 8.062222222222221e-07,
+ "logits/chosen": 1.532488465309143,
+ "logits/rejected": 1.3316718339920044,
+ "logps/chosen": -163.06858825683594,
+ "logps/rejected": -136.57989501953125,
+ "loss": 0.6789252281188964,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.11086118221282959,
+ "rewards/margins": 0.033117204904556274,
+ "rewards/rejected": 0.07774396985769272,
+ "step": 1030
+ },
+ {
+ "epoch": 0.832,
+ "grad_norm": 1.1019353866577148,
+ "learning_rate": 8.032592592592592e-07,
+ "logits/chosen": 1.4798121452331543,
+ "logits/rejected": 1.5203083753585815,
+ "logps/chosen": -140.3281707763672,
+ "logps/rejected": -141.06216430664062,
+ "loss": 0.6724937915802002,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.1202714815735817,
+ "rewards/margins": 0.046712182462215424,
+ "rewards/rejected": 0.07355931401252747,
+ "step": 1040
+ },
+ {
+ "epoch": 0.84,
+ "grad_norm": 1.2966418266296387,
+ "learning_rate": 8.002962962962963e-07,
+ "logits/chosen": 1.3034439086914062,
+ "logits/rejected": 1.4363505840301514,
+ "logps/chosen": -142.1992645263672,
+ "logps/rejected": -132.0046844482422,
+ "loss": 0.6848338603973388,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.11126589775085449,
+ "rewards/margins": 0.019870441406965256,
+ "rewards/rejected": 0.09139545261859894,
+ "step": 1050
+ },
+ {
+ "epoch": 0.848,
+ "grad_norm": 1.0321130752563477,
+ "learning_rate": 7.973333333333333e-07,
+ "logits/chosen": 1.3257265090942383,
+ "logits/rejected": 1.4649587869644165,
+ "logps/chosen": -112.98075103759766,
+ "logps/rejected": -127.4059829711914,
+ "loss": 0.6679783344268799,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.17311830818653107,
+ "rewards/margins": 0.05631983280181885,
+ "rewards/rejected": 0.11679847538471222,
+ "step": 1060
+ },
+ {
+ "epoch": 0.856,
+ "grad_norm": 1.2037222385406494,
+ "learning_rate": 7.943703703703704e-07,
+ "logits/chosen": 1.3766987323760986,
+ "logits/rejected": 1.2802103757858276,
+ "logps/chosen": -154.8142547607422,
+ "logps/rejected": -148.680908203125,
+ "loss": 0.668931531906128,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1449051946401596,
+ "rewards/margins": 0.05721765011548996,
+ "rewards/rejected": 0.08768755197525024,
+ "step": 1070
+ },
+ {
+ "epoch": 0.864,
+ "grad_norm": 0.9230766296386719,
+ "learning_rate": 7.914074074074074e-07,
+ "logits/chosen": 1.447328805923462,
+ "logits/rejected": 1.5235106945037842,
+ "logps/chosen": -150.21812438964844,
+ "logps/rejected": -123.29923248291016,
+ "loss": 0.6716424465179444,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.16066041588783264,
+ "rewards/margins": 0.04824484512209892,
+ "rewards/rejected": 0.11241557449102402,
+ "step": 1080
+ },
+ {
+ "epoch": 0.872,
+ "grad_norm": 1.0423877239227295,
+ "learning_rate": 7.884444444444443e-07,
+ "logits/chosen": 1.6920299530029297,
+ "logits/rejected": 1.6123542785644531,
+ "logps/chosen": -175.1676025390625,
+ "logps/rejected": -185.07223510742188,
+ "loss": 0.6723968505859375,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.16490009427070618,
+ "rewards/margins": 0.046577490866184235,
+ "rewards/rejected": 0.11832261085510254,
+ "step": 1090
+ },
+ {
+ "epoch": 0.88,
+ "grad_norm": 1.4142779111862183,
+ "learning_rate": 7.854814814814814e-07,
+ "logits/chosen": 1.453111171722412,
+ "logits/rejected": 1.4981731176376343,
+ "logps/chosen": -139.9765167236328,
+ "logps/rejected": -153.63058471679688,
+ "loss": 0.6872457027435303,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.13096000254154205,
+ "rewards/margins": 0.01702108606696129,
+ "rewards/rejected": 0.11393891274929047,
+ "step": 1100
+ },
+ {
+ "epoch": 0.88,
+ "eval_logits/chosen": 1.5076195001602173,
+ "eval_logits/rejected": 1.5373249053955078,
+ "eval_logps/chosen": -152.1697998046875,
+ "eval_logps/rejected": -146.83502197265625,
+ "eval_loss": 0.6873304843902588,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.13790392875671387,
+ "eval_rewards/margins": 0.019191108644008636,
+ "eval_rewards/rejected": 0.11871281266212463,
+ "eval_runtime": 91.0157,
+ "eval_samples_per_second": 5.494,
+ "eval_steps_per_second": 2.747,
+ "step": 1100
+ },
+ {
+ "epoch": 0.888,
+ "grad_norm": 1.413996696472168,
+ "learning_rate": 7.825185185185185e-07,
+ "logits/chosen": 1.5183905363082886,
+ "logits/rejected": 1.4942539930343628,
+ "logps/chosen": -156.383544921875,
+ "logps/rejected": -145.61398315429688,
+ "loss": 0.6810199737548828,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.14243564009666443,
+ "rewards/margins": 0.03136260434985161,
+ "rewards/rejected": 0.11107305437326431,
+ "step": 1110
+ },
+ {
+ "epoch": 0.896,
+ "grad_norm": 1.0046707391738892,
+ "learning_rate": 7.795555555555555e-07,
+ "logits/chosen": 1.4982213973999023,
+ "logits/rejected": 1.5465660095214844,
+ "logps/chosen": -156.79293823242188,
+ "logps/rejected": -150.77853393554688,
+ "loss": 0.6808318614959716,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1551930010318756,
+ "rewards/margins": 0.028187204152345657,
+ "rewards/rejected": 0.12700578570365906,
+ "step": 1120
+ },
+ {
+ "epoch": 0.904,
+ "grad_norm": 1.125625729560852,
+ "learning_rate": 7.765925925925925e-07,
+ "logits/chosen": 1.6053575277328491,
+ "logits/rejected": 1.47335946559906,
+ "logps/chosen": -148.05690002441406,
+ "logps/rejected": -135.36598205566406,
+ "loss": 0.6889908790588379,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.15128108859062195,
+ "rewards/margins": 0.015811312943696976,
+ "rewards/rejected": 0.13546979427337646,
+ "step": 1130
+ },
+ {
+ "epoch": 0.912,
+ "grad_norm": 1.0933659076690674,
+ "learning_rate": 7.736296296296297e-07,
+ "logits/chosen": 1.5862042903900146,
+ "logits/rejected": 1.5827430486679077,
+ "logps/chosen": -156.10145568847656,
+ "logps/rejected": -142.94851684570312,
+ "loss": 0.6753206253051758,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1547432243824005,
+ "rewards/margins": 0.04229574277997017,
+ "rewards/rejected": 0.11244747787714005,
+ "step": 1140
+ },
+ {
+ "epoch": 0.92,
+ "grad_norm": 1.0774686336517334,
+ "learning_rate": 7.706666666666667e-07,
+ "logits/chosen": 1.2369152307510376,
+ "logits/rejected": 1.2727023363113403,
+ "logps/chosen": -139.23147583007812,
+ "logps/rejected": -130.53038024902344,
+ "loss": 0.6967329025268555,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": 0.10801626741886139,
+ "rewards/margins": 0.004670143127441406,
+ "rewards/rejected": 0.10334610939025879,
+ "step": 1150
+ },
+ {
+ "epoch": 0.928,
+ "grad_norm": 1.1111780405044556,
+ "learning_rate": 7.677037037037036e-07,
+ "logits/chosen": 1.501250982284546,
+ "logits/rejected": 1.4272868633270264,
+ "logps/chosen": -127.09858703613281,
+ "logps/rejected": -114.9041748046875,
+ "loss": 0.6853152275085449,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.1366150826215744,
+ "rewards/margins": 0.021945688873529434,
+ "rewards/rejected": 0.11466939747333527,
+ "step": 1160
+ },
+ {
+ "epoch": 0.936,
+ "grad_norm": 1.5593854188919067,
+ "learning_rate": 7.647407407407407e-07,
+ "logits/chosen": 1.727927803993225,
+ "logits/rejected": 1.6510727405548096,
+ "logps/chosen": -156.70993041992188,
+ "logps/rejected": -167.07266235351562,
+ "loss": 0.6884279251098633,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.14902853965759277,
+ "rewards/margins": 0.01685408689081669,
+ "rewards/rejected": 0.13217447698116302,
+ "step": 1170
+ },
+ {
+ "epoch": 0.944,
+ "grad_norm": 2.0064918994903564,
+ "learning_rate": 7.617777777777778e-07,
+ "logits/chosen": 1.6262975931167603,
+ "logits/rejected": 1.58951997756958,
+ "logps/chosen": -186.59210205078125,
+ "logps/rejected": -151.2965850830078,
+ "loss": 0.6906405448913574,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.14898104965686798,
+ "rewards/margins": 0.013303476385772228,
+ "rewards/rejected": 0.13567757606506348,
+ "step": 1180
+ },
+ {
+ "epoch": 0.952,
+ "grad_norm": 1.209291696548462,
+ "learning_rate": 7.588148148148148e-07,
+ "logits/chosen": 1.639369249343872,
+ "logits/rejected": 1.6400636434555054,
+ "logps/chosen": -166.24684143066406,
+ "logps/rejected": -196.35305786132812,
+ "loss": 0.6789956092834473,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1348457634449005,
+ "rewards/margins": 0.03475186973810196,
+ "rewards/rejected": 0.10009388625621796,
+ "step": 1190
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 1.209230899810791,
+ "learning_rate": 7.558518518518519e-07,
+ "logits/chosen": 1.542942762374878,
+ "logits/rejected": 1.5128916501998901,
+ "logps/chosen": -138.994384765625,
+ "logps/rejected": -145.42880249023438,
+ "loss": 0.6936234951019287,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.13895569741725922,
+ "rewards/margins": 0.005582497920840979,
+ "rewards/rejected": 0.1333732008934021,
+ "step": 1200
+ },
+ {
+ "epoch": 0.96,
+ "eval_logits/chosen": 1.5055651664733887,
+ "eval_logits/rejected": 1.5351229906082153,
+ "eval_logps/chosen": -152.2256622314453,
+ "eval_logps/rejected": -146.909912109375,
+ "eval_loss": 0.6866763830184937,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.13231882452964783,
+ "eval_rewards/margins": 0.02109396643936634,
+ "eval_rewards/rejected": 0.11122485250234604,
+ "eval_runtime": 90.8141,
+ "eval_samples_per_second": 5.506,
+ "eval_steps_per_second": 2.753,
+ "step": 1200
+ },
+ {
+ "epoch": 0.968,
+ "grad_norm": 1.2058950662612915,
+ "learning_rate": 7.528888888888889e-07,
+ "logits/chosen": 1.3591210842132568,
+ "logits/rejected": 1.3811699151992798,
+ "logps/chosen": -155.1759033203125,
+ "logps/rejected": -141.97772216796875,
+ "loss": 0.6902606964111329,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.153532013297081,
+ "rewards/margins": 0.011359233409166336,
+ "rewards/rejected": 0.14217278361320496,
+ "step": 1210
+ },
+ {
+ "epoch": 0.976,
+ "grad_norm": 1.366702914237976,
+ "learning_rate": 7.499259259259258e-07,
+ "logits/chosen": 1.3841370344161987,
+ "logits/rejected": 1.3791372776031494,
+ "logps/chosen": -145.47613525390625,
+ "logps/rejected": -123.9173583984375,
+ "loss": 0.6971709251403808,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.10521620512008667,
+ "rewards/margins": -0.0030749079305678606,
+ "rewards/rejected": 0.1082911267876625,
+ "step": 1220
+ },
+ {
+ "epoch": 0.984,
+ "grad_norm": 1.477455496788025,
+ "learning_rate": 7.469629629629629e-07,
+ "logits/chosen": 1.4447228908538818,
+ "logits/rejected": 1.6022417545318604,
+ "logps/chosen": -137.9462127685547,
+ "logps/rejected": -150.25559997558594,
+ "loss": 0.6915127277374268,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.15108394622802734,
+ "rewards/margins": 0.009388584643602371,
+ "rewards/rejected": 0.14169538021087646,
+ "step": 1230
+ },
+ {
+ "epoch": 0.992,
+ "grad_norm": 1.4407854080200195,
+ "learning_rate": 7.44e-07,
+ "logits/chosen": 1.3998825550079346,
+ "logits/rejected": 1.5924925804138184,
+ "logps/chosen": -144.93775939941406,
+ "logps/rejected": -148.02542114257812,
+ "loss": 0.6699621677398682,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.146137997508049,
+ "rewards/margins": 0.05326627567410469,
+ "rewards/rejected": 0.09287171810865402,
+ "step": 1240
+ },
+ {
+ "epoch": 1.0,
+ "grad_norm": 1.183438777923584,
+ "learning_rate": 7.41037037037037e-07,
+ "logits/chosen": 1.5393705368041992,
+ "logits/rejected": 1.6312555074691772,
+ "logps/chosen": -145.61672973632812,
+ "logps/rejected": -158.9256591796875,
+ "loss": 0.6811168670654297,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.1304798424243927,
+ "rewards/margins": 0.03360626846551895,
+ "rewards/rejected": 0.09687358886003494,
+ "step": 1250
+ },
+ {
+ "epoch": 1.008,
+ "grad_norm": 1.2436553239822388,
+ "learning_rate": 7.38074074074074e-07,
+ "logits/chosen": 1.4758602380752563,
+ "logits/rejected": 1.534609079360962,
+ "logps/chosen": -124.26841735839844,
+ "logps/rejected": -132.41867065429688,
+ "loss": 0.688177490234375,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.12570872902870178,
+ "rewards/margins": 0.014875045046210289,
+ "rewards/rejected": 0.11083368957042694,
+ "step": 1260
+ },
+ {
+ "epoch": 1.016,
+ "grad_norm": 1.3853868246078491,
+ "learning_rate": 7.351111111111112e-07,
+ "logits/chosen": 1.7195091247558594,
+ "logits/rejected": 1.632851004600525,
+ "logps/chosen": -160.8590850830078,
+ "logps/rejected": -153.91671752929688,
+ "loss": 0.6669504165649414,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.181864932179451,
+ "rewards/margins": 0.06175640970468521,
+ "rewards/rejected": 0.12010850757360458,
+ "step": 1270
+ },
+ {
+ "epoch": 1.024,
+ "grad_norm": 1.2765302658081055,
+ "learning_rate": 7.321481481481481e-07,
+ "logits/chosen": 1.5782036781311035,
+ "logits/rejected": 1.5713088512420654,
+ "logps/chosen": -154.3319549560547,
+ "logps/rejected": -141.43345642089844,
+ "loss": 0.6734190464019776,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.12361389398574829,
+ "rewards/margins": 0.04626934230327606,
+ "rewards/rejected": 0.07734455168247223,
+ "step": 1280
+ },
+ {
+ "epoch": 1.032,
+ "grad_norm": 1.0421397686004639,
+ "learning_rate": 7.291851851851851e-07,
+ "logits/chosen": 1.5199967622756958,
+ "logits/rejected": 1.5965670347213745,
+ "logps/chosen": -145.28416442871094,
+ "logps/rejected": -133.72256469726562,
+ "loss": 0.6804514408111573,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.11794980615377426,
+ "rewards/margins": 0.030916064977645874,
+ "rewards/rejected": 0.08703374862670898,
+ "step": 1290
+ },
+ {
+ "epoch": 1.04,
+ "grad_norm": 1.032158613204956,
+ "learning_rate": 7.262222222222222e-07,
+ "logits/chosen": 1.6042639017105103,
+ "logits/rejected": 1.4255346059799194,
+ "logps/chosen": -150.4803009033203,
+ "logps/rejected": -130.2232208251953,
+ "loss": 0.6676507472991944,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.15466037392616272,
+ "rewards/margins": 0.05808692425489426,
+ "rewards/rejected": 0.09657344967126846,
+ "step": 1300
+ },
+ {
+ "epoch": 1.04,
+ "eval_logits/chosen": 1.5118356943130493,
+ "eval_logits/rejected": 1.5418950319290161,
+ "eval_logps/chosen": -152.23300170898438,
+ "eval_logps/rejected": -146.9257049560547,
+ "eval_loss": 0.6859617233276367,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.13158410787582397,
+ "eval_rewards/margins": 0.021939631551504135,
+ "eval_rewards/rejected": 0.10964448750019073,
+ "eval_runtime": 90.9624,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 1300
+ },
+ {
+ "epoch": 1.048,
+ "grad_norm": 1.4942574501037598,
+ "learning_rate": 7.232592592592593e-07,
+ "logits/chosen": 1.689648985862732,
+ "logits/rejected": 1.691888451576233,
+ "logps/chosen": -166.02809143066406,
+ "logps/rejected": -160.30601501464844,
+ "loss": 0.6770285606384278,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.14377735555171967,
+ "rewards/margins": 0.03825496509671211,
+ "rewards/rejected": 0.10552239418029785,
+ "step": 1310
+ },
+ {
+ "epoch": 1.056,
+ "grad_norm": 1.3656011819839478,
+ "learning_rate": 7.202962962962963e-07,
+ "logits/chosen": 1.3065154552459717,
+ "logits/rejected": 1.5681203603744507,
+ "logps/chosen": -129.1146240234375,
+ "logps/rejected": -151.2584991455078,
+ "loss": 0.6844655036926269,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.12841743230819702,
+ "rewards/margins": 0.02178655005991459,
+ "rewards/rejected": 0.10663086175918579,
+ "step": 1320
+ },
+ {
+ "epoch": 1.064,
+ "grad_norm": 1.1723405122756958,
+ "learning_rate": 7.173333333333333e-07,
+ "logits/chosen": 1.6219158172607422,
+ "logits/rejected": 1.6032555103302002,
+ "logps/chosen": -156.4185028076172,
+ "logps/rejected": -146.28067016601562,
+ "loss": 0.6650479316711426,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1538492739200592,
+ "rewards/margins": 0.06132146716117859,
+ "rewards/rejected": 0.09252782166004181,
+ "step": 1330
+ },
+ {
+ "epoch": 1.072,
+ "grad_norm": 1.2311245203018188,
+ "learning_rate": 7.143703703703703e-07,
+ "logits/chosen": 1.4625260829925537,
+ "logits/rejected": 1.392937421798706,
+ "logps/chosen": -153.05941772460938,
+ "logps/rejected": -145.37841796875,
+ "loss": 0.6570021152496338,
+ "rewards/accuracies": 0.762499988079071,
+ "rewards/chosen": 0.16768500208854675,
+ "rewards/margins": 0.0787494108080864,
+ "rewards/rejected": 0.08893559873104095,
+ "step": 1340
+ },
+ {
+ "epoch": 1.08,
+ "grad_norm": 1.314666748046875,
+ "learning_rate": 7.114074074074073e-07,
+ "logits/chosen": 1.4812809228897095,
+ "logits/rejected": 1.3796050548553467,
+ "logps/chosen": -165.70205688476562,
+ "logps/rejected": -140.13150024414062,
+ "loss": 0.6535200595855712,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.20691323280334473,
+ "rewards/margins": 0.0911526307463646,
+ "rewards/rejected": 0.11576057970523834,
+ "step": 1350
+ },
+ {
+ "epoch": 1.088,
+ "grad_norm": 0.9556720852851868,
+ "learning_rate": 7.084444444444444e-07,
+ "logits/chosen": 1.6368863582611084,
+ "logits/rejected": 1.678134560585022,
+ "logps/chosen": -166.56253051757812,
+ "logps/rejected": -164.36416625976562,
+ "loss": 0.6742628574371338,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.15145304799079895,
+ "rewards/margins": 0.04319921135902405,
+ "rewards/rejected": 0.1082538366317749,
+ "step": 1360
+ },
+ {
+ "epoch": 1.096,
+ "grad_norm": 0.945633053779602,
+ "learning_rate": 7.054814814814815e-07,
+ "logits/chosen": 1.4285938739776611,
+ "logits/rejected": 1.5576260089874268,
+ "logps/chosen": -131.5234832763672,
+ "logps/rejected": -128.02340698242188,
+ "loss": 0.6670140743255615,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.16399362683296204,
+ "rewards/margins": 0.05922812223434448,
+ "rewards/rejected": 0.10476551204919815,
+ "step": 1370
+ },
+ {
+ "epoch": 1.104,
+ "grad_norm": 1.1189475059509277,
+ "learning_rate": 7.025185185185185e-07,
+ "logits/chosen": 1.5019981861114502,
+ "logits/rejected": 1.667614221572876,
+ "logps/chosen": -161.61854553222656,
+ "logps/rejected": -166.39996337890625,
+ "loss": 0.6742690086364747,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.15786050260066986,
+ "rewards/margins": 0.044500257819890976,
+ "rewards/rejected": 0.11336024850606918,
+ "step": 1380
+ },
+ {
+ "epoch": 1.112,
+ "grad_norm": 1.3382912874221802,
+ "learning_rate": 6.995555555555555e-07,
+ "logits/chosen": 1.4972480535507202,
+ "logits/rejected": 1.4946129322052002,
+ "logps/chosen": -166.43682861328125,
+ "logps/rejected": -162.3004913330078,
+ "loss": 0.6557814121246338,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.18964914977550507,
+ "rewards/margins": 0.08500295877456665,
+ "rewards/rejected": 0.10464620590209961,
+ "step": 1390
+ },
+ {
+ "epoch": 1.12,
+ "grad_norm": 1.2123515605926514,
+ "learning_rate": 6.965925925925926e-07,
+ "logits/chosen": 1.456635594367981,
+ "logits/rejected": 1.5964475870132446,
+ "logps/chosen": -152.08261108398438,
+ "logps/rejected": -161.51023864746094,
+ "loss": 0.6708066940307618,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.216823011636734,
+ "rewards/margins": 0.050363689661026,
+ "rewards/rejected": 0.166459321975708,
+ "step": 1400
+ },
+ {
+ "epoch": 1.12,
+ "eval_logits/chosen": 1.521518588066101,
+ "eval_logits/rejected": 1.5510718822479248,
+ "eval_logps/chosen": -151.858154296875,
+ "eval_logps/rejected": -146.5819549560547,
+ "eval_loss": 0.6853657364845276,
+ "eval_rewards/accuracies": 0.5440000295639038,
+ "eval_rewards/chosen": 0.16906805336475372,
+ "eval_rewards/margins": 0.025049500167369843,
+ "eval_rewards/rejected": 0.14401854574680328,
+ "eval_runtime": 90.8779,
+ "eval_samples_per_second": 5.502,
+ "eval_steps_per_second": 2.751,
+ "step": 1400
+ },
+ {
+ "epoch": 1.1280000000000001,
+ "grad_norm": 1.3488291501998901,
+ "learning_rate": 6.936296296296296e-07,
+ "logits/chosen": 1.6009830236434937,
+ "logits/rejected": 1.6035544872283936,
+ "logps/chosen": -145.44662475585938,
+ "logps/rejected": -149.9284210205078,
+ "loss": 0.679435396194458,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.1647307425737381,
+ "rewards/margins": 0.03348377346992493,
+ "rewards/rejected": 0.13124695420265198,
+ "step": 1410
+ },
+ {
+ "epoch": 1.1360000000000001,
+ "grad_norm": 1.6453970670700073,
+ "learning_rate": 6.906666666666666e-07,
+ "logits/chosen": 1.593658447265625,
+ "logits/rejected": 1.648276925086975,
+ "logps/chosen": -135.2926788330078,
+ "logps/rejected": -150.9585723876953,
+ "loss": 0.6875734329223633,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.15669891238212585,
+ "rewards/margins": 0.01690087839961052,
+ "rewards/rejected": 0.13979804515838623,
+ "step": 1420
+ },
+ {
+ "epoch": 1.144,
+ "grad_norm": 1.6117807626724243,
+ "learning_rate": 6.877037037037036e-07,
+ "logits/chosen": 1.4510356187820435,
+ "logits/rejected": 1.2334567308425903,
+ "logps/chosen": -153.94854736328125,
+ "logps/rejected": -147.78079223632812,
+ "loss": 0.6689413070678711,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1819056123495102,
+ "rewards/margins": 0.058564942330121994,
+ "rewards/rejected": 0.1233406662940979,
+ "step": 1430
+ },
+ {
+ "epoch": 1.152,
+ "grad_norm": 1.5533809661865234,
+ "learning_rate": 6.847407407407408e-07,
+ "logits/chosen": 1.4535889625549316,
+ "logits/rejected": 1.4273865222930908,
+ "logps/chosen": -170.42904663085938,
+ "logps/rejected": -151.9979705810547,
+ "loss": 0.6845246315002441,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.14223966002464294,
+ "rewards/margins": 0.022389881312847137,
+ "rewards/rejected": 0.1198497861623764,
+ "step": 1440
+ },
+ {
+ "epoch": 1.16,
+ "grad_norm": 1.5696005821228027,
+ "learning_rate": 6.817777777777778e-07,
+ "logits/chosen": 1.4418725967407227,
+ "logits/rejected": 1.4354021549224854,
+ "logps/chosen": -153.962158203125,
+ "logps/rejected": -148.40512084960938,
+ "loss": 0.6761108875274658,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.15529735386371613,
+ "rewards/margins": 0.040999218821525574,
+ "rewards/rejected": 0.11429812759160995,
+ "step": 1450
+ },
+ {
+ "epoch": 1.168,
+ "grad_norm": 1.0711781978607178,
+ "learning_rate": 6.788148148148147e-07,
+ "logits/chosen": 1.5659481287002563,
+ "logits/rejected": 1.5640677213668823,
+ "logps/chosen": -149.816162109375,
+ "logps/rejected": -174.68643188476562,
+ "loss": 0.677489185333252,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1510745733976364,
+ "rewards/margins": 0.03914733976125717,
+ "rewards/rejected": 0.11192724853754044,
+ "step": 1460
+ },
+ {
+ "epoch": 1.176,
+ "grad_norm": 1.2336673736572266,
+ "learning_rate": 6.758518518518518e-07,
+ "logits/chosen": 1.4275394678115845,
+ "logits/rejected": 1.6394407749176025,
+ "logps/chosen": -137.93788146972656,
+ "logps/rejected": -154.97195434570312,
+ "loss": 0.6785802364349365,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10557998716831207,
+ "rewards/margins": 0.0352485254406929,
+ "rewards/rejected": 0.07033145427703857,
+ "step": 1470
+ },
+ {
+ "epoch": 1.184,
+ "grad_norm": 1.0030723810195923,
+ "learning_rate": 6.728888888888888e-07,
+ "logits/chosen": 1.4652725458145142,
+ "logits/rejected": 1.4992272853851318,
+ "logps/chosen": -143.87374877929688,
+ "logps/rejected": -139.5693817138672,
+ "loss": 0.6700259208679199,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.13436684012413025,
+ "rewards/margins": 0.05286343768239021,
+ "rewards/rejected": 0.08150340616703033,
+ "step": 1480
+ },
+ {
+ "epoch": 1.192,
+ "grad_norm": 1.0897104740142822,
+ "learning_rate": 6.699259259259259e-07,
+ "logits/chosen": 1.7033601999282837,
+ "logits/rejected": 1.617489218711853,
+ "logps/chosen": -140.43667602539062,
+ "logps/rejected": -146.59683227539062,
+ "loss": 0.6685672283172608,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.11724893748760223,
+ "rewards/margins": 0.055138181895017624,
+ "rewards/rejected": 0.06211075931787491,
+ "step": 1490
+ },
+ {
+ "epoch": 1.2,
+ "grad_norm": 0.8573256731033325,
+ "learning_rate": 6.66962962962963e-07,
+ "logits/chosen": 1.5181013345718384,
+ "logits/rejected": 1.5965152978897095,
+ "logps/chosen": -147.78506469726562,
+ "logps/rejected": -142.18930053710938,
+ "loss": 0.6564288139343262,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.14890792965888977,
+ "rewards/margins": 0.08058460801839828,
+ "rewards/rejected": 0.06832331418991089,
+ "step": 1500
+ },
+ {
+ "epoch": 1.2,
+ "eval_logits/chosen": 1.488415002822876,
+ "eval_logits/rejected": 1.5179662704467773,
+ "eval_logps/chosen": -152.60232543945312,
+ "eval_logps/rejected": -147.2954864501953,
+ "eval_loss": 0.6861968040466309,
+ "eval_rewards/accuracies": 0.5400000214576721,
+ "eval_rewards/chosen": 0.09465231001377106,
+ "eval_rewards/margins": 0.021986663341522217,
+ "eval_rewards/rejected": 0.07266565412282944,
+ "eval_runtime": 90.8951,
+ "eval_samples_per_second": 5.501,
+ "eval_steps_per_second": 2.75,
+ "step": 1500
+ },
+ {
+ "epoch": 1.208,
+ "grad_norm": 1.29599928855896,
+ "learning_rate": 6.64e-07,
+ "logits/chosen": 1.4851064682006836,
+ "logits/rejected": 1.4355719089508057,
+ "logps/chosen": -140.07186889648438,
+ "logps/rejected": -155.5626983642578,
+ "loss": 0.6695255756378173,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08214708417654037,
+ "rewards/margins": 0.05618969723582268,
+ "rewards/rejected": 0.025957385078072548,
+ "step": 1510
+ },
+ {
+ "epoch": 1.216,
+ "grad_norm": 1.3030505180358887,
+ "learning_rate": 6.61037037037037e-07,
+ "logits/chosen": 1.3785858154296875,
+ "logits/rejected": 1.3867576122283936,
+ "logps/chosen": -143.06137084960938,
+ "logps/rejected": -152.8805694580078,
+ "loss": 0.6704819202423096,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.08582031726837158,
+ "rewards/margins": 0.052868831902742386,
+ "rewards/rejected": 0.0329514816403389,
+ "step": 1520
+ },
+ {
+ "epoch": 1.224,
+ "grad_norm": 1.2935470342636108,
+ "learning_rate": 6.58074074074074e-07,
+ "logits/chosen": 1.4631284475326538,
+ "logits/rejected": 1.4908760786056519,
+ "logps/chosen": -181.2951202392578,
+ "logps/rejected": -173.83523559570312,
+ "loss": 0.6680389881134033,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.11432291567325592,
+ "rewards/margins": 0.05878107622265816,
+ "rewards/rejected": 0.05554184317588806,
+ "step": 1530
+ },
+ {
+ "epoch": 1.232,
+ "grad_norm": 1.4426783323287964,
+ "learning_rate": 6.551111111111111e-07,
+ "logits/chosen": 1.4765878915786743,
+ "logits/rejected": 1.4511891603469849,
+ "logps/chosen": -167.4827117919922,
+ "logps/rejected": -153.88833618164062,
+ "loss": 0.6684586524963378,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.08437791466712952,
+ "rewards/margins": 0.05823448300361633,
+ "rewards/rejected": 0.026143427938222885,
+ "step": 1540
+ },
+ {
+ "epoch": 1.24,
+ "grad_norm": 1.3253812789916992,
+ "learning_rate": 6.521481481481481e-07,
+ "logits/chosen": 1.2496798038482666,
+ "logits/rejected": 1.5193806886672974,
+ "logps/chosen": -130.33685302734375,
+ "logps/rejected": -132.5361328125,
+ "loss": 0.6695978164672851,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.10806921869516373,
+ "rewards/margins": 0.05488727241754532,
+ "rewards/rejected": 0.05318193882703781,
+ "step": 1550
+ },
+ {
+ "epoch": 1.248,
+ "grad_norm": 1.1890907287597656,
+ "learning_rate": 6.491851851851851e-07,
+ "logits/chosen": 1.2053601741790771,
+ "logits/rejected": 1.2588948011398315,
+ "logps/chosen": -132.61026000976562,
+ "logps/rejected": -150.66477966308594,
+ "loss": 0.673579216003418,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.062341660261154175,
+ "rewards/margins": 0.04531678557395935,
+ "rewards/rejected": 0.017024870961904526,
+ "step": 1560
+ },
+ {
+ "epoch": 1.256,
+ "grad_norm": 1.2076733112335205,
+ "learning_rate": 6.462222222222223e-07,
+ "logits/chosen": 1.5180461406707764,
+ "logits/rejected": 1.4118196964263916,
+ "logps/chosen": -120.61619567871094,
+ "logps/rejected": -146.42056274414062,
+ "loss": 0.6747167587280274,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.0764162540435791,
+ "rewards/margins": 0.04275941476225853,
+ "rewards/rejected": 0.03365683928132057,
+ "step": 1570
+ },
+ {
+ "epoch": 1.264,
+ "grad_norm": 1.3394429683685303,
+ "learning_rate": 6.432592592592593e-07,
+ "logits/chosen": 1.2879914045333862,
+ "logits/rejected": 1.3957126140594482,
+ "logps/chosen": -150.98355102539062,
+ "logps/rejected": -172.91090393066406,
+ "loss": 0.6560922622680664,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.09853875637054443,
+ "rewards/margins": 0.08982758224010468,
+ "rewards/rejected": 0.008711177855730057,
+ "step": 1580
+ },
+ {
+ "epoch": 1.272,
+ "grad_norm": 1.303663730621338,
+ "learning_rate": 6.402962962962962e-07,
+ "logits/chosen": 1.5617700815200806,
+ "logits/rejected": 1.6307615041732788,
+ "logps/chosen": -157.00338745117188,
+ "logps/rejected": -159.2900848388672,
+ "loss": 0.6639400959014893,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.07477572560310364,
+ "rewards/margins": 0.06561199575662613,
+ "rewards/rejected": 0.009163733571767807,
+ "step": 1590
+ },
+ {
+ "epoch": 1.28,
+ "grad_norm": 1.4180800914764404,
+ "learning_rate": 6.373333333333333e-07,
+ "logits/chosen": 1.3017181158065796,
+ "logits/rejected": 1.3270524740219116,
+ "logps/chosen": -134.52029418945312,
+ "logps/rejected": -135.45938110351562,
+ "loss": 0.6564930438995361,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.08265326917171478,
+ "rewards/margins": 0.09048662334680557,
+ "rewards/rejected": -0.007833337411284447,
+ "step": 1600
+ },
+ {
+ "epoch": 1.28,
+ "eval_logits/chosen": 1.4600619077682495,
+ "eval_logits/rejected": 1.489198923110962,
+ "eval_logps/chosen": -152.86001586914062,
+ "eval_logps/rejected": -147.54771423339844,
+ "eval_loss": 0.6871278882026672,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": 0.0688827633857727,
+ "eval_rewards/margins": 0.021439960226416588,
+ "eval_rewards/rejected": 0.04744280129671097,
+ "eval_runtime": 91.0555,
+ "eval_samples_per_second": 5.491,
+ "eval_steps_per_second": 2.746,
+ "step": 1600
+ },
+ {
+ "epoch": 1.288,
+ "grad_norm": 1.1791326999664307,
+ "learning_rate": 6.343703703703703e-07,
+ "logits/chosen": 1.4442191123962402,
+ "logits/rejected": 1.5532598495483398,
+ "logps/chosen": -141.93701171875,
+ "logps/rejected": -147.68682861328125,
+ "loss": 0.6539246082305908,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.09121527522802353,
+ "rewards/margins": 0.08849794417619705,
+ "rewards/rejected": 0.002717340365052223,
+ "step": 1610
+ },
+ {
+ "epoch": 1.296,
+ "grad_norm": 1.1131889820098877,
+ "learning_rate": 6.314074074074074e-07,
+ "logits/chosen": 1.3450567722320557,
+ "logits/rejected": 1.4408035278320312,
+ "logps/chosen": -157.0798797607422,
+ "logps/rejected": -161.1500701904297,
+ "loss": 0.6505428314208984,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.11321337521076202,
+ "rewards/margins": 0.09579396992921829,
+ "rewards/rejected": 0.017419395968317986,
+ "step": 1620
+ },
+ {
+ "epoch": 1.304,
+ "grad_norm": 1.6172596216201782,
+ "learning_rate": 6.284444444444445e-07,
+ "logits/chosen": 1.478847622871399,
+ "logits/rejected": 1.3894659280776978,
+ "logps/chosen": -140.17453002929688,
+ "logps/rejected": -169.06289672851562,
+ "loss": 0.6720306396484375,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.08244314789772034,
+ "rewards/margins": 0.062018074095249176,
+ "rewards/rejected": 0.020425086840987206,
+ "step": 1630
+ },
+ {
+ "epoch": 1.312,
+ "grad_norm": 1.1743669509887695,
+ "learning_rate": 6.254814814814815e-07,
+ "logits/chosen": 1.4197287559509277,
+ "logits/rejected": 1.3319528102874756,
+ "logps/chosen": -157.23072814941406,
+ "logps/rejected": -139.50074768066406,
+ "loss": 0.6712700843811035,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.04650931805372238,
+ "rewards/margins": 0.057532649487257004,
+ "rewards/rejected": -0.0110233249142766,
+ "step": 1640
+ },
+ {
+ "epoch": 1.32,
+ "grad_norm": 1.3225634098052979,
+ "learning_rate": 6.225185185185184e-07,
+ "logits/chosen": 1.35457444190979,
+ "logits/rejected": 1.3751428127288818,
+ "logps/chosen": -137.4834747314453,
+ "logps/rejected": -134.4203643798828,
+ "loss": 0.6679590225219727,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.07574301958084106,
+ "rewards/margins": 0.057132184505462646,
+ "rewards/rejected": 0.01861082948744297,
+ "step": 1650
+ },
+ {
+ "epoch": 1.328,
+ "grad_norm": 1.5913130044937134,
+ "learning_rate": 6.195555555555555e-07,
+ "logits/chosen": 1.5328034162521362,
+ "logits/rejected": 1.3161242008209229,
+ "logps/chosen": -147.09475708007812,
+ "logps/rejected": -135.3501434326172,
+ "loss": 0.6575756072998047,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.1077059879899025,
+ "rewards/margins": 0.08081348240375519,
+ "rewards/rejected": 0.026892507448792458,
+ "step": 1660
+ },
+ {
+ "epoch": 1.336,
+ "grad_norm": 0.9880369901657104,
+ "learning_rate": 6.165925925925926e-07,
+ "logits/chosen": 1.3911128044128418,
+ "logits/rejected": 1.3688428401947021,
+ "logps/chosen": -152.17483520507812,
+ "logps/rejected": -149.79383850097656,
+ "loss": 0.6656455516815185,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.10711929947137833,
+ "rewards/margins": 0.0669747143983841,
+ "rewards/rejected": 0.04014457389712334,
+ "step": 1670
+ },
+ {
+ "epoch": 1.3439999999999999,
+ "grad_norm": 1.125959873199463,
+ "learning_rate": 6.136296296296296e-07,
+ "logits/chosen": 1.4232629537582397,
+ "logits/rejected": 1.5579107999801636,
+ "logps/chosen": -160.2217559814453,
+ "logps/rejected": -148.1455535888672,
+ "loss": 0.6710956573486329,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10255889594554901,
+ "rewards/margins": 0.05178038403391838,
+ "rewards/rejected": 0.05077851936221123,
+ "step": 1680
+ },
+ {
+ "epoch": 1.3519999999999999,
+ "grad_norm": 1.3627047538757324,
+ "learning_rate": 6.106666666666666e-07,
+ "logits/chosen": 1.4259792566299438,
+ "logits/rejected": 1.4750150442123413,
+ "logps/chosen": -152.88381958007812,
+ "logps/rejected": -153.72708129882812,
+ "loss": 0.6512804508209229,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.14147944748401642,
+ "rewards/margins": 0.09307968616485596,
+ "rewards/rejected": 0.04839975759387016,
+ "step": 1690
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "grad_norm": 1.0653239488601685,
+ "learning_rate": 6.077037037037038e-07,
+ "logits/chosen": 1.340463399887085,
+ "logits/rejected": 1.3752565383911133,
+ "logps/chosen": -150.36248779296875,
+ "logps/rejected": -151.90725708007812,
+ "loss": 0.6768710613250732,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.08575484901666641,
+ "rewards/margins": 0.043545305728912354,
+ "rewards/rejected": 0.04220953956246376,
+ "step": 1700
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "eval_logits/chosen": 1.4521433115005493,
+ "eval_logits/rejected": 1.4804331064224243,
+ "eval_logps/chosen": -152.77755737304688,
+ "eval_logps/rejected": -147.48025512695312,
+ "eval_loss": 0.6873273849487305,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": 0.07712872326374054,
+ "eval_rewards/margins": 0.022939365357160568,
+ "eval_rewards/rejected": 0.05418936163187027,
+ "eval_runtime": 90.962,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 1700
+ },
+ {
+ "epoch": 1.3679999999999999,
+ "grad_norm": 1.3996928930282593,
+ "learning_rate": 6.047407407407407e-07,
+ "logits/chosen": 1.41148042678833,
+ "logits/rejected": 1.4520132541656494,
+ "logps/chosen": -143.23252868652344,
+ "logps/rejected": -180.05331420898438,
+ "loss": 0.655043363571167,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10279543697834015,
+ "rewards/margins": 0.0895618349313736,
+ "rewards/rejected": 0.013233604840934277,
+ "step": 1710
+ },
+ {
+ "epoch": 1.376,
+ "grad_norm": 1.4253696203231812,
+ "learning_rate": 6.017777777777777e-07,
+ "logits/chosen": 1.3833668231964111,
+ "logits/rejected": 1.409301519393921,
+ "logps/chosen": -149.3744659423828,
+ "logps/rejected": -151.1966552734375,
+ "loss": 0.6865286827087402,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.06293162703514099,
+ "rewards/margins": 0.02295227348804474,
+ "rewards/rejected": 0.03997935354709625,
+ "step": 1720
+ },
+ {
+ "epoch": 1.384,
+ "grad_norm": 1.185731053352356,
+ "learning_rate": 5.988148148148148e-07,
+ "logits/chosen": 1.528174877166748,
+ "logits/rejected": 1.6338379383087158,
+ "logps/chosen": -160.7788543701172,
+ "logps/rejected": -159.85354614257812,
+ "loss": 0.6458437442779541,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.12789444625377655,
+ "rewards/margins": 0.1114773377776146,
+ "rewards/rejected": 0.016417117789387703,
+ "step": 1730
+ },
+ {
+ "epoch": 1.392,
+ "grad_norm": 1.2963365316390991,
+ "learning_rate": 5.958518518518518e-07,
+ "logits/chosen": 1.4939125776290894,
+ "logits/rejected": 1.4197361469268799,
+ "logps/chosen": -162.34744262695312,
+ "logps/rejected": -177.93087768554688,
+ "loss": 0.6620248317718506,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.11072772741317749,
+ "rewards/margins": 0.07320726662874222,
+ "rewards/rejected": 0.03752044588327408,
+ "step": 1740
+ },
+ {
+ "epoch": 1.4,
+ "grad_norm": 1.3535394668579102,
+ "learning_rate": 5.928888888888889e-07,
+ "logits/chosen": 1.3506572246551514,
+ "logits/rejected": 1.424142837524414,
+ "logps/chosen": -139.69985961914062,
+ "logps/rejected": -144.9062042236328,
+ "loss": 0.6815268516540527,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.07917657494544983,
+ "rewards/margins": 0.03688482195138931,
+ "rewards/rejected": 0.04229176044464111,
+ "step": 1750
+ },
+ {
+ "epoch": 1.408,
+ "grad_norm": 1.471380352973938,
+ "learning_rate": 5.899259259259259e-07,
+ "logits/chosen": 1.5949152708053589,
+ "logits/rejected": 1.710013747215271,
+ "logps/chosen": -170.3776092529297,
+ "logps/rejected": -177.1616973876953,
+ "loss": 0.6533854961395263,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.11633266508579254,
+ "rewards/margins": 0.09301649034023285,
+ "rewards/rejected": 0.023316185921430588,
+ "step": 1760
+ },
+ {
+ "epoch": 1.416,
+ "grad_norm": 1.3164308071136475,
+ "learning_rate": 5.869629629629629e-07,
+ "logits/chosen": 1.5159170627593994,
+ "logits/rejected": 1.4434025287628174,
+ "logps/chosen": -161.05157470703125,
+ "logps/rejected": -140.06948852539062,
+ "loss": 0.660222864151001,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.10675680637359619,
+ "rewards/margins": 0.07750208675861359,
+ "rewards/rejected": 0.029254715889692307,
+ "step": 1770
+ },
+ {
+ "epoch": 1.424,
+ "grad_norm": 1.102005124092102,
+ "learning_rate": 5.839999999999999e-07,
+ "logits/chosen": 1.349440336227417,
+ "logits/rejected": 1.1565381288528442,
+ "logps/chosen": -150.61328125,
+ "logps/rejected": -151.36001586914062,
+ "loss": 0.660042142868042,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.09433183073997498,
+ "rewards/margins": 0.07778269052505493,
+ "rewards/rejected": 0.016549136489629745,
+ "step": 1780
+ },
+ {
+ "epoch": 1.432,
+ "grad_norm": 1.2305164337158203,
+ "learning_rate": 5.81037037037037e-07,
+ "logits/chosen": 1.4631009101867676,
+ "logits/rejected": 1.497757911682129,
+ "logps/chosen": -156.7335968017578,
+ "logps/rejected": -150.91065979003906,
+ "loss": 0.6564284324645996,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.13712921738624573,
+ "rewards/margins": 0.09247532486915588,
+ "rewards/rejected": 0.04465390369296074,
+ "step": 1790
+ },
+ {
+ "epoch": 1.44,
+ "grad_norm": 1.3459731340408325,
+ "learning_rate": 5.780740740740741e-07,
+ "logits/chosen": 1.5344784259796143,
+ "logits/rejected": 1.5709785223007202,
+ "logps/chosen": -166.15830993652344,
+ "logps/rejected": -158.678955078125,
+ "loss": 0.6511439800262451,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1227751225233078,
+ "rewards/margins": 0.0968184620141983,
+ "rewards/rejected": 0.025956671684980392,
+ "step": 1800
+ },
+ {
+ "epoch": 1.44,
+ "eval_logits/chosen": 1.4476159811019897,
+ "eval_logits/rejected": 1.4754574298858643,
+ "eval_logps/chosen": -152.70677185058594,
+ "eval_logps/rejected": -147.41787719726562,
+ "eval_loss": 0.6876539587974548,
+ "eval_rewards/accuracies": 0.5199999809265137,
+ "eval_rewards/chosen": 0.08420863002538681,
+ "eval_rewards/margins": 0.023782728239893913,
+ "eval_rewards/rejected": 0.06042589247226715,
+ "eval_runtime": 91.0223,
+ "eval_samples_per_second": 5.493,
+ "eval_steps_per_second": 2.747,
+ "step": 1800
+ },
+ {
+ "epoch": 1.448,
+ "grad_norm": 1.3158259391784668,
+ "learning_rate": 5.751111111111111e-07,
+ "logits/chosen": 1.425363302230835,
+ "logits/rejected": 1.350799798965454,
+ "logps/chosen": -151.4562530517578,
+ "logps/rejected": -134.9906005859375,
+ "loss": 0.6759338855743409,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1467444747686386,
+ "rewards/margins": 0.047227732837200165,
+ "rewards/rejected": 0.09951673448085785,
+ "step": 1810
+ },
+ {
+ "epoch": 1.456,
+ "grad_norm": 1.179150938987732,
+ "learning_rate": 5.721481481481481e-07,
+ "logits/chosen": 1.6323440074920654,
+ "logits/rejected": 1.4893152713775635,
+ "logps/chosen": -149.57675170898438,
+ "logps/rejected": -146.96237182617188,
+ "loss": 0.657456636428833,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1445317566394806,
+ "rewards/margins": 0.0896807312965393,
+ "rewards/rejected": 0.05485100671648979,
+ "step": 1820
+ },
+ {
+ "epoch": 1.464,
+ "grad_norm": 1.0987558364868164,
+ "learning_rate": 5.691851851851853e-07,
+ "logits/chosen": 1.3851706981658936,
+ "logits/rejected": 1.381049633026123,
+ "logps/chosen": -163.92648315429688,
+ "logps/rejected": -142.801025390625,
+ "loss": 0.6631375312805176,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.14093783497810364,
+ "rewards/margins": 0.07281022518873215,
+ "rewards/rejected": 0.06812760978937149,
+ "step": 1830
+ },
+ {
+ "epoch": 1.472,
+ "grad_norm": 1.1079164743423462,
+ "learning_rate": 5.662222222222222e-07,
+ "logits/chosen": 1.4228973388671875,
+ "logits/rejected": 1.3453642129898071,
+ "logps/chosen": -148.5561981201172,
+ "logps/rejected": -141.28009033203125,
+ "loss": 0.6612800121307373,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.10088491439819336,
+ "rewards/margins": 0.08228114247322083,
+ "rewards/rejected": 0.018603766337037086,
+ "step": 1840
+ },
+ {
+ "epoch": 1.48,
+ "grad_norm": 1.0415616035461426,
+ "learning_rate": 5.632592592592592e-07,
+ "logits/chosen": 1.4923865795135498,
+ "logits/rejected": 1.536971092224121,
+ "logps/chosen": -142.55166625976562,
+ "logps/rejected": -149.22906494140625,
+ "loss": 0.6690784931182862,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.10785870254039764,
+ "rewards/margins": 0.05975862219929695,
+ "rewards/rejected": 0.04810008406639099,
+ "step": 1850
+ },
+ {
+ "epoch": 1.488,
+ "grad_norm": 1.4004729986190796,
+ "learning_rate": 5.602962962962962e-07,
+ "logits/chosen": 1.4783391952514648,
+ "logits/rejected": 1.5112550258636475,
+ "logps/chosen": -153.54034423828125,
+ "logps/rejected": -118.77142333984375,
+ "loss": 0.6666333675384521,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.12789246439933777,
+ "rewards/margins": 0.06238863617181778,
+ "rewards/rejected": 0.06550382077693939,
+ "step": 1860
+ },
+ {
+ "epoch": 1.496,
+ "grad_norm": 1.1908437013626099,
+ "learning_rate": 5.573333333333333e-07,
+ "logits/chosen": 1.5748203992843628,
+ "logits/rejected": 1.4827988147735596,
+ "logps/chosen": -162.0931854248047,
+ "logps/rejected": -156.00779724121094,
+ "loss": 0.6626818656921387,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.11168892681598663,
+ "rewards/margins": 0.07420919090509415,
+ "rewards/rejected": 0.03747973591089249,
+ "step": 1870
+ },
+ {
+ "epoch": 1.504,
+ "grad_norm": 1.288218379020691,
+ "learning_rate": 5.543703703703704e-07,
+ "logits/chosen": 1.5260225534439087,
+ "logits/rejected": 1.5843703746795654,
+ "logps/chosen": -148.2488555908203,
+ "logps/rejected": -161.31570434570312,
+ "loss": 0.6677760124206543,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.12626001238822937,
+ "rewards/margins": 0.06881437450647354,
+ "rewards/rejected": 0.05744565278291702,
+ "step": 1880
+ },
+ {
+ "epoch": 1.512,
+ "grad_norm": 1.2899844646453857,
+ "learning_rate": 5.514074074074074e-07,
+ "logits/chosen": 1.417141318321228,
+ "logits/rejected": 1.389639973640442,
+ "logps/chosen": -160.5532989501953,
+ "logps/rejected": -158.82923889160156,
+ "loss": 0.6510573863983155,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.15593740344047546,
+ "rewards/margins": 0.09999983012676239,
+ "rewards/rejected": 0.05593757703900337,
+ "step": 1890
+ },
+ {
+ "epoch": 1.52,
+ "grad_norm": 1.3312058448791504,
+ "learning_rate": 5.484444444444444e-07,
+ "logits/chosen": 1.4996871948242188,
+ "logits/rejected": 1.6273930072784424,
+ "logps/chosen": -154.84347534179688,
+ "logps/rejected": -130.4560089111328,
+ "loss": 0.6844823360443115,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.13181835412979126,
+ "rewards/margins": 0.029756689444184303,
+ "rewards/rejected": 0.10206165164709091,
+ "step": 1900
+ },
+ {
+ "epoch": 1.52,
+ "eval_logits/chosen": 1.4582936763763428,
+ "eval_logits/rejected": 1.4853557348251343,
+ "eval_logps/chosen": -152.36231994628906,
+ "eval_logps/rejected": -147.1087188720703,
+ "eval_loss": 0.6870284676551819,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": 0.1186550110578537,
+ "eval_rewards/margins": 0.027311701327562332,
+ "eval_rewards/rejected": 0.09134332090616226,
+ "eval_runtime": 90.913,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 1900
+ },
+ {
+ "epoch": 1.528,
+ "grad_norm": 1.2257205247879028,
+ "learning_rate": 5.454814814814814e-07,
+ "logits/chosen": 1.4909894466400146,
+ "logits/rejected": 1.4579002857208252,
+ "logps/chosen": -129.2428741455078,
+ "logps/rejected": -149.32220458984375,
+ "loss": 0.6677265644073487,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.13237449526786804,
+ "rewards/margins": 0.059117116034030914,
+ "rewards/rejected": 0.07325738668441772,
+ "step": 1910
+ },
+ {
+ "epoch": 1.536,
+ "grad_norm": 1.3048031330108643,
+ "learning_rate": 5.425185185185185e-07,
+ "logits/chosen": 1.5237693786621094,
+ "logits/rejected": 1.4846075773239136,
+ "logps/chosen": -159.10055541992188,
+ "logps/rejected": -155.69888305664062,
+ "loss": 0.6709336280822754,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.1088198646903038,
+ "rewards/margins": 0.05951980873942375,
+ "rewards/rejected": 0.04930006340146065,
+ "step": 1920
+ },
+ {
+ "epoch": 1.544,
+ "grad_norm": 1.3463319540023804,
+ "learning_rate": 5.395555555555556e-07,
+ "logits/chosen": 1.6476523876190186,
+ "logits/rejected": 1.6236053705215454,
+ "logps/chosen": -171.18319702148438,
+ "logps/rejected": -172.63796997070312,
+ "loss": 0.6670461177825928,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.13160961866378784,
+ "rewards/margins": 0.06587684154510498,
+ "rewards/rejected": 0.06573276221752167,
+ "step": 1930
+ },
+ {
+ "epoch": 1.552,
+ "grad_norm": 1.2326266765594482,
+ "learning_rate": 5.365925925925926e-07,
+ "logits/chosen": 1.4379268884658813,
+ "logits/rejected": 1.5206880569458008,
+ "logps/chosen": -157.2953643798828,
+ "logps/rejected": -153.21218872070312,
+ "loss": 0.6558516502380372,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.17594724893569946,
+ "rewards/margins": 0.09524654597043991,
+ "rewards/rejected": 0.08070070296525955,
+ "step": 1940
+ },
+ {
+ "epoch": 1.56,
+ "grad_norm": 1.1362297534942627,
+ "learning_rate": 5.336296296296296e-07,
+ "logits/chosen": 1.524285078048706,
+ "logits/rejected": 1.3964688777923584,
+ "logps/chosen": -161.221435546875,
+ "logps/rejected": -154.8270263671875,
+ "loss": 0.6746312141418457,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.12519574165344238,
+ "rewards/margins": 0.05550859495997429,
+ "rewards/rejected": 0.06968715041875839,
+ "step": 1950
+ },
+ {
+ "epoch": 1.568,
+ "grad_norm": 1.5411877632141113,
+ "learning_rate": 5.306666666666665e-07,
+ "logits/chosen": 1.2426170110702515,
+ "logits/rejected": 1.3256272077560425,
+ "logps/chosen": -137.97537231445312,
+ "logps/rejected": -159.09580993652344,
+ "loss": 0.6492344856262207,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.16253654658794403,
+ "rewards/margins": 0.12099182605743408,
+ "rewards/rejected": 0.04154472425580025,
+ "step": 1960
+ },
+ {
+ "epoch": 1.576,
+ "grad_norm": 1.2111555337905884,
+ "learning_rate": 5.277037037037037e-07,
+ "logits/chosen": 1.3337433338165283,
+ "logits/rejected": 1.479209065437317,
+ "logps/chosen": -149.0251007080078,
+ "logps/rejected": -140.0357208251953,
+ "loss": 0.6559633731842041,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.13673515617847443,
+ "rewards/margins": 0.0912124291062355,
+ "rewards/rejected": 0.04552273079752922,
+ "step": 1970
+ },
+ {
+ "epoch": 1.584,
+ "grad_norm": 1.1228832006454468,
+ "learning_rate": 5.247407407407407e-07,
+ "logits/chosen": 1.4376949071884155,
+ "logits/rejected": 1.5033822059631348,
+ "logps/chosen": -144.91751098632812,
+ "logps/rejected": -147.37057495117188,
+ "loss": 0.6593158721923829,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.1573731154203415,
+ "rewards/margins": 0.08546318858861923,
+ "rewards/rejected": 0.07190994173288345,
+ "step": 1980
+ },
+ {
+ "epoch": 1.592,
+ "grad_norm": 0.9601779580116272,
+ "learning_rate": 5.217777777777777e-07,
+ "logits/chosen": 1.2674219608306885,
+ "logits/rejected": 1.3877508640289307,
+ "logps/chosen": -116.023193359375,
+ "logps/rejected": -122.81617736816406,
+ "loss": 0.6842892646789551,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.13847123086452484,
+ "rewards/margins": 0.026916390284895897,
+ "rewards/rejected": 0.11155486106872559,
+ "step": 1990
+ },
+ {
+ "epoch": 1.6,
+ "grad_norm": 1.2849960327148438,
+ "learning_rate": 5.188148148148148e-07,
+ "logits/chosen": 1.6982529163360596,
+ "logits/rejected": 1.6600332260131836,
+ "logps/chosen": -143.352294921875,
+ "logps/rejected": -150.0862274169922,
+ "loss": 0.6666803359985352,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.13707581162452698,
+ "rewards/margins": 0.06178014352917671,
+ "rewards/rejected": 0.07529564201831818,
+ "step": 2000
+ },
+ {
+ "epoch": 1.6,
+ "eval_logits/chosen": 1.4610096216201782,
+ "eval_logits/rejected": 1.4881619215011597,
+ "eval_logps/chosen": -152.36614990234375,
+ "eval_logps/rejected": -147.1319580078125,
+ "eval_loss": 0.6861760020256042,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": 0.11826877295970917,
+ "eval_rewards/margins": 0.02924923785030842,
+ "eval_rewards/rejected": 0.0890195295214653,
+ "eval_runtime": 91.0197,
+ "eval_samples_per_second": 5.493,
+ "eval_steps_per_second": 2.747,
+ "step": 2000
+ }
+ ],
+ "logging_steps": 10,
+ "max_steps": 3750,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2000/training_args.bin b/v3/DPO/DPO_10k/lora/checkpoint-2000/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..b3fc3d76093802087869ab1fee5e37b9c830fe77
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4f90e4a7053b7fb440d68cb64a3d198396af09db942d08839c5f4f0f0b8a0c8
+size 6097
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2500/README.md b/v3/DPO/DPO_10k/lora/checkpoint-2500/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..250500f798b4682b2cd2c35cd1fc366677c215dd
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2500/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2500/adapter_config.json b/v3/DPO/DPO_10k/lora/checkpoint-2500/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..039869f8d79ca6c48a793a29a79267da6d8f81c2
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2500/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "gate_proj",
+ "down_proj",
+ "k_proj",
+ "o_proj",
+ "up_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2500/adapter_model.safetensors b/v3/DPO/DPO_10k/lora/checkpoint-2500/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..587628093f97273416fcb2c9386c3e305981778d
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2500/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db88ddde8c0acc4b815db6e5df510e53f7bfc3771ddef9d9ad28dea0508259af
+size 180385008
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2500/chat_template.jinja b/v3/DPO/DPO_10k/lora/checkpoint-2500/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2500/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2500/optimizer.pt b/v3/DPO/DPO_10k/lora/checkpoint-2500/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..972825c744af6426b8ccf5b89d1ca53928ee585a
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2500/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f1b4059726a476893f9847b0dac5300b878982294231aa483ff44a5dbc3fa2e
+size 360902475
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2500/rng_state.pth b/v3/DPO/DPO_10k/lora/checkpoint-2500/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..2275456c138024633720ad98c6c63acbe9777008
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2500/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a95215f64b02d62fb58ace326ad670f1d16eb1761f7fa3b3478d43d2b8d6108
+size 14645
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2500/scaler.pt b/v3/DPO/DPO_10k/lora/checkpoint-2500/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f6bccf2b99239cf26ef4ea2b6a5f9f897042b61f
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2500/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:861ce13e6ca091acee9a68ebfc5ca38479baf4b537c37b3949f071f77b81e9f0
+size 1383
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2500/scheduler.pt b/v3/DPO/DPO_10k/lora/checkpoint-2500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b79250d9f940e161d00721448982feb0899bb288
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b591a42ef57235439a22a8a57079a30fbe40d1594624128816b83bc57b23410c
+size 1465
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2500/tokenizer.json b/v3/DPO/DPO_10k/lora/checkpoint-2500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2500/tokenizer_config.json b/v3/DPO/DPO_10k/lora/checkpoint-2500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2500/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2500/trainer_state.json b/v3/DPO/DPO_10k/lora/checkpoint-2500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..2655b49b2ae3082f08bfc13604120285455ce121
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2500/trainer_state.json
@@ -0,0 +1,4184 @@
+{
+ "best_global_step": 2400,
+ "best_metric": 0.6834847331047058,
+ "best_model_checkpoint": "output/lora/checkpoint-1000",
+ "epoch": 2.0,
+ "eval_steps": 100,
+ "global_step": 2500,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.008,
+ "grad_norm": 1.1179773807525635,
+ "learning_rate": 2.4e-08,
+ "logits/chosen": 1.4545083045959473,
+ "logits/rejected": 1.425490379333496,
+ "logps/chosen": -132.46189880371094,
+ "logps/rejected": -148.20260620117188,
+ "loss": 0.6930626392364502,
+ "rewards/accuracies": 0.21250000596046448,
+ "rewards/chosen": -6.165739614516497e-05,
+ "rewards/margins": 0.00017333509458694607,
+ "rewards/rejected": -0.00023499250528402627,
+ "step": 10
+ },
+ {
+ "epoch": 0.016,
+ "grad_norm": 1.3502839803695679,
+ "learning_rate": 5.0666666666666664e-08,
+ "logits/chosen": 1.5668801069259644,
+ "logits/rejected": 1.5191389322280884,
+ "logps/chosen": -157.63624572753906,
+ "logps/rejected": -144.2716827392578,
+ "loss": 0.6930380821228027,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.0009244013344869018,
+ "rewards/margins": 0.00023505210992880166,
+ "rewards/rejected": 0.0006893490790389478,
+ "step": 20
+ },
+ {
+ "epoch": 0.024,
+ "grad_norm": 1.0219553709030151,
+ "learning_rate": 7.733333333333334e-08,
+ "logits/chosen": 1.8921797275543213,
+ "logits/rejected": 1.7342685461044312,
+ "logps/chosen": -164.55799865722656,
+ "logps/rejected": -158.92752075195312,
+ "loss": 0.69290189743042,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0009391927160322666,
+ "rewards/margins": 0.0005032324115745723,
+ "rewards/rejected": 0.0004359603044576943,
+ "step": 30
+ },
+ {
+ "epoch": 0.032,
+ "grad_norm": 1.2056113481521606,
+ "learning_rate": 1.0399999999999999e-07,
+ "logits/chosen": 1.4950047731399536,
+ "logits/rejected": 1.6163270473480225,
+ "logps/chosen": -145.54183959960938,
+ "logps/rejected": -134.19979858398438,
+ "loss": 0.6935500621795654,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.0006421065190806985,
+ "rewards/margins": -0.0007943272357806563,
+ "rewards/rejected": 0.0014364338712766767,
+ "step": 40
+ },
+ {
+ "epoch": 0.04,
+ "grad_norm": 1.1029725074768066,
+ "learning_rate": 1.3066666666666665e-07,
+ "logits/chosen": 1.4076533317565918,
+ "logits/rejected": 1.4791977405548096,
+ "logps/chosen": -133.23324584960938,
+ "logps/rejected": -138.3593292236328,
+ "loss": 0.6925833702087403,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.0012372838100418448,
+ "rewards/margins": 0.001139522879384458,
+ "rewards/rejected": 9.776116348803043e-05,
+ "step": 50
+ },
+ {
+ "epoch": 0.048,
+ "grad_norm": 0.9960266351699829,
+ "learning_rate": 1.573333333333333e-07,
+ "logits/chosen": 1.407166600227356,
+ "logits/rejected": 1.376042127609253,
+ "logps/chosen": -137.8921661376953,
+ "logps/rejected": -127.97320556640625,
+ "loss": 0.693614149093628,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.00021316049969755113,
+ "rewards/margins": -0.0009249996510334313,
+ "rewards/rejected": 0.0007118391804397106,
+ "step": 60
+ },
+ {
+ "epoch": 0.056,
+ "grad_norm": 1.0946824550628662,
+ "learning_rate": 1.8399999999999998e-07,
+ "logits/chosen": 1.5944501161575317,
+ "logits/rejected": 1.7248146533966064,
+ "logps/chosen": -153.5526123046875,
+ "logps/rejected": -157.19735717773438,
+ "loss": 0.6928449153900147,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0026370862033218145,
+ "rewards/margins": 0.0006150150438770652,
+ "rewards/rejected": 0.0020220710430294275,
+ "step": 70
+ },
+ {
+ "epoch": 0.064,
+ "grad_norm": 1.0322489738464355,
+ "learning_rate": 2.1066666666666665e-07,
+ "logits/chosen": 1.4586353302001953,
+ "logits/rejected": 1.4842995405197144,
+ "logps/chosen": -149.03665161132812,
+ "logps/rejected": -136.24081420898438,
+ "loss": 0.6932015895843506,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.004061593674123287,
+ "rewards/margins": -9.99594121822156e-05,
+ "rewards/rejected": 0.004161553457379341,
+ "step": 80
+ },
+ {
+ "epoch": 0.072,
+ "grad_norm": 1.0012447834014893,
+ "learning_rate": 2.3733333333333334e-07,
+ "logits/chosen": 1.5075902938842773,
+ "logits/rejected": 1.5986944437026978,
+ "logps/chosen": -159.3433074951172,
+ "logps/rejected": -151.85072326660156,
+ "loss": 0.6928757190704345,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.006978762801736593,
+ "rewards/margins": 0.0005627512582577765,
+ "rewards/rejected": 0.006416010670363903,
+ "step": 90
+ },
+ {
+ "epoch": 0.08,
+ "grad_norm": 1.280081033706665,
+ "learning_rate": 2.64e-07,
+ "logits/chosen": 1.448286533355713,
+ "logits/rejected": 1.5480695962905884,
+ "logps/chosen": -151.12625122070312,
+ "logps/rejected": -171.24545288085938,
+ "loss": 0.6931126117706299,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.00693341251462698,
+ "rewards/margins": 9.057526767719537e-05,
+ "rewards/rejected": 0.006842837668955326,
+ "step": 100
+ },
+ {
+ "epoch": 0.08,
+ "eval_logits/chosen": 1.518183946609497,
+ "eval_logits/rejected": 1.5525447130203247,
+ "eval_logps/chosen": -153.46630859375,
+ "eval_logps/rejected": -147.95005798339844,
+ "eval_loss": 0.6926330327987671,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.008253541775047779,
+ "eval_rewards/margins": 0.0010449704714119434,
+ "eval_rewards/rejected": 0.007208569906651974,
+ "eval_runtime": 91.0812,
+ "eval_samples_per_second": 5.49,
+ "eval_steps_per_second": 2.745,
+ "step": 100
+ },
+ {
+ "epoch": 0.088,
+ "grad_norm": 1.3912373781204224,
+ "learning_rate": 2.906666666666667e-07,
+ "logits/chosen": 1.6896222829818726,
+ "logits/rejected": 1.6516921520233154,
+ "logps/chosen": -173.70030212402344,
+ "logps/rejected": -155.5503692626953,
+ "loss": 0.6926521778106689,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.010179834440350533,
+ "rewards/margins": 0.0010137532372027636,
+ "rewards/rejected": 0.009166081435978413,
+ "step": 110
+ },
+ {
+ "epoch": 0.096,
+ "grad_norm": 1.1518888473510742,
+ "learning_rate": 3.173333333333333e-07,
+ "logits/chosen": 1.5166860818862915,
+ "logits/rejected": 1.5136979818344116,
+ "logps/chosen": -136.39906311035156,
+ "logps/rejected": -140.86561584472656,
+ "loss": 0.6932918548583984,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": 0.00907944981008768,
+ "rewards/margins": -0.00026998287648893893,
+ "rewards/rejected": 0.009349432773888111,
+ "step": 120
+ },
+ {
+ "epoch": 0.104,
+ "grad_norm": 1.3213376998901367,
+ "learning_rate": 3.4399999999999996e-07,
+ "logits/chosen": 1.4598973989486694,
+ "logits/rejected": 1.6739647388458252,
+ "logps/chosen": -159.12461853027344,
+ "logps/rejected": -163.39346313476562,
+ "loss": 0.693086051940918,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.010477779433131218,
+ "rewards/margins": 0.00014539004769176245,
+ "rewards/rejected": 0.010332388803362846,
+ "step": 130
+ },
+ {
+ "epoch": 0.112,
+ "grad_norm": 0.8833863735198975,
+ "learning_rate": 3.7066666666666665e-07,
+ "logits/chosen": 1.3829412460327148,
+ "logits/rejected": 1.4100173711776733,
+ "logps/chosen": -132.7149658203125,
+ "logps/rejected": -126.85478210449219,
+ "loss": 0.6932382583618164,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": 0.010262183845043182,
+ "rewards/margins": -0.0001672578218858689,
+ "rewards/rejected": 0.010429441928863525,
+ "step": 140
+ },
+ {
+ "epoch": 0.12,
+ "grad_norm": 1.1472455263137817,
+ "learning_rate": 3.973333333333333e-07,
+ "logits/chosen": 1.4954869747161865,
+ "logits/rejected": 1.5857340097427368,
+ "logps/chosen": -165.10568237304688,
+ "logps/rejected": -158.97433471679688,
+ "loss": 0.6916103363037109,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.014177520759403706,
+ "rewards/margins": 0.003108317730948329,
+ "rewards/rejected": 0.011069201864302158,
+ "step": 150
+ },
+ {
+ "epoch": 0.128,
+ "grad_norm": 1.0628798007965088,
+ "learning_rate": 4.24e-07,
+ "logits/chosen": 1.6717243194580078,
+ "logits/rejected": 1.705092191696167,
+ "logps/chosen": -159.332275390625,
+ "logps/rejected": -147.21646118164062,
+ "loss": 0.6935629844665527,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": 0.016994481906294823,
+ "rewards/margins": -0.0007874655420891941,
+ "rewards/rejected": 0.017781946808099747,
+ "step": 160
+ },
+ {
+ "epoch": 0.136,
+ "grad_norm": 1.390810251235962,
+ "learning_rate": 4.506666666666666e-07,
+ "logits/chosen": 1.4719234704971313,
+ "logits/rejected": 1.499526023864746,
+ "logps/chosen": -148.67251586914062,
+ "logps/rejected": -161.58021545410156,
+ "loss": 0.6926599979400635,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.021239640191197395,
+ "rewards/margins": 0.0010346340714022517,
+ "rewards/rejected": 0.020205006003379822,
+ "step": 170
+ },
+ {
+ "epoch": 0.144,
+ "grad_norm": 1.139707326889038,
+ "learning_rate": 4.773333333333333e-07,
+ "logits/chosen": 1.4833905696868896,
+ "logits/rejected": 1.637584924697876,
+ "logps/chosen": -158.96731567382812,
+ "logps/rejected": -161.9830322265625,
+ "loss": 0.6919547557830811,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.019416773691773415,
+ "rewards/margins": 0.0024296878837049007,
+ "rewards/rejected": 0.016987085342407227,
+ "step": 180
+ },
+ {
+ "epoch": 0.152,
+ "grad_norm": 1.2029796838760376,
+ "learning_rate": 5.04e-07,
+ "logits/chosen": 1.5398027896881104,
+ "logits/rejected": 1.6678705215454102,
+ "logps/chosen": -143.00283813476562,
+ "logps/rejected": -144.89378356933594,
+ "loss": 0.692262840270996,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.018960200250148773,
+ "rewards/margins": 0.0018175147706642747,
+ "rewards/rejected": 0.017142686992883682,
+ "step": 190
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 1.0527437925338745,
+ "learning_rate": 5.306666666666665e-07,
+ "logits/chosen": 1.6087623834609985,
+ "logits/rejected": 1.5181543827056885,
+ "logps/chosen": -159.35073852539062,
+ "logps/rejected": -133.59890747070312,
+ "loss": 0.6924069881439209,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.01926913857460022,
+ "rewards/margins": 0.0015339398523792624,
+ "rewards/rejected": 0.0177351962774992,
+ "step": 200
+ },
+ {
+ "epoch": 0.16,
+ "eval_logits/chosen": 1.5193672180175781,
+ "eval_logits/rejected": 1.5534639358520508,
+ "eval_logps/chosen": -153.34857177734375,
+ "eval_logps/rejected": -147.8425750732422,
+ "eval_loss": 0.6921458840370178,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.020027177408337593,
+ "eval_rewards/margins": 0.0020714527927339077,
+ "eval_rewards/rejected": 0.017955724149942398,
+ "eval_runtime": 90.9762,
+ "eval_samples_per_second": 5.496,
+ "eval_steps_per_second": 2.748,
+ "step": 200
+ },
+ {
+ "epoch": 0.168,
+ "grad_norm": 1.1897239685058594,
+ "learning_rate": 5.573333333333333e-07,
+ "logits/chosen": 1.6267732381820679,
+ "logits/rejected": 1.5775268077850342,
+ "logps/chosen": -159.51437377929688,
+ "logps/rejected": -170.17762756347656,
+ "loss": 0.6925770759582519,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.02078000269830227,
+ "rewards/margins": 0.0012053751852363348,
+ "rewards/rejected": 0.01957462914288044,
+ "step": 210
+ },
+ {
+ "epoch": 0.176,
+ "grad_norm": 0.9995236396789551,
+ "learning_rate": 5.839999999999999e-07,
+ "logits/chosen": 1.5781934261322021,
+ "logits/rejected": 1.642251968383789,
+ "logps/chosen": -159.95309448242188,
+ "logps/rejected": -144.36058044433594,
+ "loss": 0.6931475639343262,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.023322442546486855,
+ "rewards/margins": 6.392716750269756e-05,
+ "rewards/rejected": 0.02325851283967495,
+ "step": 220
+ },
+ {
+ "epoch": 0.184,
+ "grad_norm": 0.8819260597229004,
+ "learning_rate": 6.106666666666666e-07,
+ "logits/chosen": 1.459991693496704,
+ "logits/rejected": 1.6060739755630493,
+ "logps/chosen": -162.04745483398438,
+ "logps/rejected": -139.5411834716797,
+ "loss": 0.6919463634490967,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.026052182540297508,
+ "rewards/margins": 0.002481849165633321,
+ "rewards/rejected": 0.023570330813527107,
+ "step": 230
+ },
+ {
+ "epoch": 0.192,
+ "grad_norm": 1.1055208444595337,
+ "learning_rate": 6.373333333333333e-07,
+ "logits/chosen": 1.558739185333252,
+ "logits/rejected": 1.5965116024017334,
+ "logps/chosen": -171.9166259765625,
+ "logps/rejected": -145.66433715820312,
+ "loss": 0.6912176132202148,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.033868782222270966,
+ "rewards/margins": 0.004017929546535015,
+ "rewards/rejected": 0.029850851744413376,
+ "step": 240
+ },
+ {
+ "epoch": 0.2,
+ "grad_norm": 1.1372733116149902,
+ "learning_rate": 6.64e-07,
+ "logits/chosen": 1.5322860479354858,
+ "logits/rejected": 1.4695795774459839,
+ "logps/chosen": -161.7803955078125,
+ "logps/rejected": -148.91949462890625,
+ "loss": 0.6940969944000244,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.034065864980220795,
+ "rewards/margins": -0.0016761379083618522,
+ "rewards/rejected": 0.035742007195949554,
+ "step": 250
+ },
+ {
+ "epoch": 0.208,
+ "grad_norm": 1.0917680263519287,
+ "learning_rate": 6.906666666666666e-07,
+ "logits/chosen": 1.4094547033309937,
+ "logits/rejected": 1.50186026096344,
+ "logps/chosen": -132.91455078125,
+ "logps/rejected": -146.79714965820312,
+ "loss": 0.6930203914642334,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.03089061938226223,
+ "rewards/margins": 0.00040971505222842097,
+ "rewards/rejected": 0.030480902642011642,
+ "step": 260
+ },
+ {
+ "epoch": 0.216,
+ "grad_norm": 1.0280259847640991,
+ "learning_rate": 7.173333333333333e-07,
+ "logits/chosen": 1.531285285949707,
+ "logits/rejected": 1.6163132190704346,
+ "logps/chosen": -146.902587890625,
+ "logps/rejected": -147.8347625732422,
+ "loss": 0.6944749355316162,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.028514739125967026,
+ "rewards/margins": -0.0025452517438679934,
+ "rewards/rejected": 0.0310599897056818,
+ "step": 270
+ },
+ {
+ "epoch": 0.224,
+ "grad_norm": 1.398708701133728,
+ "learning_rate": 7.44e-07,
+ "logits/chosen": 1.563852071762085,
+ "logits/rejected": 1.396703839302063,
+ "logps/chosen": -172.68153381347656,
+ "logps/rejected": -155.80215454101562,
+ "loss": 0.689476490020752,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03224216774106026,
+ "rewards/margins": 0.007473687641322613,
+ "rewards/rejected": 0.02476847730576992,
+ "step": 280
+ },
+ {
+ "epoch": 0.232,
+ "grad_norm": 1.0394141674041748,
+ "learning_rate": 7.706666666666667e-07,
+ "logits/chosen": 1.5581729412078857,
+ "logits/rejected": 1.5819021463394165,
+ "logps/chosen": -144.41465759277344,
+ "logps/rejected": -153.55406188964844,
+ "loss": 0.6927877902984619,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.03272664546966553,
+ "rewards/margins": 0.0008557910332456231,
+ "rewards/rejected": 0.031870849430561066,
+ "step": 290
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 1.2557185888290405,
+ "learning_rate": 7.973333333333333e-07,
+ "logits/chosen": 1.5062446594238281,
+ "logits/rejected": 1.5786281824111938,
+ "logps/chosen": -153.63558959960938,
+ "logps/rejected": -165.46617126464844,
+ "loss": 0.6897892951965332,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03914078325033188,
+ "rewards/margins": 0.006873926613479853,
+ "rewards/rejected": 0.032266855239868164,
+ "step": 300
+ },
+ {
+ "epoch": 0.24,
+ "eval_logits/chosen": 1.5237834453582764,
+ "eval_logits/rejected": 1.557741641998291,
+ "eval_logps/chosen": -153.19883728027344,
+ "eval_logps/rejected": -147.7100830078125,
+ "eval_loss": 0.6913501024246216,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.03499976545572281,
+ "eval_rewards/margins": 0.0037925743963569403,
+ "eval_rewards/rejected": 0.031207194551825523,
+ "eval_runtime": 90.9133,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 300
+ },
+ {
+ "epoch": 0.248,
+ "grad_norm": 1.1200144290924072,
+ "learning_rate": 8.24e-07,
+ "logits/chosen": 1.618949294090271,
+ "logits/rejected": 1.6625702381134033,
+ "logps/chosen": -141.8953857421875,
+ "logps/rejected": -156.6927032470703,
+ "loss": 0.694425106048584,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.03162423521280289,
+ "rewards/margins": -0.0023833750747144222,
+ "rewards/rejected": 0.034007612615823746,
+ "step": 310
+ },
+ {
+ "epoch": 0.256,
+ "grad_norm": 1.0823743343353271,
+ "learning_rate": 8.506666666666667e-07,
+ "logits/chosen": 1.5974153280258179,
+ "logits/rejected": 1.6237560510635376,
+ "logps/chosen": -139.2996368408203,
+ "logps/rejected": -141.94407653808594,
+ "loss": 0.6914949893951416,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.029445698484778404,
+ "rewards/margins": 0.0034329916816204786,
+ "rewards/rejected": 0.026012707501649857,
+ "step": 320
+ },
+ {
+ "epoch": 0.264,
+ "grad_norm": 1.1765722036361694,
+ "learning_rate": 8.773333333333332e-07,
+ "logits/chosen": 1.465215802192688,
+ "logits/rejected": 1.4882014989852905,
+ "logps/chosen": -158.4329833984375,
+ "logps/rejected": -183.94259643554688,
+ "loss": 0.6925934791564942,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.026302698999643326,
+ "rewards/margins": 0.0012350418837741017,
+ "rewards/rejected": 0.025067657232284546,
+ "step": 330
+ },
+ {
+ "epoch": 0.272,
+ "grad_norm": 1.0667294263839722,
+ "learning_rate": 9.039999999999999e-07,
+ "logits/chosen": 1.7538875341415405,
+ "logits/rejected": 1.6171789169311523,
+ "logps/chosen": -159.63790893554688,
+ "logps/rejected": -151.1880340576172,
+ "loss": 0.6922041416168213,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.026619985699653625,
+ "rewards/margins": 0.002031774492934346,
+ "rewards/rejected": 0.024588212370872498,
+ "step": 340
+ },
+ {
+ "epoch": 0.28,
+ "grad_norm": 1.2210006713867188,
+ "learning_rate": 9.306666666666666e-07,
+ "logits/chosen": 1.5815479755401611,
+ "logits/rejected": 1.710217833518982,
+ "logps/chosen": -167.21743774414062,
+ "logps/rejected": -196.77989196777344,
+ "loss": 0.6899692058563233,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.03636014088988304,
+ "rewards/margins": 0.006536121014505625,
+ "rewards/rejected": 0.029824022203683853,
+ "step": 350
+ },
+ {
+ "epoch": 0.288,
+ "grad_norm": 0.9623845219612122,
+ "learning_rate": 9.573333333333333e-07,
+ "logits/chosen": 1.4416790008544922,
+ "logits/rejected": 1.3618693351745605,
+ "logps/chosen": -163.27401733398438,
+ "logps/rejected": -169.3351287841797,
+ "loss": 0.6910821437835694,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.037200987339019775,
+ "rewards/margins": 0.004338678438216448,
+ "rewards/rejected": 0.032862309366464615,
+ "step": 360
+ },
+ {
+ "epoch": 0.296,
+ "grad_norm": 1.1005645990371704,
+ "learning_rate": 9.84e-07,
+ "logits/chosen": 1.6214616298675537,
+ "logits/rejected": 1.6572606563568115,
+ "logps/chosen": -162.95504760742188,
+ "logps/rejected": -154.91807556152344,
+ "loss": 0.6895030021667481,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.047198958694934845,
+ "rewards/margins": 0.0075958347879350185,
+ "rewards/rejected": 0.03960312157869339,
+ "step": 370
+ },
+ {
+ "epoch": 0.304,
+ "grad_norm": 1.1126338243484497,
+ "learning_rate": 9.988148148148148e-07,
+ "logits/chosen": 1.4842126369476318,
+ "logits/rejected": 1.410930871963501,
+ "logps/chosen": -143.09182739257812,
+ "logps/rejected": -153.48135375976562,
+ "loss": 0.6885370254516602,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.05975018069148064,
+ "rewards/margins": 0.00981281790882349,
+ "rewards/rejected": 0.04993735998868942,
+ "step": 380
+ },
+ {
+ "epoch": 0.312,
+ "grad_norm": 1.1146304607391357,
+ "learning_rate": 9.95851851851852e-07,
+ "logits/chosen": 1.608332633972168,
+ "logits/rejected": 1.5848472118377686,
+ "logps/chosen": -152.25250244140625,
+ "logps/rejected": -138.348876953125,
+ "loss": 0.6851807117462159,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.07630724459886551,
+ "rewards/margins": 0.016460279002785683,
+ "rewards/rejected": 0.059846967458724976,
+ "step": 390
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 1.2686601877212524,
+ "learning_rate": 9.928888888888889e-07,
+ "logits/chosen": 1.5840070247650146,
+ "logits/rejected": 1.5679785013198853,
+ "logps/chosen": -155.90219116210938,
+ "logps/rejected": -148.89549255371094,
+ "loss": 0.6891039371490478,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.08129279315471649,
+ "rewards/margins": 0.008997146971523762,
+ "rewards/rejected": 0.072295643389225,
+ "step": 400
+ },
+ {
+ "epoch": 0.32,
+ "eval_logits/chosen": 1.5358340740203857,
+ "eval_logits/rejected": 1.5689741373062134,
+ "eval_logps/chosen": -152.6895294189453,
+ "eval_logps/rejected": -147.23422241210938,
+ "eval_loss": 0.6900544166564941,
+ "eval_rewards/accuracies": 0.5419999957084656,
+ "eval_rewards/chosen": 0.08593228459358215,
+ "eval_rewards/margins": 0.0071397931315004826,
+ "eval_rewards/rejected": 0.0787924975156784,
+ "eval_runtime": 90.9072,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 400
+ },
+ {
+ "epoch": 0.328,
+ "grad_norm": 1.1653108596801758,
+ "learning_rate": 9.899259259259258e-07,
+ "logits/chosen": 1.5450490713119507,
+ "logits/rejected": 1.4428151845932007,
+ "logps/chosen": -148.61581420898438,
+ "logps/rejected": -188.90919494628906,
+ "loss": 0.6918989181518554,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0946170911192894,
+ "rewards/margins": 0.003271549940109253,
+ "rewards/rejected": 0.09134554117918015,
+ "step": 410
+ },
+ {
+ "epoch": 0.336,
+ "grad_norm": 1.3114937543869019,
+ "learning_rate": 9.86962962962963e-07,
+ "logits/chosen": 1.5342715978622437,
+ "logits/rejected": 1.5572118759155273,
+ "logps/chosen": -152.65264892578125,
+ "logps/rejected": -168.03384399414062,
+ "loss": 0.6972955703735352,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.08869597315788269,
+ "rewards/margins": -0.007436127867549658,
+ "rewards/rejected": 0.09613210707902908,
+ "step": 420
+ },
+ {
+ "epoch": 0.344,
+ "grad_norm": 1.0040781497955322,
+ "learning_rate": 9.84e-07,
+ "logits/chosen": 1.5979712009429932,
+ "logits/rejected": 1.6713664531707764,
+ "logps/chosen": -164.2566375732422,
+ "logps/rejected": -152.1428680419922,
+ "loss": 0.6897010326385498,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.08101107180118561,
+ "rewards/margins": 0.007684691343456507,
+ "rewards/rejected": 0.07332637906074524,
+ "step": 430
+ },
+ {
+ "epoch": 0.352,
+ "grad_norm": 1.0801892280578613,
+ "learning_rate": 9.81037037037037e-07,
+ "logits/chosen": 1.439635992050171,
+ "logits/rejected": 1.3282365798950195,
+ "logps/chosen": -157.23403930664062,
+ "logps/rejected": -146.54806518554688,
+ "loss": 0.6881664752960205,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.08112544566392899,
+ "rewards/margins": 0.010971959680318832,
+ "rewards/rejected": 0.07015348225831985,
+ "step": 440
+ },
+ {
+ "epoch": 0.36,
+ "grad_norm": 1.1139949560165405,
+ "learning_rate": 9.78074074074074e-07,
+ "logits/chosen": 1.6345123052597046,
+ "logits/rejected": 1.5876491069793701,
+ "logps/chosen": -157.6160125732422,
+ "logps/rejected": -160.02040100097656,
+ "loss": 0.6868602275848389,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.07638200372457504,
+ "rewards/margins": 0.013464884832501411,
+ "rewards/rejected": 0.06291711330413818,
+ "step": 450
+ },
+ {
+ "epoch": 0.368,
+ "grad_norm": 1.158035397529602,
+ "learning_rate": 9.751111111111112e-07,
+ "logits/chosen": 1.5430586338043213,
+ "logits/rejected": 1.6022913455963135,
+ "logps/chosen": -151.98919677734375,
+ "logps/rejected": -164.90565490722656,
+ "loss": 0.6891930580139161,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06889279186725616,
+ "rewards/margins": 0.008791089057922363,
+ "rewards/rejected": 0.0601017065346241,
+ "step": 460
+ },
+ {
+ "epoch": 0.376,
+ "grad_norm": 1.0903704166412354,
+ "learning_rate": 9.721481481481481e-07,
+ "logits/chosen": 1.313881754875183,
+ "logits/rejected": 1.3182194232940674,
+ "logps/chosen": -131.5592041015625,
+ "logps/rejected": -123.84442138671875,
+ "loss": 0.6889429569244385,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.08150745928287506,
+ "rewards/margins": 0.009235726669430733,
+ "rewards/rejected": 0.07227173447608948,
+ "step": 470
+ },
+ {
+ "epoch": 0.384,
+ "grad_norm": 1.0766969919204712,
+ "learning_rate": 9.69185185185185e-07,
+ "logits/chosen": 1.425756812095642,
+ "logits/rejected": 1.476854920387268,
+ "logps/chosen": -148.72279357910156,
+ "logps/rejected": -164.40951538085938,
+ "loss": 0.6854116439819335,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1009502187371254,
+ "rewards/margins": 0.0166789498180151,
+ "rewards/rejected": 0.08427127450704575,
+ "step": 480
+ },
+ {
+ "epoch": 0.392,
+ "grad_norm": 1.2078129053115845,
+ "learning_rate": 9.662222222222222e-07,
+ "logits/chosen": 1.5511282682418823,
+ "logits/rejected": 1.611383080482483,
+ "logps/chosen": -143.22006225585938,
+ "logps/rejected": -159.70176696777344,
+ "loss": 0.6968960762023926,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.08779363334178925,
+ "rewards/margins": -0.00598606513813138,
+ "rewards/rejected": 0.09377971291542053,
+ "step": 490
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.9106306433677673,
+ "learning_rate": 9.632592592592593e-07,
+ "logits/chosen": 1.4217239618301392,
+ "logits/rejected": 1.505446195602417,
+ "logps/chosen": -144.34109497070312,
+ "logps/rejected": -135.52430725097656,
+ "loss": 0.691196870803833,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06902458518743515,
+ "rewards/margins": 0.005059319548308849,
+ "rewards/rejected": 0.06396526843309402,
+ "step": 500
+ },
+ {
+ "epoch": 0.4,
+ "eval_logits/chosen": 1.5185325145721436,
+ "eval_logits/rejected": 1.5510345697402954,
+ "eval_logps/chosen": -152.7921905517578,
+ "eval_logps/rejected": -147.3463134765625,
+ "eval_loss": 0.6898502111434937,
+ "eval_rewards/accuracies": 0.5099999904632568,
+ "eval_rewards/chosen": 0.07566512376070023,
+ "eval_rewards/margins": 0.008083080872893333,
+ "eval_rewards/rejected": 0.06758204847574234,
+ "eval_runtime": 90.8294,
+ "eval_samples_per_second": 5.505,
+ "eval_steps_per_second": 2.752,
+ "step": 500
+ },
+ {
+ "epoch": 0.408,
+ "grad_norm": 1.1637762784957886,
+ "learning_rate": 9.602962962962962e-07,
+ "logits/chosen": 1.5965330600738525,
+ "logits/rejected": 1.5338464975357056,
+ "logps/chosen": -131.49871826171875,
+ "logps/rejected": -140.05995178222656,
+ "loss": 0.6868171215057373,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.08385131508111954,
+ "rewards/margins": 0.013950209133327007,
+ "rewards/rejected": 0.06990110129117966,
+ "step": 510
+ },
+ {
+ "epoch": 0.416,
+ "grad_norm": 1.229719638824463,
+ "learning_rate": 9.573333333333333e-07,
+ "logits/chosen": 1.7057275772094727,
+ "logits/rejected": 1.5898948907852173,
+ "logps/chosen": -166.4267120361328,
+ "logps/rejected": -172.22337341308594,
+ "loss": 0.6912094116210937,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.07427279651165009,
+ "rewards/margins": 0.004661654122173786,
+ "rewards/rejected": 0.06961113959550858,
+ "step": 520
+ },
+ {
+ "epoch": 0.424,
+ "grad_norm": 1.136383295059204,
+ "learning_rate": 9.543703703703705e-07,
+ "logits/chosen": 1.6207077503204346,
+ "logits/rejected": 1.5417520999908447,
+ "logps/chosen": -166.81509399414062,
+ "logps/rejected": -168.5408172607422,
+ "loss": 0.6772748470306397,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.0953373983502388,
+ "rewards/margins": 0.033697597682476044,
+ "rewards/rejected": 0.06163979694247246,
+ "step": 530
+ },
+ {
+ "epoch": 0.432,
+ "grad_norm": 1.0640541315078735,
+ "learning_rate": 9.514074074074074e-07,
+ "logits/chosen": 1.6825854778289795,
+ "logits/rejected": 1.6327041387557983,
+ "logps/chosen": -181.39956665039062,
+ "logps/rejected": -158.49647521972656,
+ "loss": 0.6883615493774414,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.10187923908233643,
+ "rewards/margins": 0.010873330757021904,
+ "rewards/rejected": 0.09100590646266937,
+ "step": 540
+ },
+ {
+ "epoch": 0.44,
+ "grad_norm": 1.477909803390503,
+ "learning_rate": 9.484444444444444e-07,
+ "logits/chosen": 1.5402686595916748,
+ "logits/rejected": 1.6487388610839844,
+ "logps/chosen": -151.53814697265625,
+ "logps/rejected": -167.8573455810547,
+ "loss": 0.6915213584899902,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09472338110208511,
+ "rewards/margins": 0.005475065670907497,
+ "rewards/rejected": 0.08924831449985504,
+ "step": 550
+ },
+ {
+ "epoch": 0.448,
+ "grad_norm": 1.0138317346572876,
+ "learning_rate": 9.454814814814814e-07,
+ "logits/chosen": 1.6286503076553345,
+ "logits/rejected": 1.5223101377487183,
+ "logps/chosen": -154.72079467773438,
+ "logps/rejected": -139.2535858154297,
+ "loss": 0.6835216522216797,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.09194488823413849,
+ "rewards/margins": 0.02140027843415737,
+ "rewards/rejected": 0.07054460793733597,
+ "step": 560
+ },
+ {
+ "epoch": 0.456,
+ "grad_norm": 0.896689772605896,
+ "learning_rate": 9.425185185185184e-07,
+ "logits/chosen": 1.4857146739959717,
+ "logits/rejected": 1.4369819164276123,
+ "logps/chosen": -158.31809997558594,
+ "logps/rejected": -152.33676147460938,
+ "loss": 0.6863360404968262,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.07895292341709137,
+ "rewards/margins": 0.016260787844657898,
+ "rewards/rejected": 0.06269213557243347,
+ "step": 570
+ },
+ {
+ "epoch": 0.464,
+ "grad_norm": 1.4772330522537231,
+ "learning_rate": 9.395555555555556e-07,
+ "logits/chosen": 1.315298318862915,
+ "logits/rejected": 1.4757534265518188,
+ "logps/chosen": -136.4202117919922,
+ "logps/rejected": -154.99362182617188,
+ "loss": 0.6916167259216308,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.04677204787731171,
+ "rewards/margins": 0.005649724509567022,
+ "rewards/rejected": 0.041122324764728546,
+ "step": 580
+ },
+ {
+ "epoch": 0.472,
+ "grad_norm": 0.9806157350540161,
+ "learning_rate": 9.365925925925926e-07,
+ "logits/chosen": 1.465427279472351,
+ "logits/rejected": 1.459496259689331,
+ "logps/chosen": -145.07470703125,
+ "logps/rejected": -131.71974182128906,
+ "loss": 0.6857181549072265,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.07979898899793625,
+ "rewards/margins": 0.016965944319963455,
+ "rewards/rejected": 0.0628330409526825,
+ "step": 590
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 1.4313396215438843,
+ "learning_rate": 9.336296296296295e-07,
+ "logits/chosen": 1.5190367698669434,
+ "logits/rejected": 1.563504934310913,
+ "logps/chosen": -132.0201416015625,
+ "logps/rejected": -158.89865112304688,
+ "loss": 0.6964848041534424,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.06544725596904755,
+ "rewards/margins": -0.004280097782611847,
+ "rewards/rejected": 0.0697273463010788,
+ "step": 600
+ },
+ {
+ "epoch": 0.48,
+ "eval_logits/chosen": 1.5031698942184448,
+ "eval_logits/rejected": 1.5348584651947021,
+ "eval_logps/chosen": -152.76596069335938,
+ "eval_logps/rejected": -147.3241729736328,
+ "eval_loss": 0.6903045773506165,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": 0.07828804850578308,
+ "eval_rewards/margins": 0.008492168970406055,
+ "eval_rewards/rejected": 0.0697958841919899,
+ "eval_runtime": 90.9881,
+ "eval_samples_per_second": 5.495,
+ "eval_steps_per_second": 2.748,
+ "step": 600
+ },
+ {
+ "epoch": 0.488,
+ "grad_norm": 0.9467533230781555,
+ "learning_rate": 9.306666666666666e-07,
+ "logits/chosen": 1.5242283344268799,
+ "logits/rejected": 1.5798314809799194,
+ "logps/chosen": -131.90048217773438,
+ "logps/rejected": -149.93222045898438,
+ "loss": 0.6899854183197022,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.08419154584407806,
+ "rewards/margins": 0.00830511748790741,
+ "rewards/rejected": 0.07588644325733185,
+ "step": 610
+ },
+ {
+ "epoch": 0.496,
+ "grad_norm": 1.8463492393493652,
+ "learning_rate": 9.277037037037037e-07,
+ "logits/chosen": 1.4965155124664307,
+ "logits/rejected": 1.6336272954940796,
+ "logps/chosen": -156.1863555908203,
+ "logps/rejected": -167.74075317382812,
+ "loss": 0.6977802276611328,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.05518193915486336,
+ "rewards/margins": -0.006808985956013203,
+ "rewards/rejected": 0.061990927904844284,
+ "step": 620
+ },
+ {
+ "epoch": 0.504,
+ "grad_norm": 1.278316617012024,
+ "learning_rate": 9.247407407407407e-07,
+ "logits/chosen": 1.5074065923690796,
+ "logits/rejected": 1.535696029663086,
+ "logps/chosen": -167.56777954101562,
+ "logps/rejected": -147.04922485351562,
+ "loss": 0.6806219100952149,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.0745314508676529,
+ "rewards/margins": 0.027104835957288742,
+ "rewards/rejected": 0.047426607459783554,
+ "step": 630
+ },
+ {
+ "epoch": 0.512,
+ "grad_norm": 1.0424679517745972,
+ "learning_rate": 9.217777777777778e-07,
+ "logits/chosen": 1.4299286603927612,
+ "logits/rejected": 1.4484076499938965,
+ "logps/chosen": -168.305908203125,
+ "logps/rejected": -152.61854553222656,
+ "loss": 0.6871783256530761,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07751411944627762,
+ "rewards/margins": 0.017045259475708008,
+ "rewards/rejected": 0.06046885997056961,
+ "step": 640
+ },
+ {
+ "epoch": 0.52,
+ "grad_norm": 1.1143999099731445,
+ "learning_rate": 9.188148148148148e-07,
+ "logits/chosen": 1.5520436763763428,
+ "logits/rejected": 1.531718134880066,
+ "logps/chosen": -154.92115783691406,
+ "logps/rejected": -126.78369140625,
+ "loss": 0.6889286518096924,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.0945608839392662,
+ "rewards/margins": 0.01080345083028078,
+ "rewards/rejected": 0.0837574377655983,
+ "step": 650
+ },
+ {
+ "epoch": 0.528,
+ "grad_norm": 1.2762326002120972,
+ "learning_rate": 9.158518518518517e-07,
+ "logits/chosen": 1.5747522115707397,
+ "logits/rejected": 1.4972833395004272,
+ "logps/chosen": -151.13914489746094,
+ "logps/rejected": -132.013916015625,
+ "loss": 0.6900454998016358,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.07303085923194885,
+ "rewards/margins": 0.008073865436017513,
+ "rewards/rejected": 0.06495700031518936,
+ "step": 660
+ },
+ {
+ "epoch": 0.536,
+ "grad_norm": 1.5207557678222656,
+ "learning_rate": 9.128888888888888e-07,
+ "logits/chosen": 1.6594665050506592,
+ "logits/rejected": 1.468492031097412,
+ "logps/chosen": -166.0108184814453,
+ "logps/rejected": -151.38555908203125,
+ "loss": 0.6934059143066407,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.09294265508651733,
+ "rewards/margins": 0.0021180796902626753,
+ "rewards/rejected": 0.09082455933094025,
+ "step": 670
+ },
+ {
+ "epoch": 0.544,
+ "grad_norm": 1.1073335409164429,
+ "learning_rate": 9.099259259259259e-07,
+ "logits/chosen": 1.5321934223175049,
+ "logits/rejected": 1.5313541889190674,
+ "logps/chosen": -144.5046844482422,
+ "logps/rejected": -133.05470275878906,
+ "loss": 0.6824594974517822,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.10782947391271591,
+ "rewards/margins": 0.0230915155261755,
+ "rewards/rejected": 0.08473796397447586,
+ "step": 680
+ },
+ {
+ "epoch": 0.552,
+ "grad_norm": 1.096970558166504,
+ "learning_rate": 9.069629629629629e-07,
+ "logits/chosen": 1.438305139541626,
+ "logits/rejected": 1.4338308572769165,
+ "logps/chosen": -173.1280517578125,
+ "logps/rejected": -160.20950317382812,
+ "loss": 0.6878445625305176,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09615986794233322,
+ "rewards/margins": 0.012350928969681263,
+ "rewards/rejected": 0.08380892872810364,
+ "step": 690
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 1.3664878606796265,
+ "learning_rate": 9.039999999999999e-07,
+ "logits/chosen": 1.530369758605957,
+ "logits/rejected": 1.5520622730255127,
+ "logps/chosen": -159.60740661621094,
+ "logps/rejected": -137.77352905273438,
+ "loss": 0.6991429805755616,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07249583303928375,
+ "rewards/margins": -0.009333941154181957,
+ "rewards/rejected": 0.08182977139949799,
+ "step": 700
+ },
+ {
+ "epoch": 0.56,
+ "eval_logits/chosen": 1.5136796236038208,
+ "eval_logits/rejected": 1.5456002950668335,
+ "eval_logps/chosen": -152.5748748779297,
+ "eval_logps/rejected": -147.17152404785156,
+ "eval_loss": 0.688511848449707,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.09739598631858826,
+ "eval_rewards/margins": 0.012335418723523617,
+ "eval_rewards/rejected": 0.08506056666374207,
+ "eval_runtime": 90.8734,
+ "eval_samples_per_second": 5.502,
+ "eval_steps_per_second": 2.751,
+ "step": 700
+ },
+ {
+ "epoch": 0.568,
+ "grad_norm": 1.224107265472412,
+ "learning_rate": 9.010370370370371e-07,
+ "logits/chosen": 1.6023967266082764,
+ "logits/rejected": 1.558224081993103,
+ "logps/chosen": -148.47120666503906,
+ "logps/rejected": -163.5648651123047,
+ "loss": 0.6865221500396729,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.12494877725839615,
+ "rewards/margins": 0.01545787788927555,
+ "rewards/rejected": 0.10949089378118515,
+ "step": 710
+ },
+ {
+ "epoch": 0.576,
+ "grad_norm": 1.2135164737701416,
+ "learning_rate": 8.98074074074074e-07,
+ "logits/chosen": 1.5902750492095947,
+ "logits/rejected": 1.7012746334075928,
+ "logps/chosen": -139.7012939453125,
+ "logps/rejected": -135.42225646972656,
+ "loss": 0.7014039039611817,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.09102506935596466,
+ "rewards/margins": -0.014695463702082634,
+ "rewards/rejected": 0.10572052001953125,
+ "step": 720
+ },
+ {
+ "epoch": 0.584,
+ "grad_norm": 1.1953649520874023,
+ "learning_rate": 8.95111111111111e-07,
+ "logits/chosen": 1.6540731191635132,
+ "logits/rejected": 1.6386349201202393,
+ "logps/chosen": -153.12847900390625,
+ "logps/rejected": -149.66319274902344,
+ "loss": 0.6952126502990723,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.1124584898352623,
+ "rewards/margins": -0.0012063594767823815,
+ "rewards/rejected": 0.11366486549377441,
+ "step": 730
+ },
+ {
+ "epoch": 0.592,
+ "grad_norm": 1.147875428199768,
+ "learning_rate": 8.921481481481481e-07,
+ "logits/chosen": 1.5578330755233765,
+ "logits/rejected": 1.4689290523529053,
+ "logps/chosen": -156.93788146972656,
+ "logps/rejected": -134.1832733154297,
+ "loss": 0.6919830799102783,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09214536100625992,
+ "rewards/margins": 0.004291228950023651,
+ "rewards/rejected": 0.08785412460565567,
+ "step": 740
+ },
+ {
+ "epoch": 0.6,
+ "grad_norm": 1.21685791015625,
+ "learning_rate": 8.891851851851852e-07,
+ "logits/chosen": 1.4241983890533447,
+ "logits/rejected": 1.4537718296051025,
+ "logps/chosen": -136.9163055419922,
+ "logps/rejected": -158.44491577148438,
+ "loss": 0.691422986984253,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.09173480421304703,
+ "rewards/margins": 0.00533579895272851,
+ "rewards/rejected": 0.08639900386333466,
+ "step": 750
+ },
+ {
+ "epoch": 0.608,
+ "grad_norm": 1.3629519939422607,
+ "learning_rate": 8.862222222222222e-07,
+ "logits/chosen": 1.565716028213501,
+ "logits/rejected": 1.3824570178985596,
+ "logps/chosen": -138.8521728515625,
+ "logps/rejected": -129.51028442382812,
+ "loss": 0.6931031227111817,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0763697624206543,
+ "rewards/margins": 0.0021251302678138018,
+ "rewards/rejected": 0.07424463331699371,
+ "step": 760
+ },
+ {
+ "epoch": 0.616,
+ "grad_norm": 1.1030747890472412,
+ "learning_rate": 8.832592592592593e-07,
+ "logits/chosen": 1.5781702995300293,
+ "logits/rejected": 1.5428775548934937,
+ "logps/chosen": -141.01480102539062,
+ "logps/rejected": -136.47189331054688,
+ "loss": 0.6777657032012939,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.10366280376911163,
+ "rewards/margins": 0.0333605632185936,
+ "rewards/rejected": 0.07030224055051804,
+ "step": 770
+ },
+ {
+ "epoch": 0.624,
+ "grad_norm": 1.3228758573532104,
+ "learning_rate": 8.802962962962962e-07,
+ "logits/chosen": 1.451596736907959,
+ "logits/rejected": 1.5489904880523682,
+ "logps/chosen": -153.74618530273438,
+ "logps/rejected": -159.26333618164062,
+ "loss": 0.6870471000671386,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.10403706133365631,
+ "rewards/margins": 0.014494165778160095,
+ "rewards/rejected": 0.08954289555549622,
+ "step": 780
+ },
+ {
+ "epoch": 0.632,
+ "grad_norm": 1.3267385959625244,
+ "learning_rate": 8.773333333333332e-07,
+ "logits/chosen": 1.4153831005096436,
+ "logits/rejected": 1.4497296810150146,
+ "logps/chosen": -148.05368041992188,
+ "logps/rejected": -151.3020477294922,
+ "loss": 0.6877872943878174,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.07300253212451935,
+ "rewards/margins": 0.013981563039124012,
+ "rewards/rejected": 0.05902096629142761,
+ "step": 790
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 1.2227483987808228,
+ "learning_rate": 8.743703703703703e-07,
+ "logits/chosen": 1.6773914098739624,
+ "logits/rejected": 1.6918518543243408,
+ "logps/chosen": -180.06863403320312,
+ "logps/rejected": -194.30752563476562,
+ "loss": 0.7015917778015137,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07480724155902863,
+ "rewards/margins": -0.013628068380057812,
+ "rewards/rejected": 0.08843531459569931,
+ "step": 800
+ },
+ {
+ "epoch": 0.64,
+ "eval_logits/chosen": 1.5051140785217285,
+ "eval_logits/rejected": 1.5367870330810547,
+ "eval_logps/chosen": -152.8026885986328,
+ "eval_logps/rejected": -147.39437866210938,
+ "eval_loss": 0.688633918762207,
+ "eval_rewards/accuracies": 0.5220000147819519,
+ "eval_rewards/chosen": 0.07461711764335632,
+ "eval_rewards/margins": 0.011839868500828743,
+ "eval_rewards/rejected": 0.06277725100517273,
+ "eval_runtime": 90.9206,
+ "eval_samples_per_second": 5.499,
+ "eval_steps_per_second": 2.75,
+ "step": 800
+ },
+ {
+ "epoch": 0.648,
+ "grad_norm": 1.209770679473877,
+ "learning_rate": 8.714074074074074e-07,
+ "logits/chosen": 1.3785041570663452,
+ "logits/rejected": 1.60085928440094,
+ "logps/chosen": -114.5325927734375,
+ "logps/rejected": -151.88900756835938,
+ "loss": 0.6890413761138916,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0640551745891571,
+ "rewards/margins": 0.010126596316695213,
+ "rewards/rejected": 0.05392857640981674,
+ "step": 810
+ },
+ {
+ "epoch": 0.656,
+ "grad_norm": 1.0250163078308105,
+ "learning_rate": 8.684444444444444e-07,
+ "logits/chosen": 1.5827982425689697,
+ "logits/rejected": 1.6239618062973022,
+ "logps/chosen": -149.01904296875,
+ "logps/rejected": -141.9372100830078,
+ "loss": 0.6728014469146728,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.0830039232969284,
+ "rewards/margins": 0.043090131133794785,
+ "rewards/rejected": 0.03991378843784332,
+ "step": 820
+ },
+ {
+ "epoch": 0.664,
+ "grad_norm": 1.0627655982971191,
+ "learning_rate": 8.654814814814814e-07,
+ "logits/chosen": 1.501873254776001,
+ "logits/rejected": 1.537386417388916,
+ "logps/chosen": -138.75169372558594,
+ "logps/rejected": -134.00970458984375,
+ "loss": 0.6801187992095947,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0640982910990715,
+ "rewards/margins": 0.02855527400970459,
+ "rewards/rejected": 0.03554301708936691,
+ "step": 830
+ },
+ {
+ "epoch": 0.672,
+ "grad_norm": 1.2453433275222778,
+ "learning_rate": 8.625185185185186e-07,
+ "logits/chosen": 1.4766005277633667,
+ "logits/rejected": 1.5185787677764893,
+ "logps/chosen": -148.41250610351562,
+ "logps/rejected": -157.73989868164062,
+ "loss": 0.6788079738616943,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.06551454961299896,
+ "rewards/margins": 0.03301671892404556,
+ "rewards/rejected": 0.0324978306889534,
+ "step": 840
+ },
+ {
+ "epoch": 0.68,
+ "grad_norm": 1.191698670387268,
+ "learning_rate": 8.595555555555555e-07,
+ "logits/chosen": 1.555551528930664,
+ "logits/rejected": 1.5438499450683594,
+ "logps/chosen": -160.77488708496094,
+ "logps/rejected": -141.9876251220703,
+ "loss": 0.6801597595214843,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.053498052060604095,
+ "rewards/margins": 0.028470590710639954,
+ "rewards/rejected": 0.025027472525835037,
+ "step": 850
+ },
+ {
+ "epoch": 0.688,
+ "grad_norm": 0.9382092356681824,
+ "learning_rate": 8.565925925925925e-07,
+ "logits/chosen": 1.5482738018035889,
+ "logits/rejected": 1.5833700895309448,
+ "logps/chosen": -139.68118286132812,
+ "logps/rejected": -130.05947875976562,
+ "loss": 0.6825921535491943,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.06382572650909424,
+ "rewards/margins": 0.023111844435334206,
+ "rewards/rejected": 0.04071388021111488,
+ "step": 860
+ },
+ {
+ "epoch": 0.696,
+ "grad_norm": 1.2248382568359375,
+ "learning_rate": 8.536296296296296e-07,
+ "logits/chosen": 1.4529926776885986,
+ "logits/rejected": 1.4423902034759521,
+ "logps/chosen": -154.4588165283203,
+ "logps/rejected": -153.44949340820312,
+ "loss": 0.6915592193603516,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.03207828477025032,
+ "rewards/margins": 0.006785523146390915,
+ "rewards/rejected": 0.025292763486504555,
+ "step": 870
+ },
+ {
+ "epoch": 0.704,
+ "grad_norm": 1.2174893617630005,
+ "learning_rate": 8.506666666666667e-07,
+ "logits/chosen": 1.4073113203048706,
+ "logits/rejected": 1.668140172958374,
+ "logps/chosen": -145.18460083007812,
+ "logps/rejected": -157.0777587890625,
+ "loss": 0.6922356605529785,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.049450717866420746,
+ "rewards/margins": 0.004049716051667929,
+ "rewards/rejected": 0.04540099948644638,
+ "step": 880
+ },
+ {
+ "epoch": 0.712,
+ "grad_norm": 1.3186850547790527,
+ "learning_rate": 8.477037037037037e-07,
+ "logits/chosen": 1.2923892736434937,
+ "logits/rejected": 1.491051435470581,
+ "logps/chosen": -149.03018188476562,
+ "logps/rejected": -161.61715698242188,
+ "loss": 0.6841150760650635,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.038961172103881836,
+ "rewards/margins": 0.020759906619787216,
+ "rewards/rejected": 0.01820126175880432,
+ "step": 890
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 1.1123124361038208,
+ "learning_rate": 8.447407407407407e-07,
+ "logits/chosen": 1.546547293663025,
+ "logits/rejected": 1.682992696762085,
+ "logps/chosen": -166.2056884765625,
+ "logps/rejected": -180.73330688476562,
+ "loss": 0.6826215744018554,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.07279330492019653,
+ "rewards/margins": 0.025264525786042213,
+ "rewards/rejected": 0.04752878472208977,
+ "step": 900
+ },
+ {
+ "epoch": 0.72,
+ "eval_logits/chosen": 1.4819762706756592,
+ "eval_logits/rejected": 1.513027548789978,
+ "eval_logps/chosen": -153.1695556640625,
+ "eval_logps/rejected": -147.7560577392578,
+ "eval_loss": 0.689373254776001,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": 0.03792775049805641,
+ "eval_rewards/margins": 0.011320074088871479,
+ "eval_rewards/rejected": 0.026607677340507507,
+ "eval_runtime": 90.9603,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 900
+ },
+ {
+ "epoch": 0.728,
+ "grad_norm": 1.1507582664489746,
+ "learning_rate": 8.417777777777777e-07,
+ "logits/chosen": 1.5831308364868164,
+ "logits/rejected": 1.3737132549285889,
+ "logps/chosen": -140.00253295898438,
+ "logps/rejected": -138.45155334472656,
+ "loss": 0.6873391628265381,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.05066479369997978,
+ "rewards/margins": 0.015466553159058094,
+ "rewards/rejected": 0.03519824147224426,
+ "step": 910
+ },
+ {
+ "epoch": 0.736,
+ "grad_norm": 1.415456771850586,
+ "learning_rate": 8.388148148148147e-07,
+ "logits/chosen": 1.3000686168670654,
+ "logits/rejected": 1.2642180919647217,
+ "logps/chosen": -160.14027404785156,
+ "logps/rejected": -163.32827758789062,
+ "loss": 0.687142276763916,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.06618930399417877,
+ "rewards/margins": 0.014807088300585747,
+ "rewards/rejected": 0.05138222128152847,
+ "step": 920
+ },
+ {
+ "epoch": 0.744,
+ "grad_norm": 1.3228634595870972,
+ "learning_rate": 8.358518518518518e-07,
+ "logits/chosen": 1.5204170942306519,
+ "logits/rejected": 1.6129217147827148,
+ "logps/chosen": -157.99832153320312,
+ "logps/rejected": -162.09567260742188,
+ "loss": 0.6790886878967285,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.05940447002649307,
+ "rewards/margins": 0.03161252290010452,
+ "rewards/rejected": 0.027791941538453102,
+ "step": 930
+ },
+ {
+ "epoch": 0.752,
+ "grad_norm": 1.048064947128296,
+ "learning_rate": 8.328888888888889e-07,
+ "logits/chosen": 1.5515320301055908,
+ "logits/rejected": 1.676857352256775,
+ "logps/chosen": -148.4853515625,
+ "logps/rejected": -174.4320526123047,
+ "loss": 0.6935453414916992,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.010225490666925907,
+ "rewards/margins": 0.0021693313028663397,
+ "rewards/rejected": 0.008056161925196648,
+ "step": 940
+ },
+ {
+ "epoch": 0.76,
+ "grad_norm": 1.247799038887024,
+ "learning_rate": 8.299259259259259e-07,
+ "logits/chosen": 1.384231686592102,
+ "logits/rejected": 1.4611561298370361,
+ "logps/chosen": -140.81443786621094,
+ "logps/rejected": -153.81326293945312,
+ "loss": 0.6805226325988769,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.053223587572574615,
+ "rewards/margins": 0.02933613583445549,
+ "rewards/rejected": 0.023887457326054573,
+ "step": 950
+ },
+ {
+ "epoch": 0.768,
+ "grad_norm": 1.1611264944076538,
+ "learning_rate": 8.269629629629629e-07,
+ "logits/chosen": 1.4987528324127197,
+ "logits/rejected": 1.3684321641921997,
+ "logps/chosen": -135.81056213378906,
+ "logps/rejected": -119.05133056640625,
+ "loss": 0.6954316616058349,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.0197703056037426,
+ "rewards/margins": -0.0012637012405321002,
+ "rewards/rejected": 0.021034007892012596,
+ "step": 960
+ },
+ {
+ "epoch": 0.776,
+ "grad_norm": 0.964948832988739,
+ "learning_rate": 8.24e-07,
+ "logits/chosen": 1.612053632736206,
+ "logits/rejected": 1.7114235162734985,
+ "logps/chosen": -153.56137084960938,
+ "logps/rejected": -149.17323303222656,
+ "loss": 0.6916662693023682,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.05082285404205322,
+ "rewards/margins": 0.0056606316938996315,
+ "rewards/rejected": 0.04516221210360527,
+ "step": 970
+ },
+ {
+ "epoch": 0.784,
+ "grad_norm": 1.409449815750122,
+ "learning_rate": 8.21037037037037e-07,
+ "logits/chosen": 1.384060025215149,
+ "logits/rejected": 1.316361904144287,
+ "logps/chosen": -168.75360107421875,
+ "logps/rejected": -141.79537963867188,
+ "loss": 0.6845529556274415,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.0886571854352951,
+ "rewards/margins": 0.02084263041615486,
+ "rewards/rejected": 0.06781454384326935,
+ "step": 980
+ },
+ {
+ "epoch": 0.792,
+ "grad_norm": 1.0246562957763672,
+ "learning_rate": 8.18074074074074e-07,
+ "logits/chosen": 1.4986507892608643,
+ "logits/rejected": 1.571103811264038,
+ "logps/chosen": -165.22254943847656,
+ "logps/rejected": -168.63540649414062,
+ "loss": 0.6790967464447022,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.07517905533313751,
+ "rewards/margins": 0.03272219002246857,
+ "rewards/rejected": 0.04245685786008835,
+ "step": 990
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 1.014426827430725,
+ "learning_rate": 8.15111111111111e-07,
+ "logits/chosen": 1.4708755016326904,
+ "logits/rejected": 1.4550955295562744,
+ "logps/chosen": -149.49874877929688,
+ "logps/rejected": -125.0966567993164,
+ "loss": 0.6892098903656005,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.07825140655040741,
+ "rewards/margins": 0.01007872074842453,
+ "rewards/rejected": 0.06817268580198288,
+ "step": 1000
+ },
+ {
+ "epoch": 0.8,
+ "eval_logits/chosen": 1.4972777366638184,
+ "eval_logits/rejected": 1.5283576250076294,
+ "eval_logps/chosen": -152.8240966796875,
+ "eval_logps/rejected": -147.44175720214844,
+ "eval_loss": 0.6880542039871216,
+ "eval_rewards/accuracies": 0.5419999957084656,
+ "eval_rewards/chosen": 0.07247376441955566,
+ "eval_rewards/margins": 0.01443479023873806,
+ "eval_rewards/rejected": 0.058038972318172455,
+ "eval_runtime": 90.827,
+ "eval_samples_per_second": 5.505,
+ "eval_steps_per_second": 2.752,
+ "step": 1000
+ },
+ {
+ "epoch": 0.808,
+ "grad_norm": 1.1155726909637451,
+ "learning_rate": 8.121481481481482e-07,
+ "logits/chosen": 1.547481894493103,
+ "logits/rejected": 1.5461527109146118,
+ "logps/chosen": -154.89791870117188,
+ "logps/rejected": -142.05111694335938,
+ "loss": 0.6809016704559326,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.10093016922473907,
+ "rewards/margins": 0.027038773521780968,
+ "rewards/rejected": 0.07389138638973236,
+ "step": 1010
+ },
+ {
+ "epoch": 0.816,
+ "grad_norm": 1.6716786623001099,
+ "learning_rate": 8.091851851851852e-07,
+ "logits/chosen": 1.3954919576644897,
+ "logits/rejected": 1.5088036060333252,
+ "logps/chosen": -143.11846923828125,
+ "logps/rejected": -146.41482543945312,
+ "loss": 0.6849985122680664,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.05970728397369385,
+ "rewards/margins": 0.021999523043632507,
+ "rewards/rejected": 0.03770776465535164,
+ "step": 1020
+ },
+ {
+ "epoch": 0.824,
+ "grad_norm": 1.308182716369629,
+ "learning_rate": 8.062222222222221e-07,
+ "logits/chosen": 1.532488465309143,
+ "logits/rejected": 1.3316718339920044,
+ "logps/chosen": -163.06858825683594,
+ "logps/rejected": -136.57989501953125,
+ "loss": 0.6789252281188964,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.11086118221282959,
+ "rewards/margins": 0.033117204904556274,
+ "rewards/rejected": 0.07774396985769272,
+ "step": 1030
+ },
+ {
+ "epoch": 0.832,
+ "grad_norm": 1.1019353866577148,
+ "learning_rate": 8.032592592592592e-07,
+ "logits/chosen": 1.4798121452331543,
+ "logits/rejected": 1.5203083753585815,
+ "logps/chosen": -140.3281707763672,
+ "logps/rejected": -141.06216430664062,
+ "loss": 0.6724937915802002,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.1202714815735817,
+ "rewards/margins": 0.046712182462215424,
+ "rewards/rejected": 0.07355931401252747,
+ "step": 1040
+ },
+ {
+ "epoch": 0.84,
+ "grad_norm": 1.2966418266296387,
+ "learning_rate": 8.002962962962963e-07,
+ "logits/chosen": 1.3034439086914062,
+ "logits/rejected": 1.4363505840301514,
+ "logps/chosen": -142.1992645263672,
+ "logps/rejected": -132.0046844482422,
+ "loss": 0.6848338603973388,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.11126589775085449,
+ "rewards/margins": 0.019870441406965256,
+ "rewards/rejected": 0.09139545261859894,
+ "step": 1050
+ },
+ {
+ "epoch": 0.848,
+ "grad_norm": 1.0321130752563477,
+ "learning_rate": 7.973333333333333e-07,
+ "logits/chosen": 1.3257265090942383,
+ "logits/rejected": 1.4649587869644165,
+ "logps/chosen": -112.98075103759766,
+ "logps/rejected": -127.4059829711914,
+ "loss": 0.6679783344268799,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.17311830818653107,
+ "rewards/margins": 0.05631983280181885,
+ "rewards/rejected": 0.11679847538471222,
+ "step": 1060
+ },
+ {
+ "epoch": 0.856,
+ "grad_norm": 1.2037222385406494,
+ "learning_rate": 7.943703703703704e-07,
+ "logits/chosen": 1.3766987323760986,
+ "logits/rejected": 1.2802103757858276,
+ "logps/chosen": -154.8142547607422,
+ "logps/rejected": -148.680908203125,
+ "loss": 0.668931531906128,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1449051946401596,
+ "rewards/margins": 0.05721765011548996,
+ "rewards/rejected": 0.08768755197525024,
+ "step": 1070
+ },
+ {
+ "epoch": 0.864,
+ "grad_norm": 0.9230766296386719,
+ "learning_rate": 7.914074074074074e-07,
+ "logits/chosen": 1.447328805923462,
+ "logits/rejected": 1.5235106945037842,
+ "logps/chosen": -150.21812438964844,
+ "logps/rejected": -123.29923248291016,
+ "loss": 0.6716424465179444,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.16066041588783264,
+ "rewards/margins": 0.04824484512209892,
+ "rewards/rejected": 0.11241557449102402,
+ "step": 1080
+ },
+ {
+ "epoch": 0.872,
+ "grad_norm": 1.0423877239227295,
+ "learning_rate": 7.884444444444443e-07,
+ "logits/chosen": 1.6920299530029297,
+ "logits/rejected": 1.6123542785644531,
+ "logps/chosen": -175.1676025390625,
+ "logps/rejected": -185.07223510742188,
+ "loss": 0.6723968505859375,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.16490009427070618,
+ "rewards/margins": 0.046577490866184235,
+ "rewards/rejected": 0.11832261085510254,
+ "step": 1090
+ },
+ {
+ "epoch": 0.88,
+ "grad_norm": 1.4142779111862183,
+ "learning_rate": 7.854814814814814e-07,
+ "logits/chosen": 1.453111171722412,
+ "logits/rejected": 1.4981731176376343,
+ "logps/chosen": -139.9765167236328,
+ "logps/rejected": -153.63058471679688,
+ "loss": 0.6872457027435303,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.13096000254154205,
+ "rewards/margins": 0.01702108606696129,
+ "rewards/rejected": 0.11393891274929047,
+ "step": 1100
+ },
+ {
+ "epoch": 0.88,
+ "eval_logits/chosen": 1.5076195001602173,
+ "eval_logits/rejected": 1.5373249053955078,
+ "eval_logps/chosen": -152.1697998046875,
+ "eval_logps/rejected": -146.83502197265625,
+ "eval_loss": 0.6873304843902588,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.13790392875671387,
+ "eval_rewards/margins": 0.019191108644008636,
+ "eval_rewards/rejected": 0.11871281266212463,
+ "eval_runtime": 91.0157,
+ "eval_samples_per_second": 5.494,
+ "eval_steps_per_second": 2.747,
+ "step": 1100
+ },
+ {
+ "epoch": 0.888,
+ "grad_norm": 1.413996696472168,
+ "learning_rate": 7.825185185185185e-07,
+ "logits/chosen": 1.5183905363082886,
+ "logits/rejected": 1.4942539930343628,
+ "logps/chosen": -156.383544921875,
+ "logps/rejected": -145.61398315429688,
+ "loss": 0.6810199737548828,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.14243564009666443,
+ "rewards/margins": 0.03136260434985161,
+ "rewards/rejected": 0.11107305437326431,
+ "step": 1110
+ },
+ {
+ "epoch": 0.896,
+ "grad_norm": 1.0046707391738892,
+ "learning_rate": 7.795555555555555e-07,
+ "logits/chosen": 1.4982213973999023,
+ "logits/rejected": 1.5465660095214844,
+ "logps/chosen": -156.79293823242188,
+ "logps/rejected": -150.77853393554688,
+ "loss": 0.6808318614959716,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1551930010318756,
+ "rewards/margins": 0.028187204152345657,
+ "rewards/rejected": 0.12700578570365906,
+ "step": 1120
+ },
+ {
+ "epoch": 0.904,
+ "grad_norm": 1.125625729560852,
+ "learning_rate": 7.765925925925925e-07,
+ "logits/chosen": 1.6053575277328491,
+ "logits/rejected": 1.47335946559906,
+ "logps/chosen": -148.05690002441406,
+ "logps/rejected": -135.36598205566406,
+ "loss": 0.6889908790588379,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.15128108859062195,
+ "rewards/margins": 0.015811312943696976,
+ "rewards/rejected": 0.13546979427337646,
+ "step": 1130
+ },
+ {
+ "epoch": 0.912,
+ "grad_norm": 1.0933659076690674,
+ "learning_rate": 7.736296296296297e-07,
+ "logits/chosen": 1.5862042903900146,
+ "logits/rejected": 1.5827430486679077,
+ "logps/chosen": -156.10145568847656,
+ "logps/rejected": -142.94851684570312,
+ "loss": 0.6753206253051758,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1547432243824005,
+ "rewards/margins": 0.04229574277997017,
+ "rewards/rejected": 0.11244747787714005,
+ "step": 1140
+ },
+ {
+ "epoch": 0.92,
+ "grad_norm": 1.0774686336517334,
+ "learning_rate": 7.706666666666667e-07,
+ "logits/chosen": 1.2369152307510376,
+ "logits/rejected": 1.2727023363113403,
+ "logps/chosen": -139.23147583007812,
+ "logps/rejected": -130.53038024902344,
+ "loss": 0.6967329025268555,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": 0.10801626741886139,
+ "rewards/margins": 0.004670143127441406,
+ "rewards/rejected": 0.10334610939025879,
+ "step": 1150
+ },
+ {
+ "epoch": 0.928,
+ "grad_norm": 1.1111780405044556,
+ "learning_rate": 7.677037037037036e-07,
+ "logits/chosen": 1.501250982284546,
+ "logits/rejected": 1.4272868633270264,
+ "logps/chosen": -127.09858703613281,
+ "logps/rejected": -114.9041748046875,
+ "loss": 0.6853152275085449,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.1366150826215744,
+ "rewards/margins": 0.021945688873529434,
+ "rewards/rejected": 0.11466939747333527,
+ "step": 1160
+ },
+ {
+ "epoch": 0.936,
+ "grad_norm": 1.5593854188919067,
+ "learning_rate": 7.647407407407407e-07,
+ "logits/chosen": 1.727927803993225,
+ "logits/rejected": 1.6510727405548096,
+ "logps/chosen": -156.70993041992188,
+ "logps/rejected": -167.07266235351562,
+ "loss": 0.6884279251098633,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.14902853965759277,
+ "rewards/margins": 0.01685408689081669,
+ "rewards/rejected": 0.13217447698116302,
+ "step": 1170
+ },
+ {
+ "epoch": 0.944,
+ "grad_norm": 2.0064918994903564,
+ "learning_rate": 7.617777777777778e-07,
+ "logits/chosen": 1.6262975931167603,
+ "logits/rejected": 1.58951997756958,
+ "logps/chosen": -186.59210205078125,
+ "logps/rejected": -151.2965850830078,
+ "loss": 0.6906405448913574,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.14898104965686798,
+ "rewards/margins": 0.013303476385772228,
+ "rewards/rejected": 0.13567757606506348,
+ "step": 1180
+ },
+ {
+ "epoch": 0.952,
+ "grad_norm": 1.209291696548462,
+ "learning_rate": 7.588148148148148e-07,
+ "logits/chosen": 1.639369249343872,
+ "logits/rejected": 1.6400636434555054,
+ "logps/chosen": -166.24684143066406,
+ "logps/rejected": -196.35305786132812,
+ "loss": 0.6789956092834473,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1348457634449005,
+ "rewards/margins": 0.03475186973810196,
+ "rewards/rejected": 0.10009388625621796,
+ "step": 1190
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 1.209230899810791,
+ "learning_rate": 7.558518518518519e-07,
+ "logits/chosen": 1.542942762374878,
+ "logits/rejected": 1.5128916501998901,
+ "logps/chosen": -138.994384765625,
+ "logps/rejected": -145.42880249023438,
+ "loss": 0.6936234951019287,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.13895569741725922,
+ "rewards/margins": 0.005582497920840979,
+ "rewards/rejected": 0.1333732008934021,
+ "step": 1200
+ },
+ {
+ "epoch": 0.96,
+ "eval_logits/chosen": 1.5055651664733887,
+ "eval_logits/rejected": 1.5351229906082153,
+ "eval_logps/chosen": -152.2256622314453,
+ "eval_logps/rejected": -146.909912109375,
+ "eval_loss": 0.6866763830184937,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.13231882452964783,
+ "eval_rewards/margins": 0.02109396643936634,
+ "eval_rewards/rejected": 0.11122485250234604,
+ "eval_runtime": 90.8141,
+ "eval_samples_per_second": 5.506,
+ "eval_steps_per_second": 2.753,
+ "step": 1200
+ },
+ {
+ "epoch": 0.968,
+ "grad_norm": 1.2058950662612915,
+ "learning_rate": 7.528888888888889e-07,
+ "logits/chosen": 1.3591210842132568,
+ "logits/rejected": 1.3811699151992798,
+ "logps/chosen": -155.1759033203125,
+ "logps/rejected": -141.97772216796875,
+ "loss": 0.6902606964111329,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.153532013297081,
+ "rewards/margins": 0.011359233409166336,
+ "rewards/rejected": 0.14217278361320496,
+ "step": 1210
+ },
+ {
+ "epoch": 0.976,
+ "grad_norm": 1.366702914237976,
+ "learning_rate": 7.499259259259258e-07,
+ "logits/chosen": 1.3841370344161987,
+ "logits/rejected": 1.3791372776031494,
+ "logps/chosen": -145.47613525390625,
+ "logps/rejected": -123.9173583984375,
+ "loss": 0.6971709251403808,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.10521620512008667,
+ "rewards/margins": -0.0030749079305678606,
+ "rewards/rejected": 0.1082911267876625,
+ "step": 1220
+ },
+ {
+ "epoch": 0.984,
+ "grad_norm": 1.477455496788025,
+ "learning_rate": 7.469629629629629e-07,
+ "logits/chosen": 1.4447228908538818,
+ "logits/rejected": 1.6022417545318604,
+ "logps/chosen": -137.9462127685547,
+ "logps/rejected": -150.25559997558594,
+ "loss": 0.6915127277374268,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.15108394622802734,
+ "rewards/margins": 0.009388584643602371,
+ "rewards/rejected": 0.14169538021087646,
+ "step": 1230
+ },
+ {
+ "epoch": 0.992,
+ "grad_norm": 1.4407854080200195,
+ "learning_rate": 7.44e-07,
+ "logits/chosen": 1.3998825550079346,
+ "logits/rejected": 1.5924925804138184,
+ "logps/chosen": -144.93775939941406,
+ "logps/rejected": -148.02542114257812,
+ "loss": 0.6699621677398682,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.146137997508049,
+ "rewards/margins": 0.05326627567410469,
+ "rewards/rejected": 0.09287171810865402,
+ "step": 1240
+ },
+ {
+ "epoch": 1.0,
+ "grad_norm": 1.183438777923584,
+ "learning_rate": 7.41037037037037e-07,
+ "logits/chosen": 1.5393705368041992,
+ "logits/rejected": 1.6312555074691772,
+ "logps/chosen": -145.61672973632812,
+ "logps/rejected": -158.9256591796875,
+ "loss": 0.6811168670654297,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.1304798424243927,
+ "rewards/margins": 0.03360626846551895,
+ "rewards/rejected": 0.09687358886003494,
+ "step": 1250
+ },
+ {
+ "epoch": 1.008,
+ "grad_norm": 1.2436553239822388,
+ "learning_rate": 7.38074074074074e-07,
+ "logits/chosen": 1.4758602380752563,
+ "logits/rejected": 1.534609079360962,
+ "logps/chosen": -124.26841735839844,
+ "logps/rejected": -132.41867065429688,
+ "loss": 0.688177490234375,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.12570872902870178,
+ "rewards/margins": 0.014875045046210289,
+ "rewards/rejected": 0.11083368957042694,
+ "step": 1260
+ },
+ {
+ "epoch": 1.016,
+ "grad_norm": 1.3853868246078491,
+ "learning_rate": 7.351111111111112e-07,
+ "logits/chosen": 1.7195091247558594,
+ "logits/rejected": 1.632851004600525,
+ "logps/chosen": -160.8590850830078,
+ "logps/rejected": -153.91671752929688,
+ "loss": 0.6669504165649414,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.181864932179451,
+ "rewards/margins": 0.06175640970468521,
+ "rewards/rejected": 0.12010850757360458,
+ "step": 1270
+ },
+ {
+ "epoch": 1.024,
+ "grad_norm": 1.2765302658081055,
+ "learning_rate": 7.321481481481481e-07,
+ "logits/chosen": 1.5782036781311035,
+ "logits/rejected": 1.5713088512420654,
+ "logps/chosen": -154.3319549560547,
+ "logps/rejected": -141.43345642089844,
+ "loss": 0.6734190464019776,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.12361389398574829,
+ "rewards/margins": 0.04626934230327606,
+ "rewards/rejected": 0.07734455168247223,
+ "step": 1280
+ },
+ {
+ "epoch": 1.032,
+ "grad_norm": 1.0421397686004639,
+ "learning_rate": 7.291851851851851e-07,
+ "logits/chosen": 1.5199967622756958,
+ "logits/rejected": 1.5965670347213745,
+ "logps/chosen": -145.28416442871094,
+ "logps/rejected": -133.72256469726562,
+ "loss": 0.6804514408111573,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.11794980615377426,
+ "rewards/margins": 0.030916064977645874,
+ "rewards/rejected": 0.08703374862670898,
+ "step": 1290
+ },
+ {
+ "epoch": 1.04,
+ "grad_norm": 1.032158613204956,
+ "learning_rate": 7.262222222222222e-07,
+ "logits/chosen": 1.6042639017105103,
+ "logits/rejected": 1.4255346059799194,
+ "logps/chosen": -150.4803009033203,
+ "logps/rejected": -130.2232208251953,
+ "loss": 0.6676507472991944,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.15466037392616272,
+ "rewards/margins": 0.05808692425489426,
+ "rewards/rejected": 0.09657344967126846,
+ "step": 1300
+ },
+ {
+ "epoch": 1.04,
+ "eval_logits/chosen": 1.5118356943130493,
+ "eval_logits/rejected": 1.5418950319290161,
+ "eval_logps/chosen": -152.23300170898438,
+ "eval_logps/rejected": -146.9257049560547,
+ "eval_loss": 0.6859617233276367,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.13158410787582397,
+ "eval_rewards/margins": 0.021939631551504135,
+ "eval_rewards/rejected": 0.10964448750019073,
+ "eval_runtime": 90.9624,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 1300
+ },
+ {
+ "epoch": 1.048,
+ "grad_norm": 1.4942574501037598,
+ "learning_rate": 7.232592592592593e-07,
+ "logits/chosen": 1.689648985862732,
+ "logits/rejected": 1.691888451576233,
+ "logps/chosen": -166.02809143066406,
+ "logps/rejected": -160.30601501464844,
+ "loss": 0.6770285606384278,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.14377735555171967,
+ "rewards/margins": 0.03825496509671211,
+ "rewards/rejected": 0.10552239418029785,
+ "step": 1310
+ },
+ {
+ "epoch": 1.056,
+ "grad_norm": 1.3656011819839478,
+ "learning_rate": 7.202962962962963e-07,
+ "logits/chosen": 1.3065154552459717,
+ "logits/rejected": 1.5681203603744507,
+ "logps/chosen": -129.1146240234375,
+ "logps/rejected": -151.2584991455078,
+ "loss": 0.6844655036926269,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.12841743230819702,
+ "rewards/margins": 0.02178655005991459,
+ "rewards/rejected": 0.10663086175918579,
+ "step": 1320
+ },
+ {
+ "epoch": 1.064,
+ "grad_norm": 1.1723405122756958,
+ "learning_rate": 7.173333333333333e-07,
+ "logits/chosen": 1.6219158172607422,
+ "logits/rejected": 1.6032555103302002,
+ "logps/chosen": -156.4185028076172,
+ "logps/rejected": -146.28067016601562,
+ "loss": 0.6650479316711426,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1538492739200592,
+ "rewards/margins": 0.06132146716117859,
+ "rewards/rejected": 0.09252782166004181,
+ "step": 1330
+ },
+ {
+ "epoch": 1.072,
+ "grad_norm": 1.2311245203018188,
+ "learning_rate": 7.143703703703703e-07,
+ "logits/chosen": 1.4625260829925537,
+ "logits/rejected": 1.392937421798706,
+ "logps/chosen": -153.05941772460938,
+ "logps/rejected": -145.37841796875,
+ "loss": 0.6570021152496338,
+ "rewards/accuracies": 0.762499988079071,
+ "rewards/chosen": 0.16768500208854675,
+ "rewards/margins": 0.0787494108080864,
+ "rewards/rejected": 0.08893559873104095,
+ "step": 1340
+ },
+ {
+ "epoch": 1.08,
+ "grad_norm": 1.314666748046875,
+ "learning_rate": 7.114074074074073e-07,
+ "logits/chosen": 1.4812809228897095,
+ "logits/rejected": 1.3796050548553467,
+ "logps/chosen": -165.70205688476562,
+ "logps/rejected": -140.13150024414062,
+ "loss": 0.6535200595855712,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.20691323280334473,
+ "rewards/margins": 0.0911526307463646,
+ "rewards/rejected": 0.11576057970523834,
+ "step": 1350
+ },
+ {
+ "epoch": 1.088,
+ "grad_norm": 0.9556720852851868,
+ "learning_rate": 7.084444444444444e-07,
+ "logits/chosen": 1.6368863582611084,
+ "logits/rejected": 1.678134560585022,
+ "logps/chosen": -166.56253051757812,
+ "logps/rejected": -164.36416625976562,
+ "loss": 0.6742628574371338,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.15145304799079895,
+ "rewards/margins": 0.04319921135902405,
+ "rewards/rejected": 0.1082538366317749,
+ "step": 1360
+ },
+ {
+ "epoch": 1.096,
+ "grad_norm": 0.945633053779602,
+ "learning_rate": 7.054814814814815e-07,
+ "logits/chosen": 1.4285938739776611,
+ "logits/rejected": 1.5576260089874268,
+ "logps/chosen": -131.5234832763672,
+ "logps/rejected": -128.02340698242188,
+ "loss": 0.6670140743255615,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.16399362683296204,
+ "rewards/margins": 0.05922812223434448,
+ "rewards/rejected": 0.10476551204919815,
+ "step": 1370
+ },
+ {
+ "epoch": 1.104,
+ "grad_norm": 1.1189475059509277,
+ "learning_rate": 7.025185185185185e-07,
+ "logits/chosen": 1.5019981861114502,
+ "logits/rejected": 1.667614221572876,
+ "logps/chosen": -161.61854553222656,
+ "logps/rejected": -166.39996337890625,
+ "loss": 0.6742690086364747,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.15786050260066986,
+ "rewards/margins": 0.044500257819890976,
+ "rewards/rejected": 0.11336024850606918,
+ "step": 1380
+ },
+ {
+ "epoch": 1.112,
+ "grad_norm": 1.3382912874221802,
+ "learning_rate": 6.995555555555555e-07,
+ "logits/chosen": 1.4972480535507202,
+ "logits/rejected": 1.4946129322052002,
+ "logps/chosen": -166.43682861328125,
+ "logps/rejected": -162.3004913330078,
+ "loss": 0.6557814121246338,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.18964914977550507,
+ "rewards/margins": 0.08500295877456665,
+ "rewards/rejected": 0.10464620590209961,
+ "step": 1390
+ },
+ {
+ "epoch": 1.12,
+ "grad_norm": 1.2123515605926514,
+ "learning_rate": 6.965925925925926e-07,
+ "logits/chosen": 1.456635594367981,
+ "logits/rejected": 1.5964475870132446,
+ "logps/chosen": -152.08261108398438,
+ "logps/rejected": -161.51023864746094,
+ "loss": 0.6708066940307618,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.216823011636734,
+ "rewards/margins": 0.050363689661026,
+ "rewards/rejected": 0.166459321975708,
+ "step": 1400
+ },
+ {
+ "epoch": 1.12,
+ "eval_logits/chosen": 1.521518588066101,
+ "eval_logits/rejected": 1.5510718822479248,
+ "eval_logps/chosen": -151.858154296875,
+ "eval_logps/rejected": -146.5819549560547,
+ "eval_loss": 0.6853657364845276,
+ "eval_rewards/accuracies": 0.5440000295639038,
+ "eval_rewards/chosen": 0.16906805336475372,
+ "eval_rewards/margins": 0.025049500167369843,
+ "eval_rewards/rejected": 0.14401854574680328,
+ "eval_runtime": 90.8779,
+ "eval_samples_per_second": 5.502,
+ "eval_steps_per_second": 2.751,
+ "step": 1400
+ },
+ {
+ "epoch": 1.1280000000000001,
+ "grad_norm": 1.3488291501998901,
+ "learning_rate": 6.936296296296296e-07,
+ "logits/chosen": 1.6009830236434937,
+ "logits/rejected": 1.6035544872283936,
+ "logps/chosen": -145.44662475585938,
+ "logps/rejected": -149.9284210205078,
+ "loss": 0.679435396194458,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.1647307425737381,
+ "rewards/margins": 0.03348377346992493,
+ "rewards/rejected": 0.13124695420265198,
+ "step": 1410
+ },
+ {
+ "epoch": 1.1360000000000001,
+ "grad_norm": 1.6453970670700073,
+ "learning_rate": 6.906666666666666e-07,
+ "logits/chosen": 1.593658447265625,
+ "logits/rejected": 1.648276925086975,
+ "logps/chosen": -135.2926788330078,
+ "logps/rejected": -150.9585723876953,
+ "loss": 0.6875734329223633,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.15669891238212585,
+ "rewards/margins": 0.01690087839961052,
+ "rewards/rejected": 0.13979804515838623,
+ "step": 1420
+ },
+ {
+ "epoch": 1.144,
+ "grad_norm": 1.6117807626724243,
+ "learning_rate": 6.877037037037036e-07,
+ "logits/chosen": 1.4510356187820435,
+ "logits/rejected": 1.2334567308425903,
+ "logps/chosen": -153.94854736328125,
+ "logps/rejected": -147.78079223632812,
+ "loss": 0.6689413070678711,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1819056123495102,
+ "rewards/margins": 0.058564942330121994,
+ "rewards/rejected": 0.1233406662940979,
+ "step": 1430
+ },
+ {
+ "epoch": 1.152,
+ "grad_norm": 1.5533809661865234,
+ "learning_rate": 6.847407407407408e-07,
+ "logits/chosen": 1.4535889625549316,
+ "logits/rejected": 1.4273865222930908,
+ "logps/chosen": -170.42904663085938,
+ "logps/rejected": -151.9979705810547,
+ "loss": 0.6845246315002441,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.14223966002464294,
+ "rewards/margins": 0.022389881312847137,
+ "rewards/rejected": 0.1198497861623764,
+ "step": 1440
+ },
+ {
+ "epoch": 1.16,
+ "grad_norm": 1.5696005821228027,
+ "learning_rate": 6.817777777777778e-07,
+ "logits/chosen": 1.4418725967407227,
+ "logits/rejected": 1.4354021549224854,
+ "logps/chosen": -153.962158203125,
+ "logps/rejected": -148.40512084960938,
+ "loss": 0.6761108875274658,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.15529735386371613,
+ "rewards/margins": 0.040999218821525574,
+ "rewards/rejected": 0.11429812759160995,
+ "step": 1450
+ },
+ {
+ "epoch": 1.168,
+ "grad_norm": 1.0711781978607178,
+ "learning_rate": 6.788148148148147e-07,
+ "logits/chosen": 1.5659481287002563,
+ "logits/rejected": 1.5640677213668823,
+ "logps/chosen": -149.816162109375,
+ "logps/rejected": -174.68643188476562,
+ "loss": 0.677489185333252,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1510745733976364,
+ "rewards/margins": 0.03914733976125717,
+ "rewards/rejected": 0.11192724853754044,
+ "step": 1460
+ },
+ {
+ "epoch": 1.176,
+ "grad_norm": 1.2336673736572266,
+ "learning_rate": 6.758518518518518e-07,
+ "logits/chosen": 1.4275394678115845,
+ "logits/rejected": 1.6394407749176025,
+ "logps/chosen": -137.93788146972656,
+ "logps/rejected": -154.97195434570312,
+ "loss": 0.6785802364349365,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10557998716831207,
+ "rewards/margins": 0.0352485254406929,
+ "rewards/rejected": 0.07033145427703857,
+ "step": 1470
+ },
+ {
+ "epoch": 1.184,
+ "grad_norm": 1.0030723810195923,
+ "learning_rate": 6.728888888888888e-07,
+ "logits/chosen": 1.4652725458145142,
+ "logits/rejected": 1.4992272853851318,
+ "logps/chosen": -143.87374877929688,
+ "logps/rejected": -139.5693817138672,
+ "loss": 0.6700259208679199,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.13436684012413025,
+ "rewards/margins": 0.05286343768239021,
+ "rewards/rejected": 0.08150340616703033,
+ "step": 1480
+ },
+ {
+ "epoch": 1.192,
+ "grad_norm": 1.0897104740142822,
+ "learning_rate": 6.699259259259259e-07,
+ "logits/chosen": 1.7033601999282837,
+ "logits/rejected": 1.617489218711853,
+ "logps/chosen": -140.43667602539062,
+ "logps/rejected": -146.59683227539062,
+ "loss": 0.6685672283172608,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.11724893748760223,
+ "rewards/margins": 0.055138181895017624,
+ "rewards/rejected": 0.06211075931787491,
+ "step": 1490
+ },
+ {
+ "epoch": 1.2,
+ "grad_norm": 0.8573256731033325,
+ "learning_rate": 6.66962962962963e-07,
+ "logits/chosen": 1.5181013345718384,
+ "logits/rejected": 1.5965152978897095,
+ "logps/chosen": -147.78506469726562,
+ "logps/rejected": -142.18930053710938,
+ "loss": 0.6564288139343262,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.14890792965888977,
+ "rewards/margins": 0.08058460801839828,
+ "rewards/rejected": 0.06832331418991089,
+ "step": 1500
+ },
+ {
+ "epoch": 1.2,
+ "eval_logits/chosen": 1.488415002822876,
+ "eval_logits/rejected": 1.5179662704467773,
+ "eval_logps/chosen": -152.60232543945312,
+ "eval_logps/rejected": -147.2954864501953,
+ "eval_loss": 0.6861968040466309,
+ "eval_rewards/accuracies": 0.5400000214576721,
+ "eval_rewards/chosen": 0.09465231001377106,
+ "eval_rewards/margins": 0.021986663341522217,
+ "eval_rewards/rejected": 0.07266565412282944,
+ "eval_runtime": 90.8951,
+ "eval_samples_per_second": 5.501,
+ "eval_steps_per_second": 2.75,
+ "step": 1500
+ },
+ {
+ "epoch": 1.208,
+ "grad_norm": 1.29599928855896,
+ "learning_rate": 6.64e-07,
+ "logits/chosen": 1.4851064682006836,
+ "logits/rejected": 1.4355719089508057,
+ "logps/chosen": -140.07186889648438,
+ "logps/rejected": -155.5626983642578,
+ "loss": 0.6695255756378173,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08214708417654037,
+ "rewards/margins": 0.05618969723582268,
+ "rewards/rejected": 0.025957385078072548,
+ "step": 1510
+ },
+ {
+ "epoch": 1.216,
+ "grad_norm": 1.3030505180358887,
+ "learning_rate": 6.61037037037037e-07,
+ "logits/chosen": 1.3785858154296875,
+ "logits/rejected": 1.3867576122283936,
+ "logps/chosen": -143.06137084960938,
+ "logps/rejected": -152.8805694580078,
+ "loss": 0.6704819202423096,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.08582031726837158,
+ "rewards/margins": 0.052868831902742386,
+ "rewards/rejected": 0.0329514816403389,
+ "step": 1520
+ },
+ {
+ "epoch": 1.224,
+ "grad_norm": 1.2935470342636108,
+ "learning_rate": 6.58074074074074e-07,
+ "logits/chosen": 1.4631284475326538,
+ "logits/rejected": 1.4908760786056519,
+ "logps/chosen": -181.2951202392578,
+ "logps/rejected": -173.83523559570312,
+ "loss": 0.6680389881134033,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.11432291567325592,
+ "rewards/margins": 0.05878107622265816,
+ "rewards/rejected": 0.05554184317588806,
+ "step": 1530
+ },
+ {
+ "epoch": 1.232,
+ "grad_norm": 1.4426783323287964,
+ "learning_rate": 6.551111111111111e-07,
+ "logits/chosen": 1.4765878915786743,
+ "logits/rejected": 1.4511891603469849,
+ "logps/chosen": -167.4827117919922,
+ "logps/rejected": -153.88833618164062,
+ "loss": 0.6684586524963378,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.08437791466712952,
+ "rewards/margins": 0.05823448300361633,
+ "rewards/rejected": 0.026143427938222885,
+ "step": 1540
+ },
+ {
+ "epoch": 1.24,
+ "grad_norm": 1.3253812789916992,
+ "learning_rate": 6.521481481481481e-07,
+ "logits/chosen": 1.2496798038482666,
+ "logits/rejected": 1.5193806886672974,
+ "logps/chosen": -130.33685302734375,
+ "logps/rejected": -132.5361328125,
+ "loss": 0.6695978164672851,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.10806921869516373,
+ "rewards/margins": 0.05488727241754532,
+ "rewards/rejected": 0.05318193882703781,
+ "step": 1550
+ },
+ {
+ "epoch": 1.248,
+ "grad_norm": 1.1890907287597656,
+ "learning_rate": 6.491851851851851e-07,
+ "logits/chosen": 1.2053601741790771,
+ "logits/rejected": 1.2588948011398315,
+ "logps/chosen": -132.61026000976562,
+ "logps/rejected": -150.66477966308594,
+ "loss": 0.673579216003418,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.062341660261154175,
+ "rewards/margins": 0.04531678557395935,
+ "rewards/rejected": 0.017024870961904526,
+ "step": 1560
+ },
+ {
+ "epoch": 1.256,
+ "grad_norm": 1.2076733112335205,
+ "learning_rate": 6.462222222222223e-07,
+ "logits/chosen": 1.5180461406707764,
+ "logits/rejected": 1.4118196964263916,
+ "logps/chosen": -120.61619567871094,
+ "logps/rejected": -146.42056274414062,
+ "loss": 0.6747167587280274,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.0764162540435791,
+ "rewards/margins": 0.04275941476225853,
+ "rewards/rejected": 0.03365683928132057,
+ "step": 1570
+ },
+ {
+ "epoch": 1.264,
+ "grad_norm": 1.3394429683685303,
+ "learning_rate": 6.432592592592593e-07,
+ "logits/chosen": 1.2879914045333862,
+ "logits/rejected": 1.3957126140594482,
+ "logps/chosen": -150.98355102539062,
+ "logps/rejected": -172.91090393066406,
+ "loss": 0.6560922622680664,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.09853875637054443,
+ "rewards/margins": 0.08982758224010468,
+ "rewards/rejected": 0.008711177855730057,
+ "step": 1580
+ },
+ {
+ "epoch": 1.272,
+ "grad_norm": 1.303663730621338,
+ "learning_rate": 6.402962962962962e-07,
+ "logits/chosen": 1.5617700815200806,
+ "logits/rejected": 1.6307615041732788,
+ "logps/chosen": -157.00338745117188,
+ "logps/rejected": -159.2900848388672,
+ "loss": 0.6639400959014893,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.07477572560310364,
+ "rewards/margins": 0.06561199575662613,
+ "rewards/rejected": 0.009163733571767807,
+ "step": 1590
+ },
+ {
+ "epoch": 1.28,
+ "grad_norm": 1.4180800914764404,
+ "learning_rate": 6.373333333333333e-07,
+ "logits/chosen": 1.3017181158065796,
+ "logits/rejected": 1.3270524740219116,
+ "logps/chosen": -134.52029418945312,
+ "logps/rejected": -135.45938110351562,
+ "loss": 0.6564930438995361,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.08265326917171478,
+ "rewards/margins": 0.09048662334680557,
+ "rewards/rejected": -0.007833337411284447,
+ "step": 1600
+ },
+ {
+ "epoch": 1.28,
+ "eval_logits/chosen": 1.4600619077682495,
+ "eval_logits/rejected": 1.489198923110962,
+ "eval_logps/chosen": -152.86001586914062,
+ "eval_logps/rejected": -147.54771423339844,
+ "eval_loss": 0.6871278882026672,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": 0.0688827633857727,
+ "eval_rewards/margins": 0.021439960226416588,
+ "eval_rewards/rejected": 0.04744280129671097,
+ "eval_runtime": 91.0555,
+ "eval_samples_per_second": 5.491,
+ "eval_steps_per_second": 2.746,
+ "step": 1600
+ },
+ {
+ "epoch": 1.288,
+ "grad_norm": 1.1791326999664307,
+ "learning_rate": 6.343703703703703e-07,
+ "logits/chosen": 1.4442191123962402,
+ "logits/rejected": 1.5532598495483398,
+ "logps/chosen": -141.93701171875,
+ "logps/rejected": -147.68682861328125,
+ "loss": 0.6539246082305908,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.09121527522802353,
+ "rewards/margins": 0.08849794417619705,
+ "rewards/rejected": 0.002717340365052223,
+ "step": 1610
+ },
+ {
+ "epoch": 1.296,
+ "grad_norm": 1.1131889820098877,
+ "learning_rate": 6.314074074074074e-07,
+ "logits/chosen": 1.3450567722320557,
+ "logits/rejected": 1.4408035278320312,
+ "logps/chosen": -157.0798797607422,
+ "logps/rejected": -161.1500701904297,
+ "loss": 0.6505428314208984,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.11321337521076202,
+ "rewards/margins": 0.09579396992921829,
+ "rewards/rejected": 0.017419395968317986,
+ "step": 1620
+ },
+ {
+ "epoch": 1.304,
+ "grad_norm": 1.6172596216201782,
+ "learning_rate": 6.284444444444445e-07,
+ "logits/chosen": 1.478847622871399,
+ "logits/rejected": 1.3894659280776978,
+ "logps/chosen": -140.17453002929688,
+ "logps/rejected": -169.06289672851562,
+ "loss": 0.6720306396484375,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.08244314789772034,
+ "rewards/margins": 0.062018074095249176,
+ "rewards/rejected": 0.020425086840987206,
+ "step": 1630
+ },
+ {
+ "epoch": 1.312,
+ "grad_norm": 1.1743669509887695,
+ "learning_rate": 6.254814814814815e-07,
+ "logits/chosen": 1.4197287559509277,
+ "logits/rejected": 1.3319528102874756,
+ "logps/chosen": -157.23072814941406,
+ "logps/rejected": -139.50074768066406,
+ "loss": 0.6712700843811035,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.04650931805372238,
+ "rewards/margins": 0.057532649487257004,
+ "rewards/rejected": -0.0110233249142766,
+ "step": 1640
+ },
+ {
+ "epoch": 1.32,
+ "grad_norm": 1.3225634098052979,
+ "learning_rate": 6.225185185185184e-07,
+ "logits/chosen": 1.35457444190979,
+ "logits/rejected": 1.3751428127288818,
+ "logps/chosen": -137.4834747314453,
+ "logps/rejected": -134.4203643798828,
+ "loss": 0.6679590225219727,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.07574301958084106,
+ "rewards/margins": 0.057132184505462646,
+ "rewards/rejected": 0.01861082948744297,
+ "step": 1650
+ },
+ {
+ "epoch": 1.328,
+ "grad_norm": 1.5913130044937134,
+ "learning_rate": 6.195555555555555e-07,
+ "logits/chosen": 1.5328034162521362,
+ "logits/rejected": 1.3161242008209229,
+ "logps/chosen": -147.09475708007812,
+ "logps/rejected": -135.3501434326172,
+ "loss": 0.6575756072998047,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.1077059879899025,
+ "rewards/margins": 0.08081348240375519,
+ "rewards/rejected": 0.026892507448792458,
+ "step": 1660
+ },
+ {
+ "epoch": 1.336,
+ "grad_norm": 0.9880369901657104,
+ "learning_rate": 6.165925925925926e-07,
+ "logits/chosen": 1.3911128044128418,
+ "logits/rejected": 1.3688428401947021,
+ "logps/chosen": -152.17483520507812,
+ "logps/rejected": -149.79383850097656,
+ "loss": 0.6656455516815185,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.10711929947137833,
+ "rewards/margins": 0.0669747143983841,
+ "rewards/rejected": 0.04014457389712334,
+ "step": 1670
+ },
+ {
+ "epoch": 1.3439999999999999,
+ "grad_norm": 1.125959873199463,
+ "learning_rate": 6.136296296296296e-07,
+ "logits/chosen": 1.4232629537582397,
+ "logits/rejected": 1.5579107999801636,
+ "logps/chosen": -160.2217559814453,
+ "logps/rejected": -148.1455535888672,
+ "loss": 0.6710956573486329,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10255889594554901,
+ "rewards/margins": 0.05178038403391838,
+ "rewards/rejected": 0.05077851936221123,
+ "step": 1680
+ },
+ {
+ "epoch": 1.3519999999999999,
+ "grad_norm": 1.3627047538757324,
+ "learning_rate": 6.106666666666666e-07,
+ "logits/chosen": 1.4259792566299438,
+ "logits/rejected": 1.4750150442123413,
+ "logps/chosen": -152.88381958007812,
+ "logps/rejected": -153.72708129882812,
+ "loss": 0.6512804508209229,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.14147944748401642,
+ "rewards/margins": 0.09307968616485596,
+ "rewards/rejected": 0.04839975759387016,
+ "step": 1690
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "grad_norm": 1.0653239488601685,
+ "learning_rate": 6.077037037037038e-07,
+ "logits/chosen": 1.340463399887085,
+ "logits/rejected": 1.3752565383911133,
+ "logps/chosen": -150.36248779296875,
+ "logps/rejected": -151.90725708007812,
+ "loss": 0.6768710613250732,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.08575484901666641,
+ "rewards/margins": 0.043545305728912354,
+ "rewards/rejected": 0.04220953956246376,
+ "step": 1700
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "eval_logits/chosen": 1.4521433115005493,
+ "eval_logits/rejected": 1.4804331064224243,
+ "eval_logps/chosen": -152.77755737304688,
+ "eval_logps/rejected": -147.48025512695312,
+ "eval_loss": 0.6873273849487305,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": 0.07712872326374054,
+ "eval_rewards/margins": 0.022939365357160568,
+ "eval_rewards/rejected": 0.05418936163187027,
+ "eval_runtime": 90.962,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 1700
+ },
+ {
+ "epoch": 1.3679999999999999,
+ "grad_norm": 1.3996928930282593,
+ "learning_rate": 6.047407407407407e-07,
+ "logits/chosen": 1.41148042678833,
+ "logits/rejected": 1.4520132541656494,
+ "logps/chosen": -143.23252868652344,
+ "logps/rejected": -180.05331420898438,
+ "loss": 0.655043363571167,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10279543697834015,
+ "rewards/margins": 0.0895618349313736,
+ "rewards/rejected": 0.013233604840934277,
+ "step": 1710
+ },
+ {
+ "epoch": 1.376,
+ "grad_norm": 1.4253696203231812,
+ "learning_rate": 6.017777777777777e-07,
+ "logits/chosen": 1.3833668231964111,
+ "logits/rejected": 1.409301519393921,
+ "logps/chosen": -149.3744659423828,
+ "logps/rejected": -151.1966552734375,
+ "loss": 0.6865286827087402,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.06293162703514099,
+ "rewards/margins": 0.02295227348804474,
+ "rewards/rejected": 0.03997935354709625,
+ "step": 1720
+ },
+ {
+ "epoch": 1.384,
+ "grad_norm": 1.185731053352356,
+ "learning_rate": 5.988148148148148e-07,
+ "logits/chosen": 1.528174877166748,
+ "logits/rejected": 1.6338379383087158,
+ "logps/chosen": -160.7788543701172,
+ "logps/rejected": -159.85354614257812,
+ "loss": 0.6458437442779541,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.12789444625377655,
+ "rewards/margins": 0.1114773377776146,
+ "rewards/rejected": 0.016417117789387703,
+ "step": 1730
+ },
+ {
+ "epoch": 1.392,
+ "grad_norm": 1.2963365316390991,
+ "learning_rate": 5.958518518518518e-07,
+ "logits/chosen": 1.4939125776290894,
+ "logits/rejected": 1.4197361469268799,
+ "logps/chosen": -162.34744262695312,
+ "logps/rejected": -177.93087768554688,
+ "loss": 0.6620248317718506,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.11072772741317749,
+ "rewards/margins": 0.07320726662874222,
+ "rewards/rejected": 0.03752044588327408,
+ "step": 1740
+ },
+ {
+ "epoch": 1.4,
+ "grad_norm": 1.3535394668579102,
+ "learning_rate": 5.928888888888889e-07,
+ "logits/chosen": 1.3506572246551514,
+ "logits/rejected": 1.424142837524414,
+ "logps/chosen": -139.69985961914062,
+ "logps/rejected": -144.9062042236328,
+ "loss": 0.6815268516540527,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.07917657494544983,
+ "rewards/margins": 0.03688482195138931,
+ "rewards/rejected": 0.04229176044464111,
+ "step": 1750
+ },
+ {
+ "epoch": 1.408,
+ "grad_norm": 1.471380352973938,
+ "learning_rate": 5.899259259259259e-07,
+ "logits/chosen": 1.5949152708053589,
+ "logits/rejected": 1.710013747215271,
+ "logps/chosen": -170.3776092529297,
+ "logps/rejected": -177.1616973876953,
+ "loss": 0.6533854961395263,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.11633266508579254,
+ "rewards/margins": 0.09301649034023285,
+ "rewards/rejected": 0.023316185921430588,
+ "step": 1760
+ },
+ {
+ "epoch": 1.416,
+ "grad_norm": 1.3164308071136475,
+ "learning_rate": 5.869629629629629e-07,
+ "logits/chosen": 1.5159170627593994,
+ "logits/rejected": 1.4434025287628174,
+ "logps/chosen": -161.05157470703125,
+ "logps/rejected": -140.06948852539062,
+ "loss": 0.660222864151001,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.10675680637359619,
+ "rewards/margins": 0.07750208675861359,
+ "rewards/rejected": 0.029254715889692307,
+ "step": 1770
+ },
+ {
+ "epoch": 1.424,
+ "grad_norm": 1.102005124092102,
+ "learning_rate": 5.839999999999999e-07,
+ "logits/chosen": 1.349440336227417,
+ "logits/rejected": 1.1565381288528442,
+ "logps/chosen": -150.61328125,
+ "logps/rejected": -151.36001586914062,
+ "loss": 0.660042142868042,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.09433183073997498,
+ "rewards/margins": 0.07778269052505493,
+ "rewards/rejected": 0.016549136489629745,
+ "step": 1780
+ },
+ {
+ "epoch": 1.432,
+ "grad_norm": 1.2305164337158203,
+ "learning_rate": 5.81037037037037e-07,
+ "logits/chosen": 1.4631009101867676,
+ "logits/rejected": 1.497757911682129,
+ "logps/chosen": -156.7335968017578,
+ "logps/rejected": -150.91065979003906,
+ "loss": 0.6564284324645996,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.13712921738624573,
+ "rewards/margins": 0.09247532486915588,
+ "rewards/rejected": 0.04465390369296074,
+ "step": 1790
+ },
+ {
+ "epoch": 1.44,
+ "grad_norm": 1.3459731340408325,
+ "learning_rate": 5.780740740740741e-07,
+ "logits/chosen": 1.5344784259796143,
+ "logits/rejected": 1.5709785223007202,
+ "logps/chosen": -166.15830993652344,
+ "logps/rejected": -158.678955078125,
+ "loss": 0.6511439800262451,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1227751225233078,
+ "rewards/margins": 0.0968184620141983,
+ "rewards/rejected": 0.025956671684980392,
+ "step": 1800
+ },
+ {
+ "epoch": 1.44,
+ "eval_logits/chosen": 1.4476159811019897,
+ "eval_logits/rejected": 1.4754574298858643,
+ "eval_logps/chosen": -152.70677185058594,
+ "eval_logps/rejected": -147.41787719726562,
+ "eval_loss": 0.6876539587974548,
+ "eval_rewards/accuracies": 0.5199999809265137,
+ "eval_rewards/chosen": 0.08420863002538681,
+ "eval_rewards/margins": 0.023782728239893913,
+ "eval_rewards/rejected": 0.06042589247226715,
+ "eval_runtime": 91.0223,
+ "eval_samples_per_second": 5.493,
+ "eval_steps_per_second": 2.747,
+ "step": 1800
+ },
+ {
+ "epoch": 1.448,
+ "grad_norm": 1.3158259391784668,
+ "learning_rate": 5.751111111111111e-07,
+ "logits/chosen": 1.425363302230835,
+ "logits/rejected": 1.350799798965454,
+ "logps/chosen": -151.4562530517578,
+ "logps/rejected": -134.9906005859375,
+ "loss": 0.6759338855743409,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1467444747686386,
+ "rewards/margins": 0.047227732837200165,
+ "rewards/rejected": 0.09951673448085785,
+ "step": 1810
+ },
+ {
+ "epoch": 1.456,
+ "grad_norm": 1.179150938987732,
+ "learning_rate": 5.721481481481481e-07,
+ "logits/chosen": 1.6323440074920654,
+ "logits/rejected": 1.4893152713775635,
+ "logps/chosen": -149.57675170898438,
+ "logps/rejected": -146.96237182617188,
+ "loss": 0.657456636428833,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1445317566394806,
+ "rewards/margins": 0.0896807312965393,
+ "rewards/rejected": 0.05485100671648979,
+ "step": 1820
+ },
+ {
+ "epoch": 1.464,
+ "grad_norm": 1.0987558364868164,
+ "learning_rate": 5.691851851851853e-07,
+ "logits/chosen": 1.3851706981658936,
+ "logits/rejected": 1.381049633026123,
+ "logps/chosen": -163.92648315429688,
+ "logps/rejected": -142.801025390625,
+ "loss": 0.6631375312805176,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.14093783497810364,
+ "rewards/margins": 0.07281022518873215,
+ "rewards/rejected": 0.06812760978937149,
+ "step": 1830
+ },
+ {
+ "epoch": 1.472,
+ "grad_norm": 1.1079164743423462,
+ "learning_rate": 5.662222222222222e-07,
+ "logits/chosen": 1.4228973388671875,
+ "logits/rejected": 1.3453642129898071,
+ "logps/chosen": -148.5561981201172,
+ "logps/rejected": -141.28009033203125,
+ "loss": 0.6612800121307373,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.10088491439819336,
+ "rewards/margins": 0.08228114247322083,
+ "rewards/rejected": 0.018603766337037086,
+ "step": 1840
+ },
+ {
+ "epoch": 1.48,
+ "grad_norm": 1.0415616035461426,
+ "learning_rate": 5.632592592592592e-07,
+ "logits/chosen": 1.4923865795135498,
+ "logits/rejected": 1.536971092224121,
+ "logps/chosen": -142.55166625976562,
+ "logps/rejected": -149.22906494140625,
+ "loss": 0.6690784931182862,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.10785870254039764,
+ "rewards/margins": 0.05975862219929695,
+ "rewards/rejected": 0.04810008406639099,
+ "step": 1850
+ },
+ {
+ "epoch": 1.488,
+ "grad_norm": 1.4004729986190796,
+ "learning_rate": 5.602962962962962e-07,
+ "logits/chosen": 1.4783391952514648,
+ "logits/rejected": 1.5112550258636475,
+ "logps/chosen": -153.54034423828125,
+ "logps/rejected": -118.77142333984375,
+ "loss": 0.6666333675384521,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.12789246439933777,
+ "rewards/margins": 0.06238863617181778,
+ "rewards/rejected": 0.06550382077693939,
+ "step": 1860
+ },
+ {
+ "epoch": 1.496,
+ "grad_norm": 1.1908437013626099,
+ "learning_rate": 5.573333333333333e-07,
+ "logits/chosen": 1.5748203992843628,
+ "logits/rejected": 1.4827988147735596,
+ "logps/chosen": -162.0931854248047,
+ "logps/rejected": -156.00779724121094,
+ "loss": 0.6626818656921387,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.11168892681598663,
+ "rewards/margins": 0.07420919090509415,
+ "rewards/rejected": 0.03747973591089249,
+ "step": 1870
+ },
+ {
+ "epoch": 1.504,
+ "grad_norm": 1.288218379020691,
+ "learning_rate": 5.543703703703704e-07,
+ "logits/chosen": 1.5260225534439087,
+ "logits/rejected": 1.5843703746795654,
+ "logps/chosen": -148.2488555908203,
+ "logps/rejected": -161.31570434570312,
+ "loss": 0.6677760124206543,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.12626001238822937,
+ "rewards/margins": 0.06881437450647354,
+ "rewards/rejected": 0.05744565278291702,
+ "step": 1880
+ },
+ {
+ "epoch": 1.512,
+ "grad_norm": 1.2899844646453857,
+ "learning_rate": 5.514074074074074e-07,
+ "logits/chosen": 1.417141318321228,
+ "logits/rejected": 1.389639973640442,
+ "logps/chosen": -160.5532989501953,
+ "logps/rejected": -158.82923889160156,
+ "loss": 0.6510573863983155,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.15593740344047546,
+ "rewards/margins": 0.09999983012676239,
+ "rewards/rejected": 0.05593757703900337,
+ "step": 1890
+ },
+ {
+ "epoch": 1.52,
+ "grad_norm": 1.3312058448791504,
+ "learning_rate": 5.484444444444444e-07,
+ "logits/chosen": 1.4996871948242188,
+ "logits/rejected": 1.6273930072784424,
+ "logps/chosen": -154.84347534179688,
+ "logps/rejected": -130.4560089111328,
+ "loss": 0.6844823360443115,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.13181835412979126,
+ "rewards/margins": 0.029756689444184303,
+ "rewards/rejected": 0.10206165164709091,
+ "step": 1900
+ },
+ {
+ "epoch": 1.52,
+ "eval_logits/chosen": 1.4582936763763428,
+ "eval_logits/rejected": 1.4853557348251343,
+ "eval_logps/chosen": -152.36231994628906,
+ "eval_logps/rejected": -147.1087188720703,
+ "eval_loss": 0.6870284676551819,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": 0.1186550110578537,
+ "eval_rewards/margins": 0.027311701327562332,
+ "eval_rewards/rejected": 0.09134332090616226,
+ "eval_runtime": 90.913,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 1900
+ },
+ {
+ "epoch": 1.528,
+ "grad_norm": 1.2257205247879028,
+ "learning_rate": 5.454814814814814e-07,
+ "logits/chosen": 1.4909894466400146,
+ "logits/rejected": 1.4579002857208252,
+ "logps/chosen": -129.2428741455078,
+ "logps/rejected": -149.32220458984375,
+ "loss": 0.6677265644073487,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.13237449526786804,
+ "rewards/margins": 0.059117116034030914,
+ "rewards/rejected": 0.07325738668441772,
+ "step": 1910
+ },
+ {
+ "epoch": 1.536,
+ "grad_norm": 1.3048031330108643,
+ "learning_rate": 5.425185185185185e-07,
+ "logits/chosen": 1.5237693786621094,
+ "logits/rejected": 1.4846075773239136,
+ "logps/chosen": -159.10055541992188,
+ "logps/rejected": -155.69888305664062,
+ "loss": 0.6709336280822754,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.1088198646903038,
+ "rewards/margins": 0.05951980873942375,
+ "rewards/rejected": 0.04930006340146065,
+ "step": 1920
+ },
+ {
+ "epoch": 1.544,
+ "grad_norm": 1.3463319540023804,
+ "learning_rate": 5.395555555555556e-07,
+ "logits/chosen": 1.6476523876190186,
+ "logits/rejected": 1.6236053705215454,
+ "logps/chosen": -171.18319702148438,
+ "logps/rejected": -172.63796997070312,
+ "loss": 0.6670461177825928,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.13160961866378784,
+ "rewards/margins": 0.06587684154510498,
+ "rewards/rejected": 0.06573276221752167,
+ "step": 1930
+ },
+ {
+ "epoch": 1.552,
+ "grad_norm": 1.2326266765594482,
+ "learning_rate": 5.365925925925926e-07,
+ "logits/chosen": 1.4379268884658813,
+ "logits/rejected": 1.5206880569458008,
+ "logps/chosen": -157.2953643798828,
+ "logps/rejected": -153.21218872070312,
+ "loss": 0.6558516502380372,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.17594724893569946,
+ "rewards/margins": 0.09524654597043991,
+ "rewards/rejected": 0.08070070296525955,
+ "step": 1940
+ },
+ {
+ "epoch": 1.56,
+ "grad_norm": 1.1362297534942627,
+ "learning_rate": 5.336296296296296e-07,
+ "logits/chosen": 1.524285078048706,
+ "logits/rejected": 1.3964688777923584,
+ "logps/chosen": -161.221435546875,
+ "logps/rejected": -154.8270263671875,
+ "loss": 0.6746312141418457,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.12519574165344238,
+ "rewards/margins": 0.05550859495997429,
+ "rewards/rejected": 0.06968715041875839,
+ "step": 1950
+ },
+ {
+ "epoch": 1.568,
+ "grad_norm": 1.5411877632141113,
+ "learning_rate": 5.306666666666665e-07,
+ "logits/chosen": 1.2426170110702515,
+ "logits/rejected": 1.3256272077560425,
+ "logps/chosen": -137.97537231445312,
+ "logps/rejected": -159.09580993652344,
+ "loss": 0.6492344856262207,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.16253654658794403,
+ "rewards/margins": 0.12099182605743408,
+ "rewards/rejected": 0.04154472425580025,
+ "step": 1960
+ },
+ {
+ "epoch": 1.576,
+ "grad_norm": 1.2111555337905884,
+ "learning_rate": 5.277037037037037e-07,
+ "logits/chosen": 1.3337433338165283,
+ "logits/rejected": 1.479209065437317,
+ "logps/chosen": -149.0251007080078,
+ "logps/rejected": -140.0357208251953,
+ "loss": 0.6559633731842041,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.13673515617847443,
+ "rewards/margins": 0.0912124291062355,
+ "rewards/rejected": 0.04552273079752922,
+ "step": 1970
+ },
+ {
+ "epoch": 1.584,
+ "grad_norm": 1.1228832006454468,
+ "learning_rate": 5.247407407407407e-07,
+ "logits/chosen": 1.4376949071884155,
+ "logits/rejected": 1.5033822059631348,
+ "logps/chosen": -144.91751098632812,
+ "logps/rejected": -147.37057495117188,
+ "loss": 0.6593158721923829,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.1573731154203415,
+ "rewards/margins": 0.08546318858861923,
+ "rewards/rejected": 0.07190994173288345,
+ "step": 1980
+ },
+ {
+ "epoch": 1.592,
+ "grad_norm": 0.9601779580116272,
+ "learning_rate": 5.217777777777777e-07,
+ "logits/chosen": 1.2674219608306885,
+ "logits/rejected": 1.3877508640289307,
+ "logps/chosen": -116.023193359375,
+ "logps/rejected": -122.81617736816406,
+ "loss": 0.6842892646789551,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.13847123086452484,
+ "rewards/margins": 0.026916390284895897,
+ "rewards/rejected": 0.11155486106872559,
+ "step": 1990
+ },
+ {
+ "epoch": 1.6,
+ "grad_norm": 1.2849960327148438,
+ "learning_rate": 5.188148148148148e-07,
+ "logits/chosen": 1.6982529163360596,
+ "logits/rejected": 1.6600332260131836,
+ "logps/chosen": -143.352294921875,
+ "logps/rejected": -150.0862274169922,
+ "loss": 0.6666803359985352,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.13707581162452698,
+ "rewards/margins": 0.06178014352917671,
+ "rewards/rejected": 0.07529564201831818,
+ "step": 2000
+ },
+ {
+ "epoch": 1.6,
+ "eval_logits/chosen": 1.4610096216201782,
+ "eval_logits/rejected": 1.4881619215011597,
+ "eval_logps/chosen": -152.36614990234375,
+ "eval_logps/rejected": -147.1319580078125,
+ "eval_loss": 0.6861760020256042,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": 0.11826877295970917,
+ "eval_rewards/margins": 0.02924923785030842,
+ "eval_rewards/rejected": 0.0890195295214653,
+ "eval_runtime": 91.0197,
+ "eval_samples_per_second": 5.493,
+ "eval_steps_per_second": 2.747,
+ "step": 2000
+ },
+ {
+ "epoch": 1.608,
+ "grad_norm": 1.731399655342102,
+ "learning_rate": 5.158518518518519e-07,
+ "logits/chosen": 1.4297409057617188,
+ "logits/rejected": 1.5017236471176147,
+ "logps/chosen": -164.93292236328125,
+ "logps/rejected": -164.80677795410156,
+ "loss": 0.6650710582733155,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.20246899127960205,
+ "rewards/margins": 0.0675087422132492,
+ "rewards/rejected": 0.13496026396751404,
+ "step": 2010
+ },
+ {
+ "epoch": 1.616,
+ "grad_norm": 1.2728053331375122,
+ "learning_rate": 5.128888888888888e-07,
+ "logits/chosen": 1.305499792098999,
+ "logits/rejected": 1.436187744140625,
+ "logps/chosen": -143.90879821777344,
+ "logps/rejected": -158.8233184814453,
+ "loss": 0.6744787693023682,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.17535606026649475,
+ "rewards/margins": 0.04918169230222702,
+ "rewards/rejected": 0.12617434561252594,
+ "step": 2020
+ },
+ {
+ "epoch": 1.624,
+ "grad_norm": 1.4291980266571045,
+ "learning_rate": 5.099259259259259e-07,
+ "logits/chosen": 1.4624104499816895,
+ "logits/rejected": 1.4936870336532593,
+ "logps/chosen": -151.03884887695312,
+ "logps/rejected": -127.40641784667969,
+ "loss": 0.6721909046173096,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.1530950516462326,
+ "rewards/margins": 0.0540502592921257,
+ "rewards/rejected": 0.0990448072552681,
+ "step": 2030
+ },
+ {
+ "epoch": 1.6320000000000001,
+ "grad_norm": 1.5909459590911865,
+ "learning_rate": 5.069629629629629e-07,
+ "logits/chosen": 1.468285322189331,
+ "logits/rejected": 1.3470795154571533,
+ "logps/chosen": -158.12046813964844,
+ "logps/rejected": -160.971923828125,
+ "loss": 0.6752189636230469,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.12029504776000977,
+ "rewards/margins": 0.05528820678591728,
+ "rewards/rejected": 0.06500682979822159,
+ "step": 2040
+ },
+ {
+ "epoch": 1.6400000000000001,
+ "grad_norm": 1.2923390865325928,
+ "learning_rate": 5.04e-07,
+ "logits/chosen": 1.530002236366272,
+ "logits/rejected": 1.5821081399917603,
+ "logps/chosen": -157.1008758544922,
+ "logps/rejected": -158.16775512695312,
+ "loss": 0.6889143943786621,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.14681917428970337,
+ "rewards/margins": 0.01898271217942238,
+ "rewards/rejected": 0.1278364509344101,
+ "step": 2050
+ },
+ {
+ "epoch": 1.6480000000000001,
+ "grad_norm": 1.2025870084762573,
+ "learning_rate": 5.010370370370371e-07,
+ "logits/chosen": 1.4749194383621216,
+ "logits/rejected": 1.3301353454589844,
+ "logps/chosen": -119.29649353027344,
+ "logps/rejected": -145.0949249267578,
+ "loss": 0.6332630634307861,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.1944357454776764,
+ "rewards/margins": 0.14449670910835266,
+ "rewards/rejected": 0.049939028918743134,
+ "step": 2060
+ },
+ {
+ "epoch": 1.6560000000000001,
+ "grad_norm": 1.3136606216430664,
+ "learning_rate": 4.98074074074074e-07,
+ "logits/chosen": 1.5436251163482666,
+ "logits/rejected": 1.4338414669036865,
+ "logps/chosen": -157.49942016601562,
+ "logps/rejected": -140.4504852294922,
+ "loss": 0.6923964977264404,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.13830263912677765,
+ "rewards/margins": 0.010729723609983921,
+ "rewards/rejected": 0.1275729238986969,
+ "step": 2070
+ },
+ {
+ "epoch": 1.6640000000000001,
+ "grad_norm": 1.2378884553909302,
+ "learning_rate": 4.951111111111111e-07,
+ "logits/chosen": 1.5672242641448975,
+ "logits/rejected": 1.4682090282440186,
+ "logps/chosen": -158.99301147460938,
+ "logps/rejected": -149.21221923828125,
+ "loss": 0.6473859786987305,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.1993202269077301,
+ "rewards/margins": 0.10525582730770111,
+ "rewards/rejected": 0.0940643846988678,
+ "step": 2080
+ },
+ {
+ "epoch": 1.6720000000000002,
+ "grad_norm": 1.1976169347763062,
+ "learning_rate": 4.921481481481482e-07,
+ "logits/chosen": 1.4160411357879639,
+ "logits/rejected": 1.3142106533050537,
+ "logps/chosen": -153.40621948242188,
+ "logps/rejected": -154.57308959960938,
+ "loss": 0.6821109771728515,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.14698973298072815,
+ "rewards/margins": 0.034348614513874054,
+ "rewards/rejected": 0.1126411110162735,
+ "step": 2090
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "grad_norm": 1.3028976917266846,
+ "learning_rate": 4.891851851851852e-07,
+ "logits/chosen": 1.61053466796875,
+ "logits/rejected": 1.526831030845642,
+ "logps/chosen": -164.3324737548828,
+ "logps/rejected": -149.382080078125,
+ "loss": 0.6684560298919677,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.22455796599388123,
+ "rewards/margins": 0.06641162931919098,
+ "rewards/rejected": 0.15814633667469025,
+ "step": 2100
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "eval_logits/chosen": 1.4865608215332031,
+ "eval_logits/rejected": 1.5141921043395996,
+ "eval_logps/chosen": -151.979248046875,
+ "eval_logps/rejected": -146.76809692382812,
+ "eval_loss": 0.6849560141563416,
+ "eval_rewards/accuracies": 0.5379999876022339,
+ "eval_rewards/chosen": 0.15696033835411072,
+ "eval_rewards/margins": 0.0315566249191761,
+ "eval_rewards/rejected": 0.1254037320613861,
+ "eval_runtime": 90.9172,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 2100
+ },
+ {
+ "epoch": 1.688,
+ "grad_norm": 1.3767266273498535,
+ "learning_rate": 4.862222222222222e-07,
+ "logits/chosen": 1.458231806755066,
+ "logits/rejected": 1.3821945190429688,
+ "logps/chosen": -166.67698669433594,
+ "logps/rejected": -156.74313354492188,
+ "loss": 0.6658836364746094,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.17039546370506287,
+ "rewards/margins": 0.07431827485561371,
+ "rewards/rejected": 0.09607716649770737,
+ "step": 2110
+ },
+ {
+ "epoch": 1.696,
+ "grad_norm": 1.4396629333496094,
+ "learning_rate": 4.832592592592592e-07,
+ "logits/chosen": 1.5568702220916748,
+ "logits/rejected": 1.5459610223770142,
+ "logps/chosen": -133.0255889892578,
+ "logps/rejected": -138.2047882080078,
+ "loss": 0.6704818725585937,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.173842191696167,
+ "rewards/margins": 0.06081608682870865,
+ "rewards/rejected": 0.11302611976861954,
+ "step": 2120
+ },
+ {
+ "epoch": 1.704,
+ "grad_norm": 2.3225553035736084,
+ "learning_rate": 4.802962962962962e-07,
+ "logits/chosen": 1.5432292222976685,
+ "logits/rejected": 1.521958589553833,
+ "logps/chosen": -165.04318237304688,
+ "logps/rejected": -163.95855712890625,
+ "loss": 0.6567724704742431,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.20300237834453583,
+ "rewards/margins": 0.09381169825792313,
+ "rewards/rejected": 0.1091906875371933,
+ "step": 2130
+ },
+ {
+ "epoch": 1.712,
+ "grad_norm": 1.3185398578643799,
+ "learning_rate": 4.773333333333333e-07,
+ "logits/chosen": 1.2340335845947266,
+ "logits/rejected": 1.3610973358154297,
+ "logps/chosen": -139.11878967285156,
+ "logps/rejected": -146.57260131835938,
+ "loss": 0.671626091003418,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.17043188214302063,
+ "rewards/margins": 0.054147154092788696,
+ "rewards/rejected": 0.11628471314907074,
+ "step": 2140
+ },
+ {
+ "epoch": 1.72,
+ "grad_norm": 1.618586540222168,
+ "learning_rate": 4.7437037037037034e-07,
+ "logits/chosen": 1.5895780324935913,
+ "logits/rejected": 1.613523244857788,
+ "logps/chosen": -156.53970336914062,
+ "logps/rejected": -154.35116577148438,
+ "loss": 0.6798102855682373,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.20041461288928986,
+ "rewards/margins": 0.03897023946046829,
+ "rewards/rejected": 0.16144435107707977,
+ "step": 2150
+ },
+ {
+ "epoch": 1.728,
+ "grad_norm": 0.8500939607620239,
+ "learning_rate": 4.7140740740740736e-07,
+ "logits/chosen": 1.3736517429351807,
+ "logits/rejected": 1.361998438835144,
+ "logps/chosen": -137.701904296875,
+ "logps/rejected": -139.91567993164062,
+ "loss": 0.6568773269653321,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.16317322850227356,
+ "rewards/margins": 0.08119454979896545,
+ "rewards/rejected": 0.0819786787033081,
+ "step": 2160
+ },
+ {
+ "epoch": 1.736,
+ "grad_norm": 1.2291405200958252,
+ "learning_rate": 4.6844444444444444e-07,
+ "logits/chosen": 1.474157452583313,
+ "logits/rejected": 1.4488928318023682,
+ "logps/chosen": -152.36732482910156,
+ "logps/rejected": -149.70980834960938,
+ "loss": 0.6551193714141845,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.2117660939693451,
+ "rewards/margins": 0.0950237363576889,
+ "rewards/rejected": 0.11674235761165619,
+ "step": 2170
+ },
+ {
+ "epoch": 1.744,
+ "grad_norm": 1.2946466207504272,
+ "learning_rate": 4.6548148148148146e-07,
+ "logits/chosen": 1.257048487663269,
+ "logits/rejected": 1.3120336532592773,
+ "logps/chosen": -143.43148803710938,
+ "logps/rejected": -169.9266815185547,
+ "loss": 0.6712408542633057,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.1496409922838211,
+ "rewards/margins": 0.05779259279370308,
+ "rewards/rejected": 0.09184839576482773,
+ "step": 2180
+ },
+ {
+ "epoch": 1.752,
+ "grad_norm": 1.0697646141052246,
+ "learning_rate": 4.625185185185185e-07,
+ "logits/chosen": 1.5544202327728271,
+ "logits/rejected": 1.5186070203781128,
+ "logps/chosen": -170.5447235107422,
+ "logps/rejected": -165.55209350585938,
+ "loss": 0.6674113750457764,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.12943358719348907,
+ "rewards/margins": 0.07018069922924042,
+ "rewards/rejected": 0.05925289914011955,
+ "step": 2190
+ },
+ {
+ "epoch": 1.76,
+ "grad_norm": 1.228698968887329,
+ "learning_rate": 4.595555555555555e-07,
+ "logits/chosen": 1.46980881690979,
+ "logits/rejected": 1.2382839918136597,
+ "logps/chosen": -133.5864715576172,
+ "logps/rejected": -135.65968322753906,
+ "loss": 0.6756285667419434,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.18240585923194885,
+ "rewards/margins": 0.052959512919187546,
+ "rewards/rejected": 0.1294463574886322,
+ "step": 2200
+ },
+ {
+ "epoch": 1.76,
+ "eval_logits/chosen": 1.4826997518539429,
+ "eval_logits/rejected": 1.5107221603393555,
+ "eval_logps/chosen": -152.19204711914062,
+ "eval_logps/rejected": -146.98675537109375,
+ "eval_loss": 0.6843143105506897,
+ "eval_rewards/accuracies": 0.5379999876022339,
+ "eval_rewards/chosen": 0.13567964732646942,
+ "eval_rewards/margins": 0.03213911131024361,
+ "eval_rewards/rejected": 0.10354051738977432,
+ "eval_runtime": 91.0116,
+ "eval_samples_per_second": 5.494,
+ "eval_steps_per_second": 2.747,
+ "step": 2200
+ },
+ {
+ "epoch": 1.768,
+ "grad_norm": 1.4145433902740479,
+ "learning_rate": 4.565925925925926e-07,
+ "logits/chosen": 1.49526047706604,
+ "logits/rejected": 1.519241213798523,
+ "logps/chosen": -152.7605438232422,
+ "logps/rejected": -140.98629760742188,
+ "loss": 0.6503778457641601,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.16126050055027008,
+ "rewards/margins": 0.0973939448595047,
+ "rewards/rejected": 0.06386655569076538,
+ "step": 2210
+ },
+ {
+ "epoch": 1.776,
+ "grad_norm": 1.4429577589035034,
+ "learning_rate": 4.536296296296296e-07,
+ "logits/chosen": 1.4706772565841675,
+ "logits/rejected": 1.3714580535888672,
+ "logps/chosen": -135.95150756835938,
+ "logps/rejected": -165.23587036132812,
+ "loss": 0.6639570713043212,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.18487155437469482,
+ "rewards/margins": 0.07484755665063858,
+ "rewards/rejected": 0.11002399027347565,
+ "step": 2220
+ },
+ {
+ "epoch": 1.784,
+ "grad_norm": 1.0622409582138062,
+ "learning_rate": 4.506666666666666e-07,
+ "logits/chosen": 1.2506721019744873,
+ "logits/rejected": 1.4802286624908447,
+ "logps/chosen": -129.645751953125,
+ "logps/rejected": -131.7567596435547,
+ "loss": 0.6788074493408203,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.12520259618759155,
+ "rewards/margins": 0.03735826909542084,
+ "rewards/rejected": 0.08784433454275131,
+ "step": 2230
+ },
+ {
+ "epoch": 1.792,
+ "grad_norm": 1.0001399517059326,
+ "learning_rate": 4.477037037037037e-07,
+ "logits/chosen": 1.4206364154815674,
+ "logits/rejected": 1.5338770151138306,
+ "logps/chosen": -143.95074462890625,
+ "logps/rejected": -160.1469268798828,
+ "loss": 0.6695474147796631,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.13798081874847412,
+ "rewards/margins": 0.05759159475564957,
+ "rewards/rejected": 0.08038921654224396,
+ "step": 2240
+ },
+ {
+ "epoch": 1.8,
+ "grad_norm": 1.3531230688095093,
+ "learning_rate": 4.4474074074074067e-07,
+ "logits/chosen": 1.5709412097930908,
+ "logits/rejected": 1.5301862955093384,
+ "logps/chosen": -161.30087280273438,
+ "logps/rejected": -150.31236267089844,
+ "loss": 0.6771373271942138,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.1488293707370758,
+ "rewards/margins": 0.04127617925405502,
+ "rewards/rejected": 0.10755319893360138,
+ "step": 2250
+ },
+ {
+ "epoch": 1.808,
+ "grad_norm": 1.37367582321167,
+ "learning_rate": 4.4177777777777775e-07,
+ "logits/chosen": 1.36393141746521,
+ "logits/rejected": 1.484881043434143,
+ "logps/chosen": -125.18086242675781,
+ "logps/rejected": -148.78546142578125,
+ "loss": 0.6596842288970948,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.15397249162197113,
+ "rewards/margins": 0.07893262803554535,
+ "rewards/rejected": 0.07503987848758698,
+ "step": 2260
+ },
+ {
+ "epoch": 1.8159999999999998,
+ "grad_norm": 1.4511909484863281,
+ "learning_rate": 4.388148148148148e-07,
+ "logits/chosen": 1.3140859603881836,
+ "logits/rejected": 1.3879334926605225,
+ "logps/chosen": -131.48226928710938,
+ "logps/rejected": -123.94035339355469,
+ "loss": 0.6698450088500977,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.12767434120178223,
+ "rewards/margins": 0.06716184318065643,
+ "rewards/rejected": 0.060512494295835495,
+ "step": 2270
+ },
+ {
+ "epoch": 1.8239999999999998,
+ "grad_norm": 1.3375639915466309,
+ "learning_rate": 4.3585185185185184e-07,
+ "logits/chosen": 1.5821285247802734,
+ "logits/rejected": 1.43180251121521,
+ "logps/chosen": -167.2334747314453,
+ "logps/rejected": -145.28871154785156,
+ "loss": 0.658758020401001,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.12011411041021347,
+ "rewards/margins": 0.08019578456878662,
+ "rewards/rejected": 0.039918314665555954,
+ "step": 2280
+ },
+ {
+ "epoch": 1.8319999999999999,
+ "grad_norm": 1.2121729850769043,
+ "learning_rate": 4.3288888888888887e-07,
+ "logits/chosen": 1.3520320653915405,
+ "logits/rejected": 1.18943452835083,
+ "logps/chosen": -147.15621948242188,
+ "logps/rejected": -132.015625,
+ "loss": 0.6655458450317383,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08098682016134262,
+ "rewards/margins": 0.07385818660259247,
+ "rewards/rejected": 0.007128633558750153,
+ "step": 2290
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "grad_norm": 1.1618680953979492,
+ "learning_rate": 4.2992592592592594e-07,
+ "logits/chosen": 1.4779785871505737,
+ "logits/rejected": 1.448462724685669,
+ "logps/chosen": -157.75070190429688,
+ "logps/rejected": -141.3148956298828,
+ "loss": 0.6813467979431153,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.12400279194116592,
+ "rewards/margins": 0.034347161650657654,
+ "rewards/rejected": 0.08965563774108887,
+ "step": 2300
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "eval_logits/chosen": 1.4821760654449463,
+ "eval_logits/rejected": 1.5101500749588013,
+ "eval_logps/chosen": -152.2089080810547,
+ "eval_logps/rejected": -147.0087432861328,
+ "eval_loss": 0.6842693090438843,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.13399286568164825,
+ "eval_rewards/margins": 0.032653141766786575,
+ "eval_rewards/rejected": 0.10133973509073257,
+ "eval_runtime": 90.8665,
+ "eval_samples_per_second": 5.503,
+ "eval_steps_per_second": 2.751,
+ "step": 2300
+ },
+ {
+ "epoch": 1.8479999999999999,
+ "grad_norm": 1.1095342636108398,
+ "learning_rate": 4.2696296296296296e-07,
+ "logits/chosen": 1.6475988626480103,
+ "logits/rejected": 1.6654495000839233,
+ "logps/chosen": -165.965087890625,
+ "logps/rejected": -153.98214721679688,
+ "loss": 0.6605272769927979,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.2209581881761551,
+ "rewards/margins": 0.07579347491264343,
+ "rewards/rejected": 0.14516471326351166,
+ "step": 2310
+ },
+ {
+ "epoch": 1.8559999999999999,
+ "grad_norm": 1.1993674039840698,
+ "learning_rate": 4.24e-07,
+ "logits/chosen": 1.5783694982528687,
+ "logits/rejected": 1.406640648841858,
+ "logps/chosen": -139.3554229736328,
+ "logps/rejected": -126.39006042480469,
+ "loss": 0.6817263126373291,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1467253863811493,
+ "rewards/margins": 0.03637007623910904,
+ "rewards/rejected": 0.11035531759262085,
+ "step": 2320
+ },
+ {
+ "epoch": 1.8639999999999999,
+ "grad_norm": 1.4875127077102661,
+ "learning_rate": 4.21037037037037e-07,
+ "logits/chosen": 1.4171388149261475,
+ "logits/rejected": 1.4270119667053223,
+ "logps/chosen": -145.1273651123047,
+ "logps/rejected": -157.85816955566406,
+ "loss": 0.6678358554840088,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.14816439151763916,
+ "rewards/margins": 0.06963086873292923,
+ "rewards/rejected": 0.07853354513645172,
+ "step": 2330
+ },
+ {
+ "epoch": 1.8719999999999999,
+ "grad_norm": 1.2899141311645508,
+ "learning_rate": 4.180740740740741e-07,
+ "logits/chosen": 1.3682881593704224,
+ "logits/rejected": 1.3383492231369019,
+ "logps/chosen": -167.96450805664062,
+ "logps/rejected": -158.2183837890625,
+ "loss": 0.6754446029663086,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.1453542709350586,
+ "rewards/margins": 0.047066912055015564,
+ "rewards/rejected": 0.09828735888004303,
+ "step": 2340
+ },
+ {
+ "epoch": 1.88,
+ "grad_norm": 1.4864925146102905,
+ "learning_rate": 4.151111111111111e-07,
+ "logits/chosen": 1.5181632041931152,
+ "logits/rejected": 1.6898539066314697,
+ "logps/chosen": -146.9915771484375,
+ "logps/rejected": -156.66282653808594,
+ "loss": 0.6776726245880127,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.1270596832036972,
+ "rewards/margins": 0.04481229558587074,
+ "rewards/rejected": 0.08224737644195557,
+ "step": 2350
+ },
+ {
+ "epoch": 1.888,
+ "grad_norm": 1.3912899494171143,
+ "learning_rate": 4.1214814814814813e-07,
+ "logits/chosen": 1.279984474182129,
+ "logits/rejected": 1.258424997329712,
+ "logps/chosen": -162.26925659179688,
+ "logps/rejected": -153.519287109375,
+ "loss": 0.680011796951294,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1301378756761551,
+ "rewards/margins": 0.03849628567695618,
+ "rewards/rejected": 0.09164158999919891,
+ "step": 2360
+ },
+ {
+ "epoch": 1.896,
+ "grad_norm": 1.2498691082000732,
+ "learning_rate": 4.091851851851852e-07,
+ "logits/chosen": 1.5644304752349854,
+ "logits/rejected": 1.5302040576934814,
+ "logps/chosen": -181.20506286621094,
+ "logps/rejected": -164.98670959472656,
+ "loss": 0.6632030963897705,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.16487805545330048,
+ "rewards/margins": 0.08321762084960938,
+ "rewards/rejected": 0.0816604346036911,
+ "step": 2370
+ },
+ {
+ "epoch": 1.904,
+ "grad_norm": 1.3421261310577393,
+ "learning_rate": 4.0622222222222217e-07,
+ "logits/chosen": 1.5388444662094116,
+ "logits/rejected": 1.5939340591430664,
+ "logps/chosen": -151.34434509277344,
+ "logps/rejected": -149.8802490234375,
+ "loss": 0.691883659362793,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.11643080413341522,
+ "rewards/margins": 0.017300304025411606,
+ "rewards/rejected": 0.09913049638271332,
+ "step": 2380
+ },
+ {
+ "epoch": 1.912,
+ "grad_norm": 1.4378565549850464,
+ "learning_rate": 4.0325925925925925e-07,
+ "logits/chosen": 1.4750030040740967,
+ "logits/rejected": 1.385709524154663,
+ "logps/chosen": -159.98593139648438,
+ "logps/rejected": -144.0281219482422,
+ "loss": 0.6825335502624512,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.1235724687576294,
+ "rewards/margins": 0.03918968886137009,
+ "rewards/rejected": 0.0843827873468399,
+ "step": 2390
+ },
+ {
+ "epoch": 1.92,
+ "grad_norm": 1.1941649913787842,
+ "learning_rate": 4.002962962962963e-07,
+ "logits/chosen": 1.3726251125335693,
+ "logits/rejected": 1.5375208854675293,
+ "logps/chosen": -149.9639129638672,
+ "logps/rejected": -142.49295043945312,
+ "loss": 0.6461881160736084,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.12773704528808594,
+ "rewards/margins": 0.1104915589094162,
+ "rewards/rejected": 0.017245473340153694,
+ "step": 2400
+ },
+ {
+ "epoch": 1.92,
+ "eval_logits/chosen": 1.4780151844024658,
+ "eval_logits/rejected": 1.506591558456421,
+ "eval_logps/chosen": -152.50015258789062,
+ "eval_logps/rejected": -147.30506896972656,
+ "eval_loss": 0.6834847331047058,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": 0.10486816614866257,
+ "eval_rewards/margins": 0.03315889462828636,
+ "eval_rewards/rejected": 0.0717092752456665,
+ "eval_runtime": 90.9835,
+ "eval_samples_per_second": 5.496,
+ "eval_steps_per_second": 2.748,
+ "step": 2400
+ },
+ {
+ "epoch": 1.928,
+ "grad_norm": 1.1473901271820068,
+ "learning_rate": 3.973333333333333e-07,
+ "logits/chosen": 1.3499658107757568,
+ "logits/rejected": 1.479353666305542,
+ "logps/chosen": -153.15272521972656,
+ "logps/rejected": -170.32418823242188,
+ "loss": 0.6756062507629395,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.09217077493667603,
+ "rewards/margins": 0.05041539669036865,
+ "rewards/rejected": 0.041755370795726776,
+ "step": 2410
+ },
+ {
+ "epoch": 1.936,
+ "grad_norm": 1.4407395124435425,
+ "learning_rate": 3.9437037037037037e-07,
+ "logits/chosen": 1.5876386165618896,
+ "logits/rejected": 1.5701191425323486,
+ "logps/chosen": -144.0182342529297,
+ "logps/rejected": -130.75460815429688,
+ "loss": 0.6733745574951172,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.1180434376001358,
+ "rewards/margins": 0.050890080630779266,
+ "rewards/rejected": 0.06715335696935654,
+ "step": 2420
+ },
+ {
+ "epoch": 1.944,
+ "grad_norm": 1.46868097782135,
+ "learning_rate": 3.914074074074074e-07,
+ "logits/chosen": 1.4809507131576538,
+ "logits/rejected": 1.4793384075164795,
+ "logps/chosen": -144.4136199951172,
+ "logps/rejected": -140.72645568847656,
+ "loss": 0.65852952003479,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.13238464295864105,
+ "rewards/margins": 0.08860992640256882,
+ "rewards/rejected": 0.043774720281362534,
+ "step": 2430
+ },
+ {
+ "epoch": 1.952,
+ "grad_norm": 1.2951419353485107,
+ "learning_rate": 3.884444444444444e-07,
+ "logits/chosen": 1.4519541263580322,
+ "logits/rejected": 1.6060655117034912,
+ "logps/chosen": -150.94955444335938,
+ "logps/rejected": -146.84732055664062,
+ "loss": 0.650700569152832,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.14693069458007812,
+ "rewards/margins": 0.09734389185905457,
+ "rewards/rejected": 0.04958680272102356,
+ "step": 2440
+ },
+ {
+ "epoch": 1.96,
+ "grad_norm": 1.2181923389434814,
+ "learning_rate": 3.854814814814815e-07,
+ "logits/chosen": 1.5463917255401611,
+ "logits/rejected": 1.4773080348968506,
+ "logps/chosen": -150.07005310058594,
+ "logps/rejected": -163.2088165283203,
+ "loss": 0.6521890163421631,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.15517377853393555,
+ "rewards/margins": 0.09567751735448837,
+ "rewards/rejected": 0.05949625372886658,
+ "step": 2450
+ },
+ {
+ "epoch": 1.968,
+ "grad_norm": 1.2294185161590576,
+ "learning_rate": 3.825185185185185e-07,
+ "logits/chosen": 1.4533751010894775,
+ "logits/rejected": 1.4323598146438599,
+ "logps/chosen": -156.0291290283203,
+ "logps/rejected": -161.64663696289062,
+ "loss": 0.6698451042175293,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.07660157233476639,
+ "rewards/margins": 0.05659238249063492,
+ "rewards/rejected": 0.020009174942970276,
+ "step": 2460
+ },
+ {
+ "epoch": 1.976,
+ "grad_norm": 1.0025874376296997,
+ "learning_rate": 3.7955555555555553e-07,
+ "logits/chosen": 1.6077829599380493,
+ "logits/rejected": 1.502532720565796,
+ "logps/chosen": -168.87271118164062,
+ "logps/rejected": -160.9680633544922,
+ "loss": 0.6690214157104493,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.11292171478271484,
+ "rewards/margins": 0.06120762228965759,
+ "rewards/rejected": 0.05171409994363785,
+ "step": 2470
+ },
+ {
+ "epoch": 1.984,
+ "grad_norm": 1.1663116216659546,
+ "learning_rate": 3.7659259259259255e-07,
+ "logits/chosen": 1.6099679470062256,
+ "logits/rejected": 1.6698023080825806,
+ "logps/chosen": -147.36190795898438,
+ "logps/rejected": -145.37986755371094,
+ "loss": 0.666702127456665,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1266266405582428,
+ "rewards/margins": 0.0683312863111496,
+ "rewards/rejected": 0.0582953505218029,
+ "step": 2480
+ },
+ {
+ "epoch": 1.992,
+ "grad_norm": 1.1562272310256958,
+ "learning_rate": 3.7362962962962963e-07,
+ "logits/chosen": 1.5222467184066772,
+ "logits/rejected": 1.592881441116333,
+ "logps/chosen": -145.1186065673828,
+ "logps/rejected": -158.30718994140625,
+ "loss": 0.6763694763183594,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.09335663169622421,
+ "rewards/margins": 0.04241436719894409,
+ "rewards/rejected": 0.05094226449728012,
+ "step": 2490
+ },
+ {
+ "epoch": 2.0,
+ "grad_norm": 1.5228629112243652,
+ "learning_rate": 3.7066666666666665e-07,
+ "logits/chosen": 1.5318562984466553,
+ "logits/rejected": 1.5440677404403687,
+ "logps/chosen": -178.16098022460938,
+ "logps/rejected": -143.88893127441406,
+ "loss": 0.6654858112335205,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0874411091208458,
+ "rewards/margins": 0.07056570053100586,
+ "rewards/rejected": 0.01687539741396904,
+ "step": 2500
+ },
+ {
+ "epoch": 2.0,
+ "eval_logits/chosen": 1.4583442211151123,
+ "eval_logits/rejected": 1.486719012260437,
+ "eval_logps/chosen": -152.8921661376953,
+ "eval_logps/rejected": -147.6856231689453,
+ "eval_loss": 0.6842741370201111,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.06566743552684784,
+ "eval_rewards/margins": 0.032015930861234665,
+ "eval_rewards/rejected": 0.03365150839090347,
+ "eval_runtime": 90.8716,
+ "eval_samples_per_second": 5.502,
+ "eval_steps_per_second": 2.751,
+ "step": 2500
+ }
+ ],
+ "logging_steps": 10,
+ "max_steps": 3750,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-2500/training_args.bin b/v3/DPO/DPO_10k/lora/checkpoint-2500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..b3fc3d76093802087869ab1fee5e37b9c830fe77
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-2500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4f90e4a7053b7fb440d68cb64a3d198396af09db942d08839c5f4f0f0b8a0c8
+size 6097
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3000/README.md b/v3/DPO/DPO_10k/lora/checkpoint-3000/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..250500f798b4682b2cd2c35cd1fc366677c215dd
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3000/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3000/adapter_config.json b/v3/DPO/DPO_10k/lora/checkpoint-3000/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..039869f8d79ca6c48a793a29a79267da6d8f81c2
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3000/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "gate_proj",
+ "down_proj",
+ "k_proj",
+ "o_proj",
+ "up_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3000/adapter_model.safetensors b/v3/DPO/DPO_10k/lora/checkpoint-3000/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ed498c87182a5634d2829dce5cae507fbb7d2c50
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3000/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e9ff81fe31a7cd29ea7c74f92830690e59d6d11f974233cf1b0ddf5a3982332
+size 180385008
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3000/chat_template.jinja b/v3/DPO/DPO_10k/lora/checkpoint-3000/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3000/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3000/optimizer.pt b/v3/DPO/DPO_10k/lora/checkpoint-3000/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..843e35987db8606dfc41f7da46975e2cf50e6bc6
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3000/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed53f06b6576bfdd8980021882e3eab0834e4f3ea31639e14895880e02187047
+size 360902475
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3000/rng_state.pth b/v3/DPO/DPO_10k/lora/checkpoint-3000/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..33ab1511c4dbfc6e61dcb7f70233792670b01e19
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3000/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:428aed83766f09a6a2d9ccfb86972eeda646225b3b0cd6accec4a90e4f900161
+size 14645
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3000/scaler.pt b/v3/DPO/DPO_10k/lora/checkpoint-3000/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..84673265a8f8292da7ed0b31c5c2f284eeb9ab95
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3000/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ac1c46a2776d12775d23d0f587efc112188137ce2140da35bc15d301c9f620e
+size 1383
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3000/scheduler.pt b/v3/DPO/DPO_10k/lora/checkpoint-3000/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..45f7b92c251ac6237a77ae3319fe897fac0e107e
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3000/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc9bf509db739d921a0910005a8a83efa2601a567bd1c9652058812b1fcd4225
+size 1465
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3000/tokenizer.json b/v3/DPO/DPO_10k/lora/checkpoint-3000/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3000/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3000/tokenizer_config.json b/v3/DPO/DPO_10k/lora/checkpoint-3000/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3000/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3000/trainer_state.json b/v3/DPO/DPO_10k/lora/checkpoint-3000/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..9767bc68845c6087f6bdae03ec69e50684f3e068
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3000/trainer_state.json
@@ -0,0 +1,5014 @@
+{
+ "best_global_step": 2400,
+ "best_metric": 0.6834847331047058,
+ "best_model_checkpoint": "output/lora/checkpoint-1000",
+ "epoch": 2.4,
+ "eval_steps": 100,
+ "global_step": 3000,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.008,
+ "grad_norm": 1.1179773807525635,
+ "learning_rate": 2.4e-08,
+ "logits/chosen": 1.4545083045959473,
+ "logits/rejected": 1.425490379333496,
+ "logps/chosen": -132.46189880371094,
+ "logps/rejected": -148.20260620117188,
+ "loss": 0.6930626392364502,
+ "rewards/accuracies": 0.21250000596046448,
+ "rewards/chosen": -6.165739614516497e-05,
+ "rewards/margins": 0.00017333509458694607,
+ "rewards/rejected": -0.00023499250528402627,
+ "step": 10
+ },
+ {
+ "epoch": 0.016,
+ "grad_norm": 1.3502839803695679,
+ "learning_rate": 5.0666666666666664e-08,
+ "logits/chosen": 1.5668801069259644,
+ "logits/rejected": 1.5191389322280884,
+ "logps/chosen": -157.63624572753906,
+ "logps/rejected": -144.2716827392578,
+ "loss": 0.6930380821228027,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.0009244013344869018,
+ "rewards/margins": 0.00023505210992880166,
+ "rewards/rejected": 0.0006893490790389478,
+ "step": 20
+ },
+ {
+ "epoch": 0.024,
+ "grad_norm": 1.0219553709030151,
+ "learning_rate": 7.733333333333334e-08,
+ "logits/chosen": 1.8921797275543213,
+ "logits/rejected": 1.7342685461044312,
+ "logps/chosen": -164.55799865722656,
+ "logps/rejected": -158.92752075195312,
+ "loss": 0.69290189743042,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0009391927160322666,
+ "rewards/margins": 0.0005032324115745723,
+ "rewards/rejected": 0.0004359603044576943,
+ "step": 30
+ },
+ {
+ "epoch": 0.032,
+ "grad_norm": 1.2056113481521606,
+ "learning_rate": 1.0399999999999999e-07,
+ "logits/chosen": 1.4950047731399536,
+ "logits/rejected": 1.6163270473480225,
+ "logps/chosen": -145.54183959960938,
+ "logps/rejected": -134.19979858398438,
+ "loss": 0.6935500621795654,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.0006421065190806985,
+ "rewards/margins": -0.0007943272357806563,
+ "rewards/rejected": 0.0014364338712766767,
+ "step": 40
+ },
+ {
+ "epoch": 0.04,
+ "grad_norm": 1.1029725074768066,
+ "learning_rate": 1.3066666666666665e-07,
+ "logits/chosen": 1.4076533317565918,
+ "logits/rejected": 1.4791977405548096,
+ "logps/chosen": -133.23324584960938,
+ "logps/rejected": -138.3593292236328,
+ "loss": 0.6925833702087403,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.0012372838100418448,
+ "rewards/margins": 0.001139522879384458,
+ "rewards/rejected": 9.776116348803043e-05,
+ "step": 50
+ },
+ {
+ "epoch": 0.048,
+ "grad_norm": 0.9960266351699829,
+ "learning_rate": 1.573333333333333e-07,
+ "logits/chosen": 1.407166600227356,
+ "logits/rejected": 1.376042127609253,
+ "logps/chosen": -137.8921661376953,
+ "logps/rejected": -127.97320556640625,
+ "loss": 0.693614149093628,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.00021316049969755113,
+ "rewards/margins": -0.0009249996510334313,
+ "rewards/rejected": 0.0007118391804397106,
+ "step": 60
+ },
+ {
+ "epoch": 0.056,
+ "grad_norm": 1.0946824550628662,
+ "learning_rate": 1.8399999999999998e-07,
+ "logits/chosen": 1.5944501161575317,
+ "logits/rejected": 1.7248146533966064,
+ "logps/chosen": -153.5526123046875,
+ "logps/rejected": -157.19735717773438,
+ "loss": 0.6928449153900147,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0026370862033218145,
+ "rewards/margins": 0.0006150150438770652,
+ "rewards/rejected": 0.0020220710430294275,
+ "step": 70
+ },
+ {
+ "epoch": 0.064,
+ "grad_norm": 1.0322489738464355,
+ "learning_rate": 2.1066666666666665e-07,
+ "logits/chosen": 1.4586353302001953,
+ "logits/rejected": 1.4842995405197144,
+ "logps/chosen": -149.03665161132812,
+ "logps/rejected": -136.24081420898438,
+ "loss": 0.6932015895843506,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.004061593674123287,
+ "rewards/margins": -9.99594121822156e-05,
+ "rewards/rejected": 0.004161553457379341,
+ "step": 80
+ },
+ {
+ "epoch": 0.072,
+ "grad_norm": 1.0012447834014893,
+ "learning_rate": 2.3733333333333334e-07,
+ "logits/chosen": 1.5075902938842773,
+ "logits/rejected": 1.5986944437026978,
+ "logps/chosen": -159.3433074951172,
+ "logps/rejected": -151.85072326660156,
+ "loss": 0.6928757190704345,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.006978762801736593,
+ "rewards/margins": 0.0005627512582577765,
+ "rewards/rejected": 0.006416010670363903,
+ "step": 90
+ },
+ {
+ "epoch": 0.08,
+ "grad_norm": 1.280081033706665,
+ "learning_rate": 2.64e-07,
+ "logits/chosen": 1.448286533355713,
+ "logits/rejected": 1.5480695962905884,
+ "logps/chosen": -151.12625122070312,
+ "logps/rejected": -171.24545288085938,
+ "loss": 0.6931126117706299,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.00693341251462698,
+ "rewards/margins": 9.057526767719537e-05,
+ "rewards/rejected": 0.006842837668955326,
+ "step": 100
+ },
+ {
+ "epoch": 0.08,
+ "eval_logits/chosen": 1.518183946609497,
+ "eval_logits/rejected": 1.5525447130203247,
+ "eval_logps/chosen": -153.46630859375,
+ "eval_logps/rejected": -147.95005798339844,
+ "eval_loss": 0.6926330327987671,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.008253541775047779,
+ "eval_rewards/margins": 0.0010449704714119434,
+ "eval_rewards/rejected": 0.007208569906651974,
+ "eval_runtime": 91.0812,
+ "eval_samples_per_second": 5.49,
+ "eval_steps_per_second": 2.745,
+ "step": 100
+ },
+ {
+ "epoch": 0.088,
+ "grad_norm": 1.3912373781204224,
+ "learning_rate": 2.906666666666667e-07,
+ "logits/chosen": 1.6896222829818726,
+ "logits/rejected": 1.6516921520233154,
+ "logps/chosen": -173.70030212402344,
+ "logps/rejected": -155.5503692626953,
+ "loss": 0.6926521778106689,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.010179834440350533,
+ "rewards/margins": 0.0010137532372027636,
+ "rewards/rejected": 0.009166081435978413,
+ "step": 110
+ },
+ {
+ "epoch": 0.096,
+ "grad_norm": 1.1518888473510742,
+ "learning_rate": 3.173333333333333e-07,
+ "logits/chosen": 1.5166860818862915,
+ "logits/rejected": 1.5136979818344116,
+ "logps/chosen": -136.39906311035156,
+ "logps/rejected": -140.86561584472656,
+ "loss": 0.6932918548583984,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": 0.00907944981008768,
+ "rewards/margins": -0.00026998287648893893,
+ "rewards/rejected": 0.009349432773888111,
+ "step": 120
+ },
+ {
+ "epoch": 0.104,
+ "grad_norm": 1.3213376998901367,
+ "learning_rate": 3.4399999999999996e-07,
+ "logits/chosen": 1.4598973989486694,
+ "logits/rejected": 1.6739647388458252,
+ "logps/chosen": -159.12461853027344,
+ "logps/rejected": -163.39346313476562,
+ "loss": 0.693086051940918,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.010477779433131218,
+ "rewards/margins": 0.00014539004769176245,
+ "rewards/rejected": 0.010332388803362846,
+ "step": 130
+ },
+ {
+ "epoch": 0.112,
+ "grad_norm": 0.8833863735198975,
+ "learning_rate": 3.7066666666666665e-07,
+ "logits/chosen": 1.3829412460327148,
+ "logits/rejected": 1.4100173711776733,
+ "logps/chosen": -132.7149658203125,
+ "logps/rejected": -126.85478210449219,
+ "loss": 0.6932382583618164,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": 0.010262183845043182,
+ "rewards/margins": -0.0001672578218858689,
+ "rewards/rejected": 0.010429441928863525,
+ "step": 140
+ },
+ {
+ "epoch": 0.12,
+ "grad_norm": 1.1472455263137817,
+ "learning_rate": 3.973333333333333e-07,
+ "logits/chosen": 1.4954869747161865,
+ "logits/rejected": 1.5857340097427368,
+ "logps/chosen": -165.10568237304688,
+ "logps/rejected": -158.97433471679688,
+ "loss": 0.6916103363037109,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.014177520759403706,
+ "rewards/margins": 0.003108317730948329,
+ "rewards/rejected": 0.011069201864302158,
+ "step": 150
+ },
+ {
+ "epoch": 0.128,
+ "grad_norm": 1.0628798007965088,
+ "learning_rate": 4.24e-07,
+ "logits/chosen": 1.6717243194580078,
+ "logits/rejected": 1.705092191696167,
+ "logps/chosen": -159.332275390625,
+ "logps/rejected": -147.21646118164062,
+ "loss": 0.6935629844665527,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": 0.016994481906294823,
+ "rewards/margins": -0.0007874655420891941,
+ "rewards/rejected": 0.017781946808099747,
+ "step": 160
+ },
+ {
+ "epoch": 0.136,
+ "grad_norm": 1.390810251235962,
+ "learning_rate": 4.506666666666666e-07,
+ "logits/chosen": 1.4719234704971313,
+ "logits/rejected": 1.499526023864746,
+ "logps/chosen": -148.67251586914062,
+ "logps/rejected": -161.58021545410156,
+ "loss": 0.6926599979400635,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.021239640191197395,
+ "rewards/margins": 0.0010346340714022517,
+ "rewards/rejected": 0.020205006003379822,
+ "step": 170
+ },
+ {
+ "epoch": 0.144,
+ "grad_norm": 1.139707326889038,
+ "learning_rate": 4.773333333333333e-07,
+ "logits/chosen": 1.4833905696868896,
+ "logits/rejected": 1.637584924697876,
+ "logps/chosen": -158.96731567382812,
+ "logps/rejected": -161.9830322265625,
+ "loss": 0.6919547557830811,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.019416773691773415,
+ "rewards/margins": 0.0024296878837049007,
+ "rewards/rejected": 0.016987085342407227,
+ "step": 180
+ },
+ {
+ "epoch": 0.152,
+ "grad_norm": 1.2029796838760376,
+ "learning_rate": 5.04e-07,
+ "logits/chosen": 1.5398027896881104,
+ "logits/rejected": 1.6678705215454102,
+ "logps/chosen": -143.00283813476562,
+ "logps/rejected": -144.89378356933594,
+ "loss": 0.692262840270996,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.018960200250148773,
+ "rewards/margins": 0.0018175147706642747,
+ "rewards/rejected": 0.017142686992883682,
+ "step": 190
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 1.0527437925338745,
+ "learning_rate": 5.306666666666665e-07,
+ "logits/chosen": 1.6087623834609985,
+ "logits/rejected": 1.5181543827056885,
+ "logps/chosen": -159.35073852539062,
+ "logps/rejected": -133.59890747070312,
+ "loss": 0.6924069881439209,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.01926913857460022,
+ "rewards/margins": 0.0015339398523792624,
+ "rewards/rejected": 0.0177351962774992,
+ "step": 200
+ },
+ {
+ "epoch": 0.16,
+ "eval_logits/chosen": 1.5193672180175781,
+ "eval_logits/rejected": 1.5534639358520508,
+ "eval_logps/chosen": -153.34857177734375,
+ "eval_logps/rejected": -147.8425750732422,
+ "eval_loss": 0.6921458840370178,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.020027177408337593,
+ "eval_rewards/margins": 0.0020714527927339077,
+ "eval_rewards/rejected": 0.017955724149942398,
+ "eval_runtime": 90.9762,
+ "eval_samples_per_second": 5.496,
+ "eval_steps_per_second": 2.748,
+ "step": 200
+ },
+ {
+ "epoch": 0.168,
+ "grad_norm": 1.1897239685058594,
+ "learning_rate": 5.573333333333333e-07,
+ "logits/chosen": 1.6267732381820679,
+ "logits/rejected": 1.5775268077850342,
+ "logps/chosen": -159.51437377929688,
+ "logps/rejected": -170.17762756347656,
+ "loss": 0.6925770759582519,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.02078000269830227,
+ "rewards/margins": 0.0012053751852363348,
+ "rewards/rejected": 0.01957462914288044,
+ "step": 210
+ },
+ {
+ "epoch": 0.176,
+ "grad_norm": 0.9995236396789551,
+ "learning_rate": 5.839999999999999e-07,
+ "logits/chosen": 1.5781934261322021,
+ "logits/rejected": 1.642251968383789,
+ "logps/chosen": -159.95309448242188,
+ "logps/rejected": -144.36058044433594,
+ "loss": 0.6931475639343262,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.023322442546486855,
+ "rewards/margins": 6.392716750269756e-05,
+ "rewards/rejected": 0.02325851283967495,
+ "step": 220
+ },
+ {
+ "epoch": 0.184,
+ "grad_norm": 0.8819260597229004,
+ "learning_rate": 6.106666666666666e-07,
+ "logits/chosen": 1.459991693496704,
+ "logits/rejected": 1.6060739755630493,
+ "logps/chosen": -162.04745483398438,
+ "logps/rejected": -139.5411834716797,
+ "loss": 0.6919463634490967,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.026052182540297508,
+ "rewards/margins": 0.002481849165633321,
+ "rewards/rejected": 0.023570330813527107,
+ "step": 230
+ },
+ {
+ "epoch": 0.192,
+ "grad_norm": 1.1055208444595337,
+ "learning_rate": 6.373333333333333e-07,
+ "logits/chosen": 1.558739185333252,
+ "logits/rejected": 1.5965116024017334,
+ "logps/chosen": -171.9166259765625,
+ "logps/rejected": -145.66433715820312,
+ "loss": 0.6912176132202148,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.033868782222270966,
+ "rewards/margins": 0.004017929546535015,
+ "rewards/rejected": 0.029850851744413376,
+ "step": 240
+ },
+ {
+ "epoch": 0.2,
+ "grad_norm": 1.1372733116149902,
+ "learning_rate": 6.64e-07,
+ "logits/chosen": 1.5322860479354858,
+ "logits/rejected": 1.4695795774459839,
+ "logps/chosen": -161.7803955078125,
+ "logps/rejected": -148.91949462890625,
+ "loss": 0.6940969944000244,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.034065864980220795,
+ "rewards/margins": -0.0016761379083618522,
+ "rewards/rejected": 0.035742007195949554,
+ "step": 250
+ },
+ {
+ "epoch": 0.208,
+ "grad_norm": 1.0917680263519287,
+ "learning_rate": 6.906666666666666e-07,
+ "logits/chosen": 1.4094547033309937,
+ "logits/rejected": 1.50186026096344,
+ "logps/chosen": -132.91455078125,
+ "logps/rejected": -146.79714965820312,
+ "loss": 0.6930203914642334,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.03089061938226223,
+ "rewards/margins": 0.00040971505222842097,
+ "rewards/rejected": 0.030480902642011642,
+ "step": 260
+ },
+ {
+ "epoch": 0.216,
+ "grad_norm": 1.0280259847640991,
+ "learning_rate": 7.173333333333333e-07,
+ "logits/chosen": 1.531285285949707,
+ "logits/rejected": 1.6163132190704346,
+ "logps/chosen": -146.902587890625,
+ "logps/rejected": -147.8347625732422,
+ "loss": 0.6944749355316162,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.028514739125967026,
+ "rewards/margins": -0.0025452517438679934,
+ "rewards/rejected": 0.0310599897056818,
+ "step": 270
+ },
+ {
+ "epoch": 0.224,
+ "grad_norm": 1.398708701133728,
+ "learning_rate": 7.44e-07,
+ "logits/chosen": 1.563852071762085,
+ "logits/rejected": 1.396703839302063,
+ "logps/chosen": -172.68153381347656,
+ "logps/rejected": -155.80215454101562,
+ "loss": 0.689476490020752,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03224216774106026,
+ "rewards/margins": 0.007473687641322613,
+ "rewards/rejected": 0.02476847730576992,
+ "step": 280
+ },
+ {
+ "epoch": 0.232,
+ "grad_norm": 1.0394141674041748,
+ "learning_rate": 7.706666666666667e-07,
+ "logits/chosen": 1.5581729412078857,
+ "logits/rejected": 1.5819021463394165,
+ "logps/chosen": -144.41465759277344,
+ "logps/rejected": -153.55406188964844,
+ "loss": 0.6927877902984619,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.03272664546966553,
+ "rewards/margins": 0.0008557910332456231,
+ "rewards/rejected": 0.031870849430561066,
+ "step": 290
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 1.2557185888290405,
+ "learning_rate": 7.973333333333333e-07,
+ "logits/chosen": 1.5062446594238281,
+ "logits/rejected": 1.5786281824111938,
+ "logps/chosen": -153.63558959960938,
+ "logps/rejected": -165.46617126464844,
+ "loss": 0.6897892951965332,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03914078325033188,
+ "rewards/margins": 0.006873926613479853,
+ "rewards/rejected": 0.032266855239868164,
+ "step": 300
+ },
+ {
+ "epoch": 0.24,
+ "eval_logits/chosen": 1.5237834453582764,
+ "eval_logits/rejected": 1.557741641998291,
+ "eval_logps/chosen": -153.19883728027344,
+ "eval_logps/rejected": -147.7100830078125,
+ "eval_loss": 0.6913501024246216,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.03499976545572281,
+ "eval_rewards/margins": 0.0037925743963569403,
+ "eval_rewards/rejected": 0.031207194551825523,
+ "eval_runtime": 90.9133,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 300
+ },
+ {
+ "epoch": 0.248,
+ "grad_norm": 1.1200144290924072,
+ "learning_rate": 8.24e-07,
+ "logits/chosen": 1.618949294090271,
+ "logits/rejected": 1.6625702381134033,
+ "logps/chosen": -141.8953857421875,
+ "logps/rejected": -156.6927032470703,
+ "loss": 0.694425106048584,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.03162423521280289,
+ "rewards/margins": -0.0023833750747144222,
+ "rewards/rejected": 0.034007612615823746,
+ "step": 310
+ },
+ {
+ "epoch": 0.256,
+ "grad_norm": 1.0823743343353271,
+ "learning_rate": 8.506666666666667e-07,
+ "logits/chosen": 1.5974153280258179,
+ "logits/rejected": 1.6237560510635376,
+ "logps/chosen": -139.2996368408203,
+ "logps/rejected": -141.94407653808594,
+ "loss": 0.6914949893951416,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.029445698484778404,
+ "rewards/margins": 0.0034329916816204786,
+ "rewards/rejected": 0.026012707501649857,
+ "step": 320
+ },
+ {
+ "epoch": 0.264,
+ "grad_norm": 1.1765722036361694,
+ "learning_rate": 8.773333333333332e-07,
+ "logits/chosen": 1.465215802192688,
+ "logits/rejected": 1.4882014989852905,
+ "logps/chosen": -158.4329833984375,
+ "logps/rejected": -183.94259643554688,
+ "loss": 0.6925934791564942,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.026302698999643326,
+ "rewards/margins": 0.0012350418837741017,
+ "rewards/rejected": 0.025067657232284546,
+ "step": 330
+ },
+ {
+ "epoch": 0.272,
+ "grad_norm": 1.0667294263839722,
+ "learning_rate": 9.039999999999999e-07,
+ "logits/chosen": 1.7538875341415405,
+ "logits/rejected": 1.6171789169311523,
+ "logps/chosen": -159.63790893554688,
+ "logps/rejected": -151.1880340576172,
+ "loss": 0.6922041416168213,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.026619985699653625,
+ "rewards/margins": 0.002031774492934346,
+ "rewards/rejected": 0.024588212370872498,
+ "step": 340
+ },
+ {
+ "epoch": 0.28,
+ "grad_norm": 1.2210006713867188,
+ "learning_rate": 9.306666666666666e-07,
+ "logits/chosen": 1.5815479755401611,
+ "logits/rejected": 1.710217833518982,
+ "logps/chosen": -167.21743774414062,
+ "logps/rejected": -196.77989196777344,
+ "loss": 0.6899692058563233,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.03636014088988304,
+ "rewards/margins": 0.006536121014505625,
+ "rewards/rejected": 0.029824022203683853,
+ "step": 350
+ },
+ {
+ "epoch": 0.288,
+ "grad_norm": 0.9623845219612122,
+ "learning_rate": 9.573333333333333e-07,
+ "logits/chosen": 1.4416790008544922,
+ "logits/rejected": 1.3618693351745605,
+ "logps/chosen": -163.27401733398438,
+ "logps/rejected": -169.3351287841797,
+ "loss": 0.6910821437835694,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.037200987339019775,
+ "rewards/margins": 0.004338678438216448,
+ "rewards/rejected": 0.032862309366464615,
+ "step": 360
+ },
+ {
+ "epoch": 0.296,
+ "grad_norm": 1.1005645990371704,
+ "learning_rate": 9.84e-07,
+ "logits/chosen": 1.6214616298675537,
+ "logits/rejected": 1.6572606563568115,
+ "logps/chosen": -162.95504760742188,
+ "logps/rejected": -154.91807556152344,
+ "loss": 0.6895030021667481,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.047198958694934845,
+ "rewards/margins": 0.0075958347879350185,
+ "rewards/rejected": 0.03960312157869339,
+ "step": 370
+ },
+ {
+ "epoch": 0.304,
+ "grad_norm": 1.1126338243484497,
+ "learning_rate": 9.988148148148148e-07,
+ "logits/chosen": 1.4842126369476318,
+ "logits/rejected": 1.410930871963501,
+ "logps/chosen": -143.09182739257812,
+ "logps/rejected": -153.48135375976562,
+ "loss": 0.6885370254516602,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.05975018069148064,
+ "rewards/margins": 0.00981281790882349,
+ "rewards/rejected": 0.04993735998868942,
+ "step": 380
+ },
+ {
+ "epoch": 0.312,
+ "grad_norm": 1.1146304607391357,
+ "learning_rate": 9.95851851851852e-07,
+ "logits/chosen": 1.608332633972168,
+ "logits/rejected": 1.5848472118377686,
+ "logps/chosen": -152.25250244140625,
+ "logps/rejected": -138.348876953125,
+ "loss": 0.6851807117462159,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.07630724459886551,
+ "rewards/margins": 0.016460279002785683,
+ "rewards/rejected": 0.059846967458724976,
+ "step": 390
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 1.2686601877212524,
+ "learning_rate": 9.928888888888889e-07,
+ "logits/chosen": 1.5840070247650146,
+ "logits/rejected": 1.5679785013198853,
+ "logps/chosen": -155.90219116210938,
+ "logps/rejected": -148.89549255371094,
+ "loss": 0.6891039371490478,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.08129279315471649,
+ "rewards/margins": 0.008997146971523762,
+ "rewards/rejected": 0.072295643389225,
+ "step": 400
+ },
+ {
+ "epoch": 0.32,
+ "eval_logits/chosen": 1.5358340740203857,
+ "eval_logits/rejected": 1.5689741373062134,
+ "eval_logps/chosen": -152.6895294189453,
+ "eval_logps/rejected": -147.23422241210938,
+ "eval_loss": 0.6900544166564941,
+ "eval_rewards/accuracies": 0.5419999957084656,
+ "eval_rewards/chosen": 0.08593228459358215,
+ "eval_rewards/margins": 0.0071397931315004826,
+ "eval_rewards/rejected": 0.0787924975156784,
+ "eval_runtime": 90.9072,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 400
+ },
+ {
+ "epoch": 0.328,
+ "grad_norm": 1.1653108596801758,
+ "learning_rate": 9.899259259259258e-07,
+ "logits/chosen": 1.5450490713119507,
+ "logits/rejected": 1.4428151845932007,
+ "logps/chosen": -148.61581420898438,
+ "logps/rejected": -188.90919494628906,
+ "loss": 0.6918989181518554,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0946170911192894,
+ "rewards/margins": 0.003271549940109253,
+ "rewards/rejected": 0.09134554117918015,
+ "step": 410
+ },
+ {
+ "epoch": 0.336,
+ "grad_norm": 1.3114937543869019,
+ "learning_rate": 9.86962962962963e-07,
+ "logits/chosen": 1.5342715978622437,
+ "logits/rejected": 1.5572118759155273,
+ "logps/chosen": -152.65264892578125,
+ "logps/rejected": -168.03384399414062,
+ "loss": 0.6972955703735352,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.08869597315788269,
+ "rewards/margins": -0.007436127867549658,
+ "rewards/rejected": 0.09613210707902908,
+ "step": 420
+ },
+ {
+ "epoch": 0.344,
+ "grad_norm": 1.0040781497955322,
+ "learning_rate": 9.84e-07,
+ "logits/chosen": 1.5979712009429932,
+ "logits/rejected": 1.6713664531707764,
+ "logps/chosen": -164.2566375732422,
+ "logps/rejected": -152.1428680419922,
+ "loss": 0.6897010326385498,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.08101107180118561,
+ "rewards/margins": 0.007684691343456507,
+ "rewards/rejected": 0.07332637906074524,
+ "step": 430
+ },
+ {
+ "epoch": 0.352,
+ "grad_norm": 1.0801892280578613,
+ "learning_rate": 9.81037037037037e-07,
+ "logits/chosen": 1.439635992050171,
+ "logits/rejected": 1.3282365798950195,
+ "logps/chosen": -157.23403930664062,
+ "logps/rejected": -146.54806518554688,
+ "loss": 0.6881664752960205,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.08112544566392899,
+ "rewards/margins": 0.010971959680318832,
+ "rewards/rejected": 0.07015348225831985,
+ "step": 440
+ },
+ {
+ "epoch": 0.36,
+ "grad_norm": 1.1139949560165405,
+ "learning_rate": 9.78074074074074e-07,
+ "logits/chosen": 1.6345123052597046,
+ "logits/rejected": 1.5876491069793701,
+ "logps/chosen": -157.6160125732422,
+ "logps/rejected": -160.02040100097656,
+ "loss": 0.6868602275848389,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.07638200372457504,
+ "rewards/margins": 0.013464884832501411,
+ "rewards/rejected": 0.06291711330413818,
+ "step": 450
+ },
+ {
+ "epoch": 0.368,
+ "grad_norm": 1.158035397529602,
+ "learning_rate": 9.751111111111112e-07,
+ "logits/chosen": 1.5430586338043213,
+ "logits/rejected": 1.6022913455963135,
+ "logps/chosen": -151.98919677734375,
+ "logps/rejected": -164.90565490722656,
+ "loss": 0.6891930580139161,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06889279186725616,
+ "rewards/margins": 0.008791089057922363,
+ "rewards/rejected": 0.0601017065346241,
+ "step": 460
+ },
+ {
+ "epoch": 0.376,
+ "grad_norm": 1.0903704166412354,
+ "learning_rate": 9.721481481481481e-07,
+ "logits/chosen": 1.313881754875183,
+ "logits/rejected": 1.3182194232940674,
+ "logps/chosen": -131.5592041015625,
+ "logps/rejected": -123.84442138671875,
+ "loss": 0.6889429569244385,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.08150745928287506,
+ "rewards/margins": 0.009235726669430733,
+ "rewards/rejected": 0.07227173447608948,
+ "step": 470
+ },
+ {
+ "epoch": 0.384,
+ "grad_norm": 1.0766969919204712,
+ "learning_rate": 9.69185185185185e-07,
+ "logits/chosen": 1.425756812095642,
+ "logits/rejected": 1.476854920387268,
+ "logps/chosen": -148.72279357910156,
+ "logps/rejected": -164.40951538085938,
+ "loss": 0.6854116439819335,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1009502187371254,
+ "rewards/margins": 0.0166789498180151,
+ "rewards/rejected": 0.08427127450704575,
+ "step": 480
+ },
+ {
+ "epoch": 0.392,
+ "grad_norm": 1.2078129053115845,
+ "learning_rate": 9.662222222222222e-07,
+ "logits/chosen": 1.5511282682418823,
+ "logits/rejected": 1.611383080482483,
+ "logps/chosen": -143.22006225585938,
+ "logps/rejected": -159.70176696777344,
+ "loss": 0.6968960762023926,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.08779363334178925,
+ "rewards/margins": -0.00598606513813138,
+ "rewards/rejected": 0.09377971291542053,
+ "step": 490
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.9106306433677673,
+ "learning_rate": 9.632592592592593e-07,
+ "logits/chosen": 1.4217239618301392,
+ "logits/rejected": 1.505446195602417,
+ "logps/chosen": -144.34109497070312,
+ "logps/rejected": -135.52430725097656,
+ "loss": 0.691196870803833,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06902458518743515,
+ "rewards/margins": 0.005059319548308849,
+ "rewards/rejected": 0.06396526843309402,
+ "step": 500
+ },
+ {
+ "epoch": 0.4,
+ "eval_logits/chosen": 1.5185325145721436,
+ "eval_logits/rejected": 1.5510345697402954,
+ "eval_logps/chosen": -152.7921905517578,
+ "eval_logps/rejected": -147.3463134765625,
+ "eval_loss": 0.6898502111434937,
+ "eval_rewards/accuracies": 0.5099999904632568,
+ "eval_rewards/chosen": 0.07566512376070023,
+ "eval_rewards/margins": 0.008083080872893333,
+ "eval_rewards/rejected": 0.06758204847574234,
+ "eval_runtime": 90.8294,
+ "eval_samples_per_second": 5.505,
+ "eval_steps_per_second": 2.752,
+ "step": 500
+ },
+ {
+ "epoch": 0.408,
+ "grad_norm": 1.1637762784957886,
+ "learning_rate": 9.602962962962962e-07,
+ "logits/chosen": 1.5965330600738525,
+ "logits/rejected": 1.5338464975357056,
+ "logps/chosen": -131.49871826171875,
+ "logps/rejected": -140.05995178222656,
+ "loss": 0.6868171215057373,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.08385131508111954,
+ "rewards/margins": 0.013950209133327007,
+ "rewards/rejected": 0.06990110129117966,
+ "step": 510
+ },
+ {
+ "epoch": 0.416,
+ "grad_norm": 1.229719638824463,
+ "learning_rate": 9.573333333333333e-07,
+ "logits/chosen": 1.7057275772094727,
+ "logits/rejected": 1.5898948907852173,
+ "logps/chosen": -166.4267120361328,
+ "logps/rejected": -172.22337341308594,
+ "loss": 0.6912094116210937,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.07427279651165009,
+ "rewards/margins": 0.004661654122173786,
+ "rewards/rejected": 0.06961113959550858,
+ "step": 520
+ },
+ {
+ "epoch": 0.424,
+ "grad_norm": 1.136383295059204,
+ "learning_rate": 9.543703703703705e-07,
+ "logits/chosen": 1.6207077503204346,
+ "logits/rejected": 1.5417520999908447,
+ "logps/chosen": -166.81509399414062,
+ "logps/rejected": -168.5408172607422,
+ "loss": 0.6772748470306397,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.0953373983502388,
+ "rewards/margins": 0.033697597682476044,
+ "rewards/rejected": 0.06163979694247246,
+ "step": 530
+ },
+ {
+ "epoch": 0.432,
+ "grad_norm": 1.0640541315078735,
+ "learning_rate": 9.514074074074074e-07,
+ "logits/chosen": 1.6825854778289795,
+ "logits/rejected": 1.6327041387557983,
+ "logps/chosen": -181.39956665039062,
+ "logps/rejected": -158.49647521972656,
+ "loss": 0.6883615493774414,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.10187923908233643,
+ "rewards/margins": 0.010873330757021904,
+ "rewards/rejected": 0.09100590646266937,
+ "step": 540
+ },
+ {
+ "epoch": 0.44,
+ "grad_norm": 1.477909803390503,
+ "learning_rate": 9.484444444444444e-07,
+ "logits/chosen": 1.5402686595916748,
+ "logits/rejected": 1.6487388610839844,
+ "logps/chosen": -151.53814697265625,
+ "logps/rejected": -167.8573455810547,
+ "loss": 0.6915213584899902,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09472338110208511,
+ "rewards/margins": 0.005475065670907497,
+ "rewards/rejected": 0.08924831449985504,
+ "step": 550
+ },
+ {
+ "epoch": 0.448,
+ "grad_norm": 1.0138317346572876,
+ "learning_rate": 9.454814814814814e-07,
+ "logits/chosen": 1.6286503076553345,
+ "logits/rejected": 1.5223101377487183,
+ "logps/chosen": -154.72079467773438,
+ "logps/rejected": -139.2535858154297,
+ "loss": 0.6835216522216797,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.09194488823413849,
+ "rewards/margins": 0.02140027843415737,
+ "rewards/rejected": 0.07054460793733597,
+ "step": 560
+ },
+ {
+ "epoch": 0.456,
+ "grad_norm": 0.896689772605896,
+ "learning_rate": 9.425185185185184e-07,
+ "logits/chosen": 1.4857146739959717,
+ "logits/rejected": 1.4369819164276123,
+ "logps/chosen": -158.31809997558594,
+ "logps/rejected": -152.33676147460938,
+ "loss": 0.6863360404968262,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.07895292341709137,
+ "rewards/margins": 0.016260787844657898,
+ "rewards/rejected": 0.06269213557243347,
+ "step": 570
+ },
+ {
+ "epoch": 0.464,
+ "grad_norm": 1.4772330522537231,
+ "learning_rate": 9.395555555555556e-07,
+ "logits/chosen": 1.315298318862915,
+ "logits/rejected": 1.4757534265518188,
+ "logps/chosen": -136.4202117919922,
+ "logps/rejected": -154.99362182617188,
+ "loss": 0.6916167259216308,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.04677204787731171,
+ "rewards/margins": 0.005649724509567022,
+ "rewards/rejected": 0.041122324764728546,
+ "step": 580
+ },
+ {
+ "epoch": 0.472,
+ "grad_norm": 0.9806157350540161,
+ "learning_rate": 9.365925925925926e-07,
+ "logits/chosen": 1.465427279472351,
+ "logits/rejected": 1.459496259689331,
+ "logps/chosen": -145.07470703125,
+ "logps/rejected": -131.71974182128906,
+ "loss": 0.6857181549072265,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.07979898899793625,
+ "rewards/margins": 0.016965944319963455,
+ "rewards/rejected": 0.0628330409526825,
+ "step": 590
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 1.4313396215438843,
+ "learning_rate": 9.336296296296295e-07,
+ "logits/chosen": 1.5190367698669434,
+ "logits/rejected": 1.563504934310913,
+ "logps/chosen": -132.0201416015625,
+ "logps/rejected": -158.89865112304688,
+ "loss": 0.6964848041534424,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.06544725596904755,
+ "rewards/margins": -0.004280097782611847,
+ "rewards/rejected": 0.0697273463010788,
+ "step": 600
+ },
+ {
+ "epoch": 0.48,
+ "eval_logits/chosen": 1.5031698942184448,
+ "eval_logits/rejected": 1.5348584651947021,
+ "eval_logps/chosen": -152.76596069335938,
+ "eval_logps/rejected": -147.3241729736328,
+ "eval_loss": 0.6903045773506165,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": 0.07828804850578308,
+ "eval_rewards/margins": 0.008492168970406055,
+ "eval_rewards/rejected": 0.0697958841919899,
+ "eval_runtime": 90.9881,
+ "eval_samples_per_second": 5.495,
+ "eval_steps_per_second": 2.748,
+ "step": 600
+ },
+ {
+ "epoch": 0.488,
+ "grad_norm": 0.9467533230781555,
+ "learning_rate": 9.306666666666666e-07,
+ "logits/chosen": 1.5242283344268799,
+ "logits/rejected": 1.5798314809799194,
+ "logps/chosen": -131.90048217773438,
+ "logps/rejected": -149.93222045898438,
+ "loss": 0.6899854183197022,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.08419154584407806,
+ "rewards/margins": 0.00830511748790741,
+ "rewards/rejected": 0.07588644325733185,
+ "step": 610
+ },
+ {
+ "epoch": 0.496,
+ "grad_norm": 1.8463492393493652,
+ "learning_rate": 9.277037037037037e-07,
+ "logits/chosen": 1.4965155124664307,
+ "logits/rejected": 1.6336272954940796,
+ "logps/chosen": -156.1863555908203,
+ "logps/rejected": -167.74075317382812,
+ "loss": 0.6977802276611328,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.05518193915486336,
+ "rewards/margins": -0.006808985956013203,
+ "rewards/rejected": 0.061990927904844284,
+ "step": 620
+ },
+ {
+ "epoch": 0.504,
+ "grad_norm": 1.278316617012024,
+ "learning_rate": 9.247407407407407e-07,
+ "logits/chosen": 1.5074065923690796,
+ "logits/rejected": 1.535696029663086,
+ "logps/chosen": -167.56777954101562,
+ "logps/rejected": -147.04922485351562,
+ "loss": 0.6806219100952149,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.0745314508676529,
+ "rewards/margins": 0.027104835957288742,
+ "rewards/rejected": 0.047426607459783554,
+ "step": 630
+ },
+ {
+ "epoch": 0.512,
+ "grad_norm": 1.0424679517745972,
+ "learning_rate": 9.217777777777778e-07,
+ "logits/chosen": 1.4299286603927612,
+ "logits/rejected": 1.4484076499938965,
+ "logps/chosen": -168.305908203125,
+ "logps/rejected": -152.61854553222656,
+ "loss": 0.6871783256530761,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07751411944627762,
+ "rewards/margins": 0.017045259475708008,
+ "rewards/rejected": 0.06046885997056961,
+ "step": 640
+ },
+ {
+ "epoch": 0.52,
+ "grad_norm": 1.1143999099731445,
+ "learning_rate": 9.188148148148148e-07,
+ "logits/chosen": 1.5520436763763428,
+ "logits/rejected": 1.531718134880066,
+ "logps/chosen": -154.92115783691406,
+ "logps/rejected": -126.78369140625,
+ "loss": 0.6889286518096924,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.0945608839392662,
+ "rewards/margins": 0.01080345083028078,
+ "rewards/rejected": 0.0837574377655983,
+ "step": 650
+ },
+ {
+ "epoch": 0.528,
+ "grad_norm": 1.2762326002120972,
+ "learning_rate": 9.158518518518517e-07,
+ "logits/chosen": 1.5747522115707397,
+ "logits/rejected": 1.4972833395004272,
+ "logps/chosen": -151.13914489746094,
+ "logps/rejected": -132.013916015625,
+ "loss": 0.6900454998016358,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.07303085923194885,
+ "rewards/margins": 0.008073865436017513,
+ "rewards/rejected": 0.06495700031518936,
+ "step": 660
+ },
+ {
+ "epoch": 0.536,
+ "grad_norm": 1.5207557678222656,
+ "learning_rate": 9.128888888888888e-07,
+ "logits/chosen": 1.6594665050506592,
+ "logits/rejected": 1.468492031097412,
+ "logps/chosen": -166.0108184814453,
+ "logps/rejected": -151.38555908203125,
+ "loss": 0.6934059143066407,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.09294265508651733,
+ "rewards/margins": 0.0021180796902626753,
+ "rewards/rejected": 0.09082455933094025,
+ "step": 670
+ },
+ {
+ "epoch": 0.544,
+ "grad_norm": 1.1073335409164429,
+ "learning_rate": 9.099259259259259e-07,
+ "logits/chosen": 1.5321934223175049,
+ "logits/rejected": 1.5313541889190674,
+ "logps/chosen": -144.5046844482422,
+ "logps/rejected": -133.05470275878906,
+ "loss": 0.6824594974517822,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.10782947391271591,
+ "rewards/margins": 0.0230915155261755,
+ "rewards/rejected": 0.08473796397447586,
+ "step": 680
+ },
+ {
+ "epoch": 0.552,
+ "grad_norm": 1.096970558166504,
+ "learning_rate": 9.069629629629629e-07,
+ "logits/chosen": 1.438305139541626,
+ "logits/rejected": 1.4338308572769165,
+ "logps/chosen": -173.1280517578125,
+ "logps/rejected": -160.20950317382812,
+ "loss": 0.6878445625305176,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09615986794233322,
+ "rewards/margins": 0.012350928969681263,
+ "rewards/rejected": 0.08380892872810364,
+ "step": 690
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 1.3664878606796265,
+ "learning_rate": 9.039999999999999e-07,
+ "logits/chosen": 1.530369758605957,
+ "logits/rejected": 1.5520622730255127,
+ "logps/chosen": -159.60740661621094,
+ "logps/rejected": -137.77352905273438,
+ "loss": 0.6991429805755616,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07249583303928375,
+ "rewards/margins": -0.009333941154181957,
+ "rewards/rejected": 0.08182977139949799,
+ "step": 700
+ },
+ {
+ "epoch": 0.56,
+ "eval_logits/chosen": 1.5136796236038208,
+ "eval_logits/rejected": 1.5456002950668335,
+ "eval_logps/chosen": -152.5748748779297,
+ "eval_logps/rejected": -147.17152404785156,
+ "eval_loss": 0.688511848449707,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.09739598631858826,
+ "eval_rewards/margins": 0.012335418723523617,
+ "eval_rewards/rejected": 0.08506056666374207,
+ "eval_runtime": 90.8734,
+ "eval_samples_per_second": 5.502,
+ "eval_steps_per_second": 2.751,
+ "step": 700
+ },
+ {
+ "epoch": 0.568,
+ "grad_norm": 1.224107265472412,
+ "learning_rate": 9.010370370370371e-07,
+ "logits/chosen": 1.6023967266082764,
+ "logits/rejected": 1.558224081993103,
+ "logps/chosen": -148.47120666503906,
+ "logps/rejected": -163.5648651123047,
+ "loss": 0.6865221500396729,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.12494877725839615,
+ "rewards/margins": 0.01545787788927555,
+ "rewards/rejected": 0.10949089378118515,
+ "step": 710
+ },
+ {
+ "epoch": 0.576,
+ "grad_norm": 1.2135164737701416,
+ "learning_rate": 8.98074074074074e-07,
+ "logits/chosen": 1.5902750492095947,
+ "logits/rejected": 1.7012746334075928,
+ "logps/chosen": -139.7012939453125,
+ "logps/rejected": -135.42225646972656,
+ "loss": 0.7014039039611817,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.09102506935596466,
+ "rewards/margins": -0.014695463702082634,
+ "rewards/rejected": 0.10572052001953125,
+ "step": 720
+ },
+ {
+ "epoch": 0.584,
+ "grad_norm": 1.1953649520874023,
+ "learning_rate": 8.95111111111111e-07,
+ "logits/chosen": 1.6540731191635132,
+ "logits/rejected": 1.6386349201202393,
+ "logps/chosen": -153.12847900390625,
+ "logps/rejected": -149.66319274902344,
+ "loss": 0.6952126502990723,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.1124584898352623,
+ "rewards/margins": -0.0012063594767823815,
+ "rewards/rejected": 0.11366486549377441,
+ "step": 730
+ },
+ {
+ "epoch": 0.592,
+ "grad_norm": 1.147875428199768,
+ "learning_rate": 8.921481481481481e-07,
+ "logits/chosen": 1.5578330755233765,
+ "logits/rejected": 1.4689290523529053,
+ "logps/chosen": -156.93788146972656,
+ "logps/rejected": -134.1832733154297,
+ "loss": 0.6919830799102783,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09214536100625992,
+ "rewards/margins": 0.004291228950023651,
+ "rewards/rejected": 0.08785412460565567,
+ "step": 740
+ },
+ {
+ "epoch": 0.6,
+ "grad_norm": 1.21685791015625,
+ "learning_rate": 8.891851851851852e-07,
+ "logits/chosen": 1.4241983890533447,
+ "logits/rejected": 1.4537718296051025,
+ "logps/chosen": -136.9163055419922,
+ "logps/rejected": -158.44491577148438,
+ "loss": 0.691422986984253,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.09173480421304703,
+ "rewards/margins": 0.00533579895272851,
+ "rewards/rejected": 0.08639900386333466,
+ "step": 750
+ },
+ {
+ "epoch": 0.608,
+ "grad_norm": 1.3629519939422607,
+ "learning_rate": 8.862222222222222e-07,
+ "logits/chosen": 1.565716028213501,
+ "logits/rejected": 1.3824570178985596,
+ "logps/chosen": -138.8521728515625,
+ "logps/rejected": -129.51028442382812,
+ "loss": 0.6931031227111817,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0763697624206543,
+ "rewards/margins": 0.0021251302678138018,
+ "rewards/rejected": 0.07424463331699371,
+ "step": 760
+ },
+ {
+ "epoch": 0.616,
+ "grad_norm": 1.1030747890472412,
+ "learning_rate": 8.832592592592593e-07,
+ "logits/chosen": 1.5781702995300293,
+ "logits/rejected": 1.5428775548934937,
+ "logps/chosen": -141.01480102539062,
+ "logps/rejected": -136.47189331054688,
+ "loss": 0.6777657032012939,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.10366280376911163,
+ "rewards/margins": 0.0333605632185936,
+ "rewards/rejected": 0.07030224055051804,
+ "step": 770
+ },
+ {
+ "epoch": 0.624,
+ "grad_norm": 1.3228758573532104,
+ "learning_rate": 8.802962962962962e-07,
+ "logits/chosen": 1.451596736907959,
+ "logits/rejected": 1.5489904880523682,
+ "logps/chosen": -153.74618530273438,
+ "logps/rejected": -159.26333618164062,
+ "loss": 0.6870471000671386,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.10403706133365631,
+ "rewards/margins": 0.014494165778160095,
+ "rewards/rejected": 0.08954289555549622,
+ "step": 780
+ },
+ {
+ "epoch": 0.632,
+ "grad_norm": 1.3267385959625244,
+ "learning_rate": 8.773333333333332e-07,
+ "logits/chosen": 1.4153831005096436,
+ "logits/rejected": 1.4497296810150146,
+ "logps/chosen": -148.05368041992188,
+ "logps/rejected": -151.3020477294922,
+ "loss": 0.6877872943878174,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.07300253212451935,
+ "rewards/margins": 0.013981563039124012,
+ "rewards/rejected": 0.05902096629142761,
+ "step": 790
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 1.2227483987808228,
+ "learning_rate": 8.743703703703703e-07,
+ "logits/chosen": 1.6773914098739624,
+ "logits/rejected": 1.6918518543243408,
+ "logps/chosen": -180.06863403320312,
+ "logps/rejected": -194.30752563476562,
+ "loss": 0.7015917778015137,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07480724155902863,
+ "rewards/margins": -0.013628068380057812,
+ "rewards/rejected": 0.08843531459569931,
+ "step": 800
+ },
+ {
+ "epoch": 0.64,
+ "eval_logits/chosen": 1.5051140785217285,
+ "eval_logits/rejected": 1.5367870330810547,
+ "eval_logps/chosen": -152.8026885986328,
+ "eval_logps/rejected": -147.39437866210938,
+ "eval_loss": 0.688633918762207,
+ "eval_rewards/accuracies": 0.5220000147819519,
+ "eval_rewards/chosen": 0.07461711764335632,
+ "eval_rewards/margins": 0.011839868500828743,
+ "eval_rewards/rejected": 0.06277725100517273,
+ "eval_runtime": 90.9206,
+ "eval_samples_per_second": 5.499,
+ "eval_steps_per_second": 2.75,
+ "step": 800
+ },
+ {
+ "epoch": 0.648,
+ "grad_norm": 1.209770679473877,
+ "learning_rate": 8.714074074074074e-07,
+ "logits/chosen": 1.3785041570663452,
+ "logits/rejected": 1.60085928440094,
+ "logps/chosen": -114.5325927734375,
+ "logps/rejected": -151.88900756835938,
+ "loss": 0.6890413761138916,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0640551745891571,
+ "rewards/margins": 0.010126596316695213,
+ "rewards/rejected": 0.05392857640981674,
+ "step": 810
+ },
+ {
+ "epoch": 0.656,
+ "grad_norm": 1.0250163078308105,
+ "learning_rate": 8.684444444444444e-07,
+ "logits/chosen": 1.5827982425689697,
+ "logits/rejected": 1.6239618062973022,
+ "logps/chosen": -149.01904296875,
+ "logps/rejected": -141.9372100830078,
+ "loss": 0.6728014469146728,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.0830039232969284,
+ "rewards/margins": 0.043090131133794785,
+ "rewards/rejected": 0.03991378843784332,
+ "step": 820
+ },
+ {
+ "epoch": 0.664,
+ "grad_norm": 1.0627655982971191,
+ "learning_rate": 8.654814814814814e-07,
+ "logits/chosen": 1.501873254776001,
+ "logits/rejected": 1.537386417388916,
+ "logps/chosen": -138.75169372558594,
+ "logps/rejected": -134.00970458984375,
+ "loss": 0.6801187992095947,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0640982910990715,
+ "rewards/margins": 0.02855527400970459,
+ "rewards/rejected": 0.03554301708936691,
+ "step": 830
+ },
+ {
+ "epoch": 0.672,
+ "grad_norm": 1.2453433275222778,
+ "learning_rate": 8.625185185185186e-07,
+ "logits/chosen": 1.4766005277633667,
+ "logits/rejected": 1.5185787677764893,
+ "logps/chosen": -148.41250610351562,
+ "logps/rejected": -157.73989868164062,
+ "loss": 0.6788079738616943,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.06551454961299896,
+ "rewards/margins": 0.03301671892404556,
+ "rewards/rejected": 0.0324978306889534,
+ "step": 840
+ },
+ {
+ "epoch": 0.68,
+ "grad_norm": 1.191698670387268,
+ "learning_rate": 8.595555555555555e-07,
+ "logits/chosen": 1.555551528930664,
+ "logits/rejected": 1.5438499450683594,
+ "logps/chosen": -160.77488708496094,
+ "logps/rejected": -141.9876251220703,
+ "loss": 0.6801597595214843,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.053498052060604095,
+ "rewards/margins": 0.028470590710639954,
+ "rewards/rejected": 0.025027472525835037,
+ "step": 850
+ },
+ {
+ "epoch": 0.688,
+ "grad_norm": 0.9382092356681824,
+ "learning_rate": 8.565925925925925e-07,
+ "logits/chosen": 1.5482738018035889,
+ "logits/rejected": 1.5833700895309448,
+ "logps/chosen": -139.68118286132812,
+ "logps/rejected": -130.05947875976562,
+ "loss": 0.6825921535491943,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.06382572650909424,
+ "rewards/margins": 0.023111844435334206,
+ "rewards/rejected": 0.04071388021111488,
+ "step": 860
+ },
+ {
+ "epoch": 0.696,
+ "grad_norm": 1.2248382568359375,
+ "learning_rate": 8.536296296296296e-07,
+ "logits/chosen": 1.4529926776885986,
+ "logits/rejected": 1.4423902034759521,
+ "logps/chosen": -154.4588165283203,
+ "logps/rejected": -153.44949340820312,
+ "loss": 0.6915592193603516,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.03207828477025032,
+ "rewards/margins": 0.006785523146390915,
+ "rewards/rejected": 0.025292763486504555,
+ "step": 870
+ },
+ {
+ "epoch": 0.704,
+ "grad_norm": 1.2174893617630005,
+ "learning_rate": 8.506666666666667e-07,
+ "logits/chosen": 1.4073113203048706,
+ "logits/rejected": 1.668140172958374,
+ "logps/chosen": -145.18460083007812,
+ "logps/rejected": -157.0777587890625,
+ "loss": 0.6922356605529785,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.049450717866420746,
+ "rewards/margins": 0.004049716051667929,
+ "rewards/rejected": 0.04540099948644638,
+ "step": 880
+ },
+ {
+ "epoch": 0.712,
+ "grad_norm": 1.3186850547790527,
+ "learning_rate": 8.477037037037037e-07,
+ "logits/chosen": 1.2923892736434937,
+ "logits/rejected": 1.491051435470581,
+ "logps/chosen": -149.03018188476562,
+ "logps/rejected": -161.61715698242188,
+ "loss": 0.6841150760650635,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.038961172103881836,
+ "rewards/margins": 0.020759906619787216,
+ "rewards/rejected": 0.01820126175880432,
+ "step": 890
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 1.1123124361038208,
+ "learning_rate": 8.447407407407407e-07,
+ "logits/chosen": 1.546547293663025,
+ "logits/rejected": 1.682992696762085,
+ "logps/chosen": -166.2056884765625,
+ "logps/rejected": -180.73330688476562,
+ "loss": 0.6826215744018554,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.07279330492019653,
+ "rewards/margins": 0.025264525786042213,
+ "rewards/rejected": 0.04752878472208977,
+ "step": 900
+ },
+ {
+ "epoch": 0.72,
+ "eval_logits/chosen": 1.4819762706756592,
+ "eval_logits/rejected": 1.513027548789978,
+ "eval_logps/chosen": -153.1695556640625,
+ "eval_logps/rejected": -147.7560577392578,
+ "eval_loss": 0.689373254776001,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": 0.03792775049805641,
+ "eval_rewards/margins": 0.011320074088871479,
+ "eval_rewards/rejected": 0.026607677340507507,
+ "eval_runtime": 90.9603,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 900
+ },
+ {
+ "epoch": 0.728,
+ "grad_norm": 1.1507582664489746,
+ "learning_rate": 8.417777777777777e-07,
+ "logits/chosen": 1.5831308364868164,
+ "logits/rejected": 1.3737132549285889,
+ "logps/chosen": -140.00253295898438,
+ "logps/rejected": -138.45155334472656,
+ "loss": 0.6873391628265381,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.05066479369997978,
+ "rewards/margins": 0.015466553159058094,
+ "rewards/rejected": 0.03519824147224426,
+ "step": 910
+ },
+ {
+ "epoch": 0.736,
+ "grad_norm": 1.415456771850586,
+ "learning_rate": 8.388148148148147e-07,
+ "logits/chosen": 1.3000686168670654,
+ "logits/rejected": 1.2642180919647217,
+ "logps/chosen": -160.14027404785156,
+ "logps/rejected": -163.32827758789062,
+ "loss": 0.687142276763916,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.06618930399417877,
+ "rewards/margins": 0.014807088300585747,
+ "rewards/rejected": 0.05138222128152847,
+ "step": 920
+ },
+ {
+ "epoch": 0.744,
+ "grad_norm": 1.3228634595870972,
+ "learning_rate": 8.358518518518518e-07,
+ "logits/chosen": 1.5204170942306519,
+ "logits/rejected": 1.6129217147827148,
+ "logps/chosen": -157.99832153320312,
+ "logps/rejected": -162.09567260742188,
+ "loss": 0.6790886878967285,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.05940447002649307,
+ "rewards/margins": 0.03161252290010452,
+ "rewards/rejected": 0.027791941538453102,
+ "step": 930
+ },
+ {
+ "epoch": 0.752,
+ "grad_norm": 1.048064947128296,
+ "learning_rate": 8.328888888888889e-07,
+ "logits/chosen": 1.5515320301055908,
+ "logits/rejected": 1.676857352256775,
+ "logps/chosen": -148.4853515625,
+ "logps/rejected": -174.4320526123047,
+ "loss": 0.6935453414916992,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.010225490666925907,
+ "rewards/margins": 0.0021693313028663397,
+ "rewards/rejected": 0.008056161925196648,
+ "step": 940
+ },
+ {
+ "epoch": 0.76,
+ "grad_norm": 1.247799038887024,
+ "learning_rate": 8.299259259259259e-07,
+ "logits/chosen": 1.384231686592102,
+ "logits/rejected": 1.4611561298370361,
+ "logps/chosen": -140.81443786621094,
+ "logps/rejected": -153.81326293945312,
+ "loss": 0.6805226325988769,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.053223587572574615,
+ "rewards/margins": 0.02933613583445549,
+ "rewards/rejected": 0.023887457326054573,
+ "step": 950
+ },
+ {
+ "epoch": 0.768,
+ "grad_norm": 1.1611264944076538,
+ "learning_rate": 8.269629629629629e-07,
+ "logits/chosen": 1.4987528324127197,
+ "logits/rejected": 1.3684321641921997,
+ "logps/chosen": -135.81056213378906,
+ "logps/rejected": -119.05133056640625,
+ "loss": 0.6954316616058349,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.0197703056037426,
+ "rewards/margins": -0.0012637012405321002,
+ "rewards/rejected": 0.021034007892012596,
+ "step": 960
+ },
+ {
+ "epoch": 0.776,
+ "grad_norm": 0.964948832988739,
+ "learning_rate": 8.24e-07,
+ "logits/chosen": 1.612053632736206,
+ "logits/rejected": 1.7114235162734985,
+ "logps/chosen": -153.56137084960938,
+ "logps/rejected": -149.17323303222656,
+ "loss": 0.6916662693023682,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.05082285404205322,
+ "rewards/margins": 0.0056606316938996315,
+ "rewards/rejected": 0.04516221210360527,
+ "step": 970
+ },
+ {
+ "epoch": 0.784,
+ "grad_norm": 1.409449815750122,
+ "learning_rate": 8.21037037037037e-07,
+ "logits/chosen": 1.384060025215149,
+ "logits/rejected": 1.316361904144287,
+ "logps/chosen": -168.75360107421875,
+ "logps/rejected": -141.79537963867188,
+ "loss": 0.6845529556274415,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.0886571854352951,
+ "rewards/margins": 0.02084263041615486,
+ "rewards/rejected": 0.06781454384326935,
+ "step": 980
+ },
+ {
+ "epoch": 0.792,
+ "grad_norm": 1.0246562957763672,
+ "learning_rate": 8.18074074074074e-07,
+ "logits/chosen": 1.4986507892608643,
+ "logits/rejected": 1.571103811264038,
+ "logps/chosen": -165.22254943847656,
+ "logps/rejected": -168.63540649414062,
+ "loss": 0.6790967464447022,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.07517905533313751,
+ "rewards/margins": 0.03272219002246857,
+ "rewards/rejected": 0.04245685786008835,
+ "step": 990
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 1.014426827430725,
+ "learning_rate": 8.15111111111111e-07,
+ "logits/chosen": 1.4708755016326904,
+ "logits/rejected": 1.4550955295562744,
+ "logps/chosen": -149.49874877929688,
+ "logps/rejected": -125.0966567993164,
+ "loss": 0.6892098903656005,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.07825140655040741,
+ "rewards/margins": 0.01007872074842453,
+ "rewards/rejected": 0.06817268580198288,
+ "step": 1000
+ },
+ {
+ "epoch": 0.8,
+ "eval_logits/chosen": 1.4972777366638184,
+ "eval_logits/rejected": 1.5283576250076294,
+ "eval_logps/chosen": -152.8240966796875,
+ "eval_logps/rejected": -147.44175720214844,
+ "eval_loss": 0.6880542039871216,
+ "eval_rewards/accuracies": 0.5419999957084656,
+ "eval_rewards/chosen": 0.07247376441955566,
+ "eval_rewards/margins": 0.01443479023873806,
+ "eval_rewards/rejected": 0.058038972318172455,
+ "eval_runtime": 90.827,
+ "eval_samples_per_second": 5.505,
+ "eval_steps_per_second": 2.752,
+ "step": 1000
+ },
+ {
+ "epoch": 0.808,
+ "grad_norm": 1.1155726909637451,
+ "learning_rate": 8.121481481481482e-07,
+ "logits/chosen": 1.547481894493103,
+ "logits/rejected": 1.5461527109146118,
+ "logps/chosen": -154.89791870117188,
+ "logps/rejected": -142.05111694335938,
+ "loss": 0.6809016704559326,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.10093016922473907,
+ "rewards/margins": 0.027038773521780968,
+ "rewards/rejected": 0.07389138638973236,
+ "step": 1010
+ },
+ {
+ "epoch": 0.816,
+ "grad_norm": 1.6716786623001099,
+ "learning_rate": 8.091851851851852e-07,
+ "logits/chosen": 1.3954919576644897,
+ "logits/rejected": 1.5088036060333252,
+ "logps/chosen": -143.11846923828125,
+ "logps/rejected": -146.41482543945312,
+ "loss": 0.6849985122680664,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.05970728397369385,
+ "rewards/margins": 0.021999523043632507,
+ "rewards/rejected": 0.03770776465535164,
+ "step": 1020
+ },
+ {
+ "epoch": 0.824,
+ "grad_norm": 1.308182716369629,
+ "learning_rate": 8.062222222222221e-07,
+ "logits/chosen": 1.532488465309143,
+ "logits/rejected": 1.3316718339920044,
+ "logps/chosen": -163.06858825683594,
+ "logps/rejected": -136.57989501953125,
+ "loss": 0.6789252281188964,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.11086118221282959,
+ "rewards/margins": 0.033117204904556274,
+ "rewards/rejected": 0.07774396985769272,
+ "step": 1030
+ },
+ {
+ "epoch": 0.832,
+ "grad_norm": 1.1019353866577148,
+ "learning_rate": 8.032592592592592e-07,
+ "logits/chosen": 1.4798121452331543,
+ "logits/rejected": 1.5203083753585815,
+ "logps/chosen": -140.3281707763672,
+ "logps/rejected": -141.06216430664062,
+ "loss": 0.6724937915802002,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.1202714815735817,
+ "rewards/margins": 0.046712182462215424,
+ "rewards/rejected": 0.07355931401252747,
+ "step": 1040
+ },
+ {
+ "epoch": 0.84,
+ "grad_norm": 1.2966418266296387,
+ "learning_rate": 8.002962962962963e-07,
+ "logits/chosen": 1.3034439086914062,
+ "logits/rejected": 1.4363505840301514,
+ "logps/chosen": -142.1992645263672,
+ "logps/rejected": -132.0046844482422,
+ "loss": 0.6848338603973388,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.11126589775085449,
+ "rewards/margins": 0.019870441406965256,
+ "rewards/rejected": 0.09139545261859894,
+ "step": 1050
+ },
+ {
+ "epoch": 0.848,
+ "grad_norm": 1.0321130752563477,
+ "learning_rate": 7.973333333333333e-07,
+ "logits/chosen": 1.3257265090942383,
+ "logits/rejected": 1.4649587869644165,
+ "logps/chosen": -112.98075103759766,
+ "logps/rejected": -127.4059829711914,
+ "loss": 0.6679783344268799,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.17311830818653107,
+ "rewards/margins": 0.05631983280181885,
+ "rewards/rejected": 0.11679847538471222,
+ "step": 1060
+ },
+ {
+ "epoch": 0.856,
+ "grad_norm": 1.2037222385406494,
+ "learning_rate": 7.943703703703704e-07,
+ "logits/chosen": 1.3766987323760986,
+ "logits/rejected": 1.2802103757858276,
+ "logps/chosen": -154.8142547607422,
+ "logps/rejected": -148.680908203125,
+ "loss": 0.668931531906128,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1449051946401596,
+ "rewards/margins": 0.05721765011548996,
+ "rewards/rejected": 0.08768755197525024,
+ "step": 1070
+ },
+ {
+ "epoch": 0.864,
+ "grad_norm": 0.9230766296386719,
+ "learning_rate": 7.914074074074074e-07,
+ "logits/chosen": 1.447328805923462,
+ "logits/rejected": 1.5235106945037842,
+ "logps/chosen": -150.21812438964844,
+ "logps/rejected": -123.29923248291016,
+ "loss": 0.6716424465179444,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.16066041588783264,
+ "rewards/margins": 0.04824484512209892,
+ "rewards/rejected": 0.11241557449102402,
+ "step": 1080
+ },
+ {
+ "epoch": 0.872,
+ "grad_norm": 1.0423877239227295,
+ "learning_rate": 7.884444444444443e-07,
+ "logits/chosen": 1.6920299530029297,
+ "logits/rejected": 1.6123542785644531,
+ "logps/chosen": -175.1676025390625,
+ "logps/rejected": -185.07223510742188,
+ "loss": 0.6723968505859375,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.16490009427070618,
+ "rewards/margins": 0.046577490866184235,
+ "rewards/rejected": 0.11832261085510254,
+ "step": 1090
+ },
+ {
+ "epoch": 0.88,
+ "grad_norm": 1.4142779111862183,
+ "learning_rate": 7.854814814814814e-07,
+ "logits/chosen": 1.453111171722412,
+ "logits/rejected": 1.4981731176376343,
+ "logps/chosen": -139.9765167236328,
+ "logps/rejected": -153.63058471679688,
+ "loss": 0.6872457027435303,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.13096000254154205,
+ "rewards/margins": 0.01702108606696129,
+ "rewards/rejected": 0.11393891274929047,
+ "step": 1100
+ },
+ {
+ "epoch": 0.88,
+ "eval_logits/chosen": 1.5076195001602173,
+ "eval_logits/rejected": 1.5373249053955078,
+ "eval_logps/chosen": -152.1697998046875,
+ "eval_logps/rejected": -146.83502197265625,
+ "eval_loss": 0.6873304843902588,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.13790392875671387,
+ "eval_rewards/margins": 0.019191108644008636,
+ "eval_rewards/rejected": 0.11871281266212463,
+ "eval_runtime": 91.0157,
+ "eval_samples_per_second": 5.494,
+ "eval_steps_per_second": 2.747,
+ "step": 1100
+ },
+ {
+ "epoch": 0.888,
+ "grad_norm": 1.413996696472168,
+ "learning_rate": 7.825185185185185e-07,
+ "logits/chosen": 1.5183905363082886,
+ "logits/rejected": 1.4942539930343628,
+ "logps/chosen": -156.383544921875,
+ "logps/rejected": -145.61398315429688,
+ "loss": 0.6810199737548828,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.14243564009666443,
+ "rewards/margins": 0.03136260434985161,
+ "rewards/rejected": 0.11107305437326431,
+ "step": 1110
+ },
+ {
+ "epoch": 0.896,
+ "grad_norm": 1.0046707391738892,
+ "learning_rate": 7.795555555555555e-07,
+ "logits/chosen": 1.4982213973999023,
+ "logits/rejected": 1.5465660095214844,
+ "logps/chosen": -156.79293823242188,
+ "logps/rejected": -150.77853393554688,
+ "loss": 0.6808318614959716,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1551930010318756,
+ "rewards/margins": 0.028187204152345657,
+ "rewards/rejected": 0.12700578570365906,
+ "step": 1120
+ },
+ {
+ "epoch": 0.904,
+ "grad_norm": 1.125625729560852,
+ "learning_rate": 7.765925925925925e-07,
+ "logits/chosen": 1.6053575277328491,
+ "logits/rejected": 1.47335946559906,
+ "logps/chosen": -148.05690002441406,
+ "logps/rejected": -135.36598205566406,
+ "loss": 0.6889908790588379,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.15128108859062195,
+ "rewards/margins": 0.015811312943696976,
+ "rewards/rejected": 0.13546979427337646,
+ "step": 1130
+ },
+ {
+ "epoch": 0.912,
+ "grad_norm": 1.0933659076690674,
+ "learning_rate": 7.736296296296297e-07,
+ "logits/chosen": 1.5862042903900146,
+ "logits/rejected": 1.5827430486679077,
+ "logps/chosen": -156.10145568847656,
+ "logps/rejected": -142.94851684570312,
+ "loss": 0.6753206253051758,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1547432243824005,
+ "rewards/margins": 0.04229574277997017,
+ "rewards/rejected": 0.11244747787714005,
+ "step": 1140
+ },
+ {
+ "epoch": 0.92,
+ "grad_norm": 1.0774686336517334,
+ "learning_rate": 7.706666666666667e-07,
+ "logits/chosen": 1.2369152307510376,
+ "logits/rejected": 1.2727023363113403,
+ "logps/chosen": -139.23147583007812,
+ "logps/rejected": -130.53038024902344,
+ "loss": 0.6967329025268555,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": 0.10801626741886139,
+ "rewards/margins": 0.004670143127441406,
+ "rewards/rejected": 0.10334610939025879,
+ "step": 1150
+ },
+ {
+ "epoch": 0.928,
+ "grad_norm": 1.1111780405044556,
+ "learning_rate": 7.677037037037036e-07,
+ "logits/chosen": 1.501250982284546,
+ "logits/rejected": 1.4272868633270264,
+ "logps/chosen": -127.09858703613281,
+ "logps/rejected": -114.9041748046875,
+ "loss": 0.6853152275085449,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.1366150826215744,
+ "rewards/margins": 0.021945688873529434,
+ "rewards/rejected": 0.11466939747333527,
+ "step": 1160
+ },
+ {
+ "epoch": 0.936,
+ "grad_norm": 1.5593854188919067,
+ "learning_rate": 7.647407407407407e-07,
+ "logits/chosen": 1.727927803993225,
+ "logits/rejected": 1.6510727405548096,
+ "logps/chosen": -156.70993041992188,
+ "logps/rejected": -167.07266235351562,
+ "loss": 0.6884279251098633,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.14902853965759277,
+ "rewards/margins": 0.01685408689081669,
+ "rewards/rejected": 0.13217447698116302,
+ "step": 1170
+ },
+ {
+ "epoch": 0.944,
+ "grad_norm": 2.0064918994903564,
+ "learning_rate": 7.617777777777778e-07,
+ "logits/chosen": 1.6262975931167603,
+ "logits/rejected": 1.58951997756958,
+ "logps/chosen": -186.59210205078125,
+ "logps/rejected": -151.2965850830078,
+ "loss": 0.6906405448913574,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.14898104965686798,
+ "rewards/margins": 0.013303476385772228,
+ "rewards/rejected": 0.13567757606506348,
+ "step": 1180
+ },
+ {
+ "epoch": 0.952,
+ "grad_norm": 1.209291696548462,
+ "learning_rate": 7.588148148148148e-07,
+ "logits/chosen": 1.639369249343872,
+ "logits/rejected": 1.6400636434555054,
+ "logps/chosen": -166.24684143066406,
+ "logps/rejected": -196.35305786132812,
+ "loss": 0.6789956092834473,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1348457634449005,
+ "rewards/margins": 0.03475186973810196,
+ "rewards/rejected": 0.10009388625621796,
+ "step": 1190
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 1.209230899810791,
+ "learning_rate": 7.558518518518519e-07,
+ "logits/chosen": 1.542942762374878,
+ "logits/rejected": 1.5128916501998901,
+ "logps/chosen": -138.994384765625,
+ "logps/rejected": -145.42880249023438,
+ "loss": 0.6936234951019287,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.13895569741725922,
+ "rewards/margins": 0.005582497920840979,
+ "rewards/rejected": 0.1333732008934021,
+ "step": 1200
+ },
+ {
+ "epoch": 0.96,
+ "eval_logits/chosen": 1.5055651664733887,
+ "eval_logits/rejected": 1.5351229906082153,
+ "eval_logps/chosen": -152.2256622314453,
+ "eval_logps/rejected": -146.909912109375,
+ "eval_loss": 0.6866763830184937,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.13231882452964783,
+ "eval_rewards/margins": 0.02109396643936634,
+ "eval_rewards/rejected": 0.11122485250234604,
+ "eval_runtime": 90.8141,
+ "eval_samples_per_second": 5.506,
+ "eval_steps_per_second": 2.753,
+ "step": 1200
+ },
+ {
+ "epoch": 0.968,
+ "grad_norm": 1.2058950662612915,
+ "learning_rate": 7.528888888888889e-07,
+ "logits/chosen": 1.3591210842132568,
+ "logits/rejected": 1.3811699151992798,
+ "logps/chosen": -155.1759033203125,
+ "logps/rejected": -141.97772216796875,
+ "loss": 0.6902606964111329,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.153532013297081,
+ "rewards/margins": 0.011359233409166336,
+ "rewards/rejected": 0.14217278361320496,
+ "step": 1210
+ },
+ {
+ "epoch": 0.976,
+ "grad_norm": 1.366702914237976,
+ "learning_rate": 7.499259259259258e-07,
+ "logits/chosen": 1.3841370344161987,
+ "logits/rejected": 1.3791372776031494,
+ "logps/chosen": -145.47613525390625,
+ "logps/rejected": -123.9173583984375,
+ "loss": 0.6971709251403808,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.10521620512008667,
+ "rewards/margins": -0.0030749079305678606,
+ "rewards/rejected": 0.1082911267876625,
+ "step": 1220
+ },
+ {
+ "epoch": 0.984,
+ "grad_norm": 1.477455496788025,
+ "learning_rate": 7.469629629629629e-07,
+ "logits/chosen": 1.4447228908538818,
+ "logits/rejected": 1.6022417545318604,
+ "logps/chosen": -137.9462127685547,
+ "logps/rejected": -150.25559997558594,
+ "loss": 0.6915127277374268,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.15108394622802734,
+ "rewards/margins": 0.009388584643602371,
+ "rewards/rejected": 0.14169538021087646,
+ "step": 1230
+ },
+ {
+ "epoch": 0.992,
+ "grad_norm": 1.4407854080200195,
+ "learning_rate": 7.44e-07,
+ "logits/chosen": 1.3998825550079346,
+ "logits/rejected": 1.5924925804138184,
+ "logps/chosen": -144.93775939941406,
+ "logps/rejected": -148.02542114257812,
+ "loss": 0.6699621677398682,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.146137997508049,
+ "rewards/margins": 0.05326627567410469,
+ "rewards/rejected": 0.09287171810865402,
+ "step": 1240
+ },
+ {
+ "epoch": 1.0,
+ "grad_norm": 1.183438777923584,
+ "learning_rate": 7.41037037037037e-07,
+ "logits/chosen": 1.5393705368041992,
+ "logits/rejected": 1.6312555074691772,
+ "logps/chosen": -145.61672973632812,
+ "logps/rejected": -158.9256591796875,
+ "loss": 0.6811168670654297,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.1304798424243927,
+ "rewards/margins": 0.03360626846551895,
+ "rewards/rejected": 0.09687358886003494,
+ "step": 1250
+ },
+ {
+ "epoch": 1.008,
+ "grad_norm": 1.2436553239822388,
+ "learning_rate": 7.38074074074074e-07,
+ "logits/chosen": 1.4758602380752563,
+ "logits/rejected": 1.534609079360962,
+ "logps/chosen": -124.26841735839844,
+ "logps/rejected": -132.41867065429688,
+ "loss": 0.688177490234375,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.12570872902870178,
+ "rewards/margins": 0.014875045046210289,
+ "rewards/rejected": 0.11083368957042694,
+ "step": 1260
+ },
+ {
+ "epoch": 1.016,
+ "grad_norm": 1.3853868246078491,
+ "learning_rate": 7.351111111111112e-07,
+ "logits/chosen": 1.7195091247558594,
+ "logits/rejected": 1.632851004600525,
+ "logps/chosen": -160.8590850830078,
+ "logps/rejected": -153.91671752929688,
+ "loss": 0.6669504165649414,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.181864932179451,
+ "rewards/margins": 0.06175640970468521,
+ "rewards/rejected": 0.12010850757360458,
+ "step": 1270
+ },
+ {
+ "epoch": 1.024,
+ "grad_norm": 1.2765302658081055,
+ "learning_rate": 7.321481481481481e-07,
+ "logits/chosen": 1.5782036781311035,
+ "logits/rejected": 1.5713088512420654,
+ "logps/chosen": -154.3319549560547,
+ "logps/rejected": -141.43345642089844,
+ "loss": 0.6734190464019776,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.12361389398574829,
+ "rewards/margins": 0.04626934230327606,
+ "rewards/rejected": 0.07734455168247223,
+ "step": 1280
+ },
+ {
+ "epoch": 1.032,
+ "grad_norm": 1.0421397686004639,
+ "learning_rate": 7.291851851851851e-07,
+ "logits/chosen": 1.5199967622756958,
+ "logits/rejected": 1.5965670347213745,
+ "logps/chosen": -145.28416442871094,
+ "logps/rejected": -133.72256469726562,
+ "loss": 0.6804514408111573,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.11794980615377426,
+ "rewards/margins": 0.030916064977645874,
+ "rewards/rejected": 0.08703374862670898,
+ "step": 1290
+ },
+ {
+ "epoch": 1.04,
+ "grad_norm": 1.032158613204956,
+ "learning_rate": 7.262222222222222e-07,
+ "logits/chosen": 1.6042639017105103,
+ "logits/rejected": 1.4255346059799194,
+ "logps/chosen": -150.4803009033203,
+ "logps/rejected": -130.2232208251953,
+ "loss": 0.6676507472991944,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.15466037392616272,
+ "rewards/margins": 0.05808692425489426,
+ "rewards/rejected": 0.09657344967126846,
+ "step": 1300
+ },
+ {
+ "epoch": 1.04,
+ "eval_logits/chosen": 1.5118356943130493,
+ "eval_logits/rejected": 1.5418950319290161,
+ "eval_logps/chosen": -152.23300170898438,
+ "eval_logps/rejected": -146.9257049560547,
+ "eval_loss": 0.6859617233276367,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.13158410787582397,
+ "eval_rewards/margins": 0.021939631551504135,
+ "eval_rewards/rejected": 0.10964448750019073,
+ "eval_runtime": 90.9624,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 1300
+ },
+ {
+ "epoch": 1.048,
+ "grad_norm": 1.4942574501037598,
+ "learning_rate": 7.232592592592593e-07,
+ "logits/chosen": 1.689648985862732,
+ "logits/rejected": 1.691888451576233,
+ "logps/chosen": -166.02809143066406,
+ "logps/rejected": -160.30601501464844,
+ "loss": 0.6770285606384278,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.14377735555171967,
+ "rewards/margins": 0.03825496509671211,
+ "rewards/rejected": 0.10552239418029785,
+ "step": 1310
+ },
+ {
+ "epoch": 1.056,
+ "grad_norm": 1.3656011819839478,
+ "learning_rate": 7.202962962962963e-07,
+ "logits/chosen": 1.3065154552459717,
+ "logits/rejected": 1.5681203603744507,
+ "logps/chosen": -129.1146240234375,
+ "logps/rejected": -151.2584991455078,
+ "loss": 0.6844655036926269,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.12841743230819702,
+ "rewards/margins": 0.02178655005991459,
+ "rewards/rejected": 0.10663086175918579,
+ "step": 1320
+ },
+ {
+ "epoch": 1.064,
+ "grad_norm": 1.1723405122756958,
+ "learning_rate": 7.173333333333333e-07,
+ "logits/chosen": 1.6219158172607422,
+ "logits/rejected": 1.6032555103302002,
+ "logps/chosen": -156.4185028076172,
+ "logps/rejected": -146.28067016601562,
+ "loss": 0.6650479316711426,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1538492739200592,
+ "rewards/margins": 0.06132146716117859,
+ "rewards/rejected": 0.09252782166004181,
+ "step": 1330
+ },
+ {
+ "epoch": 1.072,
+ "grad_norm": 1.2311245203018188,
+ "learning_rate": 7.143703703703703e-07,
+ "logits/chosen": 1.4625260829925537,
+ "logits/rejected": 1.392937421798706,
+ "logps/chosen": -153.05941772460938,
+ "logps/rejected": -145.37841796875,
+ "loss": 0.6570021152496338,
+ "rewards/accuracies": 0.762499988079071,
+ "rewards/chosen": 0.16768500208854675,
+ "rewards/margins": 0.0787494108080864,
+ "rewards/rejected": 0.08893559873104095,
+ "step": 1340
+ },
+ {
+ "epoch": 1.08,
+ "grad_norm": 1.314666748046875,
+ "learning_rate": 7.114074074074073e-07,
+ "logits/chosen": 1.4812809228897095,
+ "logits/rejected": 1.3796050548553467,
+ "logps/chosen": -165.70205688476562,
+ "logps/rejected": -140.13150024414062,
+ "loss": 0.6535200595855712,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.20691323280334473,
+ "rewards/margins": 0.0911526307463646,
+ "rewards/rejected": 0.11576057970523834,
+ "step": 1350
+ },
+ {
+ "epoch": 1.088,
+ "grad_norm": 0.9556720852851868,
+ "learning_rate": 7.084444444444444e-07,
+ "logits/chosen": 1.6368863582611084,
+ "logits/rejected": 1.678134560585022,
+ "logps/chosen": -166.56253051757812,
+ "logps/rejected": -164.36416625976562,
+ "loss": 0.6742628574371338,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.15145304799079895,
+ "rewards/margins": 0.04319921135902405,
+ "rewards/rejected": 0.1082538366317749,
+ "step": 1360
+ },
+ {
+ "epoch": 1.096,
+ "grad_norm": 0.945633053779602,
+ "learning_rate": 7.054814814814815e-07,
+ "logits/chosen": 1.4285938739776611,
+ "logits/rejected": 1.5576260089874268,
+ "logps/chosen": -131.5234832763672,
+ "logps/rejected": -128.02340698242188,
+ "loss": 0.6670140743255615,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.16399362683296204,
+ "rewards/margins": 0.05922812223434448,
+ "rewards/rejected": 0.10476551204919815,
+ "step": 1370
+ },
+ {
+ "epoch": 1.104,
+ "grad_norm": 1.1189475059509277,
+ "learning_rate": 7.025185185185185e-07,
+ "logits/chosen": 1.5019981861114502,
+ "logits/rejected": 1.667614221572876,
+ "logps/chosen": -161.61854553222656,
+ "logps/rejected": -166.39996337890625,
+ "loss": 0.6742690086364747,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.15786050260066986,
+ "rewards/margins": 0.044500257819890976,
+ "rewards/rejected": 0.11336024850606918,
+ "step": 1380
+ },
+ {
+ "epoch": 1.112,
+ "grad_norm": 1.3382912874221802,
+ "learning_rate": 6.995555555555555e-07,
+ "logits/chosen": 1.4972480535507202,
+ "logits/rejected": 1.4946129322052002,
+ "logps/chosen": -166.43682861328125,
+ "logps/rejected": -162.3004913330078,
+ "loss": 0.6557814121246338,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.18964914977550507,
+ "rewards/margins": 0.08500295877456665,
+ "rewards/rejected": 0.10464620590209961,
+ "step": 1390
+ },
+ {
+ "epoch": 1.12,
+ "grad_norm": 1.2123515605926514,
+ "learning_rate": 6.965925925925926e-07,
+ "logits/chosen": 1.456635594367981,
+ "logits/rejected": 1.5964475870132446,
+ "logps/chosen": -152.08261108398438,
+ "logps/rejected": -161.51023864746094,
+ "loss": 0.6708066940307618,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.216823011636734,
+ "rewards/margins": 0.050363689661026,
+ "rewards/rejected": 0.166459321975708,
+ "step": 1400
+ },
+ {
+ "epoch": 1.12,
+ "eval_logits/chosen": 1.521518588066101,
+ "eval_logits/rejected": 1.5510718822479248,
+ "eval_logps/chosen": -151.858154296875,
+ "eval_logps/rejected": -146.5819549560547,
+ "eval_loss": 0.6853657364845276,
+ "eval_rewards/accuracies": 0.5440000295639038,
+ "eval_rewards/chosen": 0.16906805336475372,
+ "eval_rewards/margins": 0.025049500167369843,
+ "eval_rewards/rejected": 0.14401854574680328,
+ "eval_runtime": 90.8779,
+ "eval_samples_per_second": 5.502,
+ "eval_steps_per_second": 2.751,
+ "step": 1400
+ },
+ {
+ "epoch": 1.1280000000000001,
+ "grad_norm": 1.3488291501998901,
+ "learning_rate": 6.936296296296296e-07,
+ "logits/chosen": 1.6009830236434937,
+ "logits/rejected": 1.6035544872283936,
+ "logps/chosen": -145.44662475585938,
+ "logps/rejected": -149.9284210205078,
+ "loss": 0.679435396194458,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.1647307425737381,
+ "rewards/margins": 0.03348377346992493,
+ "rewards/rejected": 0.13124695420265198,
+ "step": 1410
+ },
+ {
+ "epoch": 1.1360000000000001,
+ "grad_norm": 1.6453970670700073,
+ "learning_rate": 6.906666666666666e-07,
+ "logits/chosen": 1.593658447265625,
+ "logits/rejected": 1.648276925086975,
+ "logps/chosen": -135.2926788330078,
+ "logps/rejected": -150.9585723876953,
+ "loss": 0.6875734329223633,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.15669891238212585,
+ "rewards/margins": 0.01690087839961052,
+ "rewards/rejected": 0.13979804515838623,
+ "step": 1420
+ },
+ {
+ "epoch": 1.144,
+ "grad_norm": 1.6117807626724243,
+ "learning_rate": 6.877037037037036e-07,
+ "logits/chosen": 1.4510356187820435,
+ "logits/rejected": 1.2334567308425903,
+ "logps/chosen": -153.94854736328125,
+ "logps/rejected": -147.78079223632812,
+ "loss": 0.6689413070678711,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1819056123495102,
+ "rewards/margins": 0.058564942330121994,
+ "rewards/rejected": 0.1233406662940979,
+ "step": 1430
+ },
+ {
+ "epoch": 1.152,
+ "grad_norm": 1.5533809661865234,
+ "learning_rate": 6.847407407407408e-07,
+ "logits/chosen": 1.4535889625549316,
+ "logits/rejected": 1.4273865222930908,
+ "logps/chosen": -170.42904663085938,
+ "logps/rejected": -151.9979705810547,
+ "loss": 0.6845246315002441,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.14223966002464294,
+ "rewards/margins": 0.022389881312847137,
+ "rewards/rejected": 0.1198497861623764,
+ "step": 1440
+ },
+ {
+ "epoch": 1.16,
+ "grad_norm": 1.5696005821228027,
+ "learning_rate": 6.817777777777778e-07,
+ "logits/chosen": 1.4418725967407227,
+ "logits/rejected": 1.4354021549224854,
+ "logps/chosen": -153.962158203125,
+ "logps/rejected": -148.40512084960938,
+ "loss": 0.6761108875274658,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.15529735386371613,
+ "rewards/margins": 0.040999218821525574,
+ "rewards/rejected": 0.11429812759160995,
+ "step": 1450
+ },
+ {
+ "epoch": 1.168,
+ "grad_norm": 1.0711781978607178,
+ "learning_rate": 6.788148148148147e-07,
+ "logits/chosen": 1.5659481287002563,
+ "logits/rejected": 1.5640677213668823,
+ "logps/chosen": -149.816162109375,
+ "logps/rejected": -174.68643188476562,
+ "loss": 0.677489185333252,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1510745733976364,
+ "rewards/margins": 0.03914733976125717,
+ "rewards/rejected": 0.11192724853754044,
+ "step": 1460
+ },
+ {
+ "epoch": 1.176,
+ "grad_norm": 1.2336673736572266,
+ "learning_rate": 6.758518518518518e-07,
+ "logits/chosen": 1.4275394678115845,
+ "logits/rejected": 1.6394407749176025,
+ "logps/chosen": -137.93788146972656,
+ "logps/rejected": -154.97195434570312,
+ "loss": 0.6785802364349365,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10557998716831207,
+ "rewards/margins": 0.0352485254406929,
+ "rewards/rejected": 0.07033145427703857,
+ "step": 1470
+ },
+ {
+ "epoch": 1.184,
+ "grad_norm": 1.0030723810195923,
+ "learning_rate": 6.728888888888888e-07,
+ "logits/chosen": 1.4652725458145142,
+ "logits/rejected": 1.4992272853851318,
+ "logps/chosen": -143.87374877929688,
+ "logps/rejected": -139.5693817138672,
+ "loss": 0.6700259208679199,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.13436684012413025,
+ "rewards/margins": 0.05286343768239021,
+ "rewards/rejected": 0.08150340616703033,
+ "step": 1480
+ },
+ {
+ "epoch": 1.192,
+ "grad_norm": 1.0897104740142822,
+ "learning_rate": 6.699259259259259e-07,
+ "logits/chosen": 1.7033601999282837,
+ "logits/rejected": 1.617489218711853,
+ "logps/chosen": -140.43667602539062,
+ "logps/rejected": -146.59683227539062,
+ "loss": 0.6685672283172608,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.11724893748760223,
+ "rewards/margins": 0.055138181895017624,
+ "rewards/rejected": 0.06211075931787491,
+ "step": 1490
+ },
+ {
+ "epoch": 1.2,
+ "grad_norm": 0.8573256731033325,
+ "learning_rate": 6.66962962962963e-07,
+ "logits/chosen": 1.5181013345718384,
+ "logits/rejected": 1.5965152978897095,
+ "logps/chosen": -147.78506469726562,
+ "logps/rejected": -142.18930053710938,
+ "loss": 0.6564288139343262,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.14890792965888977,
+ "rewards/margins": 0.08058460801839828,
+ "rewards/rejected": 0.06832331418991089,
+ "step": 1500
+ },
+ {
+ "epoch": 1.2,
+ "eval_logits/chosen": 1.488415002822876,
+ "eval_logits/rejected": 1.5179662704467773,
+ "eval_logps/chosen": -152.60232543945312,
+ "eval_logps/rejected": -147.2954864501953,
+ "eval_loss": 0.6861968040466309,
+ "eval_rewards/accuracies": 0.5400000214576721,
+ "eval_rewards/chosen": 0.09465231001377106,
+ "eval_rewards/margins": 0.021986663341522217,
+ "eval_rewards/rejected": 0.07266565412282944,
+ "eval_runtime": 90.8951,
+ "eval_samples_per_second": 5.501,
+ "eval_steps_per_second": 2.75,
+ "step": 1500
+ },
+ {
+ "epoch": 1.208,
+ "grad_norm": 1.29599928855896,
+ "learning_rate": 6.64e-07,
+ "logits/chosen": 1.4851064682006836,
+ "logits/rejected": 1.4355719089508057,
+ "logps/chosen": -140.07186889648438,
+ "logps/rejected": -155.5626983642578,
+ "loss": 0.6695255756378173,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08214708417654037,
+ "rewards/margins": 0.05618969723582268,
+ "rewards/rejected": 0.025957385078072548,
+ "step": 1510
+ },
+ {
+ "epoch": 1.216,
+ "grad_norm": 1.3030505180358887,
+ "learning_rate": 6.61037037037037e-07,
+ "logits/chosen": 1.3785858154296875,
+ "logits/rejected": 1.3867576122283936,
+ "logps/chosen": -143.06137084960938,
+ "logps/rejected": -152.8805694580078,
+ "loss": 0.6704819202423096,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.08582031726837158,
+ "rewards/margins": 0.052868831902742386,
+ "rewards/rejected": 0.0329514816403389,
+ "step": 1520
+ },
+ {
+ "epoch": 1.224,
+ "grad_norm": 1.2935470342636108,
+ "learning_rate": 6.58074074074074e-07,
+ "logits/chosen": 1.4631284475326538,
+ "logits/rejected": 1.4908760786056519,
+ "logps/chosen": -181.2951202392578,
+ "logps/rejected": -173.83523559570312,
+ "loss": 0.6680389881134033,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.11432291567325592,
+ "rewards/margins": 0.05878107622265816,
+ "rewards/rejected": 0.05554184317588806,
+ "step": 1530
+ },
+ {
+ "epoch": 1.232,
+ "grad_norm": 1.4426783323287964,
+ "learning_rate": 6.551111111111111e-07,
+ "logits/chosen": 1.4765878915786743,
+ "logits/rejected": 1.4511891603469849,
+ "logps/chosen": -167.4827117919922,
+ "logps/rejected": -153.88833618164062,
+ "loss": 0.6684586524963378,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.08437791466712952,
+ "rewards/margins": 0.05823448300361633,
+ "rewards/rejected": 0.026143427938222885,
+ "step": 1540
+ },
+ {
+ "epoch": 1.24,
+ "grad_norm": 1.3253812789916992,
+ "learning_rate": 6.521481481481481e-07,
+ "logits/chosen": 1.2496798038482666,
+ "logits/rejected": 1.5193806886672974,
+ "logps/chosen": -130.33685302734375,
+ "logps/rejected": -132.5361328125,
+ "loss": 0.6695978164672851,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.10806921869516373,
+ "rewards/margins": 0.05488727241754532,
+ "rewards/rejected": 0.05318193882703781,
+ "step": 1550
+ },
+ {
+ "epoch": 1.248,
+ "grad_norm": 1.1890907287597656,
+ "learning_rate": 6.491851851851851e-07,
+ "logits/chosen": 1.2053601741790771,
+ "logits/rejected": 1.2588948011398315,
+ "logps/chosen": -132.61026000976562,
+ "logps/rejected": -150.66477966308594,
+ "loss": 0.673579216003418,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.062341660261154175,
+ "rewards/margins": 0.04531678557395935,
+ "rewards/rejected": 0.017024870961904526,
+ "step": 1560
+ },
+ {
+ "epoch": 1.256,
+ "grad_norm": 1.2076733112335205,
+ "learning_rate": 6.462222222222223e-07,
+ "logits/chosen": 1.5180461406707764,
+ "logits/rejected": 1.4118196964263916,
+ "logps/chosen": -120.61619567871094,
+ "logps/rejected": -146.42056274414062,
+ "loss": 0.6747167587280274,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.0764162540435791,
+ "rewards/margins": 0.04275941476225853,
+ "rewards/rejected": 0.03365683928132057,
+ "step": 1570
+ },
+ {
+ "epoch": 1.264,
+ "grad_norm": 1.3394429683685303,
+ "learning_rate": 6.432592592592593e-07,
+ "logits/chosen": 1.2879914045333862,
+ "logits/rejected": 1.3957126140594482,
+ "logps/chosen": -150.98355102539062,
+ "logps/rejected": -172.91090393066406,
+ "loss": 0.6560922622680664,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.09853875637054443,
+ "rewards/margins": 0.08982758224010468,
+ "rewards/rejected": 0.008711177855730057,
+ "step": 1580
+ },
+ {
+ "epoch": 1.272,
+ "grad_norm": 1.303663730621338,
+ "learning_rate": 6.402962962962962e-07,
+ "logits/chosen": 1.5617700815200806,
+ "logits/rejected": 1.6307615041732788,
+ "logps/chosen": -157.00338745117188,
+ "logps/rejected": -159.2900848388672,
+ "loss": 0.6639400959014893,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.07477572560310364,
+ "rewards/margins": 0.06561199575662613,
+ "rewards/rejected": 0.009163733571767807,
+ "step": 1590
+ },
+ {
+ "epoch": 1.28,
+ "grad_norm": 1.4180800914764404,
+ "learning_rate": 6.373333333333333e-07,
+ "logits/chosen": 1.3017181158065796,
+ "logits/rejected": 1.3270524740219116,
+ "logps/chosen": -134.52029418945312,
+ "logps/rejected": -135.45938110351562,
+ "loss": 0.6564930438995361,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.08265326917171478,
+ "rewards/margins": 0.09048662334680557,
+ "rewards/rejected": -0.007833337411284447,
+ "step": 1600
+ },
+ {
+ "epoch": 1.28,
+ "eval_logits/chosen": 1.4600619077682495,
+ "eval_logits/rejected": 1.489198923110962,
+ "eval_logps/chosen": -152.86001586914062,
+ "eval_logps/rejected": -147.54771423339844,
+ "eval_loss": 0.6871278882026672,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": 0.0688827633857727,
+ "eval_rewards/margins": 0.021439960226416588,
+ "eval_rewards/rejected": 0.04744280129671097,
+ "eval_runtime": 91.0555,
+ "eval_samples_per_second": 5.491,
+ "eval_steps_per_second": 2.746,
+ "step": 1600
+ },
+ {
+ "epoch": 1.288,
+ "grad_norm": 1.1791326999664307,
+ "learning_rate": 6.343703703703703e-07,
+ "logits/chosen": 1.4442191123962402,
+ "logits/rejected": 1.5532598495483398,
+ "logps/chosen": -141.93701171875,
+ "logps/rejected": -147.68682861328125,
+ "loss": 0.6539246082305908,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.09121527522802353,
+ "rewards/margins": 0.08849794417619705,
+ "rewards/rejected": 0.002717340365052223,
+ "step": 1610
+ },
+ {
+ "epoch": 1.296,
+ "grad_norm": 1.1131889820098877,
+ "learning_rate": 6.314074074074074e-07,
+ "logits/chosen": 1.3450567722320557,
+ "logits/rejected": 1.4408035278320312,
+ "logps/chosen": -157.0798797607422,
+ "logps/rejected": -161.1500701904297,
+ "loss": 0.6505428314208984,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.11321337521076202,
+ "rewards/margins": 0.09579396992921829,
+ "rewards/rejected": 0.017419395968317986,
+ "step": 1620
+ },
+ {
+ "epoch": 1.304,
+ "grad_norm": 1.6172596216201782,
+ "learning_rate": 6.284444444444445e-07,
+ "logits/chosen": 1.478847622871399,
+ "logits/rejected": 1.3894659280776978,
+ "logps/chosen": -140.17453002929688,
+ "logps/rejected": -169.06289672851562,
+ "loss": 0.6720306396484375,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.08244314789772034,
+ "rewards/margins": 0.062018074095249176,
+ "rewards/rejected": 0.020425086840987206,
+ "step": 1630
+ },
+ {
+ "epoch": 1.312,
+ "grad_norm": 1.1743669509887695,
+ "learning_rate": 6.254814814814815e-07,
+ "logits/chosen": 1.4197287559509277,
+ "logits/rejected": 1.3319528102874756,
+ "logps/chosen": -157.23072814941406,
+ "logps/rejected": -139.50074768066406,
+ "loss": 0.6712700843811035,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.04650931805372238,
+ "rewards/margins": 0.057532649487257004,
+ "rewards/rejected": -0.0110233249142766,
+ "step": 1640
+ },
+ {
+ "epoch": 1.32,
+ "grad_norm": 1.3225634098052979,
+ "learning_rate": 6.225185185185184e-07,
+ "logits/chosen": 1.35457444190979,
+ "logits/rejected": 1.3751428127288818,
+ "logps/chosen": -137.4834747314453,
+ "logps/rejected": -134.4203643798828,
+ "loss": 0.6679590225219727,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.07574301958084106,
+ "rewards/margins": 0.057132184505462646,
+ "rewards/rejected": 0.01861082948744297,
+ "step": 1650
+ },
+ {
+ "epoch": 1.328,
+ "grad_norm": 1.5913130044937134,
+ "learning_rate": 6.195555555555555e-07,
+ "logits/chosen": 1.5328034162521362,
+ "logits/rejected": 1.3161242008209229,
+ "logps/chosen": -147.09475708007812,
+ "logps/rejected": -135.3501434326172,
+ "loss": 0.6575756072998047,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.1077059879899025,
+ "rewards/margins": 0.08081348240375519,
+ "rewards/rejected": 0.026892507448792458,
+ "step": 1660
+ },
+ {
+ "epoch": 1.336,
+ "grad_norm": 0.9880369901657104,
+ "learning_rate": 6.165925925925926e-07,
+ "logits/chosen": 1.3911128044128418,
+ "logits/rejected": 1.3688428401947021,
+ "logps/chosen": -152.17483520507812,
+ "logps/rejected": -149.79383850097656,
+ "loss": 0.6656455516815185,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.10711929947137833,
+ "rewards/margins": 0.0669747143983841,
+ "rewards/rejected": 0.04014457389712334,
+ "step": 1670
+ },
+ {
+ "epoch": 1.3439999999999999,
+ "grad_norm": 1.125959873199463,
+ "learning_rate": 6.136296296296296e-07,
+ "logits/chosen": 1.4232629537582397,
+ "logits/rejected": 1.5579107999801636,
+ "logps/chosen": -160.2217559814453,
+ "logps/rejected": -148.1455535888672,
+ "loss": 0.6710956573486329,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10255889594554901,
+ "rewards/margins": 0.05178038403391838,
+ "rewards/rejected": 0.05077851936221123,
+ "step": 1680
+ },
+ {
+ "epoch": 1.3519999999999999,
+ "grad_norm": 1.3627047538757324,
+ "learning_rate": 6.106666666666666e-07,
+ "logits/chosen": 1.4259792566299438,
+ "logits/rejected": 1.4750150442123413,
+ "logps/chosen": -152.88381958007812,
+ "logps/rejected": -153.72708129882812,
+ "loss": 0.6512804508209229,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.14147944748401642,
+ "rewards/margins": 0.09307968616485596,
+ "rewards/rejected": 0.04839975759387016,
+ "step": 1690
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "grad_norm": 1.0653239488601685,
+ "learning_rate": 6.077037037037038e-07,
+ "logits/chosen": 1.340463399887085,
+ "logits/rejected": 1.3752565383911133,
+ "logps/chosen": -150.36248779296875,
+ "logps/rejected": -151.90725708007812,
+ "loss": 0.6768710613250732,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.08575484901666641,
+ "rewards/margins": 0.043545305728912354,
+ "rewards/rejected": 0.04220953956246376,
+ "step": 1700
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "eval_logits/chosen": 1.4521433115005493,
+ "eval_logits/rejected": 1.4804331064224243,
+ "eval_logps/chosen": -152.77755737304688,
+ "eval_logps/rejected": -147.48025512695312,
+ "eval_loss": 0.6873273849487305,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": 0.07712872326374054,
+ "eval_rewards/margins": 0.022939365357160568,
+ "eval_rewards/rejected": 0.05418936163187027,
+ "eval_runtime": 90.962,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 1700
+ },
+ {
+ "epoch": 1.3679999999999999,
+ "grad_norm": 1.3996928930282593,
+ "learning_rate": 6.047407407407407e-07,
+ "logits/chosen": 1.41148042678833,
+ "logits/rejected": 1.4520132541656494,
+ "logps/chosen": -143.23252868652344,
+ "logps/rejected": -180.05331420898438,
+ "loss": 0.655043363571167,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10279543697834015,
+ "rewards/margins": 0.0895618349313736,
+ "rewards/rejected": 0.013233604840934277,
+ "step": 1710
+ },
+ {
+ "epoch": 1.376,
+ "grad_norm": 1.4253696203231812,
+ "learning_rate": 6.017777777777777e-07,
+ "logits/chosen": 1.3833668231964111,
+ "logits/rejected": 1.409301519393921,
+ "logps/chosen": -149.3744659423828,
+ "logps/rejected": -151.1966552734375,
+ "loss": 0.6865286827087402,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.06293162703514099,
+ "rewards/margins": 0.02295227348804474,
+ "rewards/rejected": 0.03997935354709625,
+ "step": 1720
+ },
+ {
+ "epoch": 1.384,
+ "grad_norm": 1.185731053352356,
+ "learning_rate": 5.988148148148148e-07,
+ "logits/chosen": 1.528174877166748,
+ "logits/rejected": 1.6338379383087158,
+ "logps/chosen": -160.7788543701172,
+ "logps/rejected": -159.85354614257812,
+ "loss": 0.6458437442779541,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.12789444625377655,
+ "rewards/margins": 0.1114773377776146,
+ "rewards/rejected": 0.016417117789387703,
+ "step": 1730
+ },
+ {
+ "epoch": 1.392,
+ "grad_norm": 1.2963365316390991,
+ "learning_rate": 5.958518518518518e-07,
+ "logits/chosen": 1.4939125776290894,
+ "logits/rejected": 1.4197361469268799,
+ "logps/chosen": -162.34744262695312,
+ "logps/rejected": -177.93087768554688,
+ "loss": 0.6620248317718506,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.11072772741317749,
+ "rewards/margins": 0.07320726662874222,
+ "rewards/rejected": 0.03752044588327408,
+ "step": 1740
+ },
+ {
+ "epoch": 1.4,
+ "grad_norm": 1.3535394668579102,
+ "learning_rate": 5.928888888888889e-07,
+ "logits/chosen": 1.3506572246551514,
+ "logits/rejected": 1.424142837524414,
+ "logps/chosen": -139.69985961914062,
+ "logps/rejected": -144.9062042236328,
+ "loss": 0.6815268516540527,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.07917657494544983,
+ "rewards/margins": 0.03688482195138931,
+ "rewards/rejected": 0.04229176044464111,
+ "step": 1750
+ },
+ {
+ "epoch": 1.408,
+ "grad_norm": 1.471380352973938,
+ "learning_rate": 5.899259259259259e-07,
+ "logits/chosen": 1.5949152708053589,
+ "logits/rejected": 1.710013747215271,
+ "logps/chosen": -170.3776092529297,
+ "logps/rejected": -177.1616973876953,
+ "loss": 0.6533854961395263,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.11633266508579254,
+ "rewards/margins": 0.09301649034023285,
+ "rewards/rejected": 0.023316185921430588,
+ "step": 1760
+ },
+ {
+ "epoch": 1.416,
+ "grad_norm": 1.3164308071136475,
+ "learning_rate": 5.869629629629629e-07,
+ "logits/chosen": 1.5159170627593994,
+ "logits/rejected": 1.4434025287628174,
+ "logps/chosen": -161.05157470703125,
+ "logps/rejected": -140.06948852539062,
+ "loss": 0.660222864151001,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.10675680637359619,
+ "rewards/margins": 0.07750208675861359,
+ "rewards/rejected": 0.029254715889692307,
+ "step": 1770
+ },
+ {
+ "epoch": 1.424,
+ "grad_norm": 1.102005124092102,
+ "learning_rate": 5.839999999999999e-07,
+ "logits/chosen": 1.349440336227417,
+ "logits/rejected": 1.1565381288528442,
+ "logps/chosen": -150.61328125,
+ "logps/rejected": -151.36001586914062,
+ "loss": 0.660042142868042,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.09433183073997498,
+ "rewards/margins": 0.07778269052505493,
+ "rewards/rejected": 0.016549136489629745,
+ "step": 1780
+ },
+ {
+ "epoch": 1.432,
+ "grad_norm": 1.2305164337158203,
+ "learning_rate": 5.81037037037037e-07,
+ "logits/chosen": 1.4631009101867676,
+ "logits/rejected": 1.497757911682129,
+ "logps/chosen": -156.7335968017578,
+ "logps/rejected": -150.91065979003906,
+ "loss": 0.6564284324645996,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.13712921738624573,
+ "rewards/margins": 0.09247532486915588,
+ "rewards/rejected": 0.04465390369296074,
+ "step": 1790
+ },
+ {
+ "epoch": 1.44,
+ "grad_norm": 1.3459731340408325,
+ "learning_rate": 5.780740740740741e-07,
+ "logits/chosen": 1.5344784259796143,
+ "logits/rejected": 1.5709785223007202,
+ "logps/chosen": -166.15830993652344,
+ "logps/rejected": -158.678955078125,
+ "loss": 0.6511439800262451,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1227751225233078,
+ "rewards/margins": 0.0968184620141983,
+ "rewards/rejected": 0.025956671684980392,
+ "step": 1800
+ },
+ {
+ "epoch": 1.44,
+ "eval_logits/chosen": 1.4476159811019897,
+ "eval_logits/rejected": 1.4754574298858643,
+ "eval_logps/chosen": -152.70677185058594,
+ "eval_logps/rejected": -147.41787719726562,
+ "eval_loss": 0.6876539587974548,
+ "eval_rewards/accuracies": 0.5199999809265137,
+ "eval_rewards/chosen": 0.08420863002538681,
+ "eval_rewards/margins": 0.023782728239893913,
+ "eval_rewards/rejected": 0.06042589247226715,
+ "eval_runtime": 91.0223,
+ "eval_samples_per_second": 5.493,
+ "eval_steps_per_second": 2.747,
+ "step": 1800
+ },
+ {
+ "epoch": 1.448,
+ "grad_norm": 1.3158259391784668,
+ "learning_rate": 5.751111111111111e-07,
+ "logits/chosen": 1.425363302230835,
+ "logits/rejected": 1.350799798965454,
+ "logps/chosen": -151.4562530517578,
+ "logps/rejected": -134.9906005859375,
+ "loss": 0.6759338855743409,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1467444747686386,
+ "rewards/margins": 0.047227732837200165,
+ "rewards/rejected": 0.09951673448085785,
+ "step": 1810
+ },
+ {
+ "epoch": 1.456,
+ "grad_norm": 1.179150938987732,
+ "learning_rate": 5.721481481481481e-07,
+ "logits/chosen": 1.6323440074920654,
+ "logits/rejected": 1.4893152713775635,
+ "logps/chosen": -149.57675170898438,
+ "logps/rejected": -146.96237182617188,
+ "loss": 0.657456636428833,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1445317566394806,
+ "rewards/margins": 0.0896807312965393,
+ "rewards/rejected": 0.05485100671648979,
+ "step": 1820
+ },
+ {
+ "epoch": 1.464,
+ "grad_norm": 1.0987558364868164,
+ "learning_rate": 5.691851851851853e-07,
+ "logits/chosen": 1.3851706981658936,
+ "logits/rejected": 1.381049633026123,
+ "logps/chosen": -163.92648315429688,
+ "logps/rejected": -142.801025390625,
+ "loss": 0.6631375312805176,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.14093783497810364,
+ "rewards/margins": 0.07281022518873215,
+ "rewards/rejected": 0.06812760978937149,
+ "step": 1830
+ },
+ {
+ "epoch": 1.472,
+ "grad_norm": 1.1079164743423462,
+ "learning_rate": 5.662222222222222e-07,
+ "logits/chosen": 1.4228973388671875,
+ "logits/rejected": 1.3453642129898071,
+ "logps/chosen": -148.5561981201172,
+ "logps/rejected": -141.28009033203125,
+ "loss": 0.6612800121307373,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.10088491439819336,
+ "rewards/margins": 0.08228114247322083,
+ "rewards/rejected": 0.018603766337037086,
+ "step": 1840
+ },
+ {
+ "epoch": 1.48,
+ "grad_norm": 1.0415616035461426,
+ "learning_rate": 5.632592592592592e-07,
+ "logits/chosen": 1.4923865795135498,
+ "logits/rejected": 1.536971092224121,
+ "logps/chosen": -142.55166625976562,
+ "logps/rejected": -149.22906494140625,
+ "loss": 0.6690784931182862,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.10785870254039764,
+ "rewards/margins": 0.05975862219929695,
+ "rewards/rejected": 0.04810008406639099,
+ "step": 1850
+ },
+ {
+ "epoch": 1.488,
+ "grad_norm": 1.4004729986190796,
+ "learning_rate": 5.602962962962962e-07,
+ "logits/chosen": 1.4783391952514648,
+ "logits/rejected": 1.5112550258636475,
+ "logps/chosen": -153.54034423828125,
+ "logps/rejected": -118.77142333984375,
+ "loss": 0.6666333675384521,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.12789246439933777,
+ "rewards/margins": 0.06238863617181778,
+ "rewards/rejected": 0.06550382077693939,
+ "step": 1860
+ },
+ {
+ "epoch": 1.496,
+ "grad_norm": 1.1908437013626099,
+ "learning_rate": 5.573333333333333e-07,
+ "logits/chosen": 1.5748203992843628,
+ "logits/rejected": 1.4827988147735596,
+ "logps/chosen": -162.0931854248047,
+ "logps/rejected": -156.00779724121094,
+ "loss": 0.6626818656921387,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.11168892681598663,
+ "rewards/margins": 0.07420919090509415,
+ "rewards/rejected": 0.03747973591089249,
+ "step": 1870
+ },
+ {
+ "epoch": 1.504,
+ "grad_norm": 1.288218379020691,
+ "learning_rate": 5.543703703703704e-07,
+ "logits/chosen": 1.5260225534439087,
+ "logits/rejected": 1.5843703746795654,
+ "logps/chosen": -148.2488555908203,
+ "logps/rejected": -161.31570434570312,
+ "loss": 0.6677760124206543,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.12626001238822937,
+ "rewards/margins": 0.06881437450647354,
+ "rewards/rejected": 0.05744565278291702,
+ "step": 1880
+ },
+ {
+ "epoch": 1.512,
+ "grad_norm": 1.2899844646453857,
+ "learning_rate": 5.514074074074074e-07,
+ "logits/chosen": 1.417141318321228,
+ "logits/rejected": 1.389639973640442,
+ "logps/chosen": -160.5532989501953,
+ "logps/rejected": -158.82923889160156,
+ "loss": 0.6510573863983155,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.15593740344047546,
+ "rewards/margins": 0.09999983012676239,
+ "rewards/rejected": 0.05593757703900337,
+ "step": 1890
+ },
+ {
+ "epoch": 1.52,
+ "grad_norm": 1.3312058448791504,
+ "learning_rate": 5.484444444444444e-07,
+ "logits/chosen": 1.4996871948242188,
+ "logits/rejected": 1.6273930072784424,
+ "logps/chosen": -154.84347534179688,
+ "logps/rejected": -130.4560089111328,
+ "loss": 0.6844823360443115,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.13181835412979126,
+ "rewards/margins": 0.029756689444184303,
+ "rewards/rejected": 0.10206165164709091,
+ "step": 1900
+ },
+ {
+ "epoch": 1.52,
+ "eval_logits/chosen": 1.4582936763763428,
+ "eval_logits/rejected": 1.4853557348251343,
+ "eval_logps/chosen": -152.36231994628906,
+ "eval_logps/rejected": -147.1087188720703,
+ "eval_loss": 0.6870284676551819,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": 0.1186550110578537,
+ "eval_rewards/margins": 0.027311701327562332,
+ "eval_rewards/rejected": 0.09134332090616226,
+ "eval_runtime": 90.913,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 1900
+ },
+ {
+ "epoch": 1.528,
+ "grad_norm": 1.2257205247879028,
+ "learning_rate": 5.454814814814814e-07,
+ "logits/chosen": 1.4909894466400146,
+ "logits/rejected": 1.4579002857208252,
+ "logps/chosen": -129.2428741455078,
+ "logps/rejected": -149.32220458984375,
+ "loss": 0.6677265644073487,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.13237449526786804,
+ "rewards/margins": 0.059117116034030914,
+ "rewards/rejected": 0.07325738668441772,
+ "step": 1910
+ },
+ {
+ "epoch": 1.536,
+ "grad_norm": 1.3048031330108643,
+ "learning_rate": 5.425185185185185e-07,
+ "logits/chosen": 1.5237693786621094,
+ "logits/rejected": 1.4846075773239136,
+ "logps/chosen": -159.10055541992188,
+ "logps/rejected": -155.69888305664062,
+ "loss": 0.6709336280822754,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.1088198646903038,
+ "rewards/margins": 0.05951980873942375,
+ "rewards/rejected": 0.04930006340146065,
+ "step": 1920
+ },
+ {
+ "epoch": 1.544,
+ "grad_norm": 1.3463319540023804,
+ "learning_rate": 5.395555555555556e-07,
+ "logits/chosen": 1.6476523876190186,
+ "logits/rejected": 1.6236053705215454,
+ "logps/chosen": -171.18319702148438,
+ "logps/rejected": -172.63796997070312,
+ "loss": 0.6670461177825928,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.13160961866378784,
+ "rewards/margins": 0.06587684154510498,
+ "rewards/rejected": 0.06573276221752167,
+ "step": 1930
+ },
+ {
+ "epoch": 1.552,
+ "grad_norm": 1.2326266765594482,
+ "learning_rate": 5.365925925925926e-07,
+ "logits/chosen": 1.4379268884658813,
+ "logits/rejected": 1.5206880569458008,
+ "logps/chosen": -157.2953643798828,
+ "logps/rejected": -153.21218872070312,
+ "loss": 0.6558516502380372,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.17594724893569946,
+ "rewards/margins": 0.09524654597043991,
+ "rewards/rejected": 0.08070070296525955,
+ "step": 1940
+ },
+ {
+ "epoch": 1.56,
+ "grad_norm": 1.1362297534942627,
+ "learning_rate": 5.336296296296296e-07,
+ "logits/chosen": 1.524285078048706,
+ "logits/rejected": 1.3964688777923584,
+ "logps/chosen": -161.221435546875,
+ "logps/rejected": -154.8270263671875,
+ "loss": 0.6746312141418457,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.12519574165344238,
+ "rewards/margins": 0.05550859495997429,
+ "rewards/rejected": 0.06968715041875839,
+ "step": 1950
+ },
+ {
+ "epoch": 1.568,
+ "grad_norm": 1.5411877632141113,
+ "learning_rate": 5.306666666666665e-07,
+ "logits/chosen": 1.2426170110702515,
+ "logits/rejected": 1.3256272077560425,
+ "logps/chosen": -137.97537231445312,
+ "logps/rejected": -159.09580993652344,
+ "loss": 0.6492344856262207,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.16253654658794403,
+ "rewards/margins": 0.12099182605743408,
+ "rewards/rejected": 0.04154472425580025,
+ "step": 1960
+ },
+ {
+ "epoch": 1.576,
+ "grad_norm": 1.2111555337905884,
+ "learning_rate": 5.277037037037037e-07,
+ "logits/chosen": 1.3337433338165283,
+ "logits/rejected": 1.479209065437317,
+ "logps/chosen": -149.0251007080078,
+ "logps/rejected": -140.0357208251953,
+ "loss": 0.6559633731842041,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.13673515617847443,
+ "rewards/margins": 0.0912124291062355,
+ "rewards/rejected": 0.04552273079752922,
+ "step": 1970
+ },
+ {
+ "epoch": 1.584,
+ "grad_norm": 1.1228832006454468,
+ "learning_rate": 5.247407407407407e-07,
+ "logits/chosen": 1.4376949071884155,
+ "logits/rejected": 1.5033822059631348,
+ "logps/chosen": -144.91751098632812,
+ "logps/rejected": -147.37057495117188,
+ "loss": 0.6593158721923829,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.1573731154203415,
+ "rewards/margins": 0.08546318858861923,
+ "rewards/rejected": 0.07190994173288345,
+ "step": 1980
+ },
+ {
+ "epoch": 1.592,
+ "grad_norm": 0.9601779580116272,
+ "learning_rate": 5.217777777777777e-07,
+ "logits/chosen": 1.2674219608306885,
+ "logits/rejected": 1.3877508640289307,
+ "logps/chosen": -116.023193359375,
+ "logps/rejected": -122.81617736816406,
+ "loss": 0.6842892646789551,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.13847123086452484,
+ "rewards/margins": 0.026916390284895897,
+ "rewards/rejected": 0.11155486106872559,
+ "step": 1990
+ },
+ {
+ "epoch": 1.6,
+ "grad_norm": 1.2849960327148438,
+ "learning_rate": 5.188148148148148e-07,
+ "logits/chosen": 1.6982529163360596,
+ "logits/rejected": 1.6600332260131836,
+ "logps/chosen": -143.352294921875,
+ "logps/rejected": -150.0862274169922,
+ "loss": 0.6666803359985352,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.13707581162452698,
+ "rewards/margins": 0.06178014352917671,
+ "rewards/rejected": 0.07529564201831818,
+ "step": 2000
+ },
+ {
+ "epoch": 1.6,
+ "eval_logits/chosen": 1.4610096216201782,
+ "eval_logits/rejected": 1.4881619215011597,
+ "eval_logps/chosen": -152.36614990234375,
+ "eval_logps/rejected": -147.1319580078125,
+ "eval_loss": 0.6861760020256042,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": 0.11826877295970917,
+ "eval_rewards/margins": 0.02924923785030842,
+ "eval_rewards/rejected": 0.0890195295214653,
+ "eval_runtime": 91.0197,
+ "eval_samples_per_second": 5.493,
+ "eval_steps_per_second": 2.747,
+ "step": 2000
+ },
+ {
+ "epoch": 1.608,
+ "grad_norm": 1.731399655342102,
+ "learning_rate": 5.158518518518519e-07,
+ "logits/chosen": 1.4297409057617188,
+ "logits/rejected": 1.5017236471176147,
+ "logps/chosen": -164.93292236328125,
+ "logps/rejected": -164.80677795410156,
+ "loss": 0.6650710582733155,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.20246899127960205,
+ "rewards/margins": 0.0675087422132492,
+ "rewards/rejected": 0.13496026396751404,
+ "step": 2010
+ },
+ {
+ "epoch": 1.616,
+ "grad_norm": 1.2728053331375122,
+ "learning_rate": 5.128888888888888e-07,
+ "logits/chosen": 1.305499792098999,
+ "logits/rejected": 1.436187744140625,
+ "logps/chosen": -143.90879821777344,
+ "logps/rejected": -158.8233184814453,
+ "loss": 0.6744787693023682,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.17535606026649475,
+ "rewards/margins": 0.04918169230222702,
+ "rewards/rejected": 0.12617434561252594,
+ "step": 2020
+ },
+ {
+ "epoch": 1.624,
+ "grad_norm": 1.4291980266571045,
+ "learning_rate": 5.099259259259259e-07,
+ "logits/chosen": 1.4624104499816895,
+ "logits/rejected": 1.4936870336532593,
+ "logps/chosen": -151.03884887695312,
+ "logps/rejected": -127.40641784667969,
+ "loss": 0.6721909046173096,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.1530950516462326,
+ "rewards/margins": 0.0540502592921257,
+ "rewards/rejected": 0.0990448072552681,
+ "step": 2030
+ },
+ {
+ "epoch": 1.6320000000000001,
+ "grad_norm": 1.5909459590911865,
+ "learning_rate": 5.069629629629629e-07,
+ "logits/chosen": 1.468285322189331,
+ "logits/rejected": 1.3470795154571533,
+ "logps/chosen": -158.12046813964844,
+ "logps/rejected": -160.971923828125,
+ "loss": 0.6752189636230469,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.12029504776000977,
+ "rewards/margins": 0.05528820678591728,
+ "rewards/rejected": 0.06500682979822159,
+ "step": 2040
+ },
+ {
+ "epoch": 1.6400000000000001,
+ "grad_norm": 1.2923390865325928,
+ "learning_rate": 5.04e-07,
+ "logits/chosen": 1.530002236366272,
+ "logits/rejected": 1.5821081399917603,
+ "logps/chosen": -157.1008758544922,
+ "logps/rejected": -158.16775512695312,
+ "loss": 0.6889143943786621,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.14681917428970337,
+ "rewards/margins": 0.01898271217942238,
+ "rewards/rejected": 0.1278364509344101,
+ "step": 2050
+ },
+ {
+ "epoch": 1.6480000000000001,
+ "grad_norm": 1.2025870084762573,
+ "learning_rate": 5.010370370370371e-07,
+ "logits/chosen": 1.4749194383621216,
+ "logits/rejected": 1.3301353454589844,
+ "logps/chosen": -119.29649353027344,
+ "logps/rejected": -145.0949249267578,
+ "loss": 0.6332630634307861,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.1944357454776764,
+ "rewards/margins": 0.14449670910835266,
+ "rewards/rejected": 0.049939028918743134,
+ "step": 2060
+ },
+ {
+ "epoch": 1.6560000000000001,
+ "grad_norm": 1.3136606216430664,
+ "learning_rate": 4.98074074074074e-07,
+ "logits/chosen": 1.5436251163482666,
+ "logits/rejected": 1.4338414669036865,
+ "logps/chosen": -157.49942016601562,
+ "logps/rejected": -140.4504852294922,
+ "loss": 0.6923964977264404,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.13830263912677765,
+ "rewards/margins": 0.010729723609983921,
+ "rewards/rejected": 0.1275729238986969,
+ "step": 2070
+ },
+ {
+ "epoch": 1.6640000000000001,
+ "grad_norm": 1.2378884553909302,
+ "learning_rate": 4.951111111111111e-07,
+ "logits/chosen": 1.5672242641448975,
+ "logits/rejected": 1.4682090282440186,
+ "logps/chosen": -158.99301147460938,
+ "logps/rejected": -149.21221923828125,
+ "loss": 0.6473859786987305,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.1993202269077301,
+ "rewards/margins": 0.10525582730770111,
+ "rewards/rejected": 0.0940643846988678,
+ "step": 2080
+ },
+ {
+ "epoch": 1.6720000000000002,
+ "grad_norm": 1.1976169347763062,
+ "learning_rate": 4.921481481481482e-07,
+ "logits/chosen": 1.4160411357879639,
+ "logits/rejected": 1.3142106533050537,
+ "logps/chosen": -153.40621948242188,
+ "logps/rejected": -154.57308959960938,
+ "loss": 0.6821109771728515,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.14698973298072815,
+ "rewards/margins": 0.034348614513874054,
+ "rewards/rejected": 0.1126411110162735,
+ "step": 2090
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "grad_norm": 1.3028976917266846,
+ "learning_rate": 4.891851851851852e-07,
+ "logits/chosen": 1.61053466796875,
+ "logits/rejected": 1.526831030845642,
+ "logps/chosen": -164.3324737548828,
+ "logps/rejected": -149.382080078125,
+ "loss": 0.6684560298919677,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.22455796599388123,
+ "rewards/margins": 0.06641162931919098,
+ "rewards/rejected": 0.15814633667469025,
+ "step": 2100
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "eval_logits/chosen": 1.4865608215332031,
+ "eval_logits/rejected": 1.5141921043395996,
+ "eval_logps/chosen": -151.979248046875,
+ "eval_logps/rejected": -146.76809692382812,
+ "eval_loss": 0.6849560141563416,
+ "eval_rewards/accuracies": 0.5379999876022339,
+ "eval_rewards/chosen": 0.15696033835411072,
+ "eval_rewards/margins": 0.0315566249191761,
+ "eval_rewards/rejected": 0.1254037320613861,
+ "eval_runtime": 90.9172,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 2100
+ },
+ {
+ "epoch": 1.688,
+ "grad_norm": 1.3767266273498535,
+ "learning_rate": 4.862222222222222e-07,
+ "logits/chosen": 1.458231806755066,
+ "logits/rejected": 1.3821945190429688,
+ "logps/chosen": -166.67698669433594,
+ "logps/rejected": -156.74313354492188,
+ "loss": 0.6658836364746094,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.17039546370506287,
+ "rewards/margins": 0.07431827485561371,
+ "rewards/rejected": 0.09607716649770737,
+ "step": 2110
+ },
+ {
+ "epoch": 1.696,
+ "grad_norm": 1.4396629333496094,
+ "learning_rate": 4.832592592592592e-07,
+ "logits/chosen": 1.5568702220916748,
+ "logits/rejected": 1.5459610223770142,
+ "logps/chosen": -133.0255889892578,
+ "logps/rejected": -138.2047882080078,
+ "loss": 0.6704818725585937,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.173842191696167,
+ "rewards/margins": 0.06081608682870865,
+ "rewards/rejected": 0.11302611976861954,
+ "step": 2120
+ },
+ {
+ "epoch": 1.704,
+ "grad_norm": 2.3225553035736084,
+ "learning_rate": 4.802962962962962e-07,
+ "logits/chosen": 1.5432292222976685,
+ "logits/rejected": 1.521958589553833,
+ "logps/chosen": -165.04318237304688,
+ "logps/rejected": -163.95855712890625,
+ "loss": 0.6567724704742431,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.20300237834453583,
+ "rewards/margins": 0.09381169825792313,
+ "rewards/rejected": 0.1091906875371933,
+ "step": 2130
+ },
+ {
+ "epoch": 1.712,
+ "grad_norm": 1.3185398578643799,
+ "learning_rate": 4.773333333333333e-07,
+ "logits/chosen": 1.2340335845947266,
+ "logits/rejected": 1.3610973358154297,
+ "logps/chosen": -139.11878967285156,
+ "logps/rejected": -146.57260131835938,
+ "loss": 0.671626091003418,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.17043188214302063,
+ "rewards/margins": 0.054147154092788696,
+ "rewards/rejected": 0.11628471314907074,
+ "step": 2140
+ },
+ {
+ "epoch": 1.72,
+ "grad_norm": 1.618586540222168,
+ "learning_rate": 4.7437037037037034e-07,
+ "logits/chosen": 1.5895780324935913,
+ "logits/rejected": 1.613523244857788,
+ "logps/chosen": -156.53970336914062,
+ "logps/rejected": -154.35116577148438,
+ "loss": 0.6798102855682373,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.20041461288928986,
+ "rewards/margins": 0.03897023946046829,
+ "rewards/rejected": 0.16144435107707977,
+ "step": 2150
+ },
+ {
+ "epoch": 1.728,
+ "grad_norm": 0.8500939607620239,
+ "learning_rate": 4.7140740740740736e-07,
+ "logits/chosen": 1.3736517429351807,
+ "logits/rejected": 1.361998438835144,
+ "logps/chosen": -137.701904296875,
+ "logps/rejected": -139.91567993164062,
+ "loss": 0.6568773269653321,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.16317322850227356,
+ "rewards/margins": 0.08119454979896545,
+ "rewards/rejected": 0.0819786787033081,
+ "step": 2160
+ },
+ {
+ "epoch": 1.736,
+ "grad_norm": 1.2291405200958252,
+ "learning_rate": 4.6844444444444444e-07,
+ "logits/chosen": 1.474157452583313,
+ "logits/rejected": 1.4488928318023682,
+ "logps/chosen": -152.36732482910156,
+ "logps/rejected": -149.70980834960938,
+ "loss": 0.6551193714141845,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.2117660939693451,
+ "rewards/margins": 0.0950237363576889,
+ "rewards/rejected": 0.11674235761165619,
+ "step": 2170
+ },
+ {
+ "epoch": 1.744,
+ "grad_norm": 1.2946466207504272,
+ "learning_rate": 4.6548148148148146e-07,
+ "logits/chosen": 1.257048487663269,
+ "logits/rejected": 1.3120336532592773,
+ "logps/chosen": -143.43148803710938,
+ "logps/rejected": -169.9266815185547,
+ "loss": 0.6712408542633057,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.1496409922838211,
+ "rewards/margins": 0.05779259279370308,
+ "rewards/rejected": 0.09184839576482773,
+ "step": 2180
+ },
+ {
+ "epoch": 1.752,
+ "grad_norm": 1.0697646141052246,
+ "learning_rate": 4.625185185185185e-07,
+ "logits/chosen": 1.5544202327728271,
+ "logits/rejected": 1.5186070203781128,
+ "logps/chosen": -170.5447235107422,
+ "logps/rejected": -165.55209350585938,
+ "loss": 0.6674113750457764,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.12943358719348907,
+ "rewards/margins": 0.07018069922924042,
+ "rewards/rejected": 0.05925289914011955,
+ "step": 2190
+ },
+ {
+ "epoch": 1.76,
+ "grad_norm": 1.228698968887329,
+ "learning_rate": 4.595555555555555e-07,
+ "logits/chosen": 1.46980881690979,
+ "logits/rejected": 1.2382839918136597,
+ "logps/chosen": -133.5864715576172,
+ "logps/rejected": -135.65968322753906,
+ "loss": 0.6756285667419434,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.18240585923194885,
+ "rewards/margins": 0.052959512919187546,
+ "rewards/rejected": 0.1294463574886322,
+ "step": 2200
+ },
+ {
+ "epoch": 1.76,
+ "eval_logits/chosen": 1.4826997518539429,
+ "eval_logits/rejected": 1.5107221603393555,
+ "eval_logps/chosen": -152.19204711914062,
+ "eval_logps/rejected": -146.98675537109375,
+ "eval_loss": 0.6843143105506897,
+ "eval_rewards/accuracies": 0.5379999876022339,
+ "eval_rewards/chosen": 0.13567964732646942,
+ "eval_rewards/margins": 0.03213911131024361,
+ "eval_rewards/rejected": 0.10354051738977432,
+ "eval_runtime": 91.0116,
+ "eval_samples_per_second": 5.494,
+ "eval_steps_per_second": 2.747,
+ "step": 2200
+ },
+ {
+ "epoch": 1.768,
+ "grad_norm": 1.4145433902740479,
+ "learning_rate": 4.565925925925926e-07,
+ "logits/chosen": 1.49526047706604,
+ "logits/rejected": 1.519241213798523,
+ "logps/chosen": -152.7605438232422,
+ "logps/rejected": -140.98629760742188,
+ "loss": 0.6503778457641601,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.16126050055027008,
+ "rewards/margins": 0.0973939448595047,
+ "rewards/rejected": 0.06386655569076538,
+ "step": 2210
+ },
+ {
+ "epoch": 1.776,
+ "grad_norm": 1.4429577589035034,
+ "learning_rate": 4.536296296296296e-07,
+ "logits/chosen": 1.4706772565841675,
+ "logits/rejected": 1.3714580535888672,
+ "logps/chosen": -135.95150756835938,
+ "logps/rejected": -165.23587036132812,
+ "loss": 0.6639570713043212,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.18487155437469482,
+ "rewards/margins": 0.07484755665063858,
+ "rewards/rejected": 0.11002399027347565,
+ "step": 2220
+ },
+ {
+ "epoch": 1.784,
+ "grad_norm": 1.0622409582138062,
+ "learning_rate": 4.506666666666666e-07,
+ "logits/chosen": 1.2506721019744873,
+ "logits/rejected": 1.4802286624908447,
+ "logps/chosen": -129.645751953125,
+ "logps/rejected": -131.7567596435547,
+ "loss": 0.6788074493408203,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.12520259618759155,
+ "rewards/margins": 0.03735826909542084,
+ "rewards/rejected": 0.08784433454275131,
+ "step": 2230
+ },
+ {
+ "epoch": 1.792,
+ "grad_norm": 1.0001399517059326,
+ "learning_rate": 4.477037037037037e-07,
+ "logits/chosen": 1.4206364154815674,
+ "logits/rejected": 1.5338770151138306,
+ "logps/chosen": -143.95074462890625,
+ "logps/rejected": -160.1469268798828,
+ "loss": 0.6695474147796631,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.13798081874847412,
+ "rewards/margins": 0.05759159475564957,
+ "rewards/rejected": 0.08038921654224396,
+ "step": 2240
+ },
+ {
+ "epoch": 1.8,
+ "grad_norm": 1.3531230688095093,
+ "learning_rate": 4.4474074074074067e-07,
+ "logits/chosen": 1.5709412097930908,
+ "logits/rejected": 1.5301862955093384,
+ "logps/chosen": -161.30087280273438,
+ "logps/rejected": -150.31236267089844,
+ "loss": 0.6771373271942138,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.1488293707370758,
+ "rewards/margins": 0.04127617925405502,
+ "rewards/rejected": 0.10755319893360138,
+ "step": 2250
+ },
+ {
+ "epoch": 1.808,
+ "grad_norm": 1.37367582321167,
+ "learning_rate": 4.4177777777777775e-07,
+ "logits/chosen": 1.36393141746521,
+ "logits/rejected": 1.484881043434143,
+ "logps/chosen": -125.18086242675781,
+ "logps/rejected": -148.78546142578125,
+ "loss": 0.6596842288970948,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.15397249162197113,
+ "rewards/margins": 0.07893262803554535,
+ "rewards/rejected": 0.07503987848758698,
+ "step": 2260
+ },
+ {
+ "epoch": 1.8159999999999998,
+ "grad_norm": 1.4511909484863281,
+ "learning_rate": 4.388148148148148e-07,
+ "logits/chosen": 1.3140859603881836,
+ "logits/rejected": 1.3879334926605225,
+ "logps/chosen": -131.48226928710938,
+ "logps/rejected": -123.94035339355469,
+ "loss": 0.6698450088500977,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.12767434120178223,
+ "rewards/margins": 0.06716184318065643,
+ "rewards/rejected": 0.060512494295835495,
+ "step": 2270
+ },
+ {
+ "epoch": 1.8239999999999998,
+ "grad_norm": 1.3375639915466309,
+ "learning_rate": 4.3585185185185184e-07,
+ "logits/chosen": 1.5821285247802734,
+ "logits/rejected": 1.43180251121521,
+ "logps/chosen": -167.2334747314453,
+ "logps/rejected": -145.28871154785156,
+ "loss": 0.658758020401001,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.12011411041021347,
+ "rewards/margins": 0.08019578456878662,
+ "rewards/rejected": 0.039918314665555954,
+ "step": 2280
+ },
+ {
+ "epoch": 1.8319999999999999,
+ "grad_norm": 1.2121729850769043,
+ "learning_rate": 4.3288888888888887e-07,
+ "logits/chosen": 1.3520320653915405,
+ "logits/rejected": 1.18943452835083,
+ "logps/chosen": -147.15621948242188,
+ "logps/rejected": -132.015625,
+ "loss": 0.6655458450317383,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08098682016134262,
+ "rewards/margins": 0.07385818660259247,
+ "rewards/rejected": 0.007128633558750153,
+ "step": 2290
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "grad_norm": 1.1618680953979492,
+ "learning_rate": 4.2992592592592594e-07,
+ "logits/chosen": 1.4779785871505737,
+ "logits/rejected": 1.448462724685669,
+ "logps/chosen": -157.75070190429688,
+ "logps/rejected": -141.3148956298828,
+ "loss": 0.6813467979431153,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.12400279194116592,
+ "rewards/margins": 0.034347161650657654,
+ "rewards/rejected": 0.08965563774108887,
+ "step": 2300
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "eval_logits/chosen": 1.4821760654449463,
+ "eval_logits/rejected": 1.5101500749588013,
+ "eval_logps/chosen": -152.2089080810547,
+ "eval_logps/rejected": -147.0087432861328,
+ "eval_loss": 0.6842693090438843,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.13399286568164825,
+ "eval_rewards/margins": 0.032653141766786575,
+ "eval_rewards/rejected": 0.10133973509073257,
+ "eval_runtime": 90.8665,
+ "eval_samples_per_second": 5.503,
+ "eval_steps_per_second": 2.751,
+ "step": 2300
+ },
+ {
+ "epoch": 1.8479999999999999,
+ "grad_norm": 1.1095342636108398,
+ "learning_rate": 4.2696296296296296e-07,
+ "logits/chosen": 1.6475988626480103,
+ "logits/rejected": 1.6654495000839233,
+ "logps/chosen": -165.965087890625,
+ "logps/rejected": -153.98214721679688,
+ "loss": 0.6605272769927979,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.2209581881761551,
+ "rewards/margins": 0.07579347491264343,
+ "rewards/rejected": 0.14516471326351166,
+ "step": 2310
+ },
+ {
+ "epoch": 1.8559999999999999,
+ "grad_norm": 1.1993674039840698,
+ "learning_rate": 4.24e-07,
+ "logits/chosen": 1.5783694982528687,
+ "logits/rejected": 1.406640648841858,
+ "logps/chosen": -139.3554229736328,
+ "logps/rejected": -126.39006042480469,
+ "loss": 0.6817263126373291,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1467253863811493,
+ "rewards/margins": 0.03637007623910904,
+ "rewards/rejected": 0.11035531759262085,
+ "step": 2320
+ },
+ {
+ "epoch": 1.8639999999999999,
+ "grad_norm": 1.4875127077102661,
+ "learning_rate": 4.21037037037037e-07,
+ "logits/chosen": 1.4171388149261475,
+ "logits/rejected": 1.4270119667053223,
+ "logps/chosen": -145.1273651123047,
+ "logps/rejected": -157.85816955566406,
+ "loss": 0.6678358554840088,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.14816439151763916,
+ "rewards/margins": 0.06963086873292923,
+ "rewards/rejected": 0.07853354513645172,
+ "step": 2330
+ },
+ {
+ "epoch": 1.8719999999999999,
+ "grad_norm": 1.2899141311645508,
+ "learning_rate": 4.180740740740741e-07,
+ "logits/chosen": 1.3682881593704224,
+ "logits/rejected": 1.3383492231369019,
+ "logps/chosen": -167.96450805664062,
+ "logps/rejected": -158.2183837890625,
+ "loss": 0.6754446029663086,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.1453542709350586,
+ "rewards/margins": 0.047066912055015564,
+ "rewards/rejected": 0.09828735888004303,
+ "step": 2340
+ },
+ {
+ "epoch": 1.88,
+ "grad_norm": 1.4864925146102905,
+ "learning_rate": 4.151111111111111e-07,
+ "logits/chosen": 1.5181632041931152,
+ "logits/rejected": 1.6898539066314697,
+ "logps/chosen": -146.9915771484375,
+ "logps/rejected": -156.66282653808594,
+ "loss": 0.6776726245880127,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.1270596832036972,
+ "rewards/margins": 0.04481229558587074,
+ "rewards/rejected": 0.08224737644195557,
+ "step": 2350
+ },
+ {
+ "epoch": 1.888,
+ "grad_norm": 1.3912899494171143,
+ "learning_rate": 4.1214814814814813e-07,
+ "logits/chosen": 1.279984474182129,
+ "logits/rejected": 1.258424997329712,
+ "logps/chosen": -162.26925659179688,
+ "logps/rejected": -153.519287109375,
+ "loss": 0.680011796951294,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1301378756761551,
+ "rewards/margins": 0.03849628567695618,
+ "rewards/rejected": 0.09164158999919891,
+ "step": 2360
+ },
+ {
+ "epoch": 1.896,
+ "grad_norm": 1.2498691082000732,
+ "learning_rate": 4.091851851851852e-07,
+ "logits/chosen": 1.5644304752349854,
+ "logits/rejected": 1.5302040576934814,
+ "logps/chosen": -181.20506286621094,
+ "logps/rejected": -164.98670959472656,
+ "loss": 0.6632030963897705,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.16487805545330048,
+ "rewards/margins": 0.08321762084960938,
+ "rewards/rejected": 0.0816604346036911,
+ "step": 2370
+ },
+ {
+ "epoch": 1.904,
+ "grad_norm": 1.3421261310577393,
+ "learning_rate": 4.0622222222222217e-07,
+ "logits/chosen": 1.5388444662094116,
+ "logits/rejected": 1.5939340591430664,
+ "logps/chosen": -151.34434509277344,
+ "logps/rejected": -149.8802490234375,
+ "loss": 0.691883659362793,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.11643080413341522,
+ "rewards/margins": 0.017300304025411606,
+ "rewards/rejected": 0.09913049638271332,
+ "step": 2380
+ },
+ {
+ "epoch": 1.912,
+ "grad_norm": 1.4378565549850464,
+ "learning_rate": 4.0325925925925925e-07,
+ "logits/chosen": 1.4750030040740967,
+ "logits/rejected": 1.385709524154663,
+ "logps/chosen": -159.98593139648438,
+ "logps/rejected": -144.0281219482422,
+ "loss": 0.6825335502624512,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.1235724687576294,
+ "rewards/margins": 0.03918968886137009,
+ "rewards/rejected": 0.0843827873468399,
+ "step": 2390
+ },
+ {
+ "epoch": 1.92,
+ "grad_norm": 1.1941649913787842,
+ "learning_rate": 4.002962962962963e-07,
+ "logits/chosen": 1.3726251125335693,
+ "logits/rejected": 1.5375208854675293,
+ "logps/chosen": -149.9639129638672,
+ "logps/rejected": -142.49295043945312,
+ "loss": 0.6461881160736084,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.12773704528808594,
+ "rewards/margins": 0.1104915589094162,
+ "rewards/rejected": 0.017245473340153694,
+ "step": 2400
+ },
+ {
+ "epoch": 1.92,
+ "eval_logits/chosen": 1.4780151844024658,
+ "eval_logits/rejected": 1.506591558456421,
+ "eval_logps/chosen": -152.50015258789062,
+ "eval_logps/rejected": -147.30506896972656,
+ "eval_loss": 0.6834847331047058,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": 0.10486816614866257,
+ "eval_rewards/margins": 0.03315889462828636,
+ "eval_rewards/rejected": 0.0717092752456665,
+ "eval_runtime": 90.9835,
+ "eval_samples_per_second": 5.496,
+ "eval_steps_per_second": 2.748,
+ "step": 2400
+ },
+ {
+ "epoch": 1.928,
+ "grad_norm": 1.1473901271820068,
+ "learning_rate": 3.973333333333333e-07,
+ "logits/chosen": 1.3499658107757568,
+ "logits/rejected": 1.479353666305542,
+ "logps/chosen": -153.15272521972656,
+ "logps/rejected": -170.32418823242188,
+ "loss": 0.6756062507629395,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.09217077493667603,
+ "rewards/margins": 0.05041539669036865,
+ "rewards/rejected": 0.041755370795726776,
+ "step": 2410
+ },
+ {
+ "epoch": 1.936,
+ "grad_norm": 1.4407395124435425,
+ "learning_rate": 3.9437037037037037e-07,
+ "logits/chosen": 1.5876386165618896,
+ "logits/rejected": 1.5701191425323486,
+ "logps/chosen": -144.0182342529297,
+ "logps/rejected": -130.75460815429688,
+ "loss": 0.6733745574951172,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.1180434376001358,
+ "rewards/margins": 0.050890080630779266,
+ "rewards/rejected": 0.06715335696935654,
+ "step": 2420
+ },
+ {
+ "epoch": 1.944,
+ "grad_norm": 1.46868097782135,
+ "learning_rate": 3.914074074074074e-07,
+ "logits/chosen": 1.4809507131576538,
+ "logits/rejected": 1.4793384075164795,
+ "logps/chosen": -144.4136199951172,
+ "logps/rejected": -140.72645568847656,
+ "loss": 0.65852952003479,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.13238464295864105,
+ "rewards/margins": 0.08860992640256882,
+ "rewards/rejected": 0.043774720281362534,
+ "step": 2430
+ },
+ {
+ "epoch": 1.952,
+ "grad_norm": 1.2951419353485107,
+ "learning_rate": 3.884444444444444e-07,
+ "logits/chosen": 1.4519541263580322,
+ "logits/rejected": 1.6060655117034912,
+ "logps/chosen": -150.94955444335938,
+ "logps/rejected": -146.84732055664062,
+ "loss": 0.650700569152832,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.14693069458007812,
+ "rewards/margins": 0.09734389185905457,
+ "rewards/rejected": 0.04958680272102356,
+ "step": 2440
+ },
+ {
+ "epoch": 1.96,
+ "grad_norm": 1.2181923389434814,
+ "learning_rate": 3.854814814814815e-07,
+ "logits/chosen": 1.5463917255401611,
+ "logits/rejected": 1.4773080348968506,
+ "logps/chosen": -150.07005310058594,
+ "logps/rejected": -163.2088165283203,
+ "loss": 0.6521890163421631,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.15517377853393555,
+ "rewards/margins": 0.09567751735448837,
+ "rewards/rejected": 0.05949625372886658,
+ "step": 2450
+ },
+ {
+ "epoch": 1.968,
+ "grad_norm": 1.2294185161590576,
+ "learning_rate": 3.825185185185185e-07,
+ "logits/chosen": 1.4533751010894775,
+ "logits/rejected": 1.4323598146438599,
+ "logps/chosen": -156.0291290283203,
+ "logps/rejected": -161.64663696289062,
+ "loss": 0.6698451042175293,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.07660157233476639,
+ "rewards/margins": 0.05659238249063492,
+ "rewards/rejected": 0.020009174942970276,
+ "step": 2460
+ },
+ {
+ "epoch": 1.976,
+ "grad_norm": 1.0025874376296997,
+ "learning_rate": 3.7955555555555553e-07,
+ "logits/chosen": 1.6077829599380493,
+ "logits/rejected": 1.502532720565796,
+ "logps/chosen": -168.87271118164062,
+ "logps/rejected": -160.9680633544922,
+ "loss": 0.6690214157104493,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.11292171478271484,
+ "rewards/margins": 0.06120762228965759,
+ "rewards/rejected": 0.05171409994363785,
+ "step": 2470
+ },
+ {
+ "epoch": 1.984,
+ "grad_norm": 1.1663116216659546,
+ "learning_rate": 3.7659259259259255e-07,
+ "logits/chosen": 1.6099679470062256,
+ "logits/rejected": 1.6698023080825806,
+ "logps/chosen": -147.36190795898438,
+ "logps/rejected": -145.37986755371094,
+ "loss": 0.666702127456665,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1266266405582428,
+ "rewards/margins": 0.0683312863111496,
+ "rewards/rejected": 0.0582953505218029,
+ "step": 2480
+ },
+ {
+ "epoch": 1.992,
+ "grad_norm": 1.1562272310256958,
+ "learning_rate": 3.7362962962962963e-07,
+ "logits/chosen": 1.5222467184066772,
+ "logits/rejected": 1.592881441116333,
+ "logps/chosen": -145.1186065673828,
+ "logps/rejected": -158.30718994140625,
+ "loss": 0.6763694763183594,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.09335663169622421,
+ "rewards/margins": 0.04241436719894409,
+ "rewards/rejected": 0.05094226449728012,
+ "step": 2490
+ },
+ {
+ "epoch": 2.0,
+ "grad_norm": 1.5228629112243652,
+ "learning_rate": 3.7066666666666665e-07,
+ "logits/chosen": 1.5318562984466553,
+ "logits/rejected": 1.5440677404403687,
+ "logps/chosen": -178.16098022460938,
+ "logps/rejected": -143.88893127441406,
+ "loss": 0.6654858112335205,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0874411091208458,
+ "rewards/margins": 0.07056570053100586,
+ "rewards/rejected": 0.01687539741396904,
+ "step": 2500
+ },
+ {
+ "epoch": 2.0,
+ "eval_logits/chosen": 1.4583442211151123,
+ "eval_logits/rejected": 1.486719012260437,
+ "eval_logps/chosen": -152.8921661376953,
+ "eval_logps/rejected": -147.6856231689453,
+ "eval_loss": 0.6842741370201111,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.06566743552684784,
+ "eval_rewards/margins": 0.032015930861234665,
+ "eval_rewards/rejected": 0.03365150839090347,
+ "eval_runtime": 90.8716,
+ "eval_samples_per_second": 5.502,
+ "eval_steps_per_second": 2.751,
+ "step": 2500
+ },
+ {
+ "epoch": 2.008,
+ "grad_norm": 1.2679792642593384,
+ "learning_rate": 3.6770370370370367e-07,
+ "logits/chosen": 1.542234182357788,
+ "logits/rejected": 1.6291179656982422,
+ "logps/chosen": -142.90040588378906,
+ "logps/rejected": -161.39028930664062,
+ "loss": 0.6518040657043457,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.12705746293067932,
+ "rewards/margins": 0.09586121141910553,
+ "rewards/rejected": 0.031196262687444687,
+ "step": 2510
+ },
+ {
+ "epoch": 2.016,
+ "grad_norm": 1.2428960800170898,
+ "learning_rate": 3.6474074074074075e-07,
+ "logits/chosen": 1.5037320852279663,
+ "logits/rejected": 1.5008131265640259,
+ "logps/chosen": -154.43722534179688,
+ "logps/rejected": -179.9853515625,
+ "loss": 0.6436567306518555,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.09548825025558472,
+ "rewards/margins": 0.11602531373500824,
+ "rewards/rejected": -0.020537082105875015,
+ "step": 2520
+ },
+ {
+ "epoch": 2.024,
+ "grad_norm": 1.3352172374725342,
+ "learning_rate": 3.617777777777777e-07,
+ "logits/chosen": 1.5730235576629639,
+ "logits/rejected": 1.5479223728179932,
+ "logps/chosen": -143.82000732421875,
+ "logps/rejected": -154.56626892089844,
+ "loss": 0.6623670578002929,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.11983202397823334,
+ "rewards/margins": 0.07249654829502106,
+ "rewards/rejected": 0.04733546823263168,
+ "step": 2530
+ },
+ {
+ "epoch": 2.032,
+ "grad_norm": 1.2124547958374023,
+ "learning_rate": 3.588148148148148e-07,
+ "logits/chosen": 1.2967132329940796,
+ "logits/rejected": 1.198378324508667,
+ "logps/chosen": -134.57449340820312,
+ "logps/rejected": -124.48946380615234,
+ "loss": 0.6602365016937256,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.10377311706542969,
+ "rewards/margins": 0.07978598773479462,
+ "rewards/rejected": 0.023987144231796265,
+ "step": 2540
+ },
+ {
+ "epoch": 2.04,
+ "grad_norm": 1.7321484088897705,
+ "learning_rate": 3.5585185185185187e-07,
+ "logits/chosen": 1.4171000719070435,
+ "logits/rejected": 1.3644115924835205,
+ "logps/chosen": -153.2901611328125,
+ "logps/rejected": -143.07582092285156,
+ "loss": 0.6586461067199707,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.10289964824914932,
+ "rewards/margins": 0.08438605815172195,
+ "rewards/rejected": 0.018513580784201622,
+ "step": 2550
+ },
+ {
+ "epoch": 2.048,
+ "grad_norm": 1.2162495851516724,
+ "learning_rate": 3.5288888888888884e-07,
+ "logits/chosen": 1.5142430067062378,
+ "logits/rejected": 1.4998128414154053,
+ "logps/chosen": -156.86972045898438,
+ "logps/rejected": -155.01321411132812,
+ "loss": 0.633359432220459,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.12783676385879517,
+ "rewards/margins": 0.13910353183746338,
+ "rewards/rejected": -0.011266755871474743,
+ "step": 2560
+ },
+ {
+ "epoch": 2.056,
+ "grad_norm": 1.7551093101501465,
+ "learning_rate": 3.499259259259259e-07,
+ "logits/chosen": 1.3865793943405151,
+ "logits/rejected": 1.4954140186309814,
+ "logps/chosen": -156.48683166503906,
+ "logps/rejected": -148.37991333007812,
+ "loss": 0.6553828716278076,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.09534349292516708,
+ "rewards/margins": 0.08968523144721985,
+ "rewards/rejected": 0.005658247508108616,
+ "step": 2570
+ },
+ {
+ "epoch": 2.064,
+ "grad_norm": 1.282270908355713,
+ "learning_rate": 3.4696296296296293e-07,
+ "logits/chosen": 1.3754762411117554,
+ "logits/rejected": 1.3920589685440063,
+ "logps/chosen": -130.45675659179688,
+ "logps/rejected": -128.36073303222656,
+ "loss": 0.6361326217651367,
+ "rewards/accuracies": 0.8125,
+ "rewards/chosen": 0.16388167440891266,
+ "rewards/margins": 0.13015803694725037,
+ "rewards/rejected": 0.0337236113846302,
+ "step": 2580
+ },
+ {
+ "epoch": 2.072,
+ "grad_norm": 1.2549991607666016,
+ "learning_rate": 3.4399999999999996e-07,
+ "logits/chosen": 1.4334301948547363,
+ "logits/rejected": 1.4073057174682617,
+ "logps/chosen": -142.28848266601562,
+ "logps/rejected": -143.86415100097656,
+ "loss": 0.6604367733001709,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.10965392738580704,
+ "rewards/margins": 0.08058565109968185,
+ "rewards/rejected": 0.02906828001141548,
+ "step": 2590
+ },
+ {
+ "epoch": 2.08,
+ "grad_norm": 1.1193031072616577,
+ "learning_rate": 3.4103703703703703e-07,
+ "logits/chosen": 1.4255090951919556,
+ "logits/rejected": 1.3589593172073364,
+ "logps/chosen": -145.86782836914062,
+ "logps/rejected": -134.277099609375,
+ "loss": 0.6415833950042724,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.15703223645687103,
+ "rewards/margins": 0.11860913038253784,
+ "rewards/rejected": 0.03842310234904289,
+ "step": 2600
+ },
+ {
+ "epoch": 2.08,
+ "eval_logits/chosen": 1.4633173942565918,
+ "eval_logits/rejected": 1.4914942979812622,
+ "eval_logps/chosen": -152.7378387451172,
+ "eval_logps/rejected": -147.54226684570312,
+ "eval_loss": 0.684092104434967,
+ "eval_rewards/accuracies": 0.5559999942779541,
+ "eval_rewards/chosen": 0.08109957724809647,
+ "eval_rewards/margins": 0.033111557364463806,
+ "eval_rewards/rejected": 0.047988031059503555,
+ "eval_runtime": 90.9652,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 2600
+ },
+ {
+ "epoch": 2.088,
+ "grad_norm": 1.138805627822876,
+ "learning_rate": 3.3807407407407405e-07,
+ "logits/chosen": 1.6175915002822876,
+ "logits/rejected": 1.7044296264648438,
+ "logps/chosen": -161.02072143554688,
+ "logps/rejected": -150.9885711669922,
+ "loss": 0.6464156627655029,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.1512492299079895,
+ "rewards/margins": 0.11052403599023819,
+ "rewards/rejected": 0.040725212544202805,
+ "step": 2610
+ },
+ {
+ "epoch": 2.096,
+ "grad_norm": 1.2945690155029297,
+ "learning_rate": 3.3511111111111113e-07,
+ "logits/chosen": 1.7169849872589111,
+ "logits/rejected": 1.5235084295272827,
+ "logps/chosen": -178.69578552246094,
+ "logps/rejected": -147.8590850830078,
+ "loss": 0.6443929195404052,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.1698969006538391,
+ "rewards/margins": 0.11732057482004166,
+ "rewards/rejected": 0.052576325833797455,
+ "step": 2620
+ },
+ {
+ "epoch": 2.104,
+ "grad_norm": 1.4025486707687378,
+ "learning_rate": 3.321481481481481e-07,
+ "logits/chosen": 1.3389716148376465,
+ "logits/rejected": 1.4465675354003906,
+ "logps/chosen": -154.75857543945312,
+ "logps/rejected": -151.66146850585938,
+ "loss": 0.651912260055542,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.06493876874446869,
+ "rewards/margins": 0.09662672132253647,
+ "rewards/rejected": -0.03168795630335808,
+ "step": 2630
+ },
+ {
+ "epoch": 2.112,
+ "grad_norm": 1.5229434967041016,
+ "learning_rate": 3.291851851851852e-07,
+ "logits/chosen": 1.733494758605957,
+ "logits/rejected": 1.6284431219100952,
+ "logps/chosen": -159.22984313964844,
+ "logps/rejected": -148.73074340820312,
+ "loss": 0.6609207153320312,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.12649953365325928,
+ "rewards/margins": 0.07567869126796722,
+ "rewards/rejected": 0.050820838660001755,
+ "step": 2640
+ },
+ {
+ "epoch": 2.12,
+ "grad_norm": 1.4460108280181885,
+ "learning_rate": 3.2622222222222225e-07,
+ "logits/chosen": 1.4306278228759766,
+ "logits/rejected": 1.5417077541351318,
+ "logps/chosen": -158.87351989746094,
+ "logps/rejected": -174.2522430419922,
+ "loss": 0.6391911983489991,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.13457423448562622,
+ "rewards/margins": 0.12862522900104523,
+ "rewards/rejected": 0.005949012935161591,
+ "step": 2650
+ },
+ {
+ "epoch": 2.128,
+ "grad_norm": 1.3597654104232788,
+ "learning_rate": 3.232592592592592e-07,
+ "logits/chosen": 1.329627513885498,
+ "logits/rejected": 1.3987334966659546,
+ "logps/chosen": -165.3042449951172,
+ "logps/rejected": -153.50428771972656,
+ "loss": 0.6513238906860351,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.12750771641731262,
+ "rewards/margins": 0.10093003511428833,
+ "rewards/rejected": 0.026577681303024292,
+ "step": 2660
+ },
+ {
+ "epoch": 2.136,
+ "grad_norm": 1.2453831434249878,
+ "learning_rate": 3.202962962962963e-07,
+ "logits/chosen": 1.5347964763641357,
+ "logits/rejected": 1.5230233669281006,
+ "logps/chosen": -154.97242736816406,
+ "logps/rejected": -130.9337158203125,
+ "loss": 0.6872330665588379,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.08273214101791382,
+ "rewards/margins": 0.03720199316740036,
+ "rewards/rejected": 0.04553014412522316,
+ "step": 2670
+ },
+ {
+ "epoch": 2.144,
+ "grad_norm": 1.9659223556518555,
+ "learning_rate": 3.173333333333333e-07,
+ "logits/chosen": 1.5074447393417358,
+ "logits/rejected": 1.4809236526489258,
+ "logps/chosen": -146.32305908203125,
+ "logps/rejected": -143.98345947265625,
+ "loss": 0.6821617603302002,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.08092048764228821,
+ "rewards/margins": 0.03855252265930176,
+ "rewards/rejected": 0.04236796498298645,
+ "step": 2680
+ },
+ {
+ "epoch": 2.152,
+ "grad_norm": 1.2072023153305054,
+ "learning_rate": 3.1437037037037034e-07,
+ "logits/chosen": 1.3799779415130615,
+ "logits/rejected": 1.3887022733688354,
+ "logps/chosen": -143.9578094482422,
+ "logps/rejected": -159.37698364257812,
+ "loss": 0.656886100769043,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0652204304933548,
+ "rewards/margins": 0.08883848786354065,
+ "rewards/rejected": -0.02361806109547615,
+ "step": 2690
+ },
+ {
+ "epoch": 2.16,
+ "grad_norm": 1.1380783319473267,
+ "learning_rate": 3.114074074074074e-07,
+ "logits/chosen": 1.5064178705215454,
+ "logits/rejected": 1.6023409366607666,
+ "logps/chosen": -132.29075622558594,
+ "logps/rejected": -128.1918487548828,
+ "loss": 0.6357467651367188,
+ "rewards/accuracies": 0.762499988079071,
+ "rewards/chosen": 0.15728428959846497,
+ "rewards/margins": 0.13087645173072815,
+ "rewards/rejected": 0.02640785276889801,
+ "step": 2700
+ },
+ {
+ "epoch": 2.16,
+ "eval_logits/chosen": 1.4565616846084595,
+ "eval_logits/rejected": 1.4844369888305664,
+ "eval_logps/chosen": -152.74916076660156,
+ "eval_logps/rejected": -147.5604248046875,
+ "eval_loss": 0.6842489242553711,
+ "eval_rewards/accuracies": 0.5580000281333923,
+ "eval_rewards/chosen": 0.0799703598022461,
+ "eval_rewards/margins": 0.03379753977060318,
+ "eval_rewards/rejected": 0.046172816306352615,
+ "eval_runtime": 90.8529,
+ "eval_samples_per_second": 5.503,
+ "eval_steps_per_second": 2.752,
+ "step": 2700
+ },
+ {
+ "epoch": 2.168,
+ "grad_norm": 1.435728669166565,
+ "learning_rate": 3.0844444444444444e-07,
+ "logits/chosen": 1.4452109336853027,
+ "logits/rejected": 1.4124813079833984,
+ "logps/chosen": -151.26815795898438,
+ "logps/rejected": -143.44091796875,
+ "loss": 0.6585721969604492,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.0672367662191391,
+ "rewards/margins": 0.07900270074605942,
+ "rewards/rejected": -0.011765943840146065,
+ "step": 2710
+ },
+ {
+ "epoch": 2.176,
+ "grad_norm": 1.3085405826568604,
+ "learning_rate": 3.0548148148148146e-07,
+ "logits/chosen": 1.2030339241027832,
+ "logits/rejected": 1.2627733945846558,
+ "logps/chosen": -138.23812866210938,
+ "logps/rejected": -134.25238037109375,
+ "loss": 0.6320697784423828,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.10239285230636597,
+ "rewards/margins": 0.14901991188526154,
+ "rewards/rejected": -0.04662706330418587,
+ "step": 2720
+ },
+ {
+ "epoch": 2.184,
+ "grad_norm": 1.5403406620025635,
+ "learning_rate": 3.025185185185185e-07,
+ "logits/chosen": 1.4745285511016846,
+ "logits/rejected": 1.4968340396881104,
+ "logps/chosen": -153.49771118164062,
+ "logps/rejected": -148.9380645751953,
+ "loss": 0.6523497104644775,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.09372548013925552,
+ "rewards/margins": 0.09992895275354385,
+ "rewards/rejected": -0.006203479133546352,
+ "step": 2730
+ },
+ {
+ "epoch": 2.192,
+ "grad_norm": 1.5256744623184204,
+ "learning_rate": 2.9955555555555556e-07,
+ "logits/chosen": 1.39058518409729,
+ "logits/rejected": 1.3656699657440186,
+ "logps/chosen": -148.22085571289062,
+ "logps/rejected": -149.59144592285156,
+ "loss": 0.6467072010040283,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.09406543523073196,
+ "rewards/margins": 0.10628072172403336,
+ "rewards/rejected": -0.012215293012559414,
+ "step": 2740
+ },
+ {
+ "epoch": 2.2,
+ "grad_norm": 1.128711462020874,
+ "learning_rate": 2.965925925925926e-07,
+ "logits/chosen": 1.557297945022583,
+ "logits/rejected": 1.5189030170440674,
+ "logps/chosen": -160.21929931640625,
+ "logps/rejected": -151.35910034179688,
+ "loss": 0.6474220275878906,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.13500377535820007,
+ "rewards/margins": 0.10226242244243622,
+ "rewards/rejected": 0.03274135664105415,
+ "step": 2750
+ },
+ {
+ "epoch": 2.208,
+ "grad_norm": 1.3706861734390259,
+ "learning_rate": 2.936296296296296e-07,
+ "logits/chosen": 1.431715965270996,
+ "logits/rejected": 1.4564027786254883,
+ "logps/chosen": -142.4249267578125,
+ "logps/rejected": -143.3875732421875,
+ "loss": 0.6685743808746338,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.09871350228786469,
+ "rewards/margins": 0.06824100762605667,
+ "rewards/rejected": 0.03047250770032406,
+ "step": 2760
+ },
+ {
+ "epoch": 2.216,
+ "grad_norm": 1.3409324884414673,
+ "learning_rate": 2.906666666666667e-07,
+ "logits/chosen": 1.332448124885559,
+ "logits/rejected": 1.3167566061019897,
+ "logps/chosen": -146.07406616210938,
+ "logps/rejected": -138.0575408935547,
+ "loss": 0.6424362182617187,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.05286739021539688,
+ "rewards/margins": 0.12220163643360138,
+ "rewards/rejected": -0.0693342313170433,
+ "step": 2770
+ },
+ {
+ "epoch": 2.224,
+ "grad_norm": 1.5424753427505493,
+ "learning_rate": 2.8770370370370364e-07,
+ "logits/chosen": 1.7004855871200562,
+ "logits/rejected": 1.5205724239349365,
+ "logps/chosen": -180.87466430664062,
+ "logps/rejected": -163.62646484375,
+ "loss": 0.6519207000732422,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.12406478077173233,
+ "rewards/margins": 0.10040261596441269,
+ "rewards/rejected": 0.023662149906158447,
+ "step": 2780
+ },
+ {
+ "epoch": 2.232,
+ "grad_norm": 1.2184964418411255,
+ "learning_rate": 2.847407407407407e-07,
+ "logits/chosen": 1.4738783836364746,
+ "logits/rejected": 1.4357870817184448,
+ "logps/chosen": -164.53448486328125,
+ "logps/rejected": -142.24685668945312,
+ "loss": 0.6661330223083496,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.09590204805135727,
+ "rewards/margins": 0.06964156031608582,
+ "rewards/rejected": 0.026260489597916603,
+ "step": 2790
+ },
+ {
+ "epoch": 2.24,
+ "grad_norm": 1.2697800397872925,
+ "learning_rate": 2.817777777777778e-07,
+ "logits/chosen": 1.4889880418777466,
+ "logits/rejected": 1.5381649732589722,
+ "logps/chosen": -181.35275268554688,
+ "logps/rejected": -159.2748565673828,
+ "loss": 0.6470918655395508,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.10390053689479828,
+ "rewards/margins": 0.10561826080083847,
+ "rewards/rejected": -0.00171771424356848,
+ "step": 2800
+ },
+ {
+ "epoch": 2.24,
+ "eval_logits/chosen": 1.450643539428711,
+ "eval_logits/rejected": 1.478151559829712,
+ "eval_logps/chosen": -152.76634216308594,
+ "eval_logps/rejected": -147.58448791503906,
+ "eval_loss": 0.6843582987785339,
+ "eval_rewards/accuracies": 0.5619999766349792,
+ "eval_rewards/chosen": 0.07825153321027756,
+ "eval_rewards/margins": 0.03448594734072685,
+ "eval_rewards/rejected": 0.04376557841897011,
+ "eval_runtime": 90.9417,
+ "eval_samples_per_second": 5.498,
+ "eval_steps_per_second": 2.749,
+ "step": 2800
+ },
+ {
+ "epoch": 2.248,
+ "grad_norm": 1.161729097366333,
+ "learning_rate": 2.7881481481481476e-07,
+ "logits/chosen": 1.4012717008590698,
+ "logits/rejected": 1.365496277809143,
+ "logps/chosen": -142.15725708007812,
+ "logps/rejected": -131.11892700195312,
+ "loss": 0.6921159744262695,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.048748843371868134,
+ "rewards/margins": 0.018229911103844643,
+ "rewards/rejected": 0.030518928542733192,
+ "step": 2810
+ },
+ {
+ "epoch": 2.2560000000000002,
+ "grad_norm": 1.403849720954895,
+ "learning_rate": 2.7585185185185184e-07,
+ "logits/chosen": 1.6802637577056885,
+ "logits/rejected": 1.6305253505706787,
+ "logps/chosen": -162.37313842773438,
+ "logps/rejected": -165.4166717529297,
+ "loss": 0.6506373405456543,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.14859917759895325,
+ "rewards/margins": 0.09807908535003662,
+ "rewards/rejected": 0.05052009969949722,
+ "step": 2820
+ },
+ {
+ "epoch": 2.2640000000000002,
+ "grad_norm": 1.3189504146575928,
+ "learning_rate": 2.728888888888889e-07,
+ "logits/chosen": 1.4633572101593018,
+ "logits/rejected": 1.4499682188034058,
+ "logps/chosen": -156.68209838867188,
+ "logps/rejected": -155.54751586914062,
+ "loss": 0.6692873954772949,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.13907714188098907,
+ "rewards/margins": 0.06127101182937622,
+ "rewards/rejected": 0.07780613750219345,
+ "step": 2830
+ },
+ {
+ "epoch": 2.2720000000000002,
+ "grad_norm": 1.5040212869644165,
+ "learning_rate": 2.699259259259259e-07,
+ "logits/chosen": 1.4402498006820679,
+ "logits/rejected": 1.4848177433013916,
+ "logps/chosen": -145.59292602539062,
+ "logps/rejected": -142.07449340820312,
+ "loss": 0.6696759223937988,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1008831262588501,
+ "rewards/margins": 0.06455080956220627,
+ "rewards/rejected": 0.03633231669664383,
+ "step": 2840
+ },
+ {
+ "epoch": 2.2800000000000002,
+ "grad_norm": 1.4957029819488525,
+ "learning_rate": 2.6696296296296296e-07,
+ "logits/chosen": 1.309990406036377,
+ "logits/rejected": 1.2410783767700195,
+ "logps/chosen": -145.6426239013672,
+ "logps/rejected": -155.6902313232422,
+ "loss": 0.6614094257354737,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.08516388386487961,
+ "rewards/margins": 0.0843389481306076,
+ "rewards/rejected": 0.0008249226957559586,
+ "step": 2850
+ },
+ {
+ "epoch": 2.288,
+ "grad_norm": 1.1865259408950806,
+ "learning_rate": 2.64e-07,
+ "logits/chosen": 1.4638307094573975,
+ "logits/rejected": 1.5667332410812378,
+ "logps/chosen": -128.8794708251953,
+ "logps/rejected": -121.29227447509766,
+ "loss": 0.6763367652893066,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.11934898048639297,
+ "rewards/margins": 0.05003399774432182,
+ "rewards/rejected": 0.06931495666503906,
+ "step": 2860
+ },
+ {
+ "epoch": 2.296,
+ "grad_norm": 2.0794501304626465,
+ "learning_rate": 2.61037037037037e-07,
+ "logits/chosen": 1.1402711868286133,
+ "logits/rejected": 1.4019138813018799,
+ "logps/chosen": -147.0840301513672,
+ "logps/rejected": -157.21463012695312,
+ "loss": 0.6764614105224609,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.09646089375019073,
+ "rewards/margins": 0.05578025057911873,
+ "rewards/rejected": 0.04068063572049141,
+ "step": 2870
+ },
+ {
+ "epoch": 2.304,
+ "grad_norm": 1.318089485168457,
+ "learning_rate": 2.580740740740741e-07,
+ "logits/chosen": 1.3814247846603394,
+ "logits/rejected": 1.2060496807098389,
+ "logps/chosen": -131.8695068359375,
+ "logps/rejected": -124.2463607788086,
+ "loss": 0.6537458896636963,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.0847344920039177,
+ "rewards/margins": 0.10118331015110016,
+ "rewards/rejected": -0.016448814421892166,
+ "step": 2880
+ },
+ {
+ "epoch": 2.312,
+ "grad_norm": 1.1566070318222046,
+ "learning_rate": 2.551111111111111e-07,
+ "logits/chosen": 1.449655294418335,
+ "logits/rejected": 1.4626872539520264,
+ "logps/chosen": -148.62901306152344,
+ "logps/rejected": -162.71693420410156,
+ "loss": 0.6510843753814697,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.1346159428358078,
+ "rewards/margins": 0.09821879863739014,
+ "rewards/rejected": 0.03639713674783707,
+ "step": 2890
+ },
+ {
+ "epoch": 2.32,
+ "grad_norm": 1.3676846027374268,
+ "learning_rate": 2.521481481481481e-07,
+ "logits/chosen": 1.5381054878234863,
+ "logits/rejected": 1.4480576515197754,
+ "logps/chosen": -159.05850219726562,
+ "logps/rejected": -143.50048828125,
+ "loss": 0.6565275669097901,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.09088721126317978,
+ "rewards/margins": 0.08976241946220398,
+ "rewards/rejected": 0.0011247784132137895,
+ "step": 2900
+ },
+ {
+ "epoch": 2.32,
+ "eval_logits/chosen": 1.4630789756774902,
+ "eval_logits/rejected": 1.4909931421279907,
+ "eval_logps/chosen": -152.56454467773438,
+ "eval_logps/rejected": -147.39541625976562,
+ "eval_loss": 0.6835448145866394,
+ "eval_rewards/accuracies": 0.5540000200271606,
+ "eval_rewards/chosen": 0.09842933714389801,
+ "eval_rewards/margins": 0.03575750067830086,
+ "eval_rewards/rejected": 0.06267183274030685,
+ "eval_runtime": 90.9368,
+ "eval_samples_per_second": 5.498,
+ "eval_steps_per_second": 2.749,
+ "step": 2900
+ },
+ {
+ "epoch": 2.328,
+ "grad_norm": 1.2419706583023071,
+ "learning_rate": 2.4918518518518515e-07,
+ "logits/chosen": 1.3978856801986694,
+ "logits/rejected": 1.4488260746002197,
+ "logps/chosen": -131.99398803710938,
+ "logps/rejected": -155.72616577148438,
+ "loss": 0.6545028209686279,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.1015889048576355,
+ "rewards/margins": 0.0939277708530426,
+ "rewards/rejected": 0.007661131210625172,
+ "step": 2910
+ },
+ {
+ "epoch": 2.336,
+ "grad_norm": 0.8262892961502075,
+ "learning_rate": 2.462222222222222e-07,
+ "logits/chosen": 1.260616660118103,
+ "logits/rejected": 1.298372507095337,
+ "logps/chosen": -131.44737243652344,
+ "logps/rejected": -132.9801483154297,
+ "loss": 0.6633480072021485,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.11483611911535263,
+ "rewards/margins": 0.08126294612884521,
+ "rewards/rejected": 0.033573172986507416,
+ "step": 2920
+ },
+ {
+ "epoch": 2.344,
+ "grad_norm": 1.0795193910598755,
+ "learning_rate": 2.4325925925925924e-07,
+ "logits/chosen": 1.6006931066513062,
+ "logits/rejected": 1.5708739757537842,
+ "logps/chosen": -136.45252990722656,
+ "logps/rejected": -145.27377319335938,
+ "loss": 0.6421307563781739,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.1252741515636444,
+ "rewards/margins": 0.12073083221912384,
+ "rewards/rejected": 0.004543319344520569,
+ "step": 2930
+ },
+ {
+ "epoch": 2.352,
+ "grad_norm": 1.2647486925125122,
+ "learning_rate": 2.4029629629629627e-07,
+ "logits/chosen": 1.386439561843872,
+ "logits/rejected": 1.5220506191253662,
+ "logps/chosen": -160.830078125,
+ "logps/rejected": -166.8371124267578,
+ "loss": 0.6581830024719239,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.08102302998304367,
+ "rewards/margins": 0.08525596559047699,
+ "rewards/rejected": -0.004232937004417181,
+ "step": 2940
+ },
+ {
+ "epoch": 2.36,
+ "grad_norm": 1.1739935874938965,
+ "learning_rate": 2.3733333333333334e-07,
+ "logits/chosen": 1.4723713397979736,
+ "logits/rejected": 1.313079595565796,
+ "logps/chosen": -145.6066436767578,
+ "logps/rejected": -132.0267791748047,
+ "loss": 0.6555693626403809,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10856165736913681,
+ "rewards/margins": 0.10550457239151001,
+ "rewards/rejected": 0.003057088004425168,
+ "step": 2950
+ },
+ {
+ "epoch": 2.368,
+ "grad_norm": 1.1559282541275024,
+ "learning_rate": 2.3437037037037036e-07,
+ "logits/chosen": 1.6269900798797607,
+ "logits/rejected": 1.4614148139953613,
+ "logps/chosen": -166.4721221923828,
+ "logps/rejected": -142.86746215820312,
+ "loss": 0.6763554573059082,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.10653790086507797,
+ "rewards/margins": 0.04229436814785004,
+ "rewards/rejected": 0.06424354016780853,
+ "step": 2960
+ },
+ {
+ "epoch": 2.376,
+ "grad_norm": 1.315238118171692,
+ "learning_rate": 2.3140740740740739e-07,
+ "logits/chosen": 1.5281031131744385,
+ "logits/rejected": 1.6713653802871704,
+ "logps/chosen": -155.27975463867188,
+ "logps/rejected": -140.60409545898438,
+ "loss": 0.6465449333190918,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1229790598154068,
+ "rewards/margins": 0.10460098087787628,
+ "rewards/rejected": 0.018378086388111115,
+ "step": 2970
+ },
+ {
+ "epoch": 2.384,
+ "grad_norm": 1.5117779970169067,
+ "learning_rate": 2.2844444444444443e-07,
+ "logits/chosen": 1.4258384704589844,
+ "logits/rejected": 1.3107579946517944,
+ "logps/chosen": -153.65931701660156,
+ "logps/rejected": -136.7742156982422,
+ "loss": 0.6342881679534912,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.09818007051944733,
+ "rewards/margins": 0.1422421634197235,
+ "rewards/rejected": -0.04406210035085678,
+ "step": 2980
+ },
+ {
+ "epoch": 2.392,
+ "grad_norm": 1.543182134628296,
+ "learning_rate": 2.2548148148148146e-07,
+ "logits/chosen": 1.374324917793274,
+ "logits/rejected": 1.5730586051940918,
+ "logps/chosen": -137.39154052734375,
+ "logps/rejected": -156.12655639648438,
+ "loss": 0.660968017578125,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.1029653325676918,
+ "rewards/margins": 0.0822048932313919,
+ "rewards/rejected": 0.020760437473654747,
+ "step": 2990
+ },
+ {
+ "epoch": 2.4,
+ "grad_norm": 1.5507569313049316,
+ "learning_rate": 2.2251851851851853e-07,
+ "logits/chosen": 1.232704520225525,
+ "logits/rejected": 1.3583004474639893,
+ "logps/chosen": -153.07070922851562,
+ "logps/rejected": -152.47064208984375,
+ "loss": 0.6697997093200684,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.03783693164587021,
+ "rewards/margins": 0.06472334265708923,
+ "rewards/rejected": -0.02688641846179962,
+ "step": 3000
+ },
+ {
+ "epoch": 2.4,
+ "eval_logits/chosen": 1.4536765813827515,
+ "eval_logits/rejected": 1.4812932014465332,
+ "eval_logps/chosen": -152.67945861816406,
+ "eval_logps/rejected": -147.50315856933594,
+ "eval_loss": 0.6841781139373779,
+ "eval_rewards/accuracies": 0.5619999766349792,
+ "eval_rewards/chosen": 0.08694048970937729,
+ "eval_rewards/margins": 0.035040292888879776,
+ "eval_rewards/rejected": 0.05190018564462662,
+ "eval_runtime": 90.957,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.749,
+ "step": 3000
+ }
+ ],
+ "logging_steps": 10,
+ "max_steps": 3750,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3000/training_args.bin b/v3/DPO/DPO_10k/lora/checkpoint-3000/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..b3fc3d76093802087869ab1fee5e37b9c830fe77
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4f90e4a7053b7fb440d68cb64a3d198396af09db942d08839c5f4f0f0b8a0c8
+size 6097
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3500/README.md b/v3/DPO/DPO_10k/lora/checkpoint-3500/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..250500f798b4682b2cd2c35cd1fc366677c215dd
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3500/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3500/adapter_config.json b/v3/DPO/DPO_10k/lora/checkpoint-3500/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..039869f8d79ca6c48a793a29a79267da6d8f81c2
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3500/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "gate_proj",
+ "down_proj",
+ "k_proj",
+ "o_proj",
+ "up_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3500/adapter_model.safetensors b/v3/DPO/DPO_10k/lora/checkpoint-3500/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5594969be64a974e8d7a675315f1596eedb0ed8f
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3500/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:380ed5ece9d5b9f6ef45eff00729b5465e492808d082707c5a5a8e67149bb35f
+size 180385008
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3500/chat_template.jinja b/v3/DPO/DPO_10k/lora/checkpoint-3500/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3500/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3500/optimizer.pt b/v3/DPO/DPO_10k/lora/checkpoint-3500/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c16d95c09feb2b89bda5a9556f4069f25aea56cc
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3500/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c97a9b30382cb0daf32b5da0f0771e13295cf8a957461252b7dbc2c9fa6291c
+size 360902475
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3500/rng_state.pth b/v3/DPO/DPO_10k/lora/checkpoint-3500/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..761ce71e470f5f24ee03221d2ae7ede620f43487
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3500/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c0b314d655e9be31e93455deb9fbabd83362b86e9c047746bf9788399d89574
+size 14645
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3500/scaler.pt b/v3/DPO/DPO_10k/lora/checkpoint-3500/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ef467e5c4906759d9aacdeed22ae1961557aa5a1
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3500/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac1b9358ab1cb9acff90f4d1d692ac08bbdb1986a66544aed9f6fe9b801b17b3
+size 1383
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3500/scheduler.pt b/v3/DPO/DPO_10k/lora/checkpoint-3500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c85421f47b22c47ce4e0228c65c11cbc2a007e39
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e916d3ede7246b754e5cb181939ca441ff5d7efca29425ee3445fbee80348f7d
+size 1465
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3500/tokenizer.json b/v3/DPO/DPO_10k/lora/checkpoint-3500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3500/tokenizer_config.json b/v3/DPO/DPO_10k/lora/checkpoint-3500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3500/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3500/trainer_state.json b/v3/DPO/DPO_10k/lora/checkpoint-3500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..f381b657570db6562791071a56153e575ce567dd
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3500/trainer_state.json
@@ -0,0 +1,5844 @@
+{
+ "best_global_step": 3300,
+ "best_metric": 0.6834326386451721,
+ "best_model_checkpoint": "output/lora/checkpoint-1000",
+ "epoch": 2.8,
+ "eval_steps": 100,
+ "global_step": 3500,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.008,
+ "grad_norm": 1.1179773807525635,
+ "learning_rate": 2.4e-08,
+ "logits/chosen": 1.4545083045959473,
+ "logits/rejected": 1.425490379333496,
+ "logps/chosen": -132.46189880371094,
+ "logps/rejected": -148.20260620117188,
+ "loss": 0.6930626392364502,
+ "rewards/accuracies": 0.21250000596046448,
+ "rewards/chosen": -6.165739614516497e-05,
+ "rewards/margins": 0.00017333509458694607,
+ "rewards/rejected": -0.00023499250528402627,
+ "step": 10
+ },
+ {
+ "epoch": 0.016,
+ "grad_norm": 1.3502839803695679,
+ "learning_rate": 5.0666666666666664e-08,
+ "logits/chosen": 1.5668801069259644,
+ "logits/rejected": 1.5191389322280884,
+ "logps/chosen": -157.63624572753906,
+ "logps/rejected": -144.2716827392578,
+ "loss": 0.6930380821228027,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.0009244013344869018,
+ "rewards/margins": 0.00023505210992880166,
+ "rewards/rejected": 0.0006893490790389478,
+ "step": 20
+ },
+ {
+ "epoch": 0.024,
+ "grad_norm": 1.0219553709030151,
+ "learning_rate": 7.733333333333334e-08,
+ "logits/chosen": 1.8921797275543213,
+ "logits/rejected": 1.7342685461044312,
+ "logps/chosen": -164.55799865722656,
+ "logps/rejected": -158.92752075195312,
+ "loss": 0.69290189743042,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0009391927160322666,
+ "rewards/margins": 0.0005032324115745723,
+ "rewards/rejected": 0.0004359603044576943,
+ "step": 30
+ },
+ {
+ "epoch": 0.032,
+ "grad_norm": 1.2056113481521606,
+ "learning_rate": 1.0399999999999999e-07,
+ "logits/chosen": 1.4950047731399536,
+ "logits/rejected": 1.6163270473480225,
+ "logps/chosen": -145.54183959960938,
+ "logps/rejected": -134.19979858398438,
+ "loss": 0.6935500621795654,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.0006421065190806985,
+ "rewards/margins": -0.0007943272357806563,
+ "rewards/rejected": 0.0014364338712766767,
+ "step": 40
+ },
+ {
+ "epoch": 0.04,
+ "grad_norm": 1.1029725074768066,
+ "learning_rate": 1.3066666666666665e-07,
+ "logits/chosen": 1.4076533317565918,
+ "logits/rejected": 1.4791977405548096,
+ "logps/chosen": -133.23324584960938,
+ "logps/rejected": -138.3593292236328,
+ "loss": 0.6925833702087403,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.0012372838100418448,
+ "rewards/margins": 0.001139522879384458,
+ "rewards/rejected": 9.776116348803043e-05,
+ "step": 50
+ },
+ {
+ "epoch": 0.048,
+ "grad_norm": 0.9960266351699829,
+ "learning_rate": 1.573333333333333e-07,
+ "logits/chosen": 1.407166600227356,
+ "logits/rejected": 1.376042127609253,
+ "logps/chosen": -137.8921661376953,
+ "logps/rejected": -127.97320556640625,
+ "loss": 0.693614149093628,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.00021316049969755113,
+ "rewards/margins": -0.0009249996510334313,
+ "rewards/rejected": 0.0007118391804397106,
+ "step": 60
+ },
+ {
+ "epoch": 0.056,
+ "grad_norm": 1.0946824550628662,
+ "learning_rate": 1.8399999999999998e-07,
+ "logits/chosen": 1.5944501161575317,
+ "logits/rejected": 1.7248146533966064,
+ "logps/chosen": -153.5526123046875,
+ "logps/rejected": -157.19735717773438,
+ "loss": 0.6928449153900147,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0026370862033218145,
+ "rewards/margins": 0.0006150150438770652,
+ "rewards/rejected": 0.0020220710430294275,
+ "step": 70
+ },
+ {
+ "epoch": 0.064,
+ "grad_norm": 1.0322489738464355,
+ "learning_rate": 2.1066666666666665e-07,
+ "logits/chosen": 1.4586353302001953,
+ "logits/rejected": 1.4842995405197144,
+ "logps/chosen": -149.03665161132812,
+ "logps/rejected": -136.24081420898438,
+ "loss": 0.6932015895843506,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.004061593674123287,
+ "rewards/margins": -9.99594121822156e-05,
+ "rewards/rejected": 0.004161553457379341,
+ "step": 80
+ },
+ {
+ "epoch": 0.072,
+ "grad_norm": 1.0012447834014893,
+ "learning_rate": 2.3733333333333334e-07,
+ "logits/chosen": 1.5075902938842773,
+ "logits/rejected": 1.5986944437026978,
+ "logps/chosen": -159.3433074951172,
+ "logps/rejected": -151.85072326660156,
+ "loss": 0.6928757190704345,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.006978762801736593,
+ "rewards/margins": 0.0005627512582577765,
+ "rewards/rejected": 0.006416010670363903,
+ "step": 90
+ },
+ {
+ "epoch": 0.08,
+ "grad_norm": 1.280081033706665,
+ "learning_rate": 2.64e-07,
+ "logits/chosen": 1.448286533355713,
+ "logits/rejected": 1.5480695962905884,
+ "logps/chosen": -151.12625122070312,
+ "logps/rejected": -171.24545288085938,
+ "loss": 0.6931126117706299,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.00693341251462698,
+ "rewards/margins": 9.057526767719537e-05,
+ "rewards/rejected": 0.006842837668955326,
+ "step": 100
+ },
+ {
+ "epoch": 0.08,
+ "eval_logits/chosen": 1.518183946609497,
+ "eval_logits/rejected": 1.5525447130203247,
+ "eval_logps/chosen": -153.46630859375,
+ "eval_logps/rejected": -147.95005798339844,
+ "eval_loss": 0.6926330327987671,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.008253541775047779,
+ "eval_rewards/margins": 0.0010449704714119434,
+ "eval_rewards/rejected": 0.007208569906651974,
+ "eval_runtime": 91.0812,
+ "eval_samples_per_second": 5.49,
+ "eval_steps_per_second": 2.745,
+ "step": 100
+ },
+ {
+ "epoch": 0.088,
+ "grad_norm": 1.3912373781204224,
+ "learning_rate": 2.906666666666667e-07,
+ "logits/chosen": 1.6896222829818726,
+ "logits/rejected": 1.6516921520233154,
+ "logps/chosen": -173.70030212402344,
+ "logps/rejected": -155.5503692626953,
+ "loss": 0.6926521778106689,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.010179834440350533,
+ "rewards/margins": 0.0010137532372027636,
+ "rewards/rejected": 0.009166081435978413,
+ "step": 110
+ },
+ {
+ "epoch": 0.096,
+ "grad_norm": 1.1518888473510742,
+ "learning_rate": 3.173333333333333e-07,
+ "logits/chosen": 1.5166860818862915,
+ "logits/rejected": 1.5136979818344116,
+ "logps/chosen": -136.39906311035156,
+ "logps/rejected": -140.86561584472656,
+ "loss": 0.6932918548583984,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": 0.00907944981008768,
+ "rewards/margins": -0.00026998287648893893,
+ "rewards/rejected": 0.009349432773888111,
+ "step": 120
+ },
+ {
+ "epoch": 0.104,
+ "grad_norm": 1.3213376998901367,
+ "learning_rate": 3.4399999999999996e-07,
+ "logits/chosen": 1.4598973989486694,
+ "logits/rejected": 1.6739647388458252,
+ "logps/chosen": -159.12461853027344,
+ "logps/rejected": -163.39346313476562,
+ "loss": 0.693086051940918,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.010477779433131218,
+ "rewards/margins": 0.00014539004769176245,
+ "rewards/rejected": 0.010332388803362846,
+ "step": 130
+ },
+ {
+ "epoch": 0.112,
+ "grad_norm": 0.8833863735198975,
+ "learning_rate": 3.7066666666666665e-07,
+ "logits/chosen": 1.3829412460327148,
+ "logits/rejected": 1.4100173711776733,
+ "logps/chosen": -132.7149658203125,
+ "logps/rejected": -126.85478210449219,
+ "loss": 0.6932382583618164,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": 0.010262183845043182,
+ "rewards/margins": -0.0001672578218858689,
+ "rewards/rejected": 0.010429441928863525,
+ "step": 140
+ },
+ {
+ "epoch": 0.12,
+ "grad_norm": 1.1472455263137817,
+ "learning_rate": 3.973333333333333e-07,
+ "logits/chosen": 1.4954869747161865,
+ "logits/rejected": 1.5857340097427368,
+ "logps/chosen": -165.10568237304688,
+ "logps/rejected": -158.97433471679688,
+ "loss": 0.6916103363037109,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.014177520759403706,
+ "rewards/margins": 0.003108317730948329,
+ "rewards/rejected": 0.011069201864302158,
+ "step": 150
+ },
+ {
+ "epoch": 0.128,
+ "grad_norm": 1.0628798007965088,
+ "learning_rate": 4.24e-07,
+ "logits/chosen": 1.6717243194580078,
+ "logits/rejected": 1.705092191696167,
+ "logps/chosen": -159.332275390625,
+ "logps/rejected": -147.21646118164062,
+ "loss": 0.6935629844665527,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": 0.016994481906294823,
+ "rewards/margins": -0.0007874655420891941,
+ "rewards/rejected": 0.017781946808099747,
+ "step": 160
+ },
+ {
+ "epoch": 0.136,
+ "grad_norm": 1.390810251235962,
+ "learning_rate": 4.506666666666666e-07,
+ "logits/chosen": 1.4719234704971313,
+ "logits/rejected": 1.499526023864746,
+ "logps/chosen": -148.67251586914062,
+ "logps/rejected": -161.58021545410156,
+ "loss": 0.6926599979400635,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.021239640191197395,
+ "rewards/margins": 0.0010346340714022517,
+ "rewards/rejected": 0.020205006003379822,
+ "step": 170
+ },
+ {
+ "epoch": 0.144,
+ "grad_norm": 1.139707326889038,
+ "learning_rate": 4.773333333333333e-07,
+ "logits/chosen": 1.4833905696868896,
+ "logits/rejected": 1.637584924697876,
+ "logps/chosen": -158.96731567382812,
+ "logps/rejected": -161.9830322265625,
+ "loss": 0.6919547557830811,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.019416773691773415,
+ "rewards/margins": 0.0024296878837049007,
+ "rewards/rejected": 0.016987085342407227,
+ "step": 180
+ },
+ {
+ "epoch": 0.152,
+ "grad_norm": 1.2029796838760376,
+ "learning_rate": 5.04e-07,
+ "logits/chosen": 1.5398027896881104,
+ "logits/rejected": 1.6678705215454102,
+ "logps/chosen": -143.00283813476562,
+ "logps/rejected": -144.89378356933594,
+ "loss": 0.692262840270996,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.018960200250148773,
+ "rewards/margins": 0.0018175147706642747,
+ "rewards/rejected": 0.017142686992883682,
+ "step": 190
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 1.0527437925338745,
+ "learning_rate": 5.306666666666665e-07,
+ "logits/chosen": 1.6087623834609985,
+ "logits/rejected": 1.5181543827056885,
+ "logps/chosen": -159.35073852539062,
+ "logps/rejected": -133.59890747070312,
+ "loss": 0.6924069881439209,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.01926913857460022,
+ "rewards/margins": 0.0015339398523792624,
+ "rewards/rejected": 0.0177351962774992,
+ "step": 200
+ },
+ {
+ "epoch": 0.16,
+ "eval_logits/chosen": 1.5193672180175781,
+ "eval_logits/rejected": 1.5534639358520508,
+ "eval_logps/chosen": -153.34857177734375,
+ "eval_logps/rejected": -147.8425750732422,
+ "eval_loss": 0.6921458840370178,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.020027177408337593,
+ "eval_rewards/margins": 0.0020714527927339077,
+ "eval_rewards/rejected": 0.017955724149942398,
+ "eval_runtime": 90.9762,
+ "eval_samples_per_second": 5.496,
+ "eval_steps_per_second": 2.748,
+ "step": 200
+ },
+ {
+ "epoch": 0.168,
+ "grad_norm": 1.1897239685058594,
+ "learning_rate": 5.573333333333333e-07,
+ "logits/chosen": 1.6267732381820679,
+ "logits/rejected": 1.5775268077850342,
+ "logps/chosen": -159.51437377929688,
+ "logps/rejected": -170.17762756347656,
+ "loss": 0.6925770759582519,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.02078000269830227,
+ "rewards/margins": 0.0012053751852363348,
+ "rewards/rejected": 0.01957462914288044,
+ "step": 210
+ },
+ {
+ "epoch": 0.176,
+ "grad_norm": 0.9995236396789551,
+ "learning_rate": 5.839999999999999e-07,
+ "logits/chosen": 1.5781934261322021,
+ "logits/rejected": 1.642251968383789,
+ "logps/chosen": -159.95309448242188,
+ "logps/rejected": -144.36058044433594,
+ "loss": 0.6931475639343262,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.023322442546486855,
+ "rewards/margins": 6.392716750269756e-05,
+ "rewards/rejected": 0.02325851283967495,
+ "step": 220
+ },
+ {
+ "epoch": 0.184,
+ "grad_norm": 0.8819260597229004,
+ "learning_rate": 6.106666666666666e-07,
+ "logits/chosen": 1.459991693496704,
+ "logits/rejected": 1.6060739755630493,
+ "logps/chosen": -162.04745483398438,
+ "logps/rejected": -139.5411834716797,
+ "loss": 0.6919463634490967,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.026052182540297508,
+ "rewards/margins": 0.002481849165633321,
+ "rewards/rejected": 0.023570330813527107,
+ "step": 230
+ },
+ {
+ "epoch": 0.192,
+ "grad_norm": 1.1055208444595337,
+ "learning_rate": 6.373333333333333e-07,
+ "logits/chosen": 1.558739185333252,
+ "logits/rejected": 1.5965116024017334,
+ "logps/chosen": -171.9166259765625,
+ "logps/rejected": -145.66433715820312,
+ "loss": 0.6912176132202148,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.033868782222270966,
+ "rewards/margins": 0.004017929546535015,
+ "rewards/rejected": 0.029850851744413376,
+ "step": 240
+ },
+ {
+ "epoch": 0.2,
+ "grad_norm": 1.1372733116149902,
+ "learning_rate": 6.64e-07,
+ "logits/chosen": 1.5322860479354858,
+ "logits/rejected": 1.4695795774459839,
+ "logps/chosen": -161.7803955078125,
+ "logps/rejected": -148.91949462890625,
+ "loss": 0.6940969944000244,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.034065864980220795,
+ "rewards/margins": -0.0016761379083618522,
+ "rewards/rejected": 0.035742007195949554,
+ "step": 250
+ },
+ {
+ "epoch": 0.208,
+ "grad_norm": 1.0917680263519287,
+ "learning_rate": 6.906666666666666e-07,
+ "logits/chosen": 1.4094547033309937,
+ "logits/rejected": 1.50186026096344,
+ "logps/chosen": -132.91455078125,
+ "logps/rejected": -146.79714965820312,
+ "loss": 0.6930203914642334,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.03089061938226223,
+ "rewards/margins": 0.00040971505222842097,
+ "rewards/rejected": 0.030480902642011642,
+ "step": 260
+ },
+ {
+ "epoch": 0.216,
+ "grad_norm": 1.0280259847640991,
+ "learning_rate": 7.173333333333333e-07,
+ "logits/chosen": 1.531285285949707,
+ "logits/rejected": 1.6163132190704346,
+ "logps/chosen": -146.902587890625,
+ "logps/rejected": -147.8347625732422,
+ "loss": 0.6944749355316162,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.028514739125967026,
+ "rewards/margins": -0.0025452517438679934,
+ "rewards/rejected": 0.0310599897056818,
+ "step": 270
+ },
+ {
+ "epoch": 0.224,
+ "grad_norm": 1.398708701133728,
+ "learning_rate": 7.44e-07,
+ "logits/chosen": 1.563852071762085,
+ "logits/rejected": 1.396703839302063,
+ "logps/chosen": -172.68153381347656,
+ "logps/rejected": -155.80215454101562,
+ "loss": 0.689476490020752,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03224216774106026,
+ "rewards/margins": 0.007473687641322613,
+ "rewards/rejected": 0.02476847730576992,
+ "step": 280
+ },
+ {
+ "epoch": 0.232,
+ "grad_norm": 1.0394141674041748,
+ "learning_rate": 7.706666666666667e-07,
+ "logits/chosen": 1.5581729412078857,
+ "logits/rejected": 1.5819021463394165,
+ "logps/chosen": -144.41465759277344,
+ "logps/rejected": -153.55406188964844,
+ "loss": 0.6927877902984619,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.03272664546966553,
+ "rewards/margins": 0.0008557910332456231,
+ "rewards/rejected": 0.031870849430561066,
+ "step": 290
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 1.2557185888290405,
+ "learning_rate": 7.973333333333333e-07,
+ "logits/chosen": 1.5062446594238281,
+ "logits/rejected": 1.5786281824111938,
+ "logps/chosen": -153.63558959960938,
+ "logps/rejected": -165.46617126464844,
+ "loss": 0.6897892951965332,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03914078325033188,
+ "rewards/margins": 0.006873926613479853,
+ "rewards/rejected": 0.032266855239868164,
+ "step": 300
+ },
+ {
+ "epoch": 0.24,
+ "eval_logits/chosen": 1.5237834453582764,
+ "eval_logits/rejected": 1.557741641998291,
+ "eval_logps/chosen": -153.19883728027344,
+ "eval_logps/rejected": -147.7100830078125,
+ "eval_loss": 0.6913501024246216,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.03499976545572281,
+ "eval_rewards/margins": 0.0037925743963569403,
+ "eval_rewards/rejected": 0.031207194551825523,
+ "eval_runtime": 90.9133,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 300
+ },
+ {
+ "epoch": 0.248,
+ "grad_norm": 1.1200144290924072,
+ "learning_rate": 8.24e-07,
+ "logits/chosen": 1.618949294090271,
+ "logits/rejected": 1.6625702381134033,
+ "logps/chosen": -141.8953857421875,
+ "logps/rejected": -156.6927032470703,
+ "loss": 0.694425106048584,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.03162423521280289,
+ "rewards/margins": -0.0023833750747144222,
+ "rewards/rejected": 0.034007612615823746,
+ "step": 310
+ },
+ {
+ "epoch": 0.256,
+ "grad_norm": 1.0823743343353271,
+ "learning_rate": 8.506666666666667e-07,
+ "logits/chosen": 1.5974153280258179,
+ "logits/rejected": 1.6237560510635376,
+ "logps/chosen": -139.2996368408203,
+ "logps/rejected": -141.94407653808594,
+ "loss": 0.6914949893951416,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.029445698484778404,
+ "rewards/margins": 0.0034329916816204786,
+ "rewards/rejected": 0.026012707501649857,
+ "step": 320
+ },
+ {
+ "epoch": 0.264,
+ "grad_norm": 1.1765722036361694,
+ "learning_rate": 8.773333333333332e-07,
+ "logits/chosen": 1.465215802192688,
+ "logits/rejected": 1.4882014989852905,
+ "logps/chosen": -158.4329833984375,
+ "logps/rejected": -183.94259643554688,
+ "loss": 0.6925934791564942,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.026302698999643326,
+ "rewards/margins": 0.0012350418837741017,
+ "rewards/rejected": 0.025067657232284546,
+ "step": 330
+ },
+ {
+ "epoch": 0.272,
+ "grad_norm": 1.0667294263839722,
+ "learning_rate": 9.039999999999999e-07,
+ "logits/chosen": 1.7538875341415405,
+ "logits/rejected": 1.6171789169311523,
+ "logps/chosen": -159.63790893554688,
+ "logps/rejected": -151.1880340576172,
+ "loss": 0.6922041416168213,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.026619985699653625,
+ "rewards/margins": 0.002031774492934346,
+ "rewards/rejected": 0.024588212370872498,
+ "step": 340
+ },
+ {
+ "epoch": 0.28,
+ "grad_norm": 1.2210006713867188,
+ "learning_rate": 9.306666666666666e-07,
+ "logits/chosen": 1.5815479755401611,
+ "logits/rejected": 1.710217833518982,
+ "logps/chosen": -167.21743774414062,
+ "logps/rejected": -196.77989196777344,
+ "loss": 0.6899692058563233,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.03636014088988304,
+ "rewards/margins": 0.006536121014505625,
+ "rewards/rejected": 0.029824022203683853,
+ "step": 350
+ },
+ {
+ "epoch": 0.288,
+ "grad_norm": 0.9623845219612122,
+ "learning_rate": 9.573333333333333e-07,
+ "logits/chosen": 1.4416790008544922,
+ "logits/rejected": 1.3618693351745605,
+ "logps/chosen": -163.27401733398438,
+ "logps/rejected": -169.3351287841797,
+ "loss": 0.6910821437835694,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.037200987339019775,
+ "rewards/margins": 0.004338678438216448,
+ "rewards/rejected": 0.032862309366464615,
+ "step": 360
+ },
+ {
+ "epoch": 0.296,
+ "grad_norm": 1.1005645990371704,
+ "learning_rate": 9.84e-07,
+ "logits/chosen": 1.6214616298675537,
+ "logits/rejected": 1.6572606563568115,
+ "logps/chosen": -162.95504760742188,
+ "logps/rejected": -154.91807556152344,
+ "loss": 0.6895030021667481,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.047198958694934845,
+ "rewards/margins": 0.0075958347879350185,
+ "rewards/rejected": 0.03960312157869339,
+ "step": 370
+ },
+ {
+ "epoch": 0.304,
+ "grad_norm": 1.1126338243484497,
+ "learning_rate": 9.988148148148148e-07,
+ "logits/chosen": 1.4842126369476318,
+ "logits/rejected": 1.410930871963501,
+ "logps/chosen": -143.09182739257812,
+ "logps/rejected": -153.48135375976562,
+ "loss": 0.6885370254516602,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.05975018069148064,
+ "rewards/margins": 0.00981281790882349,
+ "rewards/rejected": 0.04993735998868942,
+ "step": 380
+ },
+ {
+ "epoch": 0.312,
+ "grad_norm": 1.1146304607391357,
+ "learning_rate": 9.95851851851852e-07,
+ "logits/chosen": 1.608332633972168,
+ "logits/rejected": 1.5848472118377686,
+ "logps/chosen": -152.25250244140625,
+ "logps/rejected": -138.348876953125,
+ "loss": 0.6851807117462159,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.07630724459886551,
+ "rewards/margins": 0.016460279002785683,
+ "rewards/rejected": 0.059846967458724976,
+ "step": 390
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 1.2686601877212524,
+ "learning_rate": 9.928888888888889e-07,
+ "logits/chosen": 1.5840070247650146,
+ "logits/rejected": 1.5679785013198853,
+ "logps/chosen": -155.90219116210938,
+ "logps/rejected": -148.89549255371094,
+ "loss": 0.6891039371490478,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.08129279315471649,
+ "rewards/margins": 0.008997146971523762,
+ "rewards/rejected": 0.072295643389225,
+ "step": 400
+ },
+ {
+ "epoch": 0.32,
+ "eval_logits/chosen": 1.5358340740203857,
+ "eval_logits/rejected": 1.5689741373062134,
+ "eval_logps/chosen": -152.6895294189453,
+ "eval_logps/rejected": -147.23422241210938,
+ "eval_loss": 0.6900544166564941,
+ "eval_rewards/accuracies": 0.5419999957084656,
+ "eval_rewards/chosen": 0.08593228459358215,
+ "eval_rewards/margins": 0.0071397931315004826,
+ "eval_rewards/rejected": 0.0787924975156784,
+ "eval_runtime": 90.9072,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 400
+ },
+ {
+ "epoch": 0.328,
+ "grad_norm": 1.1653108596801758,
+ "learning_rate": 9.899259259259258e-07,
+ "logits/chosen": 1.5450490713119507,
+ "logits/rejected": 1.4428151845932007,
+ "logps/chosen": -148.61581420898438,
+ "logps/rejected": -188.90919494628906,
+ "loss": 0.6918989181518554,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0946170911192894,
+ "rewards/margins": 0.003271549940109253,
+ "rewards/rejected": 0.09134554117918015,
+ "step": 410
+ },
+ {
+ "epoch": 0.336,
+ "grad_norm": 1.3114937543869019,
+ "learning_rate": 9.86962962962963e-07,
+ "logits/chosen": 1.5342715978622437,
+ "logits/rejected": 1.5572118759155273,
+ "logps/chosen": -152.65264892578125,
+ "logps/rejected": -168.03384399414062,
+ "loss": 0.6972955703735352,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.08869597315788269,
+ "rewards/margins": -0.007436127867549658,
+ "rewards/rejected": 0.09613210707902908,
+ "step": 420
+ },
+ {
+ "epoch": 0.344,
+ "grad_norm": 1.0040781497955322,
+ "learning_rate": 9.84e-07,
+ "logits/chosen": 1.5979712009429932,
+ "logits/rejected": 1.6713664531707764,
+ "logps/chosen": -164.2566375732422,
+ "logps/rejected": -152.1428680419922,
+ "loss": 0.6897010326385498,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.08101107180118561,
+ "rewards/margins": 0.007684691343456507,
+ "rewards/rejected": 0.07332637906074524,
+ "step": 430
+ },
+ {
+ "epoch": 0.352,
+ "grad_norm": 1.0801892280578613,
+ "learning_rate": 9.81037037037037e-07,
+ "logits/chosen": 1.439635992050171,
+ "logits/rejected": 1.3282365798950195,
+ "logps/chosen": -157.23403930664062,
+ "logps/rejected": -146.54806518554688,
+ "loss": 0.6881664752960205,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.08112544566392899,
+ "rewards/margins": 0.010971959680318832,
+ "rewards/rejected": 0.07015348225831985,
+ "step": 440
+ },
+ {
+ "epoch": 0.36,
+ "grad_norm": 1.1139949560165405,
+ "learning_rate": 9.78074074074074e-07,
+ "logits/chosen": 1.6345123052597046,
+ "logits/rejected": 1.5876491069793701,
+ "logps/chosen": -157.6160125732422,
+ "logps/rejected": -160.02040100097656,
+ "loss": 0.6868602275848389,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.07638200372457504,
+ "rewards/margins": 0.013464884832501411,
+ "rewards/rejected": 0.06291711330413818,
+ "step": 450
+ },
+ {
+ "epoch": 0.368,
+ "grad_norm": 1.158035397529602,
+ "learning_rate": 9.751111111111112e-07,
+ "logits/chosen": 1.5430586338043213,
+ "logits/rejected": 1.6022913455963135,
+ "logps/chosen": -151.98919677734375,
+ "logps/rejected": -164.90565490722656,
+ "loss": 0.6891930580139161,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06889279186725616,
+ "rewards/margins": 0.008791089057922363,
+ "rewards/rejected": 0.0601017065346241,
+ "step": 460
+ },
+ {
+ "epoch": 0.376,
+ "grad_norm": 1.0903704166412354,
+ "learning_rate": 9.721481481481481e-07,
+ "logits/chosen": 1.313881754875183,
+ "logits/rejected": 1.3182194232940674,
+ "logps/chosen": -131.5592041015625,
+ "logps/rejected": -123.84442138671875,
+ "loss": 0.6889429569244385,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.08150745928287506,
+ "rewards/margins": 0.009235726669430733,
+ "rewards/rejected": 0.07227173447608948,
+ "step": 470
+ },
+ {
+ "epoch": 0.384,
+ "grad_norm": 1.0766969919204712,
+ "learning_rate": 9.69185185185185e-07,
+ "logits/chosen": 1.425756812095642,
+ "logits/rejected": 1.476854920387268,
+ "logps/chosen": -148.72279357910156,
+ "logps/rejected": -164.40951538085938,
+ "loss": 0.6854116439819335,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1009502187371254,
+ "rewards/margins": 0.0166789498180151,
+ "rewards/rejected": 0.08427127450704575,
+ "step": 480
+ },
+ {
+ "epoch": 0.392,
+ "grad_norm": 1.2078129053115845,
+ "learning_rate": 9.662222222222222e-07,
+ "logits/chosen": 1.5511282682418823,
+ "logits/rejected": 1.611383080482483,
+ "logps/chosen": -143.22006225585938,
+ "logps/rejected": -159.70176696777344,
+ "loss": 0.6968960762023926,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.08779363334178925,
+ "rewards/margins": -0.00598606513813138,
+ "rewards/rejected": 0.09377971291542053,
+ "step": 490
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.9106306433677673,
+ "learning_rate": 9.632592592592593e-07,
+ "logits/chosen": 1.4217239618301392,
+ "logits/rejected": 1.505446195602417,
+ "logps/chosen": -144.34109497070312,
+ "logps/rejected": -135.52430725097656,
+ "loss": 0.691196870803833,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06902458518743515,
+ "rewards/margins": 0.005059319548308849,
+ "rewards/rejected": 0.06396526843309402,
+ "step": 500
+ },
+ {
+ "epoch": 0.4,
+ "eval_logits/chosen": 1.5185325145721436,
+ "eval_logits/rejected": 1.5510345697402954,
+ "eval_logps/chosen": -152.7921905517578,
+ "eval_logps/rejected": -147.3463134765625,
+ "eval_loss": 0.6898502111434937,
+ "eval_rewards/accuracies": 0.5099999904632568,
+ "eval_rewards/chosen": 0.07566512376070023,
+ "eval_rewards/margins": 0.008083080872893333,
+ "eval_rewards/rejected": 0.06758204847574234,
+ "eval_runtime": 90.8294,
+ "eval_samples_per_second": 5.505,
+ "eval_steps_per_second": 2.752,
+ "step": 500
+ },
+ {
+ "epoch": 0.408,
+ "grad_norm": 1.1637762784957886,
+ "learning_rate": 9.602962962962962e-07,
+ "logits/chosen": 1.5965330600738525,
+ "logits/rejected": 1.5338464975357056,
+ "logps/chosen": -131.49871826171875,
+ "logps/rejected": -140.05995178222656,
+ "loss": 0.6868171215057373,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.08385131508111954,
+ "rewards/margins": 0.013950209133327007,
+ "rewards/rejected": 0.06990110129117966,
+ "step": 510
+ },
+ {
+ "epoch": 0.416,
+ "grad_norm": 1.229719638824463,
+ "learning_rate": 9.573333333333333e-07,
+ "logits/chosen": 1.7057275772094727,
+ "logits/rejected": 1.5898948907852173,
+ "logps/chosen": -166.4267120361328,
+ "logps/rejected": -172.22337341308594,
+ "loss": 0.6912094116210937,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.07427279651165009,
+ "rewards/margins": 0.004661654122173786,
+ "rewards/rejected": 0.06961113959550858,
+ "step": 520
+ },
+ {
+ "epoch": 0.424,
+ "grad_norm": 1.136383295059204,
+ "learning_rate": 9.543703703703705e-07,
+ "logits/chosen": 1.6207077503204346,
+ "logits/rejected": 1.5417520999908447,
+ "logps/chosen": -166.81509399414062,
+ "logps/rejected": -168.5408172607422,
+ "loss": 0.6772748470306397,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.0953373983502388,
+ "rewards/margins": 0.033697597682476044,
+ "rewards/rejected": 0.06163979694247246,
+ "step": 530
+ },
+ {
+ "epoch": 0.432,
+ "grad_norm": 1.0640541315078735,
+ "learning_rate": 9.514074074074074e-07,
+ "logits/chosen": 1.6825854778289795,
+ "logits/rejected": 1.6327041387557983,
+ "logps/chosen": -181.39956665039062,
+ "logps/rejected": -158.49647521972656,
+ "loss": 0.6883615493774414,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.10187923908233643,
+ "rewards/margins": 0.010873330757021904,
+ "rewards/rejected": 0.09100590646266937,
+ "step": 540
+ },
+ {
+ "epoch": 0.44,
+ "grad_norm": 1.477909803390503,
+ "learning_rate": 9.484444444444444e-07,
+ "logits/chosen": 1.5402686595916748,
+ "logits/rejected": 1.6487388610839844,
+ "logps/chosen": -151.53814697265625,
+ "logps/rejected": -167.8573455810547,
+ "loss": 0.6915213584899902,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09472338110208511,
+ "rewards/margins": 0.005475065670907497,
+ "rewards/rejected": 0.08924831449985504,
+ "step": 550
+ },
+ {
+ "epoch": 0.448,
+ "grad_norm": 1.0138317346572876,
+ "learning_rate": 9.454814814814814e-07,
+ "logits/chosen": 1.6286503076553345,
+ "logits/rejected": 1.5223101377487183,
+ "logps/chosen": -154.72079467773438,
+ "logps/rejected": -139.2535858154297,
+ "loss": 0.6835216522216797,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.09194488823413849,
+ "rewards/margins": 0.02140027843415737,
+ "rewards/rejected": 0.07054460793733597,
+ "step": 560
+ },
+ {
+ "epoch": 0.456,
+ "grad_norm": 0.896689772605896,
+ "learning_rate": 9.425185185185184e-07,
+ "logits/chosen": 1.4857146739959717,
+ "logits/rejected": 1.4369819164276123,
+ "logps/chosen": -158.31809997558594,
+ "logps/rejected": -152.33676147460938,
+ "loss": 0.6863360404968262,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.07895292341709137,
+ "rewards/margins": 0.016260787844657898,
+ "rewards/rejected": 0.06269213557243347,
+ "step": 570
+ },
+ {
+ "epoch": 0.464,
+ "grad_norm": 1.4772330522537231,
+ "learning_rate": 9.395555555555556e-07,
+ "logits/chosen": 1.315298318862915,
+ "logits/rejected": 1.4757534265518188,
+ "logps/chosen": -136.4202117919922,
+ "logps/rejected": -154.99362182617188,
+ "loss": 0.6916167259216308,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.04677204787731171,
+ "rewards/margins": 0.005649724509567022,
+ "rewards/rejected": 0.041122324764728546,
+ "step": 580
+ },
+ {
+ "epoch": 0.472,
+ "grad_norm": 0.9806157350540161,
+ "learning_rate": 9.365925925925926e-07,
+ "logits/chosen": 1.465427279472351,
+ "logits/rejected": 1.459496259689331,
+ "logps/chosen": -145.07470703125,
+ "logps/rejected": -131.71974182128906,
+ "loss": 0.6857181549072265,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.07979898899793625,
+ "rewards/margins": 0.016965944319963455,
+ "rewards/rejected": 0.0628330409526825,
+ "step": 590
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 1.4313396215438843,
+ "learning_rate": 9.336296296296295e-07,
+ "logits/chosen": 1.5190367698669434,
+ "logits/rejected": 1.563504934310913,
+ "logps/chosen": -132.0201416015625,
+ "logps/rejected": -158.89865112304688,
+ "loss": 0.6964848041534424,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.06544725596904755,
+ "rewards/margins": -0.004280097782611847,
+ "rewards/rejected": 0.0697273463010788,
+ "step": 600
+ },
+ {
+ "epoch": 0.48,
+ "eval_logits/chosen": 1.5031698942184448,
+ "eval_logits/rejected": 1.5348584651947021,
+ "eval_logps/chosen": -152.76596069335938,
+ "eval_logps/rejected": -147.3241729736328,
+ "eval_loss": 0.6903045773506165,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": 0.07828804850578308,
+ "eval_rewards/margins": 0.008492168970406055,
+ "eval_rewards/rejected": 0.0697958841919899,
+ "eval_runtime": 90.9881,
+ "eval_samples_per_second": 5.495,
+ "eval_steps_per_second": 2.748,
+ "step": 600
+ },
+ {
+ "epoch": 0.488,
+ "grad_norm": 0.9467533230781555,
+ "learning_rate": 9.306666666666666e-07,
+ "logits/chosen": 1.5242283344268799,
+ "logits/rejected": 1.5798314809799194,
+ "logps/chosen": -131.90048217773438,
+ "logps/rejected": -149.93222045898438,
+ "loss": 0.6899854183197022,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.08419154584407806,
+ "rewards/margins": 0.00830511748790741,
+ "rewards/rejected": 0.07588644325733185,
+ "step": 610
+ },
+ {
+ "epoch": 0.496,
+ "grad_norm": 1.8463492393493652,
+ "learning_rate": 9.277037037037037e-07,
+ "logits/chosen": 1.4965155124664307,
+ "logits/rejected": 1.6336272954940796,
+ "logps/chosen": -156.1863555908203,
+ "logps/rejected": -167.74075317382812,
+ "loss": 0.6977802276611328,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.05518193915486336,
+ "rewards/margins": -0.006808985956013203,
+ "rewards/rejected": 0.061990927904844284,
+ "step": 620
+ },
+ {
+ "epoch": 0.504,
+ "grad_norm": 1.278316617012024,
+ "learning_rate": 9.247407407407407e-07,
+ "logits/chosen": 1.5074065923690796,
+ "logits/rejected": 1.535696029663086,
+ "logps/chosen": -167.56777954101562,
+ "logps/rejected": -147.04922485351562,
+ "loss": 0.6806219100952149,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.0745314508676529,
+ "rewards/margins": 0.027104835957288742,
+ "rewards/rejected": 0.047426607459783554,
+ "step": 630
+ },
+ {
+ "epoch": 0.512,
+ "grad_norm": 1.0424679517745972,
+ "learning_rate": 9.217777777777778e-07,
+ "logits/chosen": 1.4299286603927612,
+ "logits/rejected": 1.4484076499938965,
+ "logps/chosen": -168.305908203125,
+ "logps/rejected": -152.61854553222656,
+ "loss": 0.6871783256530761,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07751411944627762,
+ "rewards/margins": 0.017045259475708008,
+ "rewards/rejected": 0.06046885997056961,
+ "step": 640
+ },
+ {
+ "epoch": 0.52,
+ "grad_norm": 1.1143999099731445,
+ "learning_rate": 9.188148148148148e-07,
+ "logits/chosen": 1.5520436763763428,
+ "logits/rejected": 1.531718134880066,
+ "logps/chosen": -154.92115783691406,
+ "logps/rejected": -126.78369140625,
+ "loss": 0.6889286518096924,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.0945608839392662,
+ "rewards/margins": 0.01080345083028078,
+ "rewards/rejected": 0.0837574377655983,
+ "step": 650
+ },
+ {
+ "epoch": 0.528,
+ "grad_norm": 1.2762326002120972,
+ "learning_rate": 9.158518518518517e-07,
+ "logits/chosen": 1.5747522115707397,
+ "logits/rejected": 1.4972833395004272,
+ "logps/chosen": -151.13914489746094,
+ "logps/rejected": -132.013916015625,
+ "loss": 0.6900454998016358,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.07303085923194885,
+ "rewards/margins": 0.008073865436017513,
+ "rewards/rejected": 0.06495700031518936,
+ "step": 660
+ },
+ {
+ "epoch": 0.536,
+ "grad_norm": 1.5207557678222656,
+ "learning_rate": 9.128888888888888e-07,
+ "logits/chosen": 1.6594665050506592,
+ "logits/rejected": 1.468492031097412,
+ "logps/chosen": -166.0108184814453,
+ "logps/rejected": -151.38555908203125,
+ "loss": 0.6934059143066407,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.09294265508651733,
+ "rewards/margins": 0.0021180796902626753,
+ "rewards/rejected": 0.09082455933094025,
+ "step": 670
+ },
+ {
+ "epoch": 0.544,
+ "grad_norm": 1.1073335409164429,
+ "learning_rate": 9.099259259259259e-07,
+ "logits/chosen": 1.5321934223175049,
+ "logits/rejected": 1.5313541889190674,
+ "logps/chosen": -144.5046844482422,
+ "logps/rejected": -133.05470275878906,
+ "loss": 0.6824594974517822,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.10782947391271591,
+ "rewards/margins": 0.0230915155261755,
+ "rewards/rejected": 0.08473796397447586,
+ "step": 680
+ },
+ {
+ "epoch": 0.552,
+ "grad_norm": 1.096970558166504,
+ "learning_rate": 9.069629629629629e-07,
+ "logits/chosen": 1.438305139541626,
+ "logits/rejected": 1.4338308572769165,
+ "logps/chosen": -173.1280517578125,
+ "logps/rejected": -160.20950317382812,
+ "loss": 0.6878445625305176,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09615986794233322,
+ "rewards/margins": 0.012350928969681263,
+ "rewards/rejected": 0.08380892872810364,
+ "step": 690
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 1.3664878606796265,
+ "learning_rate": 9.039999999999999e-07,
+ "logits/chosen": 1.530369758605957,
+ "logits/rejected": 1.5520622730255127,
+ "logps/chosen": -159.60740661621094,
+ "logps/rejected": -137.77352905273438,
+ "loss": 0.6991429805755616,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07249583303928375,
+ "rewards/margins": -0.009333941154181957,
+ "rewards/rejected": 0.08182977139949799,
+ "step": 700
+ },
+ {
+ "epoch": 0.56,
+ "eval_logits/chosen": 1.5136796236038208,
+ "eval_logits/rejected": 1.5456002950668335,
+ "eval_logps/chosen": -152.5748748779297,
+ "eval_logps/rejected": -147.17152404785156,
+ "eval_loss": 0.688511848449707,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.09739598631858826,
+ "eval_rewards/margins": 0.012335418723523617,
+ "eval_rewards/rejected": 0.08506056666374207,
+ "eval_runtime": 90.8734,
+ "eval_samples_per_second": 5.502,
+ "eval_steps_per_second": 2.751,
+ "step": 700
+ },
+ {
+ "epoch": 0.568,
+ "grad_norm": 1.224107265472412,
+ "learning_rate": 9.010370370370371e-07,
+ "logits/chosen": 1.6023967266082764,
+ "logits/rejected": 1.558224081993103,
+ "logps/chosen": -148.47120666503906,
+ "logps/rejected": -163.5648651123047,
+ "loss": 0.6865221500396729,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.12494877725839615,
+ "rewards/margins": 0.01545787788927555,
+ "rewards/rejected": 0.10949089378118515,
+ "step": 710
+ },
+ {
+ "epoch": 0.576,
+ "grad_norm": 1.2135164737701416,
+ "learning_rate": 8.98074074074074e-07,
+ "logits/chosen": 1.5902750492095947,
+ "logits/rejected": 1.7012746334075928,
+ "logps/chosen": -139.7012939453125,
+ "logps/rejected": -135.42225646972656,
+ "loss": 0.7014039039611817,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.09102506935596466,
+ "rewards/margins": -0.014695463702082634,
+ "rewards/rejected": 0.10572052001953125,
+ "step": 720
+ },
+ {
+ "epoch": 0.584,
+ "grad_norm": 1.1953649520874023,
+ "learning_rate": 8.95111111111111e-07,
+ "logits/chosen": 1.6540731191635132,
+ "logits/rejected": 1.6386349201202393,
+ "logps/chosen": -153.12847900390625,
+ "logps/rejected": -149.66319274902344,
+ "loss": 0.6952126502990723,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.1124584898352623,
+ "rewards/margins": -0.0012063594767823815,
+ "rewards/rejected": 0.11366486549377441,
+ "step": 730
+ },
+ {
+ "epoch": 0.592,
+ "grad_norm": 1.147875428199768,
+ "learning_rate": 8.921481481481481e-07,
+ "logits/chosen": 1.5578330755233765,
+ "logits/rejected": 1.4689290523529053,
+ "logps/chosen": -156.93788146972656,
+ "logps/rejected": -134.1832733154297,
+ "loss": 0.6919830799102783,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09214536100625992,
+ "rewards/margins": 0.004291228950023651,
+ "rewards/rejected": 0.08785412460565567,
+ "step": 740
+ },
+ {
+ "epoch": 0.6,
+ "grad_norm": 1.21685791015625,
+ "learning_rate": 8.891851851851852e-07,
+ "logits/chosen": 1.4241983890533447,
+ "logits/rejected": 1.4537718296051025,
+ "logps/chosen": -136.9163055419922,
+ "logps/rejected": -158.44491577148438,
+ "loss": 0.691422986984253,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.09173480421304703,
+ "rewards/margins": 0.00533579895272851,
+ "rewards/rejected": 0.08639900386333466,
+ "step": 750
+ },
+ {
+ "epoch": 0.608,
+ "grad_norm": 1.3629519939422607,
+ "learning_rate": 8.862222222222222e-07,
+ "logits/chosen": 1.565716028213501,
+ "logits/rejected": 1.3824570178985596,
+ "logps/chosen": -138.8521728515625,
+ "logps/rejected": -129.51028442382812,
+ "loss": 0.6931031227111817,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0763697624206543,
+ "rewards/margins": 0.0021251302678138018,
+ "rewards/rejected": 0.07424463331699371,
+ "step": 760
+ },
+ {
+ "epoch": 0.616,
+ "grad_norm": 1.1030747890472412,
+ "learning_rate": 8.832592592592593e-07,
+ "logits/chosen": 1.5781702995300293,
+ "logits/rejected": 1.5428775548934937,
+ "logps/chosen": -141.01480102539062,
+ "logps/rejected": -136.47189331054688,
+ "loss": 0.6777657032012939,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.10366280376911163,
+ "rewards/margins": 0.0333605632185936,
+ "rewards/rejected": 0.07030224055051804,
+ "step": 770
+ },
+ {
+ "epoch": 0.624,
+ "grad_norm": 1.3228758573532104,
+ "learning_rate": 8.802962962962962e-07,
+ "logits/chosen": 1.451596736907959,
+ "logits/rejected": 1.5489904880523682,
+ "logps/chosen": -153.74618530273438,
+ "logps/rejected": -159.26333618164062,
+ "loss": 0.6870471000671386,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.10403706133365631,
+ "rewards/margins": 0.014494165778160095,
+ "rewards/rejected": 0.08954289555549622,
+ "step": 780
+ },
+ {
+ "epoch": 0.632,
+ "grad_norm": 1.3267385959625244,
+ "learning_rate": 8.773333333333332e-07,
+ "logits/chosen": 1.4153831005096436,
+ "logits/rejected": 1.4497296810150146,
+ "logps/chosen": -148.05368041992188,
+ "logps/rejected": -151.3020477294922,
+ "loss": 0.6877872943878174,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.07300253212451935,
+ "rewards/margins": 0.013981563039124012,
+ "rewards/rejected": 0.05902096629142761,
+ "step": 790
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 1.2227483987808228,
+ "learning_rate": 8.743703703703703e-07,
+ "logits/chosen": 1.6773914098739624,
+ "logits/rejected": 1.6918518543243408,
+ "logps/chosen": -180.06863403320312,
+ "logps/rejected": -194.30752563476562,
+ "loss": 0.7015917778015137,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07480724155902863,
+ "rewards/margins": -0.013628068380057812,
+ "rewards/rejected": 0.08843531459569931,
+ "step": 800
+ },
+ {
+ "epoch": 0.64,
+ "eval_logits/chosen": 1.5051140785217285,
+ "eval_logits/rejected": 1.5367870330810547,
+ "eval_logps/chosen": -152.8026885986328,
+ "eval_logps/rejected": -147.39437866210938,
+ "eval_loss": 0.688633918762207,
+ "eval_rewards/accuracies": 0.5220000147819519,
+ "eval_rewards/chosen": 0.07461711764335632,
+ "eval_rewards/margins": 0.011839868500828743,
+ "eval_rewards/rejected": 0.06277725100517273,
+ "eval_runtime": 90.9206,
+ "eval_samples_per_second": 5.499,
+ "eval_steps_per_second": 2.75,
+ "step": 800
+ },
+ {
+ "epoch": 0.648,
+ "grad_norm": 1.209770679473877,
+ "learning_rate": 8.714074074074074e-07,
+ "logits/chosen": 1.3785041570663452,
+ "logits/rejected": 1.60085928440094,
+ "logps/chosen": -114.5325927734375,
+ "logps/rejected": -151.88900756835938,
+ "loss": 0.6890413761138916,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0640551745891571,
+ "rewards/margins": 0.010126596316695213,
+ "rewards/rejected": 0.05392857640981674,
+ "step": 810
+ },
+ {
+ "epoch": 0.656,
+ "grad_norm": 1.0250163078308105,
+ "learning_rate": 8.684444444444444e-07,
+ "logits/chosen": 1.5827982425689697,
+ "logits/rejected": 1.6239618062973022,
+ "logps/chosen": -149.01904296875,
+ "logps/rejected": -141.9372100830078,
+ "loss": 0.6728014469146728,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.0830039232969284,
+ "rewards/margins": 0.043090131133794785,
+ "rewards/rejected": 0.03991378843784332,
+ "step": 820
+ },
+ {
+ "epoch": 0.664,
+ "grad_norm": 1.0627655982971191,
+ "learning_rate": 8.654814814814814e-07,
+ "logits/chosen": 1.501873254776001,
+ "logits/rejected": 1.537386417388916,
+ "logps/chosen": -138.75169372558594,
+ "logps/rejected": -134.00970458984375,
+ "loss": 0.6801187992095947,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0640982910990715,
+ "rewards/margins": 0.02855527400970459,
+ "rewards/rejected": 0.03554301708936691,
+ "step": 830
+ },
+ {
+ "epoch": 0.672,
+ "grad_norm": 1.2453433275222778,
+ "learning_rate": 8.625185185185186e-07,
+ "logits/chosen": 1.4766005277633667,
+ "logits/rejected": 1.5185787677764893,
+ "logps/chosen": -148.41250610351562,
+ "logps/rejected": -157.73989868164062,
+ "loss": 0.6788079738616943,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.06551454961299896,
+ "rewards/margins": 0.03301671892404556,
+ "rewards/rejected": 0.0324978306889534,
+ "step": 840
+ },
+ {
+ "epoch": 0.68,
+ "grad_norm": 1.191698670387268,
+ "learning_rate": 8.595555555555555e-07,
+ "logits/chosen": 1.555551528930664,
+ "logits/rejected": 1.5438499450683594,
+ "logps/chosen": -160.77488708496094,
+ "logps/rejected": -141.9876251220703,
+ "loss": 0.6801597595214843,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.053498052060604095,
+ "rewards/margins": 0.028470590710639954,
+ "rewards/rejected": 0.025027472525835037,
+ "step": 850
+ },
+ {
+ "epoch": 0.688,
+ "grad_norm": 0.9382092356681824,
+ "learning_rate": 8.565925925925925e-07,
+ "logits/chosen": 1.5482738018035889,
+ "logits/rejected": 1.5833700895309448,
+ "logps/chosen": -139.68118286132812,
+ "logps/rejected": -130.05947875976562,
+ "loss": 0.6825921535491943,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.06382572650909424,
+ "rewards/margins": 0.023111844435334206,
+ "rewards/rejected": 0.04071388021111488,
+ "step": 860
+ },
+ {
+ "epoch": 0.696,
+ "grad_norm": 1.2248382568359375,
+ "learning_rate": 8.536296296296296e-07,
+ "logits/chosen": 1.4529926776885986,
+ "logits/rejected": 1.4423902034759521,
+ "logps/chosen": -154.4588165283203,
+ "logps/rejected": -153.44949340820312,
+ "loss": 0.6915592193603516,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.03207828477025032,
+ "rewards/margins": 0.006785523146390915,
+ "rewards/rejected": 0.025292763486504555,
+ "step": 870
+ },
+ {
+ "epoch": 0.704,
+ "grad_norm": 1.2174893617630005,
+ "learning_rate": 8.506666666666667e-07,
+ "logits/chosen": 1.4073113203048706,
+ "logits/rejected": 1.668140172958374,
+ "logps/chosen": -145.18460083007812,
+ "logps/rejected": -157.0777587890625,
+ "loss": 0.6922356605529785,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.049450717866420746,
+ "rewards/margins": 0.004049716051667929,
+ "rewards/rejected": 0.04540099948644638,
+ "step": 880
+ },
+ {
+ "epoch": 0.712,
+ "grad_norm": 1.3186850547790527,
+ "learning_rate": 8.477037037037037e-07,
+ "logits/chosen": 1.2923892736434937,
+ "logits/rejected": 1.491051435470581,
+ "logps/chosen": -149.03018188476562,
+ "logps/rejected": -161.61715698242188,
+ "loss": 0.6841150760650635,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.038961172103881836,
+ "rewards/margins": 0.020759906619787216,
+ "rewards/rejected": 0.01820126175880432,
+ "step": 890
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 1.1123124361038208,
+ "learning_rate": 8.447407407407407e-07,
+ "logits/chosen": 1.546547293663025,
+ "logits/rejected": 1.682992696762085,
+ "logps/chosen": -166.2056884765625,
+ "logps/rejected": -180.73330688476562,
+ "loss": 0.6826215744018554,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.07279330492019653,
+ "rewards/margins": 0.025264525786042213,
+ "rewards/rejected": 0.04752878472208977,
+ "step": 900
+ },
+ {
+ "epoch": 0.72,
+ "eval_logits/chosen": 1.4819762706756592,
+ "eval_logits/rejected": 1.513027548789978,
+ "eval_logps/chosen": -153.1695556640625,
+ "eval_logps/rejected": -147.7560577392578,
+ "eval_loss": 0.689373254776001,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": 0.03792775049805641,
+ "eval_rewards/margins": 0.011320074088871479,
+ "eval_rewards/rejected": 0.026607677340507507,
+ "eval_runtime": 90.9603,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 900
+ },
+ {
+ "epoch": 0.728,
+ "grad_norm": 1.1507582664489746,
+ "learning_rate": 8.417777777777777e-07,
+ "logits/chosen": 1.5831308364868164,
+ "logits/rejected": 1.3737132549285889,
+ "logps/chosen": -140.00253295898438,
+ "logps/rejected": -138.45155334472656,
+ "loss": 0.6873391628265381,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.05066479369997978,
+ "rewards/margins": 0.015466553159058094,
+ "rewards/rejected": 0.03519824147224426,
+ "step": 910
+ },
+ {
+ "epoch": 0.736,
+ "grad_norm": 1.415456771850586,
+ "learning_rate": 8.388148148148147e-07,
+ "logits/chosen": 1.3000686168670654,
+ "logits/rejected": 1.2642180919647217,
+ "logps/chosen": -160.14027404785156,
+ "logps/rejected": -163.32827758789062,
+ "loss": 0.687142276763916,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.06618930399417877,
+ "rewards/margins": 0.014807088300585747,
+ "rewards/rejected": 0.05138222128152847,
+ "step": 920
+ },
+ {
+ "epoch": 0.744,
+ "grad_norm": 1.3228634595870972,
+ "learning_rate": 8.358518518518518e-07,
+ "logits/chosen": 1.5204170942306519,
+ "logits/rejected": 1.6129217147827148,
+ "logps/chosen": -157.99832153320312,
+ "logps/rejected": -162.09567260742188,
+ "loss": 0.6790886878967285,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.05940447002649307,
+ "rewards/margins": 0.03161252290010452,
+ "rewards/rejected": 0.027791941538453102,
+ "step": 930
+ },
+ {
+ "epoch": 0.752,
+ "grad_norm": 1.048064947128296,
+ "learning_rate": 8.328888888888889e-07,
+ "logits/chosen": 1.5515320301055908,
+ "logits/rejected": 1.676857352256775,
+ "logps/chosen": -148.4853515625,
+ "logps/rejected": -174.4320526123047,
+ "loss": 0.6935453414916992,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.010225490666925907,
+ "rewards/margins": 0.0021693313028663397,
+ "rewards/rejected": 0.008056161925196648,
+ "step": 940
+ },
+ {
+ "epoch": 0.76,
+ "grad_norm": 1.247799038887024,
+ "learning_rate": 8.299259259259259e-07,
+ "logits/chosen": 1.384231686592102,
+ "logits/rejected": 1.4611561298370361,
+ "logps/chosen": -140.81443786621094,
+ "logps/rejected": -153.81326293945312,
+ "loss": 0.6805226325988769,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.053223587572574615,
+ "rewards/margins": 0.02933613583445549,
+ "rewards/rejected": 0.023887457326054573,
+ "step": 950
+ },
+ {
+ "epoch": 0.768,
+ "grad_norm": 1.1611264944076538,
+ "learning_rate": 8.269629629629629e-07,
+ "logits/chosen": 1.4987528324127197,
+ "logits/rejected": 1.3684321641921997,
+ "logps/chosen": -135.81056213378906,
+ "logps/rejected": -119.05133056640625,
+ "loss": 0.6954316616058349,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.0197703056037426,
+ "rewards/margins": -0.0012637012405321002,
+ "rewards/rejected": 0.021034007892012596,
+ "step": 960
+ },
+ {
+ "epoch": 0.776,
+ "grad_norm": 0.964948832988739,
+ "learning_rate": 8.24e-07,
+ "logits/chosen": 1.612053632736206,
+ "logits/rejected": 1.7114235162734985,
+ "logps/chosen": -153.56137084960938,
+ "logps/rejected": -149.17323303222656,
+ "loss": 0.6916662693023682,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.05082285404205322,
+ "rewards/margins": 0.0056606316938996315,
+ "rewards/rejected": 0.04516221210360527,
+ "step": 970
+ },
+ {
+ "epoch": 0.784,
+ "grad_norm": 1.409449815750122,
+ "learning_rate": 8.21037037037037e-07,
+ "logits/chosen": 1.384060025215149,
+ "logits/rejected": 1.316361904144287,
+ "logps/chosen": -168.75360107421875,
+ "logps/rejected": -141.79537963867188,
+ "loss": 0.6845529556274415,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.0886571854352951,
+ "rewards/margins": 0.02084263041615486,
+ "rewards/rejected": 0.06781454384326935,
+ "step": 980
+ },
+ {
+ "epoch": 0.792,
+ "grad_norm": 1.0246562957763672,
+ "learning_rate": 8.18074074074074e-07,
+ "logits/chosen": 1.4986507892608643,
+ "logits/rejected": 1.571103811264038,
+ "logps/chosen": -165.22254943847656,
+ "logps/rejected": -168.63540649414062,
+ "loss": 0.6790967464447022,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.07517905533313751,
+ "rewards/margins": 0.03272219002246857,
+ "rewards/rejected": 0.04245685786008835,
+ "step": 990
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 1.014426827430725,
+ "learning_rate": 8.15111111111111e-07,
+ "logits/chosen": 1.4708755016326904,
+ "logits/rejected": 1.4550955295562744,
+ "logps/chosen": -149.49874877929688,
+ "logps/rejected": -125.0966567993164,
+ "loss": 0.6892098903656005,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.07825140655040741,
+ "rewards/margins": 0.01007872074842453,
+ "rewards/rejected": 0.06817268580198288,
+ "step": 1000
+ },
+ {
+ "epoch": 0.8,
+ "eval_logits/chosen": 1.4972777366638184,
+ "eval_logits/rejected": 1.5283576250076294,
+ "eval_logps/chosen": -152.8240966796875,
+ "eval_logps/rejected": -147.44175720214844,
+ "eval_loss": 0.6880542039871216,
+ "eval_rewards/accuracies": 0.5419999957084656,
+ "eval_rewards/chosen": 0.07247376441955566,
+ "eval_rewards/margins": 0.01443479023873806,
+ "eval_rewards/rejected": 0.058038972318172455,
+ "eval_runtime": 90.827,
+ "eval_samples_per_second": 5.505,
+ "eval_steps_per_second": 2.752,
+ "step": 1000
+ },
+ {
+ "epoch": 0.808,
+ "grad_norm": 1.1155726909637451,
+ "learning_rate": 8.121481481481482e-07,
+ "logits/chosen": 1.547481894493103,
+ "logits/rejected": 1.5461527109146118,
+ "logps/chosen": -154.89791870117188,
+ "logps/rejected": -142.05111694335938,
+ "loss": 0.6809016704559326,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.10093016922473907,
+ "rewards/margins": 0.027038773521780968,
+ "rewards/rejected": 0.07389138638973236,
+ "step": 1010
+ },
+ {
+ "epoch": 0.816,
+ "grad_norm": 1.6716786623001099,
+ "learning_rate": 8.091851851851852e-07,
+ "logits/chosen": 1.3954919576644897,
+ "logits/rejected": 1.5088036060333252,
+ "logps/chosen": -143.11846923828125,
+ "logps/rejected": -146.41482543945312,
+ "loss": 0.6849985122680664,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.05970728397369385,
+ "rewards/margins": 0.021999523043632507,
+ "rewards/rejected": 0.03770776465535164,
+ "step": 1020
+ },
+ {
+ "epoch": 0.824,
+ "grad_norm": 1.308182716369629,
+ "learning_rate": 8.062222222222221e-07,
+ "logits/chosen": 1.532488465309143,
+ "logits/rejected": 1.3316718339920044,
+ "logps/chosen": -163.06858825683594,
+ "logps/rejected": -136.57989501953125,
+ "loss": 0.6789252281188964,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.11086118221282959,
+ "rewards/margins": 0.033117204904556274,
+ "rewards/rejected": 0.07774396985769272,
+ "step": 1030
+ },
+ {
+ "epoch": 0.832,
+ "grad_norm": 1.1019353866577148,
+ "learning_rate": 8.032592592592592e-07,
+ "logits/chosen": 1.4798121452331543,
+ "logits/rejected": 1.5203083753585815,
+ "logps/chosen": -140.3281707763672,
+ "logps/rejected": -141.06216430664062,
+ "loss": 0.6724937915802002,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.1202714815735817,
+ "rewards/margins": 0.046712182462215424,
+ "rewards/rejected": 0.07355931401252747,
+ "step": 1040
+ },
+ {
+ "epoch": 0.84,
+ "grad_norm": 1.2966418266296387,
+ "learning_rate": 8.002962962962963e-07,
+ "logits/chosen": 1.3034439086914062,
+ "logits/rejected": 1.4363505840301514,
+ "logps/chosen": -142.1992645263672,
+ "logps/rejected": -132.0046844482422,
+ "loss": 0.6848338603973388,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.11126589775085449,
+ "rewards/margins": 0.019870441406965256,
+ "rewards/rejected": 0.09139545261859894,
+ "step": 1050
+ },
+ {
+ "epoch": 0.848,
+ "grad_norm": 1.0321130752563477,
+ "learning_rate": 7.973333333333333e-07,
+ "logits/chosen": 1.3257265090942383,
+ "logits/rejected": 1.4649587869644165,
+ "logps/chosen": -112.98075103759766,
+ "logps/rejected": -127.4059829711914,
+ "loss": 0.6679783344268799,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.17311830818653107,
+ "rewards/margins": 0.05631983280181885,
+ "rewards/rejected": 0.11679847538471222,
+ "step": 1060
+ },
+ {
+ "epoch": 0.856,
+ "grad_norm": 1.2037222385406494,
+ "learning_rate": 7.943703703703704e-07,
+ "logits/chosen": 1.3766987323760986,
+ "logits/rejected": 1.2802103757858276,
+ "logps/chosen": -154.8142547607422,
+ "logps/rejected": -148.680908203125,
+ "loss": 0.668931531906128,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1449051946401596,
+ "rewards/margins": 0.05721765011548996,
+ "rewards/rejected": 0.08768755197525024,
+ "step": 1070
+ },
+ {
+ "epoch": 0.864,
+ "grad_norm": 0.9230766296386719,
+ "learning_rate": 7.914074074074074e-07,
+ "logits/chosen": 1.447328805923462,
+ "logits/rejected": 1.5235106945037842,
+ "logps/chosen": -150.21812438964844,
+ "logps/rejected": -123.29923248291016,
+ "loss": 0.6716424465179444,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.16066041588783264,
+ "rewards/margins": 0.04824484512209892,
+ "rewards/rejected": 0.11241557449102402,
+ "step": 1080
+ },
+ {
+ "epoch": 0.872,
+ "grad_norm": 1.0423877239227295,
+ "learning_rate": 7.884444444444443e-07,
+ "logits/chosen": 1.6920299530029297,
+ "logits/rejected": 1.6123542785644531,
+ "logps/chosen": -175.1676025390625,
+ "logps/rejected": -185.07223510742188,
+ "loss": 0.6723968505859375,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.16490009427070618,
+ "rewards/margins": 0.046577490866184235,
+ "rewards/rejected": 0.11832261085510254,
+ "step": 1090
+ },
+ {
+ "epoch": 0.88,
+ "grad_norm": 1.4142779111862183,
+ "learning_rate": 7.854814814814814e-07,
+ "logits/chosen": 1.453111171722412,
+ "logits/rejected": 1.4981731176376343,
+ "logps/chosen": -139.9765167236328,
+ "logps/rejected": -153.63058471679688,
+ "loss": 0.6872457027435303,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.13096000254154205,
+ "rewards/margins": 0.01702108606696129,
+ "rewards/rejected": 0.11393891274929047,
+ "step": 1100
+ },
+ {
+ "epoch": 0.88,
+ "eval_logits/chosen": 1.5076195001602173,
+ "eval_logits/rejected": 1.5373249053955078,
+ "eval_logps/chosen": -152.1697998046875,
+ "eval_logps/rejected": -146.83502197265625,
+ "eval_loss": 0.6873304843902588,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.13790392875671387,
+ "eval_rewards/margins": 0.019191108644008636,
+ "eval_rewards/rejected": 0.11871281266212463,
+ "eval_runtime": 91.0157,
+ "eval_samples_per_second": 5.494,
+ "eval_steps_per_second": 2.747,
+ "step": 1100
+ },
+ {
+ "epoch": 0.888,
+ "grad_norm": 1.413996696472168,
+ "learning_rate": 7.825185185185185e-07,
+ "logits/chosen": 1.5183905363082886,
+ "logits/rejected": 1.4942539930343628,
+ "logps/chosen": -156.383544921875,
+ "logps/rejected": -145.61398315429688,
+ "loss": 0.6810199737548828,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.14243564009666443,
+ "rewards/margins": 0.03136260434985161,
+ "rewards/rejected": 0.11107305437326431,
+ "step": 1110
+ },
+ {
+ "epoch": 0.896,
+ "grad_norm": 1.0046707391738892,
+ "learning_rate": 7.795555555555555e-07,
+ "logits/chosen": 1.4982213973999023,
+ "logits/rejected": 1.5465660095214844,
+ "logps/chosen": -156.79293823242188,
+ "logps/rejected": -150.77853393554688,
+ "loss": 0.6808318614959716,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1551930010318756,
+ "rewards/margins": 0.028187204152345657,
+ "rewards/rejected": 0.12700578570365906,
+ "step": 1120
+ },
+ {
+ "epoch": 0.904,
+ "grad_norm": 1.125625729560852,
+ "learning_rate": 7.765925925925925e-07,
+ "logits/chosen": 1.6053575277328491,
+ "logits/rejected": 1.47335946559906,
+ "logps/chosen": -148.05690002441406,
+ "logps/rejected": -135.36598205566406,
+ "loss": 0.6889908790588379,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.15128108859062195,
+ "rewards/margins": 0.015811312943696976,
+ "rewards/rejected": 0.13546979427337646,
+ "step": 1130
+ },
+ {
+ "epoch": 0.912,
+ "grad_norm": 1.0933659076690674,
+ "learning_rate": 7.736296296296297e-07,
+ "logits/chosen": 1.5862042903900146,
+ "logits/rejected": 1.5827430486679077,
+ "logps/chosen": -156.10145568847656,
+ "logps/rejected": -142.94851684570312,
+ "loss": 0.6753206253051758,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1547432243824005,
+ "rewards/margins": 0.04229574277997017,
+ "rewards/rejected": 0.11244747787714005,
+ "step": 1140
+ },
+ {
+ "epoch": 0.92,
+ "grad_norm": 1.0774686336517334,
+ "learning_rate": 7.706666666666667e-07,
+ "logits/chosen": 1.2369152307510376,
+ "logits/rejected": 1.2727023363113403,
+ "logps/chosen": -139.23147583007812,
+ "logps/rejected": -130.53038024902344,
+ "loss": 0.6967329025268555,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": 0.10801626741886139,
+ "rewards/margins": 0.004670143127441406,
+ "rewards/rejected": 0.10334610939025879,
+ "step": 1150
+ },
+ {
+ "epoch": 0.928,
+ "grad_norm": 1.1111780405044556,
+ "learning_rate": 7.677037037037036e-07,
+ "logits/chosen": 1.501250982284546,
+ "logits/rejected": 1.4272868633270264,
+ "logps/chosen": -127.09858703613281,
+ "logps/rejected": -114.9041748046875,
+ "loss": 0.6853152275085449,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.1366150826215744,
+ "rewards/margins": 0.021945688873529434,
+ "rewards/rejected": 0.11466939747333527,
+ "step": 1160
+ },
+ {
+ "epoch": 0.936,
+ "grad_norm": 1.5593854188919067,
+ "learning_rate": 7.647407407407407e-07,
+ "logits/chosen": 1.727927803993225,
+ "logits/rejected": 1.6510727405548096,
+ "logps/chosen": -156.70993041992188,
+ "logps/rejected": -167.07266235351562,
+ "loss": 0.6884279251098633,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.14902853965759277,
+ "rewards/margins": 0.01685408689081669,
+ "rewards/rejected": 0.13217447698116302,
+ "step": 1170
+ },
+ {
+ "epoch": 0.944,
+ "grad_norm": 2.0064918994903564,
+ "learning_rate": 7.617777777777778e-07,
+ "logits/chosen": 1.6262975931167603,
+ "logits/rejected": 1.58951997756958,
+ "logps/chosen": -186.59210205078125,
+ "logps/rejected": -151.2965850830078,
+ "loss": 0.6906405448913574,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.14898104965686798,
+ "rewards/margins": 0.013303476385772228,
+ "rewards/rejected": 0.13567757606506348,
+ "step": 1180
+ },
+ {
+ "epoch": 0.952,
+ "grad_norm": 1.209291696548462,
+ "learning_rate": 7.588148148148148e-07,
+ "logits/chosen": 1.639369249343872,
+ "logits/rejected": 1.6400636434555054,
+ "logps/chosen": -166.24684143066406,
+ "logps/rejected": -196.35305786132812,
+ "loss": 0.6789956092834473,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1348457634449005,
+ "rewards/margins": 0.03475186973810196,
+ "rewards/rejected": 0.10009388625621796,
+ "step": 1190
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 1.209230899810791,
+ "learning_rate": 7.558518518518519e-07,
+ "logits/chosen": 1.542942762374878,
+ "logits/rejected": 1.5128916501998901,
+ "logps/chosen": -138.994384765625,
+ "logps/rejected": -145.42880249023438,
+ "loss": 0.6936234951019287,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.13895569741725922,
+ "rewards/margins": 0.005582497920840979,
+ "rewards/rejected": 0.1333732008934021,
+ "step": 1200
+ },
+ {
+ "epoch": 0.96,
+ "eval_logits/chosen": 1.5055651664733887,
+ "eval_logits/rejected": 1.5351229906082153,
+ "eval_logps/chosen": -152.2256622314453,
+ "eval_logps/rejected": -146.909912109375,
+ "eval_loss": 0.6866763830184937,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.13231882452964783,
+ "eval_rewards/margins": 0.02109396643936634,
+ "eval_rewards/rejected": 0.11122485250234604,
+ "eval_runtime": 90.8141,
+ "eval_samples_per_second": 5.506,
+ "eval_steps_per_second": 2.753,
+ "step": 1200
+ },
+ {
+ "epoch": 0.968,
+ "grad_norm": 1.2058950662612915,
+ "learning_rate": 7.528888888888889e-07,
+ "logits/chosen": 1.3591210842132568,
+ "logits/rejected": 1.3811699151992798,
+ "logps/chosen": -155.1759033203125,
+ "logps/rejected": -141.97772216796875,
+ "loss": 0.6902606964111329,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.153532013297081,
+ "rewards/margins": 0.011359233409166336,
+ "rewards/rejected": 0.14217278361320496,
+ "step": 1210
+ },
+ {
+ "epoch": 0.976,
+ "grad_norm": 1.366702914237976,
+ "learning_rate": 7.499259259259258e-07,
+ "logits/chosen": 1.3841370344161987,
+ "logits/rejected": 1.3791372776031494,
+ "logps/chosen": -145.47613525390625,
+ "logps/rejected": -123.9173583984375,
+ "loss": 0.6971709251403808,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.10521620512008667,
+ "rewards/margins": -0.0030749079305678606,
+ "rewards/rejected": 0.1082911267876625,
+ "step": 1220
+ },
+ {
+ "epoch": 0.984,
+ "grad_norm": 1.477455496788025,
+ "learning_rate": 7.469629629629629e-07,
+ "logits/chosen": 1.4447228908538818,
+ "logits/rejected": 1.6022417545318604,
+ "logps/chosen": -137.9462127685547,
+ "logps/rejected": -150.25559997558594,
+ "loss": 0.6915127277374268,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.15108394622802734,
+ "rewards/margins": 0.009388584643602371,
+ "rewards/rejected": 0.14169538021087646,
+ "step": 1230
+ },
+ {
+ "epoch": 0.992,
+ "grad_norm": 1.4407854080200195,
+ "learning_rate": 7.44e-07,
+ "logits/chosen": 1.3998825550079346,
+ "logits/rejected": 1.5924925804138184,
+ "logps/chosen": -144.93775939941406,
+ "logps/rejected": -148.02542114257812,
+ "loss": 0.6699621677398682,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.146137997508049,
+ "rewards/margins": 0.05326627567410469,
+ "rewards/rejected": 0.09287171810865402,
+ "step": 1240
+ },
+ {
+ "epoch": 1.0,
+ "grad_norm": 1.183438777923584,
+ "learning_rate": 7.41037037037037e-07,
+ "logits/chosen": 1.5393705368041992,
+ "logits/rejected": 1.6312555074691772,
+ "logps/chosen": -145.61672973632812,
+ "logps/rejected": -158.9256591796875,
+ "loss": 0.6811168670654297,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.1304798424243927,
+ "rewards/margins": 0.03360626846551895,
+ "rewards/rejected": 0.09687358886003494,
+ "step": 1250
+ },
+ {
+ "epoch": 1.008,
+ "grad_norm": 1.2436553239822388,
+ "learning_rate": 7.38074074074074e-07,
+ "logits/chosen": 1.4758602380752563,
+ "logits/rejected": 1.534609079360962,
+ "logps/chosen": -124.26841735839844,
+ "logps/rejected": -132.41867065429688,
+ "loss": 0.688177490234375,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.12570872902870178,
+ "rewards/margins": 0.014875045046210289,
+ "rewards/rejected": 0.11083368957042694,
+ "step": 1260
+ },
+ {
+ "epoch": 1.016,
+ "grad_norm": 1.3853868246078491,
+ "learning_rate": 7.351111111111112e-07,
+ "logits/chosen": 1.7195091247558594,
+ "logits/rejected": 1.632851004600525,
+ "logps/chosen": -160.8590850830078,
+ "logps/rejected": -153.91671752929688,
+ "loss": 0.6669504165649414,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.181864932179451,
+ "rewards/margins": 0.06175640970468521,
+ "rewards/rejected": 0.12010850757360458,
+ "step": 1270
+ },
+ {
+ "epoch": 1.024,
+ "grad_norm": 1.2765302658081055,
+ "learning_rate": 7.321481481481481e-07,
+ "logits/chosen": 1.5782036781311035,
+ "logits/rejected": 1.5713088512420654,
+ "logps/chosen": -154.3319549560547,
+ "logps/rejected": -141.43345642089844,
+ "loss": 0.6734190464019776,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.12361389398574829,
+ "rewards/margins": 0.04626934230327606,
+ "rewards/rejected": 0.07734455168247223,
+ "step": 1280
+ },
+ {
+ "epoch": 1.032,
+ "grad_norm": 1.0421397686004639,
+ "learning_rate": 7.291851851851851e-07,
+ "logits/chosen": 1.5199967622756958,
+ "logits/rejected": 1.5965670347213745,
+ "logps/chosen": -145.28416442871094,
+ "logps/rejected": -133.72256469726562,
+ "loss": 0.6804514408111573,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.11794980615377426,
+ "rewards/margins": 0.030916064977645874,
+ "rewards/rejected": 0.08703374862670898,
+ "step": 1290
+ },
+ {
+ "epoch": 1.04,
+ "grad_norm": 1.032158613204956,
+ "learning_rate": 7.262222222222222e-07,
+ "logits/chosen": 1.6042639017105103,
+ "logits/rejected": 1.4255346059799194,
+ "logps/chosen": -150.4803009033203,
+ "logps/rejected": -130.2232208251953,
+ "loss": 0.6676507472991944,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.15466037392616272,
+ "rewards/margins": 0.05808692425489426,
+ "rewards/rejected": 0.09657344967126846,
+ "step": 1300
+ },
+ {
+ "epoch": 1.04,
+ "eval_logits/chosen": 1.5118356943130493,
+ "eval_logits/rejected": 1.5418950319290161,
+ "eval_logps/chosen": -152.23300170898438,
+ "eval_logps/rejected": -146.9257049560547,
+ "eval_loss": 0.6859617233276367,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.13158410787582397,
+ "eval_rewards/margins": 0.021939631551504135,
+ "eval_rewards/rejected": 0.10964448750019073,
+ "eval_runtime": 90.9624,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 1300
+ },
+ {
+ "epoch": 1.048,
+ "grad_norm": 1.4942574501037598,
+ "learning_rate": 7.232592592592593e-07,
+ "logits/chosen": 1.689648985862732,
+ "logits/rejected": 1.691888451576233,
+ "logps/chosen": -166.02809143066406,
+ "logps/rejected": -160.30601501464844,
+ "loss": 0.6770285606384278,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.14377735555171967,
+ "rewards/margins": 0.03825496509671211,
+ "rewards/rejected": 0.10552239418029785,
+ "step": 1310
+ },
+ {
+ "epoch": 1.056,
+ "grad_norm": 1.3656011819839478,
+ "learning_rate": 7.202962962962963e-07,
+ "logits/chosen": 1.3065154552459717,
+ "logits/rejected": 1.5681203603744507,
+ "logps/chosen": -129.1146240234375,
+ "logps/rejected": -151.2584991455078,
+ "loss": 0.6844655036926269,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.12841743230819702,
+ "rewards/margins": 0.02178655005991459,
+ "rewards/rejected": 0.10663086175918579,
+ "step": 1320
+ },
+ {
+ "epoch": 1.064,
+ "grad_norm": 1.1723405122756958,
+ "learning_rate": 7.173333333333333e-07,
+ "logits/chosen": 1.6219158172607422,
+ "logits/rejected": 1.6032555103302002,
+ "logps/chosen": -156.4185028076172,
+ "logps/rejected": -146.28067016601562,
+ "loss": 0.6650479316711426,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1538492739200592,
+ "rewards/margins": 0.06132146716117859,
+ "rewards/rejected": 0.09252782166004181,
+ "step": 1330
+ },
+ {
+ "epoch": 1.072,
+ "grad_norm": 1.2311245203018188,
+ "learning_rate": 7.143703703703703e-07,
+ "logits/chosen": 1.4625260829925537,
+ "logits/rejected": 1.392937421798706,
+ "logps/chosen": -153.05941772460938,
+ "logps/rejected": -145.37841796875,
+ "loss": 0.6570021152496338,
+ "rewards/accuracies": 0.762499988079071,
+ "rewards/chosen": 0.16768500208854675,
+ "rewards/margins": 0.0787494108080864,
+ "rewards/rejected": 0.08893559873104095,
+ "step": 1340
+ },
+ {
+ "epoch": 1.08,
+ "grad_norm": 1.314666748046875,
+ "learning_rate": 7.114074074074073e-07,
+ "logits/chosen": 1.4812809228897095,
+ "logits/rejected": 1.3796050548553467,
+ "logps/chosen": -165.70205688476562,
+ "logps/rejected": -140.13150024414062,
+ "loss": 0.6535200595855712,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.20691323280334473,
+ "rewards/margins": 0.0911526307463646,
+ "rewards/rejected": 0.11576057970523834,
+ "step": 1350
+ },
+ {
+ "epoch": 1.088,
+ "grad_norm": 0.9556720852851868,
+ "learning_rate": 7.084444444444444e-07,
+ "logits/chosen": 1.6368863582611084,
+ "logits/rejected": 1.678134560585022,
+ "logps/chosen": -166.56253051757812,
+ "logps/rejected": -164.36416625976562,
+ "loss": 0.6742628574371338,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.15145304799079895,
+ "rewards/margins": 0.04319921135902405,
+ "rewards/rejected": 0.1082538366317749,
+ "step": 1360
+ },
+ {
+ "epoch": 1.096,
+ "grad_norm": 0.945633053779602,
+ "learning_rate": 7.054814814814815e-07,
+ "logits/chosen": 1.4285938739776611,
+ "logits/rejected": 1.5576260089874268,
+ "logps/chosen": -131.5234832763672,
+ "logps/rejected": -128.02340698242188,
+ "loss": 0.6670140743255615,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.16399362683296204,
+ "rewards/margins": 0.05922812223434448,
+ "rewards/rejected": 0.10476551204919815,
+ "step": 1370
+ },
+ {
+ "epoch": 1.104,
+ "grad_norm": 1.1189475059509277,
+ "learning_rate": 7.025185185185185e-07,
+ "logits/chosen": 1.5019981861114502,
+ "logits/rejected": 1.667614221572876,
+ "logps/chosen": -161.61854553222656,
+ "logps/rejected": -166.39996337890625,
+ "loss": 0.6742690086364747,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.15786050260066986,
+ "rewards/margins": 0.044500257819890976,
+ "rewards/rejected": 0.11336024850606918,
+ "step": 1380
+ },
+ {
+ "epoch": 1.112,
+ "grad_norm": 1.3382912874221802,
+ "learning_rate": 6.995555555555555e-07,
+ "logits/chosen": 1.4972480535507202,
+ "logits/rejected": 1.4946129322052002,
+ "logps/chosen": -166.43682861328125,
+ "logps/rejected": -162.3004913330078,
+ "loss": 0.6557814121246338,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.18964914977550507,
+ "rewards/margins": 0.08500295877456665,
+ "rewards/rejected": 0.10464620590209961,
+ "step": 1390
+ },
+ {
+ "epoch": 1.12,
+ "grad_norm": 1.2123515605926514,
+ "learning_rate": 6.965925925925926e-07,
+ "logits/chosen": 1.456635594367981,
+ "logits/rejected": 1.5964475870132446,
+ "logps/chosen": -152.08261108398438,
+ "logps/rejected": -161.51023864746094,
+ "loss": 0.6708066940307618,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.216823011636734,
+ "rewards/margins": 0.050363689661026,
+ "rewards/rejected": 0.166459321975708,
+ "step": 1400
+ },
+ {
+ "epoch": 1.12,
+ "eval_logits/chosen": 1.521518588066101,
+ "eval_logits/rejected": 1.5510718822479248,
+ "eval_logps/chosen": -151.858154296875,
+ "eval_logps/rejected": -146.5819549560547,
+ "eval_loss": 0.6853657364845276,
+ "eval_rewards/accuracies": 0.5440000295639038,
+ "eval_rewards/chosen": 0.16906805336475372,
+ "eval_rewards/margins": 0.025049500167369843,
+ "eval_rewards/rejected": 0.14401854574680328,
+ "eval_runtime": 90.8779,
+ "eval_samples_per_second": 5.502,
+ "eval_steps_per_second": 2.751,
+ "step": 1400
+ },
+ {
+ "epoch": 1.1280000000000001,
+ "grad_norm": 1.3488291501998901,
+ "learning_rate": 6.936296296296296e-07,
+ "logits/chosen": 1.6009830236434937,
+ "logits/rejected": 1.6035544872283936,
+ "logps/chosen": -145.44662475585938,
+ "logps/rejected": -149.9284210205078,
+ "loss": 0.679435396194458,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.1647307425737381,
+ "rewards/margins": 0.03348377346992493,
+ "rewards/rejected": 0.13124695420265198,
+ "step": 1410
+ },
+ {
+ "epoch": 1.1360000000000001,
+ "grad_norm": 1.6453970670700073,
+ "learning_rate": 6.906666666666666e-07,
+ "logits/chosen": 1.593658447265625,
+ "logits/rejected": 1.648276925086975,
+ "logps/chosen": -135.2926788330078,
+ "logps/rejected": -150.9585723876953,
+ "loss": 0.6875734329223633,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.15669891238212585,
+ "rewards/margins": 0.01690087839961052,
+ "rewards/rejected": 0.13979804515838623,
+ "step": 1420
+ },
+ {
+ "epoch": 1.144,
+ "grad_norm": 1.6117807626724243,
+ "learning_rate": 6.877037037037036e-07,
+ "logits/chosen": 1.4510356187820435,
+ "logits/rejected": 1.2334567308425903,
+ "logps/chosen": -153.94854736328125,
+ "logps/rejected": -147.78079223632812,
+ "loss": 0.6689413070678711,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1819056123495102,
+ "rewards/margins": 0.058564942330121994,
+ "rewards/rejected": 0.1233406662940979,
+ "step": 1430
+ },
+ {
+ "epoch": 1.152,
+ "grad_norm": 1.5533809661865234,
+ "learning_rate": 6.847407407407408e-07,
+ "logits/chosen": 1.4535889625549316,
+ "logits/rejected": 1.4273865222930908,
+ "logps/chosen": -170.42904663085938,
+ "logps/rejected": -151.9979705810547,
+ "loss": 0.6845246315002441,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.14223966002464294,
+ "rewards/margins": 0.022389881312847137,
+ "rewards/rejected": 0.1198497861623764,
+ "step": 1440
+ },
+ {
+ "epoch": 1.16,
+ "grad_norm": 1.5696005821228027,
+ "learning_rate": 6.817777777777778e-07,
+ "logits/chosen": 1.4418725967407227,
+ "logits/rejected": 1.4354021549224854,
+ "logps/chosen": -153.962158203125,
+ "logps/rejected": -148.40512084960938,
+ "loss": 0.6761108875274658,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.15529735386371613,
+ "rewards/margins": 0.040999218821525574,
+ "rewards/rejected": 0.11429812759160995,
+ "step": 1450
+ },
+ {
+ "epoch": 1.168,
+ "grad_norm": 1.0711781978607178,
+ "learning_rate": 6.788148148148147e-07,
+ "logits/chosen": 1.5659481287002563,
+ "logits/rejected": 1.5640677213668823,
+ "logps/chosen": -149.816162109375,
+ "logps/rejected": -174.68643188476562,
+ "loss": 0.677489185333252,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1510745733976364,
+ "rewards/margins": 0.03914733976125717,
+ "rewards/rejected": 0.11192724853754044,
+ "step": 1460
+ },
+ {
+ "epoch": 1.176,
+ "grad_norm": 1.2336673736572266,
+ "learning_rate": 6.758518518518518e-07,
+ "logits/chosen": 1.4275394678115845,
+ "logits/rejected": 1.6394407749176025,
+ "logps/chosen": -137.93788146972656,
+ "logps/rejected": -154.97195434570312,
+ "loss": 0.6785802364349365,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10557998716831207,
+ "rewards/margins": 0.0352485254406929,
+ "rewards/rejected": 0.07033145427703857,
+ "step": 1470
+ },
+ {
+ "epoch": 1.184,
+ "grad_norm": 1.0030723810195923,
+ "learning_rate": 6.728888888888888e-07,
+ "logits/chosen": 1.4652725458145142,
+ "logits/rejected": 1.4992272853851318,
+ "logps/chosen": -143.87374877929688,
+ "logps/rejected": -139.5693817138672,
+ "loss": 0.6700259208679199,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.13436684012413025,
+ "rewards/margins": 0.05286343768239021,
+ "rewards/rejected": 0.08150340616703033,
+ "step": 1480
+ },
+ {
+ "epoch": 1.192,
+ "grad_norm": 1.0897104740142822,
+ "learning_rate": 6.699259259259259e-07,
+ "logits/chosen": 1.7033601999282837,
+ "logits/rejected": 1.617489218711853,
+ "logps/chosen": -140.43667602539062,
+ "logps/rejected": -146.59683227539062,
+ "loss": 0.6685672283172608,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.11724893748760223,
+ "rewards/margins": 0.055138181895017624,
+ "rewards/rejected": 0.06211075931787491,
+ "step": 1490
+ },
+ {
+ "epoch": 1.2,
+ "grad_norm": 0.8573256731033325,
+ "learning_rate": 6.66962962962963e-07,
+ "logits/chosen": 1.5181013345718384,
+ "logits/rejected": 1.5965152978897095,
+ "logps/chosen": -147.78506469726562,
+ "logps/rejected": -142.18930053710938,
+ "loss": 0.6564288139343262,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.14890792965888977,
+ "rewards/margins": 0.08058460801839828,
+ "rewards/rejected": 0.06832331418991089,
+ "step": 1500
+ },
+ {
+ "epoch": 1.2,
+ "eval_logits/chosen": 1.488415002822876,
+ "eval_logits/rejected": 1.5179662704467773,
+ "eval_logps/chosen": -152.60232543945312,
+ "eval_logps/rejected": -147.2954864501953,
+ "eval_loss": 0.6861968040466309,
+ "eval_rewards/accuracies": 0.5400000214576721,
+ "eval_rewards/chosen": 0.09465231001377106,
+ "eval_rewards/margins": 0.021986663341522217,
+ "eval_rewards/rejected": 0.07266565412282944,
+ "eval_runtime": 90.8951,
+ "eval_samples_per_second": 5.501,
+ "eval_steps_per_second": 2.75,
+ "step": 1500
+ },
+ {
+ "epoch": 1.208,
+ "grad_norm": 1.29599928855896,
+ "learning_rate": 6.64e-07,
+ "logits/chosen": 1.4851064682006836,
+ "logits/rejected": 1.4355719089508057,
+ "logps/chosen": -140.07186889648438,
+ "logps/rejected": -155.5626983642578,
+ "loss": 0.6695255756378173,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08214708417654037,
+ "rewards/margins": 0.05618969723582268,
+ "rewards/rejected": 0.025957385078072548,
+ "step": 1510
+ },
+ {
+ "epoch": 1.216,
+ "grad_norm": 1.3030505180358887,
+ "learning_rate": 6.61037037037037e-07,
+ "logits/chosen": 1.3785858154296875,
+ "logits/rejected": 1.3867576122283936,
+ "logps/chosen": -143.06137084960938,
+ "logps/rejected": -152.8805694580078,
+ "loss": 0.6704819202423096,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.08582031726837158,
+ "rewards/margins": 0.052868831902742386,
+ "rewards/rejected": 0.0329514816403389,
+ "step": 1520
+ },
+ {
+ "epoch": 1.224,
+ "grad_norm": 1.2935470342636108,
+ "learning_rate": 6.58074074074074e-07,
+ "logits/chosen": 1.4631284475326538,
+ "logits/rejected": 1.4908760786056519,
+ "logps/chosen": -181.2951202392578,
+ "logps/rejected": -173.83523559570312,
+ "loss": 0.6680389881134033,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.11432291567325592,
+ "rewards/margins": 0.05878107622265816,
+ "rewards/rejected": 0.05554184317588806,
+ "step": 1530
+ },
+ {
+ "epoch": 1.232,
+ "grad_norm": 1.4426783323287964,
+ "learning_rate": 6.551111111111111e-07,
+ "logits/chosen": 1.4765878915786743,
+ "logits/rejected": 1.4511891603469849,
+ "logps/chosen": -167.4827117919922,
+ "logps/rejected": -153.88833618164062,
+ "loss": 0.6684586524963378,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.08437791466712952,
+ "rewards/margins": 0.05823448300361633,
+ "rewards/rejected": 0.026143427938222885,
+ "step": 1540
+ },
+ {
+ "epoch": 1.24,
+ "grad_norm": 1.3253812789916992,
+ "learning_rate": 6.521481481481481e-07,
+ "logits/chosen": 1.2496798038482666,
+ "logits/rejected": 1.5193806886672974,
+ "logps/chosen": -130.33685302734375,
+ "logps/rejected": -132.5361328125,
+ "loss": 0.6695978164672851,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.10806921869516373,
+ "rewards/margins": 0.05488727241754532,
+ "rewards/rejected": 0.05318193882703781,
+ "step": 1550
+ },
+ {
+ "epoch": 1.248,
+ "grad_norm": 1.1890907287597656,
+ "learning_rate": 6.491851851851851e-07,
+ "logits/chosen": 1.2053601741790771,
+ "logits/rejected": 1.2588948011398315,
+ "logps/chosen": -132.61026000976562,
+ "logps/rejected": -150.66477966308594,
+ "loss": 0.673579216003418,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.062341660261154175,
+ "rewards/margins": 0.04531678557395935,
+ "rewards/rejected": 0.017024870961904526,
+ "step": 1560
+ },
+ {
+ "epoch": 1.256,
+ "grad_norm": 1.2076733112335205,
+ "learning_rate": 6.462222222222223e-07,
+ "logits/chosen": 1.5180461406707764,
+ "logits/rejected": 1.4118196964263916,
+ "logps/chosen": -120.61619567871094,
+ "logps/rejected": -146.42056274414062,
+ "loss": 0.6747167587280274,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.0764162540435791,
+ "rewards/margins": 0.04275941476225853,
+ "rewards/rejected": 0.03365683928132057,
+ "step": 1570
+ },
+ {
+ "epoch": 1.264,
+ "grad_norm": 1.3394429683685303,
+ "learning_rate": 6.432592592592593e-07,
+ "logits/chosen": 1.2879914045333862,
+ "logits/rejected": 1.3957126140594482,
+ "logps/chosen": -150.98355102539062,
+ "logps/rejected": -172.91090393066406,
+ "loss": 0.6560922622680664,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.09853875637054443,
+ "rewards/margins": 0.08982758224010468,
+ "rewards/rejected": 0.008711177855730057,
+ "step": 1580
+ },
+ {
+ "epoch": 1.272,
+ "grad_norm": 1.303663730621338,
+ "learning_rate": 6.402962962962962e-07,
+ "logits/chosen": 1.5617700815200806,
+ "logits/rejected": 1.6307615041732788,
+ "logps/chosen": -157.00338745117188,
+ "logps/rejected": -159.2900848388672,
+ "loss": 0.6639400959014893,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.07477572560310364,
+ "rewards/margins": 0.06561199575662613,
+ "rewards/rejected": 0.009163733571767807,
+ "step": 1590
+ },
+ {
+ "epoch": 1.28,
+ "grad_norm": 1.4180800914764404,
+ "learning_rate": 6.373333333333333e-07,
+ "logits/chosen": 1.3017181158065796,
+ "logits/rejected": 1.3270524740219116,
+ "logps/chosen": -134.52029418945312,
+ "logps/rejected": -135.45938110351562,
+ "loss": 0.6564930438995361,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.08265326917171478,
+ "rewards/margins": 0.09048662334680557,
+ "rewards/rejected": -0.007833337411284447,
+ "step": 1600
+ },
+ {
+ "epoch": 1.28,
+ "eval_logits/chosen": 1.4600619077682495,
+ "eval_logits/rejected": 1.489198923110962,
+ "eval_logps/chosen": -152.86001586914062,
+ "eval_logps/rejected": -147.54771423339844,
+ "eval_loss": 0.6871278882026672,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": 0.0688827633857727,
+ "eval_rewards/margins": 0.021439960226416588,
+ "eval_rewards/rejected": 0.04744280129671097,
+ "eval_runtime": 91.0555,
+ "eval_samples_per_second": 5.491,
+ "eval_steps_per_second": 2.746,
+ "step": 1600
+ },
+ {
+ "epoch": 1.288,
+ "grad_norm": 1.1791326999664307,
+ "learning_rate": 6.343703703703703e-07,
+ "logits/chosen": 1.4442191123962402,
+ "logits/rejected": 1.5532598495483398,
+ "logps/chosen": -141.93701171875,
+ "logps/rejected": -147.68682861328125,
+ "loss": 0.6539246082305908,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.09121527522802353,
+ "rewards/margins": 0.08849794417619705,
+ "rewards/rejected": 0.002717340365052223,
+ "step": 1610
+ },
+ {
+ "epoch": 1.296,
+ "grad_norm": 1.1131889820098877,
+ "learning_rate": 6.314074074074074e-07,
+ "logits/chosen": 1.3450567722320557,
+ "logits/rejected": 1.4408035278320312,
+ "logps/chosen": -157.0798797607422,
+ "logps/rejected": -161.1500701904297,
+ "loss": 0.6505428314208984,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.11321337521076202,
+ "rewards/margins": 0.09579396992921829,
+ "rewards/rejected": 0.017419395968317986,
+ "step": 1620
+ },
+ {
+ "epoch": 1.304,
+ "grad_norm": 1.6172596216201782,
+ "learning_rate": 6.284444444444445e-07,
+ "logits/chosen": 1.478847622871399,
+ "logits/rejected": 1.3894659280776978,
+ "logps/chosen": -140.17453002929688,
+ "logps/rejected": -169.06289672851562,
+ "loss": 0.6720306396484375,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.08244314789772034,
+ "rewards/margins": 0.062018074095249176,
+ "rewards/rejected": 0.020425086840987206,
+ "step": 1630
+ },
+ {
+ "epoch": 1.312,
+ "grad_norm": 1.1743669509887695,
+ "learning_rate": 6.254814814814815e-07,
+ "logits/chosen": 1.4197287559509277,
+ "logits/rejected": 1.3319528102874756,
+ "logps/chosen": -157.23072814941406,
+ "logps/rejected": -139.50074768066406,
+ "loss": 0.6712700843811035,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.04650931805372238,
+ "rewards/margins": 0.057532649487257004,
+ "rewards/rejected": -0.0110233249142766,
+ "step": 1640
+ },
+ {
+ "epoch": 1.32,
+ "grad_norm": 1.3225634098052979,
+ "learning_rate": 6.225185185185184e-07,
+ "logits/chosen": 1.35457444190979,
+ "logits/rejected": 1.3751428127288818,
+ "logps/chosen": -137.4834747314453,
+ "logps/rejected": -134.4203643798828,
+ "loss": 0.6679590225219727,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.07574301958084106,
+ "rewards/margins": 0.057132184505462646,
+ "rewards/rejected": 0.01861082948744297,
+ "step": 1650
+ },
+ {
+ "epoch": 1.328,
+ "grad_norm": 1.5913130044937134,
+ "learning_rate": 6.195555555555555e-07,
+ "logits/chosen": 1.5328034162521362,
+ "logits/rejected": 1.3161242008209229,
+ "logps/chosen": -147.09475708007812,
+ "logps/rejected": -135.3501434326172,
+ "loss": 0.6575756072998047,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.1077059879899025,
+ "rewards/margins": 0.08081348240375519,
+ "rewards/rejected": 0.026892507448792458,
+ "step": 1660
+ },
+ {
+ "epoch": 1.336,
+ "grad_norm": 0.9880369901657104,
+ "learning_rate": 6.165925925925926e-07,
+ "logits/chosen": 1.3911128044128418,
+ "logits/rejected": 1.3688428401947021,
+ "logps/chosen": -152.17483520507812,
+ "logps/rejected": -149.79383850097656,
+ "loss": 0.6656455516815185,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.10711929947137833,
+ "rewards/margins": 0.0669747143983841,
+ "rewards/rejected": 0.04014457389712334,
+ "step": 1670
+ },
+ {
+ "epoch": 1.3439999999999999,
+ "grad_norm": 1.125959873199463,
+ "learning_rate": 6.136296296296296e-07,
+ "logits/chosen": 1.4232629537582397,
+ "logits/rejected": 1.5579107999801636,
+ "logps/chosen": -160.2217559814453,
+ "logps/rejected": -148.1455535888672,
+ "loss": 0.6710956573486329,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10255889594554901,
+ "rewards/margins": 0.05178038403391838,
+ "rewards/rejected": 0.05077851936221123,
+ "step": 1680
+ },
+ {
+ "epoch": 1.3519999999999999,
+ "grad_norm": 1.3627047538757324,
+ "learning_rate": 6.106666666666666e-07,
+ "logits/chosen": 1.4259792566299438,
+ "logits/rejected": 1.4750150442123413,
+ "logps/chosen": -152.88381958007812,
+ "logps/rejected": -153.72708129882812,
+ "loss": 0.6512804508209229,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.14147944748401642,
+ "rewards/margins": 0.09307968616485596,
+ "rewards/rejected": 0.04839975759387016,
+ "step": 1690
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "grad_norm": 1.0653239488601685,
+ "learning_rate": 6.077037037037038e-07,
+ "logits/chosen": 1.340463399887085,
+ "logits/rejected": 1.3752565383911133,
+ "logps/chosen": -150.36248779296875,
+ "logps/rejected": -151.90725708007812,
+ "loss": 0.6768710613250732,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.08575484901666641,
+ "rewards/margins": 0.043545305728912354,
+ "rewards/rejected": 0.04220953956246376,
+ "step": 1700
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "eval_logits/chosen": 1.4521433115005493,
+ "eval_logits/rejected": 1.4804331064224243,
+ "eval_logps/chosen": -152.77755737304688,
+ "eval_logps/rejected": -147.48025512695312,
+ "eval_loss": 0.6873273849487305,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": 0.07712872326374054,
+ "eval_rewards/margins": 0.022939365357160568,
+ "eval_rewards/rejected": 0.05418936163187027,
+ "eval_runtime": 90.962,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 1700
+ },
+ {
+ "epoch": 1.3679999999999999,
+ "grad_norm": 1.3996928930282593,
+ "learning_rate": 6.047407407407407e-07,
+ "logits/chosen": 1.41148042678833,
+ "logits/rejected": 1.4520132541656494,
+ "logps/chosen": -143.23252868652344,
+ "logps/rejected": -180.05331420898438,
+ "loss": 0.655043363571167,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10279543697834015,
+ "rewards/margins": 0.0895618349313736,
+ "rewards/rejected": 0.013233604840934277,
+ "step": 1710
+ },
+ {
+ "epoch": 1.376,
+ "grad_norm": 1.4253696203231812,
+ "learning_rate": 6.017777777777777e-07,
+ "logits/chosen": 1.3833668231964111,
+ "logits/rejected": 1.409301519393921,
+ "logps/chosen": -149.3744659423828,
+ "logps/rejected": -151.1966552734375,
+ "loss": 0.6865286827087402,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.06293162703514099,
+ "rewards/margins": 0.02295227348804474,
+ "rewards/rejected": 0.03997935354709625,
+ "step": 1720
+ },
+ {
+ "epoch": 1.384,
+ "grad_norm": 1.185731053352356,
+ "learning_rate": 5.988148148148148e-07,
+ "logits/chosen": 1.528174877166748,
+ "logits/rejected": 1.6338379383087158,
+ "logps/chosen": -160.7788543701172,
+ "logps/rejected": -159.85354614257812,
+ "loss": 0.6458437442779541,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.12789444625377655,
+ "rewards/margins": 0.1114773377776146,
+ "rewards/rejected": 0.016417117789387703,
+ "step": 1730
+ },
+ {
+ "epoch": 1.392,
+ "grad_norm": 1.2963365316390991,
+ "learning_rate": 5.958518518518518e-07,
+ "logits/chosen": 1.4939125776290894,
+ "logits/rejected": 1.4197361469268799,
+ "logps/chosen": -162.34744262695312,
+ "logps/rejected": -177.93087768554688,
+ "loss": 0.6620248317718506,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.11072772741317749,
+ "rewards/margins": 0.07320726662874222,
+ "rewards/rejected": 0.03752044588327408,
+ "step": 1740
+ },
+ {
+ "epoch": 1.4,
+ "grad_norm": 1.3535394668579102,
+ "learning_rate": 5.928888888888889e-07,
+ "logits/chosen": 1.3506572246551514,
+ "logits/rejected": 1.424142837524414,
+ "logps/chosen": -139.69985961914062,
+ "logps/rejected": -144.9062042236328,
+ "loss": 0.6815268516540527,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.07917657494544983,
+ "rewards/margins": 0.03688482195138931,
+ "rewards/rejected": 0.04229176044464111,
+ "step": 1750
+ },
+ {
+ "epoch": 1.408,
+ "grad_norm": 1.471380352973938,
+ "learning_rate": 5.899259259259259e-07,
+ "logits/chosen": 1.5949152708053589,
+ "logits/rejected": 1.710013747215271,
+ "logps/chosen": -170.3776092529297,
+ "logps/rejected": -177.1616973876953,
+ "loss": 0.6533854961395263,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.11633266508579254,
+ "rewards/margins": 0.09301649034023285,
+ "rewards/rejected": 0.023316185921430588,
+ "step": 1760
+ },
+ {
+ "epoch": 1.416,
+ "grad_norm": 1.3164308071136475,
+ "learning_rate": 5.869629629629629e-07,
+ "logits/chosen": 1.5159170627593994,
+ "logits/rejected": 1.4434025287628174,
+ "logps/chosen": -161.05157470703125,
+ "logps/rejected": -140.06948852539062,
+ "loss": 0.660222864151001,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.10675680637359619,
+ "rewards/margins": 0.07750208675861359,
+ "rewards/rejected": 0.029254715889692307,
+ "step": 1770
+ },
+ {
+ "epoch": 1.424,
+ "grad_norm": 1.102005124092102,
+ "learning_rate": 5.839999999999999e-07,
+ "logits/chosen": 1.349440336227417,
+ "logits/rejected": 1.1565381288528442,
+ "logps/chosen": -150.61328125,
+ "logps/rejected": -151.36001586914062,
+ "loss": 0.660042142868042,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.09433183073997498,
+ "rewards/margins": 0.07778269052505493,
+ "rewards/rejected": 0.016549136489629745,
+ "step": 1780
+ },
+ {
+ "epoch": 1.432,
+ "grad_norm": 1.2305164337158203,
+ "learning_rate": 5.81037037037037e-07,
+ "logits/chosen": 1.4631009101867676,
+ "logits/rejected": 1.497757911682129,
+ "logps/chosen": -156.7335968017578,
+ "logps/rejected": -150.91065979003906,
+ "loss": 0.6564284324645996,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.13712921738624573,
+ "rewards/margins": 0.09247532486915588,
+ "rewards/rejected": 0.04465390369296074,
+ "step": 1790
+ },
+ {
+ "epoch": 1.44,
+ "grad_norm": 1.3459731340408325,
+ "learning_rate": 5.780740740740741e-07,
+ "logits/chosen": 1.5344784259796143,
+ "logits/rejected": 1.5709785223007202,
+ "logps/chosen": -166.15830993652344,
+ "logps/rejected": -158.678955078125,
+ "loss": 0.6511439800262451,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1227751225233078,
+ "rewards/margins": 0.0968184620141983,
+ "rewards/rejected": 0.025956671684980392,
+ "step": 1800
+ },
+ {
+ "epoch": 1.44,
+ "eval_logits/chosen": 1.4476159811019897,
+ "eval_logits/rejected": 1.4754574298858643,
+ "eval_logps/chosen": -152.70677185058594,
+ "eval_logps/rejected": -147.41787719726562,
+ "eval_loss": 0.6876539587974548,
+ "eval_rewards/accuracies": 0.5199999809265137,
+ "eval_rewards/chosen": 0.08420863002538681,
+ "eval_rewards/margins": 0.023782728239893913,
+ "eval_rewards/rejected": 0.06042589247226715,
+ "eval_runtime": 91.0223,
+ "eval_samples_per_second": 5.493,
+ "eval_steps_per_second": 2.747,
+ "step": 1800
+ },
+ {
+ "epoch": 1.448,
+ "grad_norm": 1.3158259391784668,
+ "learning_rate": 5.751111111111111e-07,
+ "logits/chosen": 1.425363302230835,
+ "logits/rejected": 1.350799798965454,
+ "logps/chosen": -151.4562530517578,
+ "logps/rejected": -134.9906005859375,
+ "loss": 0.6759338855743409,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1467444747686386,
+ "rewards/margins": 0.047227732837200165,
+ "rewards/rejected": 0.09951673448085785,
+ "step": 1810
+ },
+ {
+ "epoch": 1.456,
+ "grad_norm": 1.179150938987732,
+ "learning_rate": 5.721481481481481e-07,
+ "logits/chosen": 1.6323440074920654,
+ "logits/rejected": 1.4893152713775635,
+ "logps/chosen": -149.57675170898438,
+ "logps/rejected": -146.96237182617188,
+ "loss": 0.657456636428833,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1445317566394806,
+ "rewards/margins": 0.0896807312965393,
+ "rewards/rejected": 0.05485100671648979,
+ "step": 1820
+ },
+ {
+ "epoch": 1.464,
+ "grad_norm": 1.0987558364868164,
+ "learning_rate": 5.691851851851853e-07,
+ "logits/chosen": 1.3851706981658936,
+ "logits/rejected": 1.381049633026123,
+ "logps/chosen": -163.92648315429688,
+ "logps/rejected": -142.801025390625,
+ "loss": 0.6631375312805176,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.14093783497810364,
+ "rewards/margins": 0.07281022518873215,
+ "rewards/rejected": 0.06812760978937149,
+ "step": 1830
+ },
+ {
+ "epoch": 1.472,
+ "grad_norm": 1.1079164743423462,
+ "learning_rate": 5.662222222222222e-07,
+ "logits/chosen": 1.4228973388671875,
+ "logits/rejected": 1.3453642129898071,
+ "logps/chosen": -148.5561981201172,
+ "logps/rejected": -141.28009033203125,
+ "loss": 0.6612800121307373,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.10088491439819336,
+ "rewards/margins": 0.08228114247322083,
+ "rewards/rejected": 0.018603766337037086,
+ "step": 1840
+ },
+ {
+ "epoch": 1.48,
+ "grad_norm": 1.0415616035461426,
+ "learning_rate": 5.632592592592592e-07,
+ "logits/chosen": 1.4923865795135498,
+ "logits/rejected": 1.536971092224121,
+ "logps/chosen": -142.55166625976562,
+ "logps/rejected": -149.22906494140625,
+ "loss": 0.6690784931182862,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.10785870254039764,
+ "rewards/margins": 0.05975862219929695,
+ "rewards/rejected": 0.04810008406639099,
+ "step": 1850
+ },
+ {
+ "epoch": 1.488,
+ "grad_norm": 1.4004729986190796,
+ "learning_rate": 5.602962962962962e-07,
+ "logits/chosen": 1.4783391952514648,
+ "logits/rejected": 1.5112550258636475,
+ "logps/chosen": -153.54034423828125,
+ "logps/rejected": -118.77142333984375,
+ "loss": 0.6666333675384521,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.12789246439933777,
+ "rewards/margins": 0.06238863617181778,
+ "rewards/rejected": 0.06550382077693939,
+ "step": 1860
+ },
+ {
+ "epoch": 1.496,
+ "grad_norm": 1.1908437013626099,
+ "learning_rate": 5.573333333333333e-07,
+ "logits/chosen": 1.5748203992843628,
+ "logits/rejected": 1.4827988147735596,
+ "logps/chosen": -162.0931854248047,
+ "logps/rejected": -156.00779724121094,
+ "loss": 0.6626818656921387,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.11168892681598663,
+ "rewards/margins": 0.07420919090509415,
+ "rewards/rejected": 0.03747973591089249,
+ "step": 1870
+ },
+ {
+ "epoch": 1.504,
+ "grad_norm": 1.288218379020691,
+ "learning_rate": 5.543703703703704e-07,
+ "logits/chosen": 1.5260225534439087,
+ "logits/rejected": 1.5843703746795654,
+ "logps/chosen": -148.2488555908203,
+ "logps/rejected": -161.31570434570312,
+ "loss": 0.6677760124206543,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.12626001238822937,
+ "rewards/margins": 0.06881437450647354,
+ "rewards/rejected": 0.05744565278291702,
+ "step": 1880
+ },
+ {
+ "epoch": 1.512,
+ "grad_norm": 1.2899844646453857,
+ "learning_rate": 5.514074074074074e-07,
+ "logits/chosen": 1.417141318321228,
+ "logits/rejected": 1.389639973640442,
+ "logps/chosen": -160.5532989501953,
+ "logps/rejected": -158.82923889160156,
+ "loss": 0.6510573863983155,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.15593740344047546,
+ "rewards/margins": 0.09999983012676239,
+ "rewards/rejected": 0.05593757703900337,
+ "step": 1890
+ },
+ {
+ "epoch": 1.52,
+ "grad_norm": 1.3312058448791504,
+ "learning_rate": 5.484444444444444e-07,
+ "logits/chosen": 1.4996871948242188,
+ "logits/rejected": 1.6273930072784424,
+ "logps/chosen": -154.84347534179688,
+ "logps/rejected": -130.4560089111328,
+ "loss": 0.6844823360443115,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.13181835412979126,
+ "rewards/margins": 0.029756689444184303,
+ "rewards/rejected": 0.10206165164709091,
+ "step": 1900
+ },
+ {
+ "epoch": 1.52,
+ "eval_logits/chosen": 1.4582936763763428,
+ "eval_logits/rejected": 1.4853557348251343,
+ "eval_logps/chosen": -152.36231994628906,
+ "eval_logps/rejected": -147.1087188720703,
+ "eval_loss": 0.6870284676551819,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": 0.1186550110578537,
+ "eval_rewards/margins": 0.027311701327562332,
+ "eval_rewards/rejected": 0.09134332090616226,
+ "eval_runtime": 90.913,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 1900
+ },
+ {
+ "epoch": 1.528,
+ "grad_norm": 1.2257205247879028,
+ "learning_rate": 5.454814814814814e-07,
+ "logits/chosen": 1.4909894466400146,
+ "logits/rejected": 1.4579002857208252,
+ "logps/chosen": -129.2428741455078,
+ "logps/rejected": -149.32220458984375,
+ "loss": 0.6677265644073487,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.13237449526786804,
+ "rewards/margins": 0.059117116034030914,
+ "rewards/rejected": 0.07325738668441772,
+ "step": 1910
+ },
+ {
+ "epoch": 1.536,
+ "grad_norm": 1.3048031330108643,
+ "learning_rate": 5.425185185185185e-07,
+ "logits/chosen": 1.5237693786621094,
+ "logits/rejected": 1.4846075773239136,
+ "logps/chosen": -159.10055541992188,
+ "logps/rejected": -155.69888305664062,
+ "loss": 0.6709336280822754,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.1088198646903038,
+ "rewards/margins": 0.05951980873942375,
+ "rewards/rejected": 0.04930006340146065,
+ "step": 1920
+ },
+ {
+ "epoch": 1.544,
+ "grad_norm": 1.3463319540023804,
+ "learning_rate": 5.395555555555556e-07,
+ "logits/chosen": 1.6476523876190186,
+ "logits/rejected": 1.6236053705215454,
+ "logps/chosen": -171.18319702148438,
+ "logps/rejected": -172.63796997070312,
+ "loss": 0.6670461177825928,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.13160961866378784,
+ "rewards/margins": 0.06587684154510498,
+ "rewards/rejected": 0.06573276221752167,
+ "step": 1930
+ },
+ {
+ "epoch": 1.552,
+ "grad_norm": 1.2326266765594482,
+ "learning_rate": 5.365925925925926e-07,
+ "logits/chosen": 1.4379268884658813,
+ "logits/rejected": 1.5206880569458008,
+ "logps/chosen": -157.2953643798828,
+ "logps/rejected": -153.21218872070312,
+ "loss": 0.6558516502380372,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.17594724893569946,
+ "rewards/margins": 0.09524654597043991,
+ "rewards/rejected": 0.08070070296525955,
+ "step": 1940
+ },
+ {
+ "epoch": 1.56,
+ "grad_norm": 1.1362297534942627,
+ "learning_rate": 5.336296296296296e-07,
+ "logits/chosen": 1.524285078048706,
+ "logits/rejected": 1.3964688777923584,
+ "logps/chosen": -161.221435546875,
+ "logps/rejected": -154.8270263671875,
+ "loss": 0.6746312141418457,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.12519574165344238,
+ "rewards/margins": 0.05550859495997429,
+ "rewards/rejected": 0.06968715041875839,
+ "step": 1950
+ },
+ {
+ "epoch": 1.568,
+ "grad_norm": 1.5411877632141113,
+ "learning_rate": 5.306666666666665e-07,
+ "logits/chosen": 1.2426170110702515,
+ "logits/rejected": 1.3256272077560425,
+ "logps/chosen": -137.97537231445312,
+ "logps/rejected": -159.09580993652344,
+ "loss": 0.6492344856262207,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.16253654658794403,
+ "rewards/margins": 0.12099182605743408,
+ "rewards/rejected": 0.04154472425580025,
+ "step": 1960
+ },
+ {
+ "epoch": 1.576,
+ "grad_norm": 1.2111555337905884,
+ "learning_rate": 5.277037037037037e-07,
+ "logits/chosen": 1.3337433338165283,
+ "logits/rejected": 1.479209065437317,
+ "logps/chosen": -149.0251007080078,
+ "logps/rejected": -140.0357208251953,
+ "loss": 0.6559633731842041,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.13673515617847443,
+ "rewards/margins": 0.0912124291062355,
+ "rewards/rejected": 0.04552273079752922,
+ "step": 1970
+ },
+ {
+ "epoch": 1.584,
+ "grad_norm": 1.1228832006454468,
+ "learning_rate": 5.247407407407407e-07,
+ "logits/chosen": 1.4376949071884155,
+ "logits/rejected": 1.5033822059631348,
+ "logps/chosen": -144.91751098632812,
+ "logps/rejected": -147.37057495117188,
+ "loss": 0.6593158721923829,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.1573731154203415,
+ "rewards/margins": 0.08546318858861923,
+ "rewards/rejected": 0.07190994173288345,
+ "step": 1980
+ },
+ {
+ "epoch": 1.592,
+ "grad_norm": 0.9601779580116272,
+ "learning_rate": 5.217777777777777e-07,
+ "logits/chosen": 1.2674219608306885,
+ "logits/rejected": 1.3877508640289307,
+ "logps/chosen": -116.023193359375,
+ "logps/rejected": -122.81617736816406,
+ "loss": 0.6842892646789551,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.13847123086452484,
+ "rewards/margins": 0.026916390284895897,
+ "rewards/rejected": 0.11155486106872559,
+ "step": 1990
+ },
+ {
+ "epoch": 1.6,
+ "grad_norm": 1.2849960327148438,
+ "learning_rate": 5.188148148148148e-07,
+ "logits/chosen": 1.6982529163360596,
+ "logits/rejected": 1.6600332260131836,
+ "logps/chosen": -143.352294921875,
+ "logps/rejected": -150.0862274169922,
+ "loss": 0.6666803359985352,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.13707581162452698,
+ "rewards/margins": 0.06178014352917671,
+ "rewards/rejected": 0.07529564201831818,
+ "step": 2000
+ },
+ {
+ "epoch": 1.6,
+ "eval_logits/chosen": 1.4610096216201782,
+ "eval_logits/rejected": 1.4881619215011597,
+ "eval_logps/chosen": -152.36614990234375,
+ "eval_logps/rejected": -147.1319580078125,
+ "eval_loss": 0.6861760020256042,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": 0.11826877295970917,
+ "eval_rewards/margins": 0.02924923785030842,
+ "eval_rewards/rejected": 0.0890195295214653,
+ "eval_runtime": 91.0197,
+ "eval_samples_per_second": 5.493,
+ "eval_steps_per_second": 2.747,
+ "step": 2000
+ },
+ {
+ "epoch": 1.608,
+ "grad_norm": 1.731399655342102,
+ "learning_rate": 5.158518518518519e-07,
+ "logits/chosen": 1.4297409057617188,
+ "logits/rejected": 1.5017236471176147,
+ "logps/chosen": -164.93292236328125,
+ "logps/rejected": -164.80677795410156,
+ "loss": 0.6650710582733155,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.20246899127960205,
+ "rewards/margins": 0.0675087422132492,
+ "rewards/rejected": 0.13496026396751404,
+ "step": 2010
+ },
+ {
+ "epoch": 1.616,
+ "grad_norm": 1.2728053331375122,
+ "learning_rate": 5.128888888888888e-07,
+ "logits/chosen": 1.305499792098999,
+ "logits/rejected": 1.436187744140625,
+ "logps/chosen": -143.90879821777344,
+ "logps/rejected": -158.8233184814453,
+ "loss": 0.6744787693023682,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.17535606026649475,
+ "rewards/margins": 0.04918169230222702,
+ "rewards/rejected": 0.12617434561252594,
+ "step": 2020
+ },
+ {
+ "epoch": 1.624,
+ "grad_norm": 1.4291980266571045,
+ "learning_rate": 5.099259259259259e-07,
+ "logits/chosen": 1.4624104499816895,
+ "logits/rejected": 1.4936870336532593,
+ "logps/chosen": -151.03884887695312,
+ "logps/rejected": -127.40641784667969,
+ "loss": 0.6721909046173096,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.1530950516462326,
+ "rewards/margins": 0.0540502592921257,
+ "rewards/rejected": 0.0990448072552681,
+ "step": 2030
+ },
+ {
+ "epoch": 1.6320000000000001,
+ "grad_norm": 1.5909459590911865,
+ "learning_rate": 5.069629629629629e-07,
+ "logits/chosen": 1.468285322189331,
+ "logits/rejected": 1.3470795154571533,
+ "logps/chosen": -158.12046813964844,
+ "logps/rejected": -160.971923828125,
+ "loss": 0.6752189636230469,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.12029504776000977,
+ "rewards/margins": 0.05528820678591728,
+ "rewards/rejected": 0.06500682979822159,
+ "step": 2040
+ },
+ {
+ "epoch": 1.6400000000000001,
+ "grad_norm": 1.2923390865325928,
+ "learning_rate": 5.04e-07,
+ "logits/chosen": 1.530002236366272,
+ "logits/rejected": 1.5821081399917603,
+ "logps/chosen": -157.1008758544922,
+ "logps/rejected": -158.16775512695312,
+ "loss": 0.6889143943786621,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.14681917428970337,
+ "rewards/margins": 0.01898271217942238,
+ "rewards/rejected": 0.1278364509344101,
+ "step": 2050
+ },
+ {
+ "epoch": 1.6480000000000001,
+ "grad_norm": 1.2025870084762573,
+ "learning_rate": 5.010370370370371e-07,
+ "logits/chosen": 1.4749194383621216,
+ "logits/rejected": 1.3301353454589844,
+ "logps/chosen": -119.29649353027344,
+ "logps/rejected": -145.0949249267578,
+ "loss": 0.6332630634307861,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.1944357454776764,
+ "rewards/margins": 0.14449670910835266,
+ "rewards/rejected": 0.049939028918743134,
+ "step": 2060
+ },
+ {
+ "epoch": 1.6560000000000001,
+ "grad_norm": 1.3136606216430664,
+ "learning_rate": 4.98074074074074e-07,
+ "logits/chosen": 1.5436251163482666,
+ "logits/rejected": 1.4338414669036865,
+ "logps/chosen": -157.49942016601562,
+ "logps/rejected": -140.4504852294922,
+ "loss": 0.6923964977264404,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.13830263912677765,
+ "rewards/margins": 0.010729723609983921,
+ "rewards/rejected": 0.1275729238986969,
+ "step": 2070
+ },
+ {
+ "epoch": 1.6640000000000001,
+ "grad_norm": 1.2378884553909302,
+ "learning_rate": 4.951111111111111e-07,
+ "logits/chosen": 1.5672242641448975,
+ "logits/rejected": 1.4682090282440186,
+ "logps/chosen": -158.99301147460938,
+ "logps/rejected": -149.21221923828125,
+ "loss": 0.6473859786987305,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.1993202269077301,
+ "rewards/margins": 0.10525582730770111,
+ "rewards/rejected": 0.0940643846988678,
+ "step": 2080
+ },
+ {
+ "epoch": 1.6720000000000002,
+ "grad_norm": 1.1976169347763062,
+ "learning_rate": 4.921481481481482e-07,
+ "logits/chosen": 1.4160411357879639,
+ "logits/rejected": 1.3142106533050537,
+ "logps/chosen": -153.40621948242188,
+ "logps/rejected": -154.57308959960938,
+ "loss": 0.6821109771728515,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.14698973298072815,
+ "rewards/margins": 0.034348614513874054,
+ "rewards/rejected": 0.1126411110162735,
+ "step": 2090
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "grad_norm": 1.3028976917266846,
+ "learning_rate": 4.891851851851852e-07,
+ "logits/chosen": 1.61053466796875,
+ "logits/rejected": 1.526831030845642,
+ "logps/chosen": -164.3324737548828,
+ "logps/rejected": -149.382080078125,
+ "loss": 0.6684560298919677,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.22455796599388123,
+ "rewards/margins": 0.06641162931919098,
+ "rewards/rejected": 0.15814633667469025,
+ "step": 2100
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "eval_logits/chosen": 1.4865608215332031,
+ "eval_logits/rejected": 1.5141921043395996,
+ "eval_logps/chosen": -151.979248046875,
+ "eval_logps/rejected": -146.76809692382812,
+ "eval_loss": 0.6849560141563416,
+ "eval_rewards/accuracies": 0.5379999876022339,
+ "eval_rewards/chosen": 0.15696033835411072,
+ "eval_rewards/margins": 0.0315566249191761,
+ "eval_rewards/rejected": 0.1254037320613861,
+ "eval_runtime": 90.9172,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 2100
+ },
+ {
+ "epoch": 1.688,
+ "grad_norm": 1.3767266273498535,
+ "learning_rate": 4.862222222222222e-07,
+ "logits/chosen": 1.458231806755066,
+ "logits/rejected": 1.3821945190429688,
+ "logps/chosen": -166.67698669433594,
+ "logps/rejected": -156.74313354492188,
+ "loss": 0.6658836364746094,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.17039546370506287,
+ "rewards/margins": 0.07431827485561371,
+ "rewards/rejected": 0.09607716649770737,
+ "step": 2110
+ },
+ {
+ "epoch": 1.696,
+ "grad_norm": 1.4396629333496094,
+ "learning_rate": 4.832592592592592e-07,
+ "logits/chosen": 1.5568702220916748,
+ "logits/rejected": 1.5459610223770142,
+ "logps/chosen": -133.0255889892578,
+ "logps/rejected": -138.2047882080078,
+ "loss": 0.6704818725585937,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.173842191696167,
+ "rewards/margins": 0.06081608682870865,
+ "rewards/rejected": 0.11302611976861954,
+ "step": 2120
+ },
+ {
+ "epoch": 1.704,
+ "grad_norm": 2.3225553035736084,
+ "learning_rate": 4.802962962962962e-07,
+ "logits/chosen": 1.5432292222976685,
+ "logits/rejected": 1.521958589553833,
+ "logps/chosen": -165.04318237304688,
+ "logps/rejected": -163.95855712890625,
+ "loss": 0.6567724704742431,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.20300237834453583,
+ "rewards/margins": 0.09381169825792313,
+ "rewards/rejected": 0.1091906875371933,
+ "step": 2130
+ },
+ {
+ "epoch": 1.712,
+ "grad_norm": 1.3185398578643799,
+ "learning_rate": 4.773333333333333e-07,
+ "logits/chosen": 1.2340335845947266,
+ "logits/rejected": 1.3610973358154297,
+ "logps/chosen": -139.11878967285156,
+ "logps/rejected": -146.57260131835938,
+ "loss": 0.671626091003418,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.17043188214302063,
+ "rewards/margins": 0.054147154092788696,
+ "rewards/rejected": 0.11628471314907074,
+ "step": 2140
+ },
+ {
+ "epoch": 1.72,
+ "grad_norm": 1.618586540222168,
+ "learning_rate": 4.7437037037037034e-07,
+ "logits/chosen": 1.5895780324935913,
+ "logits/rejected": 1.613523244857788,
+ "logps/chosen": -156.53970336914062,
+ "logps/rejected": -154.35116577148438,
+ "loss": 0.6798102855682373,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.20041461288928986,
+ "rewards/margins": 0.03897023946046829,
+ "rewards/rejected": 0.16144435107707977,
+ "step": 2150
+ },
+ {
+ "epoch": 1.728,
+ "grad_norm": 0.8500939607620239,
+ "learning_rate": 4.7140740740740736e-07,
+ "logits/chosen": 1.3736517429351807,
+ "logits/rejected": 1.361998438835144,
+ "logps/chosen": -137.701904296875,
+ "logps/rejected": -139.91567993164062,
+ "loss": 0.6568773269653321,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.16317322850227356,
+ "rewards/margins": 0.08119454979896545,
+ "rewards/rejected": 0.0819786787033081,
+ "step": 2160
+ },
+ {
+ "epoch": 1.736,
+ "grad_norm": 1.2291405200958252,
+ "learning_rate": 4.6844444444444444e-07,
+ "logits/chosen": 1.474157452583313,
+ "logits/rejected": 1.4488928318023682,
+ "logps/chosen": -152.36732482910156,
+ "logps/rejected": -149.70980834960938,
+ "loss": 0.6551193714141845,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.2117660939693451,
+ "rewards/margins": 0.0950237363576889,
+ "rewards/rejected": 0.11674235761165619,
+ "step": 2170
+ },
+ {
+ "epoch": 1.744,
+ "grad_norm": 1.2946466207504272,
+ "learning_rate": 4.6548148148148146e-07,
+ "logits/chosen": 1.257048487663269,
+ "logits/rejected": 1.3120336532592773,
+ "logps/chosen": -143.43148803710938,
+ "logps/rejected": -169.9266815185547,
+ "loss": 0.6712408542633057,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.1496409922838211,
+ "rewards/margins": 0.05779259279370308,
+ "rewards/rejected": 0.09184839576482773,
+ "step": 2180
+ },
+ {
+ "epoch": 1.752,
+ "grad_norm": 1.0697646141052246,
+ "learning_rate": 4.625185185185185e-07,
+ "logits/chosen": 1.5544202327728271,
+ "logits/rejected": 1.5186070203781128,
+ "logps/chosen": -170.5447235107422,
+ "logps/rejected": -165.55209350585938,
+ "loss": 0.6674113750457764,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.12943358719348907,
+ "rewards/margins": 0.07018069922924042,
+ "rewards/rejected": 0.05925289914011955,
+ "step": 2190
+ },
+ {
+ "epoch": 1.76,
+ "grad_norm": 1.228698968887329,
+ "learning_rate": 4.595555555555555e-07,
+ "logits/chosen": 1.46980881690979,
+ "logits/rejected": 1.2382839918136597,
+ "logps/chosen": -133.5864715576172,
+ "logps/rejected": -135.65968322753906,
+ "loss": 0.6756285667419434,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.18240585923194885,
+ "rewards/margins": 0.052959512919187546,
+ "rewards/rejected": 0.1294463574886322,
+ "step": 2200
+ },
+ {
+ "epoch": 1.76,
+ "eval_logits/chosen": 1.4826997518539429,
+ "eval_logits/rejected": 1.5107221603393555,
+ "eval_logps/chosen": -152.19204711914062,
+ "eval_logps/rejected": -146.98675537109375,
+ "eval_loss": 0.6843143105506897,
+ "eval_rewards/accuracies": 0.5379999876022339,
+ "eval_rewards/chosen": 0.13567964732646942,
+ "eval_rewards/margins": 0.03213911131024361,
+ "eval_rewards/rejected": 0.10354051738977432,
+ "eval_runtime": 91.0116,
+ "eval_samples_per_second": 5.494,
+ "eval_steps_per_second": 2.747,
+ "step": 2200
+ },
+ {
+ "epoch": 1.768,
+ "grad_norm": 1.4145433902740479,
+ "learning_rate": 4.565925925925926e-07,
+ "logits/chosen": 1.49526047706604,
+ "logits/rejected": 1.519241213798523,
+ "logps/chosen": -152.7605438232422,
+ "logps/rejected": -140.98629760742188,
+ "loss": 0.6503778457641601,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.16126050055027008,
+ "rewards/margins": 0.0973939448595047,
+ "rewards/rejected": 0.06386655569076538,
+ "step": 2210
+ },
+ {
+ "epoch": 1.776,
+ "grad_norm": 1.4429577589035034,
+ "learning_rate": 4.536296296296296e-07,
+ "logits/chosen": 1.4706772565841675,
+ "logits/rejected": 1.3714580535888672,
+ "logps/chosen": -135.95150756835938,
+ "logps/rejected": -165.23587036132812,
+ "loss": 0.6639570713043212,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.18487155437469482,
+ "rewards/margins": 0.07484755665063858,
+ "rewards/rejected": 0.11002399027347565,
+ "step": 2220
+ },
+ {
+ "epoch": 1.784,
+ "grad_norm": 1.0622409582138062,
+ "learning_rate": 4.506666666666666e-07,
+ "logits/chosen": 1.2506721019744873,
+ "logits/rejected": 1.4802286624908447,
+ "logps/chosen": -129.645751953125,
+ "logps/rejected": -131.7567596435547,
+ "loss": 0.6788074493408203,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.12520259618759155,
+ "rewards/margins": 0.03735826909542084,
+ "rewards/rejected": 0.08784433454275131,
+ "step": 2230
+ },
+ {
+ "epoch": 1.792,
+ "grad_norm": 1.0001399517059326,
+ "learning_rate": 4.477037037037037e-07,
+ "logits/chosen": 1.4206364154815674,
+ "logits/rejected": 1.5338770151138306,
+ "logps/chosen": -143.95074462890625,
+ "logps/rejected": -160.1469268798828,
+ "loss": 0.6695474147796631,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.13798081874847412,
+ "rewards/margins": 0.05759159475564957,
+ "rewards/rejected": 0.08038921654224396,
+ "step": 2240
+ },
+ {
+ "epoch": 1.8,
+ "grad_norm": 1.3531230688095093,
+ "learning_rate": 4.4474074074074067e-07,
+ "logits/chosen": 1.5709412097930908,
+ "logits/rejected": 1.5301862955093384,
+ "logps/chosen": -161.30087280273438,
+ "logps/rejected": -150.31236267089844,
+ "loss": 0.6771373271942138,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.1488293707370758,
+ "rewards/margins": 0.04127617925405502,
+ "rewards/rejected": 0.10755319893360138,
+ "step": 2250
+ },
+ {
+ "epoch": 1.808,
+ "grad_norm": 1.37367582321167,
+ "learning_rate": 4.4177777777777775e-07,
+ "logits/chosen": 1.36393141746521,
+ "logits/rejected": 1.484881043434143,
+ "logps/chosen": -125.18086242675781,
+ "logps/rejected": -148.78546142578125,
+ "loss": 0.6596842288970948,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.15397249162197113,
+ "rewards/margins": 0.07893262803554535,
+ "rewards/rejected": 0.07503987848758698,
+ "step": 2260
+ },
+ {
+ "epoch": 1.8159999999999998,
+ "grad_norm": 1.4511909484863281,
+ "learning_rate": 4.388148148148148e-07,
+ "logits/chosen": 1.3140859603881836,
+ "logits/rejected": 1.3879334926605225,
+ "logps/chosen": -131.48226928710938,
+ "logps/rejected": -123.94035339355469,
+ "loss": 0.6698450088500977,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.12767434120178223,
+ "rewards/margins": 0.06716184318065643,
+ "rewards/rejected": 0.060512494295835495,
+ "step": 2270
+ },
+ {
+ "epoch": 1.8239999999999998,
+ "grad_norm": 1.3375639915466309,
+ "learning_rate": 4.3585185185185184e-07,
+ "logits/chosen": 1.5821285247802734,
+ "logits/rejected": 1.43180251121521,
+ "logps/chosen": -167.2334747314453,
+ "logps/rejected": -145.28871154785156,
+ "loss": 0.658758020401001,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.12011411041021347,
+ "rewards/margins": 0.08019578456878662,
+ "rewards/rejected": 0.039918314665555954,
+ "step": 2280
+ },
+ {
+ "epoch": 1.8319999999999999,
+ "grad_norm": 1.2121729850769043,
+ "learning_rate": 4.3288888888888887e-07,
+ "logits/chosen": 1.3520320653915405,
+ "logits/rejected": 1.18943452835083,
+ "logps/chosen": -147.15621948242188,
+ "logps/rejected": -132.015625,
+ "loss": 0.6655458450317383,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08098682016134262,
+ "rewards/margins": 0.07385818660259247,
+ "rewards/rejected": 0.007128633558750153,
+ "step": 2290
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "grad_norm": 1.1618680953979492,
+ "learning_rate": 4.2992592592592594e-07,
+ "logits/chosen": 1.4779785871505737,
+ "logits/rejected": 1.448462724685669,
+ "logps/chosen": -157.75070190429688,
+ "logps/rejected": -141.3148956298828,
+ "loss": 0.6813467979431153,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.12400279194116592,
+ "rewards/margins": 0.034347161650657654,
+ "rewards/rejected": 0.08965563774108887,
+ "step": 2300
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "eval_logits/chosen": 1.4821760654449463,
+ "eval_logits/rejected": 1.5101500749588013,
+ "eval_logps/chosen": -152.2089080810547,
+ "eval_logps/rejected": -147.0087432861328,
+ "eval_loss": 0.6842693090438843,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.13399286568164825,
+ "eval_rewards/margins": 0.032653141766786575,
+ "eval_rewards/rejected": 0.10133973509073257,
+ "eval_runtime": 90.8665,
+ "eval_samples_per_second": 5.503,
+ "eval_steps_per_second": 2.751,
+ "step": 2300
+ },
+ {
+ "epoch": 1.8479999999999999,
+ "grad_norm": 1.1095342636108398,
+ "learning_rate": 4.2696296296296296e-07,
+ "logits/chosen": 1.6475988626480103,
+ "logits/rejected": 1.6654495000839233,
+ "logps/chosen": -165.965087890625,
+ "logps/rejected": -153.98214721679688,
+ "loss": 0.6605272769927979,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.2209581881761551,
+ "rewards/margins": 0.07579347491264343,
+ "rewards/rejected": 0.14516471326351166,
+ "step": 2310
+ },
+ {
+ "epoch": 1.8559999999999999,
+ "grad_norm": 1.1993674039840698,
+ "learning_rate": 4.24e-07,
+ "logits/chosen": 1.5783694982528687,
+ "logits/rejected": 1.406640648841858,
+ "logps/chosen": -139.3554229736328,
+ "logps/rejected": -126.39006042480469,
+ "loss": 0.6817263126373291,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1467253863811493,
+ "rewards/margins": 0.03637007623910904,
+ "rewards/rejected": 0.11035531759262085,
+ "step": 2320
+ },
+ {
+ "epoch": 1.8639999999999999,
+ "grad_norm": 1.4875127077102661,
+ "learning_rate": 4.21037037037037e-07,
+ "logits/chosen": 1.4171388149261475,
+ "logits/rejected": 1.4270119667053223,
+ "logps/chosen": -145.1273651123047,
+ "logps/rejected": -157.85816955566406,
+ "loss": 0.6678358554840088,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.14816439151763916,
+ "rewards/margins": 0.06963086873292923,
+ "rewards/rejected": 0.07853354513645172,
+ "step": 2330
+ },
+ {
+ "epoch": 1.8719999999999999,
+ "grad_norm": 1.2899141311645508,
+ "learning_rate": 4.180740740740741e-07,
+ "logits/chosen": 1.3682881593704224,
+ "logits/rejected": 1.3383492231369019,
+ "logps/chosen": -167.96450805664062,
+ "logps/rejected": -158.2183837890625,
+ "loss": 0.6754446029663086,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.1453542709350586,
+ "rewards/margins": 0.047066912055015564,
+ "rewards/rejected": 0.09828735888004303,
+ "step": 2340
+ },
+ {
+ "epoch": 1.88,
+ "grad_norm": 1.4864925146102905,
+ "learning_rate": 4.151111111111111e-07,
+ "logits/chosen": 1.5181632041931152,
+ "logits/rejected": 1.6898539066314697,
+ "logps/chosen": -146.9915771484375,
+ "logps/rejected": -156.66282653808594,
+ "loss": 0.6776726245880127,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.1270596832036972,
+ "rewards/margins": 0.04481229558587074,
+ "rewards/rejected": 0.08224737644195557,
+ "step": 2350
+ },
+ {
+ "epoch": 1.888,
+ "grad_norm": 1.3912899494171143,
+ "learning_rate": 4.1214814814814813e-07,
+ "logits/chosen": 1.279984474182129,
+ "logits/rejected": 1.258424997329712,
+ "logps/chosen": -162.26925659179688,
+ "logps/rejected": -153.519287109375,
+ "loss": 0.680011796951294,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1301378756761551,
+ "rewards/margins": 0.03849628567695618,
+ "rewards/rejected": 0.09164158999919891,
+ "step": 2360
+ },
+ {
+ "epoch": 1.896,
+ "grad_norm": 1.2498691082000732,
+ "learning_rate": 4.091851851851852e-07,
+ "logits/chosen": 1.5644304752349854,
+ "logits/rejected": 1.5302040576934814,
+ "logps/chosen": -181.20506286621094,
+ "logps/rejected": -164.98670959472656,
+ "loss": 0.6632030963897705,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.16487805545330048,
+ "rewards/margins": 0.08321762084960938,
+ "rewards/rejected": 0.0816604346036911,
+ "step": 2370
+ },
+ {
+ "epoch": 1.904,
+ "grad_norm": 1.3421261310577393,
+ "learning_rate": 4.0622222222222217e-07,
+ "logits/chosen": 1.5388444662094116,
+ "logits/rejected": 1.5939340591430664,
+ "logps/chosen": -151.34434509277344,
+ "logps/rejected": -149.8802490234375,
+ "loss": 0.691883659362793,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.11643080413341522,
+ "rewards/margins": 0.017300304025411606,
+ "rewards/rejected": 0.09913049638271332,
+ "step": 2380
+ },
+ {
+ "epoch": 1.912,
+ "grad_norm": 1.4378565549850464,
+ "learning_rate": 4.0325925925925925e-07,
+ "logits/chosen": 1.4750030040740967,
+ "logits/rejected": 1.385709524154663,
+ "logps/chosen": -159.98593139648438,
+ "logps/rejected": -144.0281219482422,
+ "loss": 0.6825335502624512,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.1235724687576294,
+ "rewards/margins": 0.03918968886137009,
+ "rewards/rejected": 0.0843827873468399,
+ "step": 2390
+ },
+ {
+ "epoch": 1.92,
+ "grad_norm": 1.1941649913787842,
+ "learning_rate": 4.002962962962963e-07,
+ "logits/chosen": 1.3726251125335693,
+ "logits/rejected": 1.5375208854675293,
+ "logps/chosen": -149.9639129638672,
+ "logps/rejected": -142.49295043945312,
+ "loss": 0.6461881160736084,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.12773704528808594,
+ "rewards/margins": 0.1104915589094162,
+ "rewards/rejected": 0.017245473340153694,
+ "step": 2400
+ },
+ {
+ "epoch": 1.92,
+ "eval_logits/chosen": 1.4780151844024658,
+ "eval_logits/rejected": 1.506591558456421,
+ "eval_logps/chosen": -152.50015258789062,
+ "eval_logps/rejected": -147.30506896972656,
+ "eval_loss": 0.6834847331047058,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": 0.10486816614866257,
+ "eval_rewards/margins": 0.03315889462828636,
+ "eval_rewards/rejected": 0.0717092752456665,
+ "eval_runtime": 90.9835,
+ "eval_samples_per_second": 5.496,
+ "eval_steps_per_second": 2.748,
+ "step": 2400
+ },
+ {
+ "epoch": 1.928,
+ "grad_norm": 1.1473901271820068,
+ "learning_rate": 3.973333333333333e-07,
+ "logits/chosen": 1.3499658107757568,
+ "logits/rejected": 1.479353666305542,
+ "logps/chosen": -153.15272521972656,
+ "logps/rejected": -170.32418823242188,
+ "loss": 0.6756062507629395,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.09217077493667603,
+ "rewards/margins": 0.05041539669036865,
+ "rewards/rejected": 0.041755370795726776,
+ "step": 2410
+ },
+ {
+ "epoch": 1.936,
+ "grad_norm": 1.4407395124435425,
+ "learning_rate": 3.9437037037037037e-07,
+ "logits/chosen": 1.5876386165618896,
+ "logits/rejected": 1.5701191425323486,
+ "logps/chosen": -144.0182342529297,
+ "logps/rejected": -130.75460815429688,
+ "loss": 0.6733745574951172,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.1180434376001358,
+ "rewards/margins": 0.050890080630779266,
+ "rewards/rejected": 0.06715335696935654,
+ "step": 2420
+ },
+ {
+ "epoch": 1.944,
+ "grad_norm": 1.46868097782135,
+ "learning_rate": 3.914074074074074e-07,
+ "logits/chosen": 1.4809507131576538,
+ "logits/rejected": 1.4793384075164795,
+ "logps/chosen": -144.4136199951172,
+ "logps/rejected": -140.72645568847656,
+ "loss": 0.65852952003479,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.13238464295864105,
+ "rewards/margins": 0.08860992640256882,
+ "rewards/rejected": 0.043774720281362534,
+ "step": 2430
+ },
+ {
+ "epoch": 1.952,
+ "grad_norm": 1.2951419353485107,
+ "learning_rate": 3.884444444444444e-07,
+ "logits/chosen": 1.4519541263580322,
+ "logits/rejected": 1.6060655117034912,
+ "logps/chosen": -150.94955444335938,
+ "logps/rejected": -146.84732055664062,
+ "loss": 0.650700569152832,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.14693069458007812,
+ "rewards/margins": 0.09734389185905457,
+ "rewards/rejected": 0.04958680272102356,
+ "step": 2440
+ },
+ {
+ "epoch": 1.96,
+ "grad_norm": 1.2181923389434814,
+ "learning_rate": 3.854814814814815e-07,
+ "logits/chosen": 1.5463917255401611,
+ "logits/rejected": 1.4773080348968506,
+ "logps/chosen": -150.07005310058594,
+ "logps/rejected": -163.2088165283203,
+ "loss": 0.6521890163421631,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.15517377853393555,
+ "rewards/margins": 0.09567751735448837,
+ "rewards/rejected": 0.05949625372886658,
+ "step": 2450
+ },
+ {
+ "epoch": 1.968,
+ "grad_norm": 1.2294185161590576,
+ "learning_rate": 3.825185185185185e-07,
+ "logits/chosen": 1.4533751010894775,
+ "logits/rejected": 1.4323598146438599,
+ "logps/chosen": -156.0291290283203,
+ "logps/rejected": -161.64663696289062,
+ "loss": 0.6698451042175293,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.07660157233476639,
+ "rewards/margins": 0.05659238249063492,
+ "rewards/rejected": 0.020009174942970276,
+ "step": 2460
+ },
+ {
+ "epoch": 1.976,
+ "grad_norm": 1.0025874376296997,
+ "learning_rate": 3.7955555555555553e-07,
+ "logits/chosen": 1.6077829599380493,
+ "logits/rejected": 1.502532720565796,
+ "logps/chosen": -168.87271118164062,
+ "logps/rejected": -160.9680633544922,
+ "loss": 0.6690214157104493,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.11292171478271484,
+ "rewards/margins": 0.06120762228965759,
+ "rewards/rejected": 0.05171409994363785,
+ "step": 2470
+ },
+ {
+ "epoch": 1.984,
+ "grad_norm": 1.1663116216659546,
+ "learning_rate": 3.7659259259259255e-07,
+ "logits/chosen": 1.6099679470062256,
+ "logits/rejected": 1.6698023080825806,
+ "logps/chosen": -147.36190795898438,
+ "logps/rejected": -145.37986755371094,
+ "loss": 0.666702127456665,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1266266405582428,
+ "rewards/margins": 0.0683312863111496,
+ "rewards/rejected": 0.0582953505218029,
+ "step": 2480
+ },
+ {
+ "epoch": 1.992,
+ "grad_norm": 1.1562272310256958,
+ "learning_rate": 3.7362962962962963e-07,
+ "logits/chosen": 1.5222467184066772,
+ "logits/rejected": 1.592881441116333,
+ "logps/chosen": -145.1186065673828,
+ "logps/rejected": -158.30718994140625,
+ "loss": 0.6763694763183594,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.09335663169622421,
+ "rewards/margins": 0.04241436719894409,
+ "rewards/rejected": 0.05094226449728012,
+ "step": 2490
+ },
+ {
+ "epoch": 2.0,
+ "grad_norm": 1.5228629112243652,
+ "learning_rate": 3.7066666666666665e-07,
+ "logits/chosen": 1.5318562984466553,
+ "logits/rejected": 1.5440677404403687,
+ "logps/chosen": -178.16098022460938,
+ "logps/rejected": -143.88893127441406,
+ "loss": 0.6654858112335205,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0874411091208458,
+ "rewards/margins": 0.07056570053100586,
+ "rewards/rejected": 0.01687539741396904,
+ "step": 2500
+ },
+ {
+ "epoch": 2.0,
+ "eval_logits/chosen": 1.4583442211151123,
+ "eval_logits/rejected": 1.486719012260437,
+ "eval_logps/chosen": -152.8921661376953,
+ "eval_logps/rejected": -147.6856231689453,
+ "eval_loss": 0.6842741370201111,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.06566743552684784,
+ "eval_rewards/margins": 0.032015930861234665,
+ "eval_rewards/rejected": 0.03365150839090347,
+ "eval_runtime": 90.8716,
+ "eval_samples_per_second": 5.502,
+ "eval_steps_per_second": 2.751,
+ "step": 2500
+ },
+ {
+ "epoch": 2.008,
+ "grad_norm": 1.2679792642593384,
+ "learning_rate": 3.6770370370370367e-07,
+ "logits/chosen": 1.542234182357788,
+ "logits/rejected": 1.6291179656982422,
+ "logps/chosen": -142.90040588378906,
+ "logps/rejected": -161.39028930664062,
+ "loss": 0.6518040657043457,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.12705746293067932,
+ "rewards/margins": 0.09586121141910553,
+ "rewards/rejected": 0.031196262687444687,
+ "step": 2510
+ },
+ {
+ "epoch": 2.016,
+ "grad_norm": 1.2428960800170898,
+ "learning_rate": 3.6474074074074075e-07,
+ "logits/chosen": 1.5037320852279663,
+ "logits/rejected": 1.5008131265640259,
+ "logps/chosen": -154.43722534179688,
+ "logps/rejected": -179.9853515625,
+ "loss": 0.6436567306518555,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.09548825025558472,
+ "rewards/margins": 0.11602531373500824,
+ "rewards/rejected": -0.020537082105875015,
+ "step": 2520
+ },
+ {
+ "epoch": 2.024,
+ "grad_norm": 1.3352172374725342,
+ "learning_rate": 3.617777777777777e-07,
+ "logits/chosen": 1.5730235576629639,
+ "logits/rejected": 1.5479223728179932,
+ "logps/chosen": -143.82000732421875,
+ "logps/rejected": -154.56626892089844,
+ "loss": 0.6623670578002929,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.11983202397823334,
+ "rewards/margins": 0.07249654829502106,
+ "rewards/rejected": 0.04733546823263168,
+ "step": 2530
+ },
+ {
+ "epoch": 2.032,
+ "grad_norm": 1.2124547958374023,
+ "learning_rate": 3.588148148148148e-07,
+ "logits/chosen": 1.2967132329940796,
+ "logits/rejected": 1.198378324508667,
+ "logps/chosen": -134.57449340820312,
+ "logps/rejected": -124.48946380615234,
+ "loss": 0.6602365016937256,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.10377311706542969,
+ "rewards/margins": 0.07978598773479462,
+ "rewards/rejected": 0.023987144231796265,
+ "step": 2540
+ },
+ {
+ "epoch": 2.04,
+ "grad_norm": 1.7321484088897705,
+ "learning_rate": 3.5585185185185187e-07,
+ "logits/chosen": 1.4171000719070435,
+ "logits/rejected": 1.3644115924835205,
+ "logps/chosen": -153.2901611328125,
+ "logps/rejected": -143.07582092285156,
+ "loss": 0.6586461067199707,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.10289964824914932,
+ "rewards/margins": 0.08438605815172195,
+ "rewards/rejected": 0.018513580784201622,
+ "step": 2550
+ },
+ {
+ "epoch": 2.048,
+ "grad_norm": 1.2162495851516724,
+ "learning_rate": 3.5288888888888884e-07,
+ "logits/chosen": 1.5142430067062378,
+ "logits/rejected": 1.4998128414154053,
+ "logps/chosen": -156.86972045898438,
+ "logps/rejected": -155.01321411132812,
+ "loss": 0.633359432220459,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.12783676385879517,
+ "rewards/margins": 0.13910353183746338,
+ "rewards/rejected": -0.011266755871474743,
+ "step": 2560
+ },
+ {
+ "epoch": 2.056,
+ "grad_norm": 1.7551093101501465,
+ "learning_rate": 3.499259259259259e-07,
+ "logits/chosen": 1.3865793943405151,
+ "logits/rejected": 1.4954140186309814,
+ "logps/chosen": -156.48683166503906,
+ "logps/rejected": -148.37991333007812,
+ "loss": 0.6553828716278076,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.09534349292516708,
+ "rewards/margins": 0.08968523144721985,
+ "rewards/rejected": 0.005658247508108616,
+ "step": 2570
+ },
+ {
+ "epoch": 2.064,
+ "grad_norm": 1.282270908355713,
+ "learning_rate": 3.4696296296296293e-07,
+ "logits/chosen": 1.3754762411117554,
+ "logits/rejected": 1.3920589685440063,
+ "logps/chosen": -130.45675659179688,
+ "logps/rejected": -128.36073303222656,
+ "loss": 0.6361326217651367,
+ "rewards/accuracies": 0.8125,
+ "rewards/chosen": 0.16388167440891266,
+ "rewards/margins": 0.13015803694725037,
+ "rewards/rejected": 0.0337236113846302,
+ "step": 2580
+ },
+ {
+ "epoch": 2.072,
+ "grad_norm": 1.2549991607666016,
+ "learning_rate": 3.4399999999999996e-07,
+ "logits/chosen": 1.4334301948547363,
+ "logits/rejected": 1.4073057174682617,
+ "logps/chosen": -142.28848266601562,
+ "logps/rejected": -143.86415100097656,
+ "loss": 0.6604367733001709,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.10965392738580704,
+ "rewards/margins": 0.08058565109968185,
+ "rewards/rejected": 0.02906828001141548,
+ "step": 2590
+ },
+ {
+ "epoch": 2.08,
+ "grad_norm": 1.1193031072616577,
+ "learning_rate": 3.4103703703703703e-07,
+ "logits/chosen": 1.4255090951919556,
+ "logits/rejected": 1.3589593172073364,
+ "logps/chosen": -145.86782836914062,
+ "logps/rejected": -134.277099609375,
+ "loss": 0.6415833950042724,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.15703223645687103,
+ "rewards/margins": 0.11860913038253784,
+ "rewards/rejected": 0.03842310234904289,
+ "step": 2600
+ },
+ {
+ "epoch": 2.08,
+ "eval_logits/chosen": 1.4633173942565918,
+ "eval_logits/rejected": 1.4914942979812622,
+ "eval_logps/chosen": -152.7378387451172,
+ "eval_logps/rejected": -147.54226684570312,
+ "eval_loss": 0.684092104434967,
+ "eval_rewards/accuracies": 0.5559999942779541,
+ "eval_rewards/chosen": 0.08109957724809647,
+ "eval_rewards/margins": 0.033111557364463806,
+ "eval_rewards/rejected": 0.047988031059503555,
+ "eval_runtime": 90.9652,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 2600
+ },
+ {
+ "epoch": 2.088,
+ "grad_norm": 1.138805627822876,
+ "learning_rate": 3.3807407407407405e-07,
+ "logits/chosen": 1.6175915002822876,
+ "logits/rejected": 1.7044296264648438,
+ "logps/chosen": -161.02072143554688,
+ "logps/rejected": -150.9885711669922,
+ "loss": 0.6464156627655029,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.1512492299079895,
+ "rewards/margins": 0.11052403599023819,
+ "rewards/rejected": 0.040725212544202805,
+ "step": 2610
+ },
+ {
+ "epoch": 2.096,
+ "grad_norm": 1.2945690155029297,
+ "learning_rate": 3.3511111111111113e-07,
+ "logits/chosen": 1.7169849872589111,
+ "logits/rejected": 1.5235084295272827,
+ "logps/chosen": -178.69578552246094,
+ "logps/rejected": -147.8590850830078,
+ "loss": 0.6443929195404052,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.1698969006538391,
+ "rewards/margins": 0.11732057482004166,
+ "rewards/rejected": 0.052576325833797455,
+ "step": 2620
+ },
+ {
+ "epoch": 2.104,
+ "grad_norm": 1.4025486707687378,
+ "learning_rate": 3.321481481481481e-07,
+ "logits/chosen": 1.3389716148376465,
+ "logits/rejected": 1.4465675354003906,
+ "logps/chosen": -154.75857543945312,
+ "logps/rejected": -151.66146850585938,
+ "loss": 0.651912260055542,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.06493876874446869,
+ "rewards/margins": 0.09662672132253647,
+ "rewards/rejected": -0.03168795630335808,
+ "step": 2630
+ },
+ {
+ "epoch": 2.112,
+ "grad_norm": 1.5229434967041016,
+ "learning_rate": 3.291851851851852e-07,
+ "logits/chosen": 1.733494758605957,
+ "logits/rejected": 1.6284431219100952,
+ "logps/chosen": -159.22984313964844,
+ "logps/rejected": -148.73074340820312,
+ "loss": 0.6609207153320312,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.12649953365325928,
+ "rewards/margins": 0.07567869126796722,
+ "rewards/rejected": 0.050820838660001755,
+ "step": 2640
+ },
+ {
+ "epoch": 2.12,
+ "grad_norm": 1.4460108280181885,
+ "learning_rate": 3.2622222222222225e-07,
+ "logits/chosen": 1.4306278228759766,
+ "logits/rejected": 1.5417077541351318,
+ "logps/chosen": -158.87351989746094,
+ "logps/rejected": -174.2522430419922,
+ "loss": 0.6391911983489991,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.13457423448562622,
+ "rewards/margins": 0.12862522900104523,
+ "rewards/rejected": 0.005949012935161591,
+ "step": 2650
+ },
+ {
+ "epoch": 2.128,
+ "grad_norm": 1.3597654104232788,
+ "learning_rate": 3.232592592592592e-07,
+ "logits/chosen": 1.329627513885498,
+ "logits/rejected": 1.3987334966659546,
+ "logps/chosen": -165.3042449951172,
+ "logps/rejected": -153.50428771972656,
+ "loss": 0.6513238906860351,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.12750771641731262,
+ "rewards/margins": 0.10093003511428833,
+ "rewards/rejected": 0.026577681303024292,
+ "step": 2660
+ },
+ {
+ "epoch": 2.136,
+ "grad_norm": 1.2453831434249878,
+ "learning_rate": 3.202962962962963e-07,
+ "logits/chosen": 1.5347964763641357,
+ "logits/rejected": 1.5230233669281006,
+ "logps/chosen": -154.97242736816406,
+ "logps/rejected": -130.9337158203125,
+ "loss": 0.6872330665588379,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.08273214101791382,
+ "rewards/margins": 0.03720199316740036,
+ "rewards/rejected": 0.04553014412522316,
+ "step": 2670
+ },
+ {
+ "epoch": 2.144,
+ "grad_norm": 1.9659223556518555,
+ "learning_rate": 3.173333333333333e-07,
+ "logits/chosen": 1.5074447393417358,
+ "logits/rejected": 1.4809236526489258,
+ "logps/chosen": -146.32305908203125,
+ "logps/rejected": -143.98345947265625,
+ "loss": 0.6821617603302002,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.08092048764228821,
+ "rewards/margins": 0.03855252265930176,
+ "rewards/rejected": 0.04236796498298645,
+ "step": 2680
+ },
+ {
+ "epoch": 2.152,
+ "grad_norm": 1.2072023153305054,
+ "learning_rate": 3.1437037037037034e-07,
+ "logits/chosen": 1.3799779415130615,
+ "logits/rejected": 1.3887022733688354,
+ "logps/chosen": -143.9578094482422,
+ "logps/rejected": -159.37698364257812,
+ "loss": 0.656886100769043,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0652204304933548,
+ "rewards/margins": 0.08883848786354065,
+ "rewards/rejected": -0.02361806109547615,
+ "step": 2690
+ },
+ {
+ "epoch": 2.16,
+ "grad_norm": 1.1380783319473267,
+ "learning_rate": 3.114074074074074e-07,
+ "logits/chosen": 1.5064178705215454,
+ "logits/rejected": 1.6023409366607666,
+ "logps/chosen": -132.29075622558594,
+ "logps/rejected": -128.1918487548828,
+ "loss": 0.6357467651367188,
+ "rewards/accuracies": 0.762499988079071,
+ "rewards/chosen": 0.15728428959846497,
+ "rewards/margins": 0.13087645173072815,
+ "rewards/rejected": 0.02640785276889801,
+ "step": 2700
+ },
+ {
+ "epoch": 2.16,
+ "eval_logits/chosen": 1.4565616846084595,
+ "eval_logits/rejected": 1.4844369888305664,
+ "eval_logps/chosen": -152.74916076660156,
+ "eval_logps/rejected": -147.5604248046875,
+ "eval_loss": 0.6842489242553711,
+ "eval_rewards/accuracies": 0.5580000281333923,
+ "eval_rewards/chosen": 0.0799703598022461,
+ "eval_rewards/margins": 0.03379753977060318,
+ "eval_rewards/rejected": 0.046172816306352615,
+ "eval_runtime": 90.8529,
+ "eval_samples_per_second": 5.503,
+ "eval_steps_per_second": 2.752,
+ "step": 2700
+ },
+ {
+ "epoch": 2.168,
+ "grad_norm": 1.435728669166565,
+ "learning_rate": 3.0844444444444444e-07,
+ "logits/chosen": 1.4452109336853027,
+ "logits/rejected": 1.4124813079833984,
+ "logps/chosen": -151.26815795898438,
+ "logps/rejected": -143.44091796875,
+ "loss": 0.6585721969604492,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.0672367662191391,
+ "rewards/margins": 0.07900270074605942,
+ "rewards/rejected": -0.011765943840146065,
+ "step": 2710
+ },
+ {
+ "epoch": 2.176,
+ "grad_norm": 1.3085405826568604,
+ "learning_rate": 3.0548148148148146e-07,
+ "logits/chosen": 1.2030339241027832,
+ "logits/rejected": 1.2627733945846558,
+ "logps/chosen": -138.23812866210938,
+ "logps/rejected": -134.25238037109375,
+ "loss": 0.6320697784423828,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.10239285230636597,
+ "rewards/margins": 0.14901991188526154,
+ "rewards/rejected": -0.04662706330418587,
+ "step": 2720
+ },
+ {
+ "epoch": 2.184,
+ "grad_norm": 1.5403406620025635,
+ "learning_rate": 3.025185185185185e-07,
+ "logits/chosen": 1.4745285511016846,
+ "logits/rejected": 1.4968340396881104,
+ "logps/chosen": -153.49771118164062,
+ "logps/rejected": -148.9380645751953,
+ "loss": 0.6523497104644775,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.09372548013925552,
+ "rewards/margins": 0.09992895275354385,
+ "rewards/rejected": -0.006203479133546352,
+ "step": 2730
+ },
+ {
+ "epoch": 2.192,
+ "grad_norm": 1.5256744623184204,
+ "learning_rate": 2.9955555555555556e-07,
+ "logits/chosen": 1.39058518409729,
+ "logits/rejected": 1.3656699657440186,
+ "logps/chosen": -148.22085571289062,
+ "logps/rejected": -149.59144592285156,
+ "loss": 0.6467072010040283,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.09406543523073196,
+ "rewards/margins": 0.10628072172403336,
+ "rewards/rejected": -0.012215293012559414,
+ "step": 2740
+ },
+ {
+ "epoch": 2.2,
+ "grad_norm": 1.128711462020874,
+ "learning_rate": 2.965925925925926e-07,
+ "logits/chosen": 1.557297945022583,
+ "logits/rejected": 1.5189030170440674,
+ "logps/chosen": -160.21929931640625,
+ "logps/rejected": -151.35910034179688,
+ "loss": 0.6474220275878906,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.13500377535820007,
+ "rewards/margins": 0.10226242244243622,
+ "rewards/rejected": 0.03274135664105415,
+ "step": 2750
+ },
+ {
+ "epoch": 2.208,
+ "grad_norm": 1.3706861734390259,
+ "learning_rate": 2.936296296296296e-07,
+ "logits/chosen": 1.431715965270996,
+ "logits/rejected": 1.4564027786254883,
+ "logps/chosen": -142.4249267578125,
+ "logps/rejected": -143.3875732421875,
+ "loss": 0.6685743808746338,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.09871350228786469,
+ "rewards/margins": 0.06824100762605667,
+ "rewards/rejected": 0.03047250770032406,
+ "step": 2760
+ },
+ {
+ "epoch": 2.216,
+ "grad_norm": 1.3409324884414673,
+ "learning_rate": 2.906666666666667e-07,
+ "logits/chosen": 1.332448124885559,
+ "logits/rejected": 1.3167566061019897,
+ "logps/chosen": -146.07406616210938,
+ "logps/rejected": -138.0575408935547,
+ "loss": 0.6424362182617187,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.05286739021539688,
+ "rewards/margins": 0.12220163643360138,
+ "rewards/rejected": -0.0693342313170433,
+ "step": 2770
+ },
+ {
+ "epoch": 2.224,
+ "grad_norm": 1.5424753427505493,
+ "learning_rate": 2.8770370370370364e-07,
+ "logits/chosen": 1.7004855871200562,
+ "logits/rejected": 1.5205724239349365,
+ "logps/chosen": -180.87466430664062,
+ "logps/rejected": -163.62646484375,
+ "loss": 0.6519207000732422,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.12406478077173233,
+ "rewards/margins": 0.10040261596441269,
+ "rewards/rejected": 0.023662149906158447,
+ "step": 2780
+ },
+ {
+ "epoch": 2.232,
+ "grad_norm": 1.2184964418411255,
+ "learning_rate": 2.847407407407407e-07,
+ "logits/chosen": 1.4738783836364746,
+ "logits/rejected": 1.4357870817184448,
+ "logps/chosen": -164.53448486328125,
+ "logps/rejected": -142.24685668945312,
+ "loss": 0.6661330223083496,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.09590204805135727,
+ "rewards/margins": 0.06964156031608582,
+ "rewards/rejected": 0.026260489597916603,
+ "step": 2790
+ },
+ {
+ "epoch": 2.24,
+ "grad_norm": 1.2697800397872925,
+ "learning_rate": 2.817777777777778e-07,
+ "logits/chosen": 1.4889880418777466,
+ "logits/rejected": 1.5381649732589722,
+ "logps/chosen": -181.35275268554688,
+ "logps/rejected": -159.2748565673828,
+ "loss": 0.6470918655395508,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.10390053689479828,
+ "rewards/margins": 0.10561826080083847,
+ "rewards/rejected": -0.00171771424356848,
+ "step": 2800
+ },
+ {
+ "epoch": 2.24,
+ "eval_logits/chosen": 1.450643539428711,
+ "eval_logits/rejected": 1.478151559829712,
+ "eval_logps/chosen": -152.76634216308594,
+ "eval_logps/rejected": -147.58448791503906,
+ "eval_loss": 0.6843582987785339,
+ "eval_rewards/accuracies": 0.5619999766349792,
+ "eval_rewards/chosen": 0.07825153321027756,
+ "eval_rewards/margins": 0.03448594734072685,
+ "eval_rewards/rejected": 0.04376557841897011,
+ "eval_runtime": 90.9417,
+ "eval_samples_per_second": 5.498,
+ "eval_steps_per_second": 2.749,
+ "step": 2800
+ },
+ {
+ "epoch": 2.248,
+ "grad_norm": 1.161729097366333,
+ "learning_rate": 2.7881481481481476e-07,
+ "logits/chosen": 1.4012717008590698,
+ "logits/rejected": 1.365496277809143,
+ "logps/chosen": -142.15725708007812,
+ "logps/rejected": -131.11892700195312,
+ "loss": 0.6921159744262695,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.048748843371868134,
+ "rewards/margins": 0.018229911103844643,
+ "rewards/rejected": 0.030518928542733192,
+ "step": 2810
+ },
+ {
+ "epoch": 2.2560000000000002,
+ "grad_norm": 1.403849720954895,
+ "learning_rate": 2.7585185185185184e-07,
+ "logits/chosen": 1.6802637577056885,
+ "logits/rejected": 1.6305253505706787,
+ "logps/chosen": -162.37313842773438,
+ "logps/rejected": -165.4166717529297,
+ "loss": 0.6506373405456543,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.14859917759895325,
+ "rewards/margins": 0.09807908535003662,
+ "rewards/rejected": 0.05052009969949722,
+ "step": 2820
+ },
+ {
+ "epoch": 2.2640000000000002,
+ "grad_norm": 1.3189504146575928,
+ "learning_rate": 2.728888888888889e-07,
+ "logits/chosen": 1.4633572101593018,
+ "logits/rejected": 1.4499682188034058,
+ "logps/chosen": -156.68209838867188,
+ "logps/rejected": -155.54751586914062,
+ "loss": 0.6692873954772949,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.13907714188098907,
+ "rewards/margins": 0.06127101182937622,
+ "rewards/rejected": 0.07780613750219345,
+ "step": 2830
+ },
+ {
+ "epoch": 2.2720000000000002,
+ "grad_norm": 1.5040212869644165,
+ "learning_rate": 2.699259259259259e-07,
+ "logits/chosen": 1.4402498006820679,
+ "logits/rejected": 1.4848177433013916,
+ "logps/chosen": -145.59292602539062,
+ "logps/rejected": -142.07449340820312,
+ "loss": 0.6696759223937988,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1008831262588501,
+ "rewards/margins": 0.06455080956220627,
+ "rewards/rejected": 0.03633231669664383,
+ "step": 2840
+ },
+ {
+ "epoch": 2.2800000000000002,
+ "grad_norm": 1.4957029819488525,
+ "learning_rate": 2.6696296296296296e-07,
+ "logits/chosen": 1.309990406036377,
+ "logits/rejected": 1.2410783767700195,
+ "logps/chosen": -145.6426239013672,
+ "logps/rejected": -155.6902313232422,
+ "loss": 0.6614094257354737,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.08516388386487961,
+ "rewards/margins": 0.0843389481306076,
+ "rewards/rejected": 0.0008249226957559586,
+ "step": 2850
+ },
+ {
+ "epoch": 2.288,
+ "grad_norm": 1.1865259408950806,
+ "learning_rate": 2.64e-07,
+ "logits/chosen": 1.4638307094573975,
+ "logits/rejected": 1.5667332410812378,
+ "logps/chosen": -128.8794708251953,
+ "logps/rejected": -121.29227447509766,
+ "loss": 0.6763367652893066,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.11934898048639297,
+ "rewards/margins": 0.05003399774432182,
+ "rewards/rejected": 0.06931495666503906,
+ "step": 2860
+ },
+ {
+ "epoch": 2.296,
+ "grad_norm": 2.0794501304626465,
+ "learning_rate": 2.61037037037037e-07,
+ "logits/chosen": 1.1402711868286133,
+ "logits/rejected": 1.4019138813018799,
+ "logps/chosen": -147.0840301513672,
+ "logps/rejected": -157.21463012695312,
+ "loss": 0.6764614105224609,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.09646089375019073,
+ "rewards/margins": 0.05578025057911873,
+ "rewards/rejected": 0.04068063572049141,
+ "step": 2870
+ },
+ {
+ "epoch": 2.304,
+ "grad_norm": 1.318089485168457,
+ "learning_rate": 2.580740740740741e-07,
+ "logits/chosen": 1.3814247846603394,
+ "logits/rejected": 1.2060496807098389,
+ "logps/chosen": -131.8695068359375,
+ "logps/rejected": -124.2463607788086,
+ "loss": 0.6537458896636963,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.0847344920039177,
+ "rewards/margins": 0.10118331015110016,
+ "rewards/rejected": -0.016448814421892166,
+ "step": 2880
+ },
+ {
+ "epoch": 2.312,
+ "grad_norm": 1.1566070318222046,
+ "learning_rate": 2.551111111111111e-07,
+ "logits/chosen": 1.449655294418335,
+ "logits/rejected": 1.4626872539520264,
+ "logps/chosen": -148.62901306152344,
+ "logps/rejected": -162.71693420410156,
+ "loss": 0.6510843753814697,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.1346159428358078,
+ "rewards/margins": 0.09821879863739014,
+ "rewards/rejected": 0.03639713674783707,
+ "step": 2890
+ },
+ {
+ "epoch": 2.32,
+ "grad_norm": 1.3676846027374268,
+ "learning_rate": 2.521481481481481e-07,
+ "logits/chosen": 1.5381054878234863,
+ "logits/rejected": 1.4480576515197754,
+ "logps/chosen": -159.05850219726562,
+ "logps/rejected": -143.50048828125,
+ "loss": 0.6565275669097901,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.09088721126317978,
+ "rewards/margins": 0.08976241946220398,
+ "rewards/rejected": 0.0011247784132137895,
+ "step": 2900
+ },
+ {
+ "epoch": 2.32,
+ "eval_logits/chosen": 1.4630789756774902,
+ "eval_logits/rejected": 1.4909931421279907,
+ "eval_logps/chosen": -152.56454467773438,
+ "eval_logps/rejected": -147.39541625976562,
+ "eval_loss": 0.6835448145866394,
+ "eval_rewards/accuracies": 0.5540000200271606,
+ "eval_rewards/chosen": 0.09842933714389801,
+ "eval_rewards/margins": 0.03575750067830086,
+ "eval_rewards/rejected": 0.06267183274030685,
+ "eval_runtime": 90.9368,
+ "eval_samples_per_second": 5.498,
+ "eval_steps_per_second": 2.749,
+ "step": 2900
+ },
+ {
+ "epoch": 2.328,
+ "grad_norm": 1.2419706583023071,
+ "learning_rate": 2.4918518518518515e-07,
+ "logits/chosen": 1.3978856801986694,
+ "logits/rejected": 1.4488260746002197,
+ "logps/chosen": -131.99398803710938,
+ "logps/rejected": -155.72616577148438,
+ "loss": 0.6545028209686279,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.1015889048576355,
+ "rewards/margins": 0.0939277708530426,
+ "rewards/rejected": 0.007661131210625172,
+ "step": 2910
+ },
+ {
+ "epoch": 2.336,
+ "grad_norm": 0.8262892961502075,
+ "learning_rate": 2.462222222222222e-07,
+ "logits/chosen": 1.260616660118103,
+ "logits/rejected": 1.298372507095337,
+ "logps/chosen": -131.44737243652344,
+ "logps/rejected": -132.9801483154297,
+ "loss": 0.6633480072021485,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.11483611911535263,
+ "rewards/margins": 0.08126294612884521,
+ "rewards/rejected": 0.033573172986507416,
+ "step": 2920
+ },
+ {
+ "epoch": 2.344,
+ "grad_norm": 1.0795193910598755,
+ "learning_rate": 2.4325925925925924e-07,
+ "logits/chosen": 1.6006931066513062,
+ "logits/rejected": 1.5708739757537842,
+ "logps/chosen": -136.45252990722656,
+ "logps/rejected": -145.27377319335938,
+ "loss": 0.6421307563781739,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.1252741515636444,
+ "rewards/margins": 0.12073083221912384,
+ "rewards/rejected": 0.004543319344520569,
+ "step": 2930
+ },
+ {
+ "epoch": 2.352,
+ "grad_norm": 1.2647486925125122,
+ "learning_rate": 2.4029629629629627e-07,
+ "logits/chosen": 1.386439561843872,
+ "logits/rejected": 1.5220506191253662,
+ "logps/chosen": -160.830078125,
+ "logps/rejected": -166.8371124267578,
+ "loss": 0.6581830024719239,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.08102302998304367,
+ "rewards/margins": 0.08525596559047699,
+ "rewards/rejected": -0.004232937004417181,
+ "step": 2940
+ },
+ {
+ "epoch": 2.36,
+ "grad_norm": 1.1739935874938965,
+ "learning_rate": 2.3733333333333334e-07,
+ "logits/chosen": 1.4723713397979736,
+ "logits/rejected": 1.313079595565796,
+ "logps/chosen": -145.6066436767578,
+ "logps/rejected": -132.0267791748047,
+ "loss": 0.6555693626403809,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10856165736913681,
+ "rewards/margins": 0.10550457239151001,
+ "rewards/rejected": 0.003057088004425168,
+ "step": 2950
+ },
+ {
+ "epoch": 2.368,
+ "grad_norm": 1.1559282541275024,
+ "learning_rate": 2.3437037037037036e-07,
+ "logits/chosen": 1.6269900798797607,
+ "logits/rejected": 1.4614148139953613,
+ "logps/chosen": -166.4721221923828,
+ "logps/rejected": -142.86746215820312,
+ "loss": 0.6763554573059082,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.10653790086507797,
+ "rewards/margins": 0.04229436814785004,
+ "rewards/rejected": 0.06424354016780853,
+ "step": 2960
+ },
+ {
+ "epoch": 2.376,
+ "grad_norm": 1.315238118171692,
+ "learning_rate": 2.3140740740740739e-07,
+ "logits/chosen": 1.5281031131744385,
+ "logits/rejected": 1.6713653802871704,
+ "logps/chosen": -155.27975463867188,
+ "logps/rejected": -140.60409545898438,
+ "loss": 0.6465449333190918,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1229790598154068,
+ "rewards/margins": 0.10460098087787628,
+ "rewards/rejected": 0.018378086388111115,
+ "step": 2970
+ },
+ {
+ "epoch": 2.384,
+ "grad_norm": 1.5117779970169067,
+ "learning_rate": 2.2844444444444443e-07,
+ "logits/chosen": 1.4258384704589844,
+ "logits/rejected": 1.3107579946517944,
+ "logps/chosen": -153.65931701660156,
+ "logps/rejected": -136.7742156982422,
+ "loss": 0.6342881679534912,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.09818007051944733,
+ "rewards/margins": 0.1422421634197235,
+ "rewards/rejected": -0.04406210035085678,
+ "step": 2980
+ },
+ {
+ "epoch": 2.392,
+ "grad_norm": 1.543182134628296,
+ "learning_rate": 2.2548148148148146e-07,
+ "logits/chosen": 1.374324917793274,
+ "logits/rejected": 1.5730586051940918,
+ "logps/chosen": -137.39154052734375,
+ "logps/rejected": -156.12655639648438,
+ "loss": 0.660968017578125,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.1029653325676918,
+ "rewards/margins": 0.0822048932313919,
+ "rewards/rejected": 0.020760437473654747,
+ "step": 2990
+ },
+ {
+ "epoch": 2.4,
+ "grad_norm": 1.5507569313049316,
+ "learning_rate": 2.2251851851851853e-07,
+ "logits/chosen": 1.232704520225525,
+ "logits/rejected": 1.3583004474639893,
+ "logps/chosen": -153.07070922851562,
+ "logps/rejected": -152.47064208984375,
+ "loss": 0.6697997093200684,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.03783693164587021,
+ "rewards/margins": 0.06472334265708923,
+ "rewards/rejected": -0.02688641846179962,
+ "step": 3000
+ },
+ {
+ "epoch": 2.4,
+ "eval_logits/chosen": 1.4536765813827515,
+ "eval_logits/rejected": 1.4812932014465332,
+ "eval_logps/chosen": -152.67945861816406,
+ "eval_logps/rejected": -147.50315856933594,
+ "eval_loss": 0.6841781139373779,
+ "eval_rewards/accuracies": 0.5619999766349792,
+ "eval_rewards/chosen": 0.08694048970937729,
+ "eval_rewards/margins": 0.035040292888879776,
+ "eval_rewards/rejected": 0.05190018564462662,
+ "eval_runtime": 90.957,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.749,
+ "step": 3000
+ },
+ {
+ "epoch": 2.408,
+ "grad_norm": 1.299101710319519,
+ "learning_rate": 2.1955555555555555e-07,
+ "logits/chosen": 1.300927996635437,
+ "logits/rejected": 1.4192984104156494,
+ "logps/chosen": -140.5328826904297,
+ "logps/rejected": -157.88247680664062,
+ "loss": 0.6539476871490478,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.04641692712903023,
+ "rewards/margins": 0.1018022894859314,
+ "rewards/rejected": -0.055385369807481766,
+ "step": 3010
+ },
+ {
+ "epoch": 2.416,
+ "grad_norm": 1.597330093383789,
+ "learning_rate": 2.1659259259259258e-07,
+ "logits/chosen": 1.410930871963501,
+ "logits/rejected": 1.4487859010696411,
+ "logps/chosen": -147.2603759765625,
+ "logps/rejected": -141.80209350585938,
+ "loss": 0.6582518577575683,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.05355765298008919,
+ "rewards/margins": 0.08682949841022491,
+ "rewards/rejected": -0.033271849155426025,
+ "step": 3020
+ },
+ {
+ "epoch": 2.424,
+ "grad_norm": 1.2904349565505981,
+ "learning_rate": 2.1362962962962962e-07,
+ "logits/chosen": 1.431049108505249,
+ "logits/rejected": 1.4038130044937134,
+ "logps/chosen": -134.06564331054688,
+ "logps/rejected": -139.49826049804688,
+ "loss": 0.6637657642364502,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.12606436014175415,
+ "rewards/margins": 0.07909566909074783,
+ "rewards/rejected": 0.04696869105100632,
+ "step": 3030
+ },
+ {
+ "epoch": 2.432,
+ "grad_norm": 1.3195490837097168,
+ "learning_rate": 2.1066666666666665e-07,
+ "logits/chosen": 1.5150330066680908,
+ "logits/rejected": 1.3475278615951538,
+ "logps/chosen": -157.96343994140625,
+ "logps/rejected": -160.56509399414062,
+ "loss": 0.6553669452667237,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.10793514549732208,
+ "rewards/margins": 0.09139807522296906,
+ "rewards/rejected": 0.01653706096112728,
+ "step": 3040
+ },
+ {
+ "epoch": 2.44,
+ "grad_norm": 1.022117018699646,
+ "learning_rate": 2.077037037037037e-07,
+ "logits/chosen": 1.411431074142456,
+ "logits/rejected": 1.5498731136322021,
+ "logps/chosen": -142.24293518066406,
+ "logps/rejected": -152.7814483642578,
+ "loss": 0.6480430603027344,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.13466320931911469,
+ "rewards/margins": 0.11154146492481232,
+ "rewards/rejected": 0.023121753707528114,
+ "step": 3050
+ },
+ {
+ "epoch": 2.448,
+ "grad_norm": 1.3372952938079834,
+ "learning_rate": 2.0474074074074074e-07,
+ "logits/chosen": 1.3349260091781616,
+ "logits/rejected": 1.36862051486969,
+ "logps/chosen": -141.4913787841797,
+ "logps/rejected": -153.81234741210938,
+ "loss": 0.6436941623687744,
+ "rewards/accuracies": 0.7749999761581421,
+ "rewards/chosen": 0.09609787166118622,
+ "rewards/margins": 0.12482483685016632,
+ "rewards/rejected": -0.02872694469988346,
+ "step": 3060
+ },
+ {
+ "epoch": 2.456,
+ "grad_norm": 1.2636305093765259,
+ "learning_rate": 2.0177777777777777e-07,
+ "logits/chosen": 1.5101096630096436,
+ "logits/rejected": 1.4085323810577393,
+ "logps/chosen": -158.89871215820312,
+ "logps/rejected": -157.28538513183594,
+ "loss": 0.6665299892425537,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.13253602385520935,
+ "rewards/margins": 0.06450225412845612,
+ "rewards/rejected": 0.06803376972675323,
+ "step": 3070
+ },
+ {
+ "epoch": 2.464,
+ "grad_norm": 1.3486571311950684,
+ "learning_rate": 1.988148148148148e-07,
+ "logits/chosen": 1.327289342880249,
+ "logits/rejected": 1.3173015117645264,
+ "logps/chosen": -160.33554077148438,
+ "logps/rejected": -153.06150817871094,
+ "loss": 0.6544498443603516,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10490100085735321,
+ "rewards/margins": 0.10563824325799942,
+ "rewards/rejected": -0.0007372468826361,
+ "step": 3080
+ },
+ {
+ "epoch": 2.472,
+ "grad_norm": 1.1544814109802246,
+ "learning_rate": 1.9585185185185186e-07,
+ "logits/chosen": 1.208981990814209,
+ "logits/rejected": 1.221260905265808,
+ "logps/chosen": -133.82061767578125,
+ "logps/rejected": -134.70327758789062,
+ "loss": 0.6506445407867432,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.07232929766178131,
+ "rewards/margins": 0.1016290932893753,
+ "rewards/rejected": -0.029299795627593994,
+ "step": 3090
+ },
+ {
+ "epoch": 2.48,
+ "grad_norm": 1.2893116474151611,
+ "learning_rate": 1.9288888888888889e-07,
+ "logits/chosen": 1.3482762575149536,
+ "logits/rejected": 1.3961074352264404,
+ "logps/chosen": -145.61500549316406,
+ "logps/rejected": -135.99945068359375,
+ "loss": 0.6463598728179931,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.16556979715824127,
+ "rewards/margins": 0.12888674437999725,
+ "rewards/rejected": 0.03668304160237312,
+ "step": 3100
+ },
+ {
+ "epoch": 2.48,
+ "eval_logits/chosen": 1.4538236856460571,
+ "eval_logits/rejected": 1.4815431833267212,
+ "eval_logps/chosen": -152.71058654785156,
+ "eval_logps/rejected": -147.53634643554688,
+ "eval_loss": 0.6841146349906921,
+ "eval_rewards/accuracies": 0.5619999766349792,
+ "eval_rewards/chosen": 0.08382504433393478,
+ "eval_rewards/margins": 0.03524418920278549,
+ "eval_rewards/rejected": 0.048580847680568695,
+ "eval_runtime": 90.8524,
+ "eval_samples_per_second": 5.503,
+ "eval_steps_per_second": 2.752,
+ "step": 3100
+ },
+ {
+ "epoch": 2.488,
+ "grad_norm": 1.2138270139694214,
+ "learning_rate": 1.899259259259259e-07,
+ "logits/chosen": 1.4320173263549805,
+ "logits/rejected": 1.379774570465088,
+ "logps/chosen": -140.01589965820312,
+ "logps/rejected": -145.36776733398438,
+ "loss": 0.6643844604492187,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.09917240589857101,
+ "rewards/margins": 0.07538153976202011,
+ "rewards/rejected": 0.023790866136550903,
+ "step": 3110
+ },
+ {
+ "epoch": 2.496,
+ "grad_norm": 1.2045173645019531,
+ "learning_rate": 1.8696296296296296e-07,
+ "logits/chosen": 1.2402794361114502,
+ "logits/rejected": 1.4080727100372314,
+ "logps/chosen": -119.64544677734375,
+ "logps/rejected": -134.0382843017578,
+ "loss": 0.6308258056640625,
+ "rewards/accuracies": 0.7749999761581421,
+ "rewards/chosen": 0.06476704776287079,
+ "rewards/margins": 0.15264371037483215,
+ "rewards/rejected": -0.08787664026021957,
+ "step": 3120
+ },
+ {
+ "epoch": 2.504,
+ "grad_norm": 1.2934551239013672,
+ "learning_rate": 1.8399999999999998e-07,
+ "logits/chosen": 1.521606206893921,
+ "logits/rejected": 1.4204734563827515,
+ "logps/chosen": -136.7379608154297,
+ "logps/rejected": -141.2797088623047,
+ "loss": 0.6567333221435547,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.14066217839717865,
+ "rewards/margins": 0.09437786787748337,
+ "rewards/rejected": 0.04628429934382439,
+ "step": 3130
+ },
+ {
+ "epoch": 2.512,
+ "grad_norm": 1.4545506238937378,
+ "learning_rate": 1.8103703703703703e-07,
+ "logits/chosen": 1.5223419666290283,
+ "logits/rejected": 1.369768738746643,
+ "logps/chosen": -157.4479217529297,
+ "logps/rejected": -146.4864501953125,
+ "loss": 0.6391398906707764,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.12953564524650574,
+ "rewards/margins": 0.13037580251693726,
+ "rewards/rejected": -0.0008401572704315186,
+ "step": 3140
+ },
+ {
+ "epoch": 2.52,
+ "grad_norm": 1.2595700025558472,
+ "learning_rate": 1.7807407407407408e-07,
+ "logits/chosen": 1.4436731338500977,
+ "logits/rejected": 1.5720274448394775,
+ "logps/chosen": -166.989013671875,
+ "logps/rejected": -162.21975708007812,
+ "loss": 0.6528693199157715,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.09743344783782959,
+ "rewards/margins": 0.09220697730779648,
+ "rewards/rejected": 0.005226473324000835,
+ "step": 3150
+ },
+ {
+ "epoch": 2.528,
+ "grad_norm": 1.2463562488555908,
+ "learning_rate": 1.751111111111111e-07,
+ "logits/chosen": 1.4497953653335571,
+ "logits/rejected": 1.4076560735702515,
+ "logps/chosen": -150.78482055664062,
+ "logps/rejected": -155.48902893066406,
+ "loss": 0.6709191799163818,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.13361872732639313,
+ "rewards/margins": 0.06404821574687958,
+ "rewards/rejected": 0.06957050412893295,
+ "step": 3160
+ },
+ {
+ "epoch": 2.536,
+ "grad_norm": 1.1280947923660278,
+ "learning_rate": 1.7214814814814815e-07,
+ "logits/chosen": 1.5186564922332764,
+ "logits/rejected": 1.5018190145492554,
+ "logps/chosen": -155.66006469726562,
+ "logps/rejected": -159.46339416503906,
+ "loss": 0.6649936676025391,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.11657847464084625,
+ "rewards/margins": 0.07113558799028397,
+ "rewards/rejected": 0.045442890375852585,
+ "step": 3170
+ },
+ {
+ "epoch": 2.544,
+ "grad_norm": 1.5582311153411865,
+ "learning_rate": 1.6918518518518517e-07,
+ "logits/chosen": 1.5405746698379517,
+ "logits/rejected": 1.5165503025054932,
+ "logps/chosen": -164.27452087402344,
+ "logps/rejected": -171.57701110839844,
+ "loss": 0.6631642818450928,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.07613547891378403,
+ "rewards/margins": 0.07466691732406616,
+ "rewards/rejected": 0.0014685706701129675,
+ "step": 3180
+ },
+ {
+ "epoch": 2.552,
+ "grad_norm": 1.3221781253814697,
+ "learning_rate": 1.6622222222222222e-07,
+ "logits/chosen": 1.59696364402771,
+ "logits/rejected": 1.3756712675094604,
+ "logps/chosen": -175.3300018310547,
+ "logps/rejected": -160.0670623779297,
+ "loss": 0.6846892833709717,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.09736991673707962,
+ "rewards/margins": 0.028879499062895775,
+ "rewards/rejected": 0.0684904232621193,
+ "step": 3190
+ },
+ {
+ "epoch": 2.56,
+ "grad_norm": 1.2958283424377441,
+ "learning_rate": 1.6325925925925927e-07,
+ "logits/chosen": 1.5975316762924194,
+ "logits/rejected": 1.3776990175247192,
+ "logps/chosen": -167.61512756347656,
+ "logps/rejected": -151.68203735351562,
+ "loss": 0.6497097969055176,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.13339915871620178,
+ "rewards/margins": 0.10488065332174301,
+ "rewards/rejected": 0.02851850353181362,
+ "step": 3200
+ },
+ {
+ "epoch": 2.56,
+ "eval_logits/chosen": 1.4546222686767578,
+ "eval_logits/rejected": 1.4822379350662231,
+ "eval_logps/chosen": -152.66015625,
+ "eval_logps/rejected": -147.49147033691406,
+ "eval_loss": 0.6839378476142883,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.08886973559856415,
+ "eval_rewards/margins": 0.03580106422305107,
+ "eval_rewards/rejected": 0.053068675100803375,
+ "eval_runtime": 90.9562,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.749,
+ "step": 3200
+ },
+ {
+ "epoch": 2.568,
+ "grad_norm": 2.3782362937927246,
+ "learning_rate": 1.602962962962963e-07,
+ "logits/chosen": 1.4312902688980103,
+ "logits/rejected": 1.6004657745361328,
+ "logps/chosen": -166.62925720214844,
+ "logps/rejected": -158.98095703125,
+ "loss": 0.6500686645507813,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.14471536874771118,
+ "rewards/margins": 0.10505994409322739,
+ "rewards/rejected": 0.039655424654483795,
+ "step": 3210
+ },
+ {
+ "epoch": 2.576,
+ "grad_norm": 1.392526388168335,
+ "learning_rate": 1.573333333333333e-07,
+ "logits/chosen": 1.453791856765747,
+ "logits/rejected": 1.5730758905410767,
+ "logps/chosen": -161.05404663085938,
+ "logps/rejected": -164.30911254882812,
+ "loss": 0.6707005977630616,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.1165541261434555,
+ "rewards/margins": 0.05886579677462578,
+ "rewards/rejected": 0.05768834426999092,
+ "step": 3220
+ },
+ {
+ "epoch": 2.584,
+ "grad_norm": 1.4355558156967163,
+ "learning_rate": 1.5437037037037036e-07,
+ "logits/chosen": 1.3674612045288086,
+ "logits/rejected": 1.5156943798065186,
+ "logps/chosen": -147.24728393554688,
+ "logps/rejected": -156.7361297607422,
+ "loss": 0.6483449935913086,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.12978991866111755,
+ "rewards/margins": 0.10252753645181656,
+ "rewards/rejected": 0.02726239338517189,
+ "step": 3230
+ },
+ {
+ "epoch": 2.592,
+ "grad_norm": 1.3843603134155273,
+ "learning_rate": 1.514074074074074e-07,
+ "logits/chosen": 1.5766382217407227,
+ "logits/rejected": 1.5126440525054932,
+ "logps/chosen": -171.20852661132812,
+ "logps/rejected": -143.60977172851562,
+ "loss": 0.6324634075164794,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.19362562894821167,
+ "rewards/margins": 0.14052848517894745,
+ "rewards/rejected": 0.05309714749455452,
+ "step": 3240
+ },
+ {
+ "epoch": 2.6,
+ "grad_norm": 1.089949369430542,
+ "learning_rate": 1.4844444444444443e-07,
+ "logits/chosen": 1.3081635236740112,
+ "logits/rejected": 1.4264672994613647,
+ "logps/chosen": -149.9308319091797,
+ "logps/rejected": -153.34234619140625,
+ "loss": 0.6382611751556396,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.1245744600892067,
+ "rewards/margins": 0.12832382321357727,
+ "rewards/rejected": -0.0037493593990802765,
+ "step": 3250
+ },
+ {
+ "epoch": 2.608,
+ "grad_norm": 1.4414907693862915,
+ "learning_rate": 1.4548148148148148e-07,
+ "logits/chosen": 1.4160754680633545,
+ "logits/rejected": 1.4785319566726685,
+ "logps/chosen": -147.95248413085938,
+ "logps/rejected": -152.06893920898438,
+ "loss": 0.6700046539306641,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.05685792490839958,
+ "rewards/margins": 0.056677937507629395,
+ "rewards/rejected": 0.00017998255498241633,
+ "step": 3260
+ },
+ {
+ "epoch": 2.616,
+ "grad_norm": 1.3908545970916748,
+ "learning_rate": 1.425185185185185e-07,
+ "logits/chosen": 1.4107372760772705,
+ "logits/rejected": 1.442772626876831,
+ "logps/chosen": -163.569091796875,
+ "logps/rejected": -153.9599609375,
+ "loss": 0.6501737117767334,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.07901959121227264,
+ "rewards/margins": 0.105490542948246,
+ "rewards/rejected": -0.02647094801068306,
+ "step": 3270
+ },
+ {
+ "epoch": 2.624,
+ "grad_norm": 1.3507336378097534,
+ "learning_rate": 1.3955555555555553e-07,
+ "logits/chosen": 1.5124311447143555,
+ "logits/rejected": 1.405656337738037,
+ "logps/chosen": -141.2902374267578,
+ "logps/rejected": -172.03335571289062,
+ "loss": 0.6489161014556885,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.13159234821796417,
+ "rewards/margins": 0.10707925260066986,
+ "rewards/rejected": 0.024513086304068565,
+ "step": 3280
+ },
+ {
+ "epoch": 2.632,
+ "grad_norm": 1.4356945753097534,
+ "learning_rate": 1.365925925925926e-07,
+ "logits/chosen": 1.4829243421554565,
+ "logits/rejected": 1.5267254114151,
+ "logps/chosen": -156.767578125,
+ "logps/rejected": -166.2552032470703,
+ "loss": 0.661461877822876,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.1381731629371643,
+ "rewards/margins": 0.08749156445264816,
+ "rewards/rejected": 0.050681598484516144,
+ "step": 3290
+ },
+ {
+ "epoch": 2.64,
+ "grad_norm": 1.2448052167892456,
+ "learning_rate": 1.3362962962962962e-07,
+ "logits/chosen": 1.4656591415405273,
+ "logits/rejected": 1.5108649730682373,
+ "logps/chosen": -151.31231689453125,
+ "logps/rejected": -135.82418823242188,
+ "loss": 0.6609601974487305,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.12814496457576752,
+ "rewards/margins": 0.0794161856174469,
+ "rewards/rejected": 0.048728786408901215,
+ "step": 3300
+ },
+ {
+ "epoch": 2.64,
+ "eval_logits/chosen": 1.4599801301956177,
+ "eval_logits/rejected": 1.4877806901931763,
+ "eval_logps/chosen": -152.58131408691406,
+ "eval_logps/rejected": -147.42189025878906,
+ "eval_loss": 0.6834326386451721,
+ "eval_rewards/accuracies": 0.5580000281333923,
+ "eval_rewards/chosen": 0.0967537984251976,
+ "eval_rewards/margins": 0.03672795370221138,
+ "eval_rewards/rejected": 0.06002584472298622,
+ "eval_runtime": 90.8651,
+ "eval_samples_per_second": 5.503,
+ "eval_steps_per_second": 2.751,
+ "step": 3300
+ },
+ {
+ "epoch": 2.648,
+ "grad_norm": 1.4104338884353638,
+ "learning_rate": 1.3066666666666665e-07,
+ "logits/chosen": 1.3793938159942627,
+ "logits/rejected": 1.4125133752822876,
+ "logps/chosen": -146.07693481445312,
+ "logps/rejected": -162.7715606689453,
+ "loss": 0.660995626449585,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.09182589501142502,
+ "rewards/margins": 0.08458633720874786,
+ "rewards/rejected": 0.00723955687135458,
+ "step": 3310
+ },
+ {
+ "epoch": 2.656,
+ "grad_norm": 1.6272552013397217,
+ "learning_rate": 1.277037037037037e-07,
+ "logits/chosen": 1.5128822326660156,
+ "logits/rejected": 1.472041368484497,
+ "logps/chosen": -168.37808227539062,
+ "logps/rejected": -138.04177856445312,
+ "loss": 0.6664434909820557,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.12670311331748962,
+ "rewards/margins": 0.06635840237140656,
+ "rewards/rejected": 0.06034470722079277,
+ "step": 3320
+ },
+ {
+ "epoch": 2.664,
+ "grad_norm": 1.6957371234893799,
+ "learning_rate": 1.2474074074074072e-07,
+ "logits/chosen": 1.6430788040161133,
+ "logits/rejected": 1.701340675354004,
+ "logps/chosen": -175.08358764648438,
+ "logps/rejected": -186.39157104492188,
+ "loss": 0.6366147041320801,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.19551560282707214,
+ "rewards/margins": 0.1353006660938263,
+ "rewards/rejected": 0.06021494418382645,
+ "step": 3330
+ },
+ {
+ "epoch": 2.672,
+ "grad_norm": 1.248181700706482,
+ "learning_rate": 1.2177777777777777e-07,
+ "logits/chosen": 1.5288277864456177,
+ "logits/rejected": 1.5558831691741943,
+ "logps/chosen": -160.19473266601562,
+ "logps/rejected": -172.4878387451172,
+ "loss": 0.6668370246887207,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.11863049119710922,
+ "rewards/margins": 0.06773928552865982,
+ "rewards/rejected": 0.05089121311903,
+ "step": 3340
+ },
+ {
+ "epoch": 2.68,
+ "grad_norm": 1.4140652418136597,
+ "learning_rate": 1.1881481481481481e-07,
+ "logits/chosen": 1.5105751752853394,
+ "logits/rejected": 1.4798310995101929,
+ "logps/chosen": -148.14773559570312,
+ "logps/rejected": -137.95657348632812,
+ "loss": 0.6380892753601074,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.15043237805366516,
+ "rewards/margins": 0.12980502843856812,
+ "rewards/rejected": 0.020627355203032494,
+ "step": 3350
+ },
+ {
+ "epoch": 2.6879999999999997,
+ "grad_norm": 1.2615612745285034,
+ "learning_rate": 1.1585185185185185e-07,
+ "logits/chosen": 1.464707374572754,
+ "logits/rejected": 1.4781959056854248,
+ "logps/chosen": -140.28860473632812,
+ "logps/rejected": -141.41860961914062,
+ "loss": 0.6501630306243896,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.13670575618743896,
+ "rewards/margins": 0.09955373406410217,
+ "rewards/rejected": 0.037152040749788284,
+ "step": 3360
+ },
+ {
+ "epoch": 2.6959999999999997,
+ "grad_norm": 1.385250210762024,
+ "learning_rate": 1.1288888888888889e-07,
+ "logits/chosen": 1.5589094161987305,
+ "logits/rejected": 1.447178840637207,
+ "logps/chosen": -148.5657958984375,
+ "logps/rejected": -169.35621643066406,
+ "loss": 0.6499518871307373,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.1626952886581421,
+ "rewards/margins": 0.10765236616134644,
+ "rewards/rejected": 0.055042944848537445,
+ "step": 3370
+ },
+ {
+ "epoch": 2.7039999999999997,
+ "grad_norm": 1.4454525709152222,
+ "learning_rate": 1.0992592592592592e-07,
+ "logits/chosen": 1.400112271308899,
+ "logits/rejected": 1.4862735271453857,
+ "logps/chosen": -150.48345947265625,
+ "logps/rejected": -168.9849395751953,
+ "loss": 0.6562737464904785,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.10018215328454971,
+ "rewards/margins": 0.08878926187753677,
+ "rewards/rejected": 0.011392900720238686,
+ "step": 3380
+ },
+ {
+ "epoch": 2.7119999999999997,
+ "grad_norm": 1.9209951162338257,
+ "learning_rate": 1.0696296296296297e-07,
+ "logits/chosen": 1.3544957637786865,
+ "logits/rejected": 1.3679192066192627,
+ "logps/chosen": -179.68997192382812,
+ "logps/rejected": -175.43161010742188,
+ "loss": 0.662507963180542,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.08724933862686157,
+ "rewards/margins": 0.08174990117549896,
+ "rewards/rejected": 0.005499442107975483,
+ "step": 3390
+ },
+ {
+ "epoch": 2.7199999999999998,
+ "grad_norm": 1.295901894569397,
+ "learning_rate": 1.0399999999999999e-07,
+ "logits/chosen": 1.2417107820510864,
+ "logits/rejected": 1.2812005281448364,
+ "logps/chosen": -149.10450744628906,
+ "logps/rejected": -141.39944458007812,
+ "loss": 0.6393757820129394,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.12547731399536133,
+ "rewards/margins": 0.12965723872184753,
+ "rewards/rejected": -0.004179902374744415,
+ "step": 3400
+ },
+ {
+ "epoch": 2.7199999999999998,
+ "eval_logits/chosen": 1.457220435142517,
+ "eval_logits/rejected": 1.4849778413772583,
+ "eval_logps/chosen": -152.62132263183594,
+ "eval_logps/rejected": -147.45970153808594,
+ "eval_loss": 0.6836552023887634,
+ "eval_rewards/accuracies": 0.5640000104904175,
+ "eval_rewards/chosen": 0.09275158494710922,
+ "eval_rewards/margins": 0.03650682047009468,
+ "eval_rewards/rejected": 0.05624476447701454,
+ "eval_runtime": 90.9464,
+ "eval_samples_per_second": 5.498,
+ "eval_steps_per_second": 2.749,
+ "step": 3400
+ },
+ {
+ "epoch": 2.7279999999999998,
+ "grad_norm": 1.3785442113876343,
+ "learning_rate": 1.0103703703703703e-07,
+ "logits/chosen": 1.502685785293579,
+ "logits/rejected": 1.5352259874343872,
+ "logps/chosen": -165.4739532470703,
+ "logps/rejected": -150.7890167236328,
+ "loss": 0.6662552833557129,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.1556030958890915,
+ "rewards/margins": 0.06861518323421478,
+ "rewards/rejected": 0.0869879275560379,
+ "step": 3410
+ },
+ {
+ "epoch": 2.7359999999999998,
+ "grad_norm": 1.2958600521087646,
+ "learning_rate": 9.807407407407408e-08,
+ "logits/chosen": 1.536089301109314,
+ "logits/rejected": 1.5964341163635254,
+ "logps/chosen": -169.01927185058594,
+ "logps/rejected": -164.82464599609375,
+ "loss": 0.6383768081665039,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1638927459716797,
+ "rewards/margins": 0.1290481686592102,
+ "rewards/rejected": 0.03484458476305008,
+ "step": 3420
+ },
+ {
+ "epoch": 2.7439999999999998,
+ "grad_norm": 1.3208509683609009,
+ "learning_rate": 9.51111111111111e-08,
+ "logits/chosen": 1.4485461711883545,
+ "logits/rejected": 1.3840972185134888,
+ "logps/chosen": -127.16844177246094,
+ "logps/rejected": -139.8473358154297,
+ "loss": 0.637468671798706,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.13635967671871185,
+ "rewards/margins": 0.12859989702701569,
+ "rewards/rejected": 0.007759793661534786,
+ "step": 3430
+ },
+ {
+ "epoch": 2.752,
+ "grad_norm": 1.6750396490097046,
+ "learning_rate": 9.214814814814815e-08,
+ "logits/chosen": 1.3725965023040771,
+ "logits/rejected": 1.2694950103759766,
+ "logps/chosen": -160.5334930419922,
+ "logps/rejected": -151.38027954101562,
+ "loss": 0.6750851631164551,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.1054903045296669,
+ "rewards/margins": 0.053377915173769,
+ "rewards/rejected": 0.0521123930811882,
+ "step": 3440
+ },
+ {
+ "epoch": 2.76,
+ "grad_norm": 1.6593612432479858,
+ "learning_rate": 8.918518518518518e-08,
+ "logits/chosen": 1.3901779651641846,
+ "logits/rejected": 1.5386005640029907,
+ "logps/chosen": -153.39671325683594,
+ "logps/rejected": -146.6219482421875,
+ "loss": 0.6775777339935303,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.08287060260772705,
+ "rewards/margins": 0.0443020761013031,
+ "rewards/rejected": 0.03856852278113365,
+ "step": 3450
+ },
+ {
+ "epoch": 2.768,
+ "grad_norm": 1.1425881385803223,
+ "learning_rate": 8.622222222222222e-08,
+ "logits/chosen": 1.6038954257965088,
+ "logits/rejected": 1.6097686290740967,
+ "logps/chosen": -157.6035614013672,
+ "logps/rejected": -149.43051147460938,
+ "loss": 0.6633115768432617,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1399986296892166,
+ "rewards/margins": 0.0743202343583107,
+ "rewards/rejected": 0.06567840278148651,
+ "step": 3460
+ },
+ {
+ "epoch": 2.776,
+ "grad_norm": 1.5609575510025024,
+ "learning_rate": 8.325925925925925e-08,
+ "logits/chosen": 1.36880624294281,
+ "logits/rejected": 1.1539722681045532,
+ "logps/chosen": -149.6429901123047,
+ "logps/rejected": -151.99984741210938,
+ "loss": 0.6565720081329346,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.09346573799848557,
+ "rewards/margins": 0.09520730376243591,
+ "rewards/rejected": -0.0017415669281035662,
+ "step": 3470
+ },
+ {
+ "epoch": 2.784,
+ "grad_norm": 1.1983734369277954,
+ "learning_rate": 8.029629629629629e-08,
+ "logits/chosen": 1.5328073501586914,
+ "logits/rejected": 1.3884016275405884,
+ "logps/chosen": -150.6219482421875,
+ "logps/rejected": -154.71389770507812,
+ "loss": 0.6380839347839355,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1845501959323883,
+ "rewards/margins": 0.13372965157032013,
+ "rewards/rejected": 0.05082052946090698,
+ "step": 3480
+ },
+ {
+ "epoch": 2.792,
+ "grad_norm": 1.2442429065704346,
+ "learning_rate": 7.733333333333334e-08,
+ "logits/chosen": 1.4287585020065308,
+ "logits/rejected": 1.6142936944961548,
+ "logps/chosen": -176.40530395507812,
+ "logps/rejected": -187.75030517578125,
+ "loss": 0.6455347537994385,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.17798057198524475,
+ "rewards/margins": 0.11338325589895248,
+ "rewards/rejected": 0.06459730863571167,
+ "step": 3490
+ },
+ {
+ "epoch": 2.8,
+ "grad_norm": 1.9635435342788696,
+ "learning_rate": 7.437037037037036e-08,
+ "logits/chosen": 1.2022366523742676,
+ "logits/rejected": 1.2814809083938599,
+ "logps/chosen": -132.84152221679688,
+ "logps/rejected": -153.19830322265625,
+ "loss": 0.6350316047668457,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.08858552575111389,
+ "rewards/margins": 0.1404990553855896,
+ "rewards/rejected": -0.05191350728273392,
+ "step": 3500
+ },
+ {
+ "epoch": 2.8,
+ "eval_logits/chosen": 1.4583321809768677,
+ "eval_logits/rejected": 1.4860731363296509,
+ "eval_logps/chosen": -152.62744140625,
+ "eval_logps/rejected": -147.46978759765625,
+ "eval_loss": 0.683444619178772,
+ "eval_rewards/accuracies": 0.5619999766349792,
+ "eval_rewards/chosen": 0.09214015305042267,
+ "eval_rewards/margins": 0.03690274432301521,
+ "eval_rewards/rejected": 0.055237412452697754,
+ "eval_runtime": 90.9644,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 3500
+ }
+ ],
+ "logging_steps": 10,
+ "max_steps": 3750,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3500/training_args.bin b/v3/DPO/DPO_10k/lora/checkpoint-3500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..b3fc3d76093802087869ab1fee5e37b9c830fe77
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4f90e4a7053b7fb440d68cb64a3d198396af09db942d08839c5f4f0f0b8a0c8
+size 6097
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3750/README.md b/v3/DPO/DPO_10k/lora/checkpoint-3750/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..250500f798b4682b2cd2c35cd1fc366677c215dd
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3750/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3750/adapter_config.json b/v3/DPO/DPO_10k/lora/checkpoint-3750/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..039869f8d79ca6c48a793a29a79267da6d8f81c2
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3750/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "gate_proj",
+ "down_proj",
+ "k_proj",
+ "o_proj",
+ "up_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3750/adapter_model.safetensors b/v3/DPO/DPO_10k/lora/checkpoint-3750/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ebbd0b288dba09879375efdb06dce9bf2ab5875e
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3750/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b9ad04c2748433cd0e0c57bc56405b9383d620d8024a4bf2549372fc4114574
+size 180385008
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3750/chat_template.jinja b/v3/DPO/DPO_10k/lora/checkpoint-3750/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3750/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3750/optimizer.pt b/v3/DPO/DPO_10k/lora/checkpoint-3750/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..afc6d5c7ab3a5c2d29af3d954b2e9e14ede14cd5
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3750/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:daafb3786377710e8aee9a27199d10e543d37f780592831e10a9c36088967524
+size 360902475
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3750/rng_state.pth b/v3/DPO/DPO_10k/lora/checkpoint-3750/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..7859360e6e5ad44f3ed47e7c17f4cafb2aebd41a
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3750/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50dd579ef1d2d932c3d7726ef0f6ec72d0e25b023ae2fd8b67112ac75e1041b9
+size 14645
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3750/scaler.pt b/v3/DPO/DPO_10k/lora/checkpoint-3750/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e9eddd8597651558cee95c401e12c817a21616ba
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3750/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e11be2bba7d7d56274441cc71cf9b23ca39e61b88b623208fd66817d0d6739f
+size 1383
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3750/scheduler.pt b/v3/DPO/DPO_10k/lora/checkpoint-3750/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dbea3f7e68381d2d0124a03011d456199478d27a
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3750/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bac579279f81460bc9717e3ffa6572190002436a90f9335566e0cd2bcc959ef2
+size 1465
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3750/tokenizer.json b/v3/DPO/DPO_10k/lora/checkpoint-3750/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3750/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3750/tokenizer_config.json b/v3/DPO/DPO_10k/lora/checkpoint-3750/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3750/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3750/trainer_state.json b/v3/DPO/DPO_10k/lora/checkpoint-3750/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..d15fd52917b94fb89b43955d10c819c7c9a87e63
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3750/trainer_state.json
@@ -0,0 +1,6251 @@
+{
+ "best_global_step": 3300,
+ "best_metric": 0.6834326386451721,
+ "best_model_checkpoint": "output/lora/checkpoint-1000",
+ "epoch": 3.0,
+ "eval_steps": 100,
+ "global_step": 3750,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.008,
+ "grad_norm": 1.1179773807525635,
+ "learning_rate": 2.4e-08,
+ "logits/chosen": 1.4545083045959473,
+ "logits/rejected": 1.425490379333496,
+ "logps/chosen": -132.46189880371094,
+ "logps/rejected": -148.20260620117188,
+ "loss": 0.6930626392364502,
+ "rewards/accuracies": 0.21250000596046448,
+ "rewards/chosen": -6.165739614516497e-05,
+ "rewards/margins": 0.00017333509458694607,
+ "rewards/rejected": -0.00023499250528402627,
+ "step": 10
+ },
+ {
+ "epoch": 0.016,
+ "grad_norm": 1.3502839803695679,
+ "learning_rate": 5.0666666666666664e-08,
+ "logits/chosen": 1.5668801069259644,
+ "logits/rejected": 1.5191389322280884,
+ "logps/chosen": -157.63624572753906,
+ "logps/rejected": -144.2716827392578,
+ "loss": 0.6930380821228027,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.0009244013344869018,
+ "rewards/margins": 0.00023505210992880166,
+ "rewards/rejected": 0.0006893490790389478,
+ "step": 20
+ },
+ {
+ "epoch": 0.024,
+ "grad_norm": 1.0219553709030151,
+ "learning_rate": 7.733333333333334e-08,
+ "logits/chosen": 1.8921797275543213,
+ "logits/rejected": 1.7342685461044312,
+ "logps/chosen": -164.55799865722656,
+ "logps/rejected": -158.92752075195312,
+ "loss": 0.69290189743042,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0009391927160322666,
+ "rewards/margins": 0.0005032324115745723,
+ "rewards/rejected": 0.0004359603044576943,
+ "step": 30
+ },
+ {
+ "epoch": 0.032,
+ "grad_norm": 1.2056113481521606,
+ "learning_rate": 1.0399999999999999e-07,
+ "logits/chosen": 1.4950047731399536,
+ "logits/rejected": 1.6163270473480225,
+ "logps/chosen": -145.54183959960938,
+ "logps/rejected": -134.19979858398438,
+ "loss": 0.6935500621795654,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.0006421065190806985,
+ "rewards/margins": -0.0007943272357806563,
+ "rewards/rejected": 0.0014364338712766767,
+ "step": 40
+ },
+ {
+ "epoch": 0.04,
+ "grad_norm": 1.1029725074768066,
+ "learning_rate": 1.3066666666666665e-07,
+ "logits/chosen": 1.4076533317565918,
+ "logits/rejected": 1.4791977405548096,
+ "logps/chosen": -133.23324584960938,
+ "logps/rejected": -138.3593292236328,
+ "loss": 0.6925833702087403,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.0012372838100418448,
+ "rewards/margins": 0.001139522879384458,
+ "rewards/rejected": 9.776116348803043e-05,
+ "step": 50
+ },
+ {
+ "epoch": 0.048,
+ "grad_norm": 0.9960266351699829,
+ "learning_rate": 1.573333333333333e-07,
+ "logits/chosen": 1.407166600227356,
+ "logits/rejected": 1.376042127609253,
+ "logps/chosen": -137.8921661376953,
+ "logps/rejected": -127.97320556640625,
+ "loss": 0.693614149093628,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.00021316049969755113,
+ "rewards/margins": -0.0009249996510334313,
+ "rewards/rejected": 0.0007118391804397106,
+ "step": 60
+ },
+ {
+ "epoch": 0.056,
+ "grad_norm": 1.0946824550628662,
+ "learning_rate": 1.8399999999999998e-07,
+ "logits/chosen": 1.5944501161575317,
+ "logits/rejected": 1.7248146533966064,
+ "logps/chosen": -153.5526123046875,
+ "logps/rejected": -157.19735717773438,
+ "loss": 0.6928449153900147,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0026370862033218145,
+ "rewards/margins": 0.0006150150438770652,
+ "rewards/rejected": 0.0020220710430294275,
+ "step": 70
+ },
+ {
+ "epoch": 0.064,
+ "grad_norm": 1.0322489738464355,
+ "learning_rate": 2.1066666666666665e-07,
+ "logits/chosen": 1.4586353302001953,
+ "logits/rejected": 1.4842995405197144,
+ "logps/chosen": -149.03665161132812,
+ "logps/rejected": -136.24081420898438,
+ "loss": 0.6932015895843506,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.004061593674123287,
+ "rewards/margins": -9.99594121822156e-05,
+ "rewards/rejected": 0.004161553457379341,
+ "step": 80
+ },
+ {
+ "epoch": 0.072,
+ "grad_norm": 1.0012447834014893,
+ "learning_rate": 2.3733333333333334e-07,
+ "logits/chosen": 1.5075902938842773,
+ "logits/rejected": 1.5986944437026978,
+ "logps/chosen": -159.3433074951172,
+ "logps/rejected": -151.85072326660156,
+ "loss": 0.6928757190704345,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.006978762801736593,
+ "rewards/margins": 0.0005627512582577765,
+ "rewards/rejected": 0.006416010670363903,
+ "step": 90
+ },
+ {
+ "epoch": 0.08,
+ "grad_norm": 1.280081033706665,
+ "learning_rate": 2.64e-07,
+ "logits/chosen": 1.448286533355713,
+ "logits/rejected": 1.5480695962905884,
+ "logps/chosen": -151.12625122070312,
+ "logps/rejected": -171.24545288085938,
+ "loss": 0.6931126117706299,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.00693341251462698,
+ "rewards/margins": 9.057526767719537e-05,
+ "rewards/rejected": 0.006842837668955326,
+ "step": 100
+ },
+ {
+ "epoch": 0.08,
+ "eval_logits/chosen": 1.518183946609497,
+ "eval_logits/rejected": 1.5525447130203247,
+ "eval_logps/chosen": -153.46630859375,
+ "eval_logps/rejected": -147.95005798339844,
+ "eval_loss": 0.6926330327987671,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.008253541775047779,
+ "eval_rewards/margins": 0.0010449704714119434,
+ "eval_rewards/rejected": 0.007208569906651974,
+ "eval_runtime": 91.0812,
+ "eval_samples_per_second": 5.49,
+ "eval_steps_per_second": 2.745,
+ "step": 100
+ },
+ {
+ "epoch": 0.088,
+ "grad_norm": 1.3912373781204224,
+ "learning_rate": 2.906666666666667e-07,
+ "logits/chosen": 1.6896222829818726,
+ "logits/rejected": 1.6516921520233154,
+ "logps/chosen": -173.70030212402344,
+ "logps/rejected": -155.5503692626953,
+ "loss": 0.6926521778106689,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.010179834440350533,
+ "rewards/margins": 0.0010137532372027636,
+ "rewards/rejected": 0.009166081435978413,
+ "step": 110
+ },
+ {
+ "epoch": 0.096,
+ "grad_norm": 1.1518888473510742,
+ "learning_rate": 3.173333333333333e-07,
+ "logits/chosen": 1.5166860818862915,
+ "logits/rejected": 1.5136979818344116,
+ "logps/chosen": -136.39906311035156,
+ "logps/rejected": -140.86561584472656,
+ "loss": 0.6932918548583984,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": 0.00907944981008768,
+ "rewards/margins": -0.00026998287648893893,
+ "rewards/rejected": 0.009349432773888111,
+ "step": 120
+ },
+ {
+ "epoch": 0.104,
+ "grad_norm": 1.3213376998901367,
+ "learning_rate": 3.4399999999999996e-07,
+ "logits/chosen": 1.4598973989486694,
+ "logits/rejected": 1.6739647388458252,
+ "logps/chosen": -159.12461853027344,
+ "logps/rejected": -163.39346313476562,
+ "loss": 0.693086051940918,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.010477779433131218,
+ "rewards/margins": 0.00014539004769176245,
+ "rewards/rejected": 0.010332388803362846,
+ "step": 130
+ },
+ {
+ "epoch": 0.112,
+ "grad_norm": 0.8833863735198975,
+ "learning_rate": 3.7066666666666665e-07,
+ "logits/chosen": 1.3829412460327148,
+ "logits/rejected": 1.4100173711776733,
+ "logps/chosen": -132.7149658203125,
+ "logps/rejected": -126.85478210449219,
+ "loss": 0.6932382583618164,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": 0.010262183845043182,
+ "rewards/margins": -0.0001672578218858689,
+ "rewards/rejected": 0.010429441928863525,
+ "step": 140
+ },
+ {
+ "epoch": 0.12,
+ "grad_norm": 1.1472455263137817,
+ "learning_rate": 3.973333333333333e-07,
+ "logits/chosen": 1.4954869747161865,
+ "logits/rejected": 1.5857340097427368,
+ "logps/chosen": -165.10568237304688,
+ "logps/rejected": -158.97433471679688,
+ "loss": 0.6916103363037109,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.014177520759403706,
+ "rewards/margins": 0.003108317730948329,
+ "rewards/rejected": 0.011069201864302158,
+ "step": 150
+ },
+ {
+ "epoch": 0.128,
+ "grad_norm": 1.0628798007965088,
+ "learning_rate": 4.24e-07,
+ "logits/chosen": 1.6717243194580078,
+ "logits/rejected": 1.705092191696167,
+ "logps/chosen": -159.332275390625,
+ "logps/rejected": -147.21646118164062,
+ "loss": 0.6935629844665527,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": 0.016994481906294823,
+ "rewards/margins": -0.0007874655420891941,
+ "rewards/rejected": 0.017781946808099747,
+ "step": 160
+ },
+ {
+ "epoch": 0.136,
+ "grad_norm": 1.390810251235962,
+ "learning_rate": 4.506666666666666e-07,
+ "logits/chosen": 1.4719234704971313,
+ "logits/rejected": 1.499526023864746,
+ "logps/chosen": -148.67251586914062,
+ "logps/rejected": -161.58021545410156,
+ "loss": 0.6926599979400635,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.021239640191197395,
+ "rewards/margins": 0.0010346340714022517,
+ "rewards/rejected": 0.020205006003379822,
+ "step": 170
+ },
+ {
+ "epoch": 0.144,
+ "grad_norm": 1.139707326889038,
+ "learning_rate": 4.773333333333333e-07,
+ "logits/chosen": 1.4833905696868896,
+ "logits/rejected": 1.637584924697876,
+ "logps/chosen": -158.96731567382812,
+ "logps/rejected": -161.9830322265625,
+ "loss": 0.6919547557830811,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.019416773691773415,
+ "rewards/margins": 0.0024296878837049007,
+ "rewards/rejected": 0.016987085342407227,
+ "step": 180
+ },
+ {
+ "epoch": 0.152,
+ "grad_norm": 1.2029796838760376,
+ "learning_rate": 5.04e-07,
+ "logits/chosen": 1.5398027896881104,
+ "logits/rejected": 1.6678705215454102,
+ "logps/chosen": -143.00283813476562,
+ "logps/rejected": -144.89378356933594,
+ "loss": 0.692262840270996,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.018960200250148773,
+ "rewards/margins": 0.0018175147706642747,
+ "rewards/rejected": 0.017142686992883682,
+ "step": 190
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 1.0527437925338745,
+ "learning_rate": 5.306666666666665e-07,
+ "logits/chosen": 1.6087623834609985,
+ "logits/rejected": 1.5181543827056885,
+ "logps/chosen": -159.35073852539062,
+ "logps/rejected": -133.59890747070312,
+ "loss": 0.6924069881439209,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.01926913857460022,
+ "rewards/margins": 0.0015339398523792624,
+ "rewards/rejected": 0.0177351962774992,
+ "step": 200
+ },
+ {
+ "epoch": 0.16,
+ "eval_logits/chosen": 1.5193672180175781,
+ "eval_logits/rejected": 1.5534639358520508,
+ "eval_logps/chosen": -153.34857177734375,
+ "eval_logps/rejected": -147.8425750732422,
+ "eval_loss": 0.6921458840370178,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.020027177408337593,
+ "eval_rewards/margins": 0.0020714527927339077,
+ "eval_rewards/rejected": 0.017955724149942398,
+ "eval_runtime": 90.9762,
+ "eval_samples_per_second": 5.496,
+ "eval_steps_per_second": 2.748,
+ "step": 200
+ },
+ {
+ "epoch": 0.168,
+ "grad_norm": 1.1897239685058594,
+ "learning_rate": 5.573333333333333e-07,
+ "logits/chosen": 1.6267732381820679,
+ "logits/rejected": 1.5775268077850342,
+ "logps/chosen": -159.51437377929688,
+ "logps/rejected": -170.17762756347656,
+ "loss": 0.6925770759582519,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.02078000269830227,
+ "rewards/margins": 0.0012053751852363348,
+ "rewards/rejected": 0.01957462914288044,
+ "step": 210
+ },
+ {
+ "epoch": 0.176,
+ "grad_norm": 0.9995236396789551,
+ "learning_rate": 5.839999999999999e-07,
+ "logits/chosen": 1.5781934261322021,
+ "logits/rejected": 1.642251968383789,
+ "logps/chosen": -159.95309448242188,
+ "logps/rejected": -144.36058044433594,
+ "loss": 0.6931475639343262,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.023322442546486855,
+ "rewards/margins": 6.392716750269756e-05,
+ "rewards/rejected": 0.02325851283967495,
+ "step": 220
+ },
+ {
+ "epoch": 0.184,
+ "grad_norm": 0.8819260597229004,
+ "learning_rate": 6.106666666666666e-07,
+ "logits/chosen": 1.459991693496704,
+ "logits/rejected": 1.6060739755630493,
+ "logps/chosen": -162.04745483398438,
+ "logps/rejected": -139.5411834716797,
+ "loss": 0.6919463634490967,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.026052182540297508,
+ "rewards/margins": 0.002481849165633321,
+ "rewards/rejected": 0.023570330813527107,
+ "step": 230
+ },
+ {
+ "epoch": 0.192,
+ "grad_norm": 1.1055208444595337,
+ "learning_rate": 6.373333333333333e-07,
+ "logits/chosen": 1.558739185333252,
+ "logits/rejected": 1.5965116024017334,
+ "logps/chosen": -171.9166259765625,
+ "logps/rejected": -145.66433715820312,
+ "loss": 0.6912176132202148,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.033868782222270966,
+ "rewards/margins": 0.004017929546535015,
+ "rewards/rejected": 0.029850851744413376,
+ "step": 240
+ },
+ {
+ "epoch": 0.2,
+ "grad_norm": 1.1372733116149902,
+ "learning_rate": 6.64e-07,
+ "logits/chosen": 1.5322860479354858,
+ "logits/rejected": 1.4695795774459839,
+ "logps/chosen": -161.7803955078125,
+ "logps/rejected": -148.91949462890625,
+ "loss": 0.6940969944000244,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.034065864980220795,
+ "rewards/margins": -0.0016761379083618522,
+ "rewards/rejected": 0.035742007195949554,
+ "step": 250
+ },
+ {
+ "epoch": 0.208,
+ "grad_norm": 1.0917680263519287,
+ "learning_rate": 6.906666666666666e-07,
+ "logits/chosen": 1.4094547033309937,
+ "logits/rejected": 1.50186026096344,
+ "logps/chosen": -132.91455078125,
+ "logps/rejected": -146.79714965820312,
+ "loss": 0.6930203914642334,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.03089061938226223,
+ "rewards/margins": 0.00040971505222842097,
+ "rewards/rejected": 0.030480902642011642,
+ "step": 260
+ },
+ {
+ "epoch": 0.216,
+ "grad_norm": 1.0280259847640991,
+ "learning_rate": 7.173333333333333e-07,
+ "logits/chosen": 1.531285285949707,
+ "logits/rejected": 1.6163132190704346,
+ "logps/chosen": -146.902587890625,
+ "logps/rejected": -147.8347625732422,
+ "loss": 0.6944749355316162,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.028514739125967026,
+ "rewards/margins": -0.0025452517438679934,
+ "rewards/rejected": 0.0310599897056818,
+ "step": 270
+ },
+ {
+ "epoch": 0.224,
+ "grad_norm": 1.398708701133728,
+ "learning_rate": 7.44e-07,
+ "logits/chosen": 1.563852071762085,
+ "logits/rejected": 1.396703839302063,
+ "logps/chosen": -172.68153381347656,
+ "logps/rejected": -155.80215454101562,
+ "loss": 0.689476490020752,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03224216774106026,
+ "rewards/margins": 0.007473687641322613,
+ "rewards/rejected": 0.02476847730576992,
+ "step": 280
+ },
+ {
+ "epoch": 0.232,
+ "grad_norm": 1.0394141674041748,
+ "learning_rate": 7.706666666666667e-07,
+ "logits/chosen": 1.5581729412078857,
+ "logits/rejected": 1.5819021463394165,
+ "logps/chosen": -144.41465759277344,
+ "logps/rejected": -153.55406188964844,
+ "loss": 0.6927877902984619,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.03272664546966553,
+ "rewards/margins": 0.0008557910332456231,
+ "rewards/rejected": 0.031870849430561066,
+ "step": 290
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 1.2557185888290405,
+ "learning_rate": 7.973333333333333e-07,
+ "logits/chosen": 1.5062446594238281,
+ "logits/rejected": 1.5786281824111938,
+ "logps/chosen": -153.63558959960938,
+ "logps/rejected": -165.46617126464844,
+ "loss": 0.6897892951965332,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03914078325033188,
+ "rewards/margins": 0.006873926613479853,
+ "rewards/rejected": 0.032266855239868164,
+ "step": 300
+ },
+ {
+ "epoch": 0.24,
+ "eval_logits/chosen": 1.5237834453582764,
+ "eval_logits/rejected": 1.557741641998291,
+ "eval_logps/chosen": -153.19883728027344,
+ "eval_logps/rejected": -147.7100830078125,
+ "eval_loss": 0.6913501024246216,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.03499976545572281,
+ "eval_rewards/margins": 0.0037925743963569403,
+ "eval_rewards/rejected": 0.031207194551825523,
+ "eval_runtime": 90.9133,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 300
+ },
+ {
+ "epoch": 0.248,
+ "grad_norm": 1.1200144290924072,
+ "learning_rate": 8.24e-07,
+ "logits/chosen": 1.618949294090271,
+ "logits/rejected": 1.6625702381134033,
+ "logps/chosen": -141.8953857421875,
+ "logps/rejected": -156.6927032470703,
+ "loss": 0.694425106048584,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.03162423521280289,
+ "rewards/margins": -0.0023833750747144222,
+ "rewards/rejected": 0.034007612615823746,
+ "step": 310
+ },
+ {
+ "epoch": 0.256,
+ "grad_norm": 1.0823743343353271,
+ "learning_rate": 8.506666666666667e-07,
+ "logits/chosen": 1.5974153280258179,
+ "logits/rejected": 1.6237560510635376,
+ "logps/chosen": -139.2996368408203,
+ "logps/rejected": -141.94407653808594,
+ "loss": 0.6914949893951416,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.029445698484778404,
+ "rewards/margins": 0.0034329916816204786,
+ "rewards/rejected": 0.026012707501649857,
+ "step": 320
+ },
+ {
+ "epoch": 0.264,
+ "grad_norm": 1.1765722036361694,
+ "learning_rate": 8.773333333333332e-07,
+ "logits/chosen": 1.465215802192688,
+ "logits/rejected": 1.4882014989852905,
+ "logps/chosen": -158.4329833984375,
+ "logps/rejected": -183.94259643554688,
+ "loss": 0.6925934791564942,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.026302698999643326,
+ "rewards/margins": 0.0012350418837741017,
+ "rewards/rejected": 0.025067657232284546,
+ "step": 330
+ },
+ {
+ "epoch": 0.272,
+ "grad_norm": 1.0667294263839722,
+ "learning_rate": 9.039999999999999e-07,
+ "logits/chosen": 1.7538875341415405,
+ "logits/rejected": 1.6171789169311523,
+ "logps/chosen": -159.63790893554688,
+ "logps/rejected": -151.1880340576172,
+ "loss": 0.6922041416168213,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.026619985699653625,
+ "rewards/margins": 0.002031774492934346,
+ "rewards/rejected": 0.024588212370872498,
+ "step": 340
+ },
+ {
+ "epoch": 0.28,
+ "grad_norm": 1.2210006713867188,
+ "learning_rate": 9.306666666666666e-07,
+ "logits/chosen": 1.5815479755401611,
+ "logits/rejected": 1.710217833518982,
+ "logps/chosen": -167.21743774414062,
+ "logps/rejected": -196.77989196777344,
+ "loss": 0.6899692058563233,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.03636014088988304,
+ "rewards/margins": 0.006536121014505625,
+ "rewards/rejected": 0.029824022203683853,
+ "step": 350
+ },
+ {
+ "epoch": 0.288,
+ "grad_norm": 0.9623845219612122,
+ "learning_rate": 9.573333333333333e-07,
+ "logits/chosen": 1.4416790008544922,
+ "logits/rejected": 1.3618693351745605,
+ "logps/chosen": -163.27401733398438,
+ "logps/rejected": -169.3351287841797,
+ "loss": 0.6910821437835694,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.037200987339019775,
+ "rewards/margins": 0.004338678438216448,
+ "rewards/rejected": 0.032862309366464615,
+ "step": 360
+ },
+ {
+ "epoch": 0.296,
+ "grad_norm": 1.1005645990371704,
+ "learning_rate": 9.84e-07,
+ "logits/chosen": 1.6214616298675537,
+ "logits/rejected": 1.6572606563568115,
+ "logps/chosen": -162.95504760742188,
+ "logps/rejected": -154.91807556152344,
+ "loss": 0.6895030021667481,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.047198958694934845,
+ "rewards/margins": 0.0075958347879350185,
+ "rewards/rejected": 0.03960312157869339,
+ "step": 370
+ },
+ {
+ "epoch": 0.304,
+ "grad_norm": 1.1126338243484497,
+ "learning_rate": 9.988148148148148e-07,
+ "logits/chosen": 1.4842126369476318,
+ "logits/rejected": 1.410930871963501,
+ "logps/chosen": -143.09182739257812,
+ "logps/rejected": -153.48135375976562,
+ "loss": 0.6885370254516602,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.05975018069148064,
+ "rewards/margins": 0.00981281790882349,
+ "rewards/rejected": 0.04993735998868942,
+ "step": 380
+ },
+ {
+ "epoch": 0.312,
+ "grad_norm": 1.1146304607391357,
+ "learning_rate": 9.95851851851852e-07,
+ "logits/chosen": 1.608332633972168,
+ "logits/rejected": 1.5848472118377686,
+ "logps/chosen": -152.25250244140625,
+ "logps/rejected": -138.348876953125,
+ "loss": 0.6851807117462159,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.07630724459886551,
+ "rewards/margins": 0.016460279002785683,
+ "rewards/rejected": 0.059846967458724976,
+ "step": 390
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 1.2686601877212524,
+ "learning_rate": 9.928888888888889e-07,
+ "logits/chosen": 1.5840070247650146,
+ "logits/rejected": 1.5679785013198853,
+ "logps/chosen": -155.90219116210938,
+ "logps/rejected": -148.89549255371094,
+ "loss": 0.6891039371490478,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.08129279315471649,
+ "rewards/margins": 0.008997146971523762,
+ "rewards/rejected": 0.072295643389225,
+ "step": 400
+ },
+ {
+ "epoch": 0.32,
+ "eval_logits/chosen": 1.5358340740203857,
+ "eval_logits/rejected": 1.5689741373062134,
+ "eval_logps/chosen": -152.6895294189453,
+ "eval_logps/rejected": -147.23422241210938,
+ "eval_loss": 0.6900544166564941,
+ "eval_rewards/accuracies": 0.5419999957084656,
+ "eval_rewards/chosen": 0.08593228459358215,
+ "eval_rewards/margins": 0.0071397931315004826,
+ "eval_rewards/rejected": 0.0787924975156784,
+ "eval_runtime": 90.9072,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 400
+ },
+ {
+ "epoch": 0.328,
+ "grad_norm": 1.1653108596801758,
+ "learning_rate": 9.899259259259258e-07,
+ "logits/chosen": 1.5450490713119507,
+ "logits/rejected": 1.4428151845932007,
+ "logps/chosen": -148.61581420898438,
+ "logps/rejected": -188.90919494628906,
+ "loss": 0.6918989181518554,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0946170911192894,
+ "rewards/margins": 0.003271549940109253,
+ "rewards/rejected": 0.09134554117918015,
+ "step": 410
+ },
+ {
+ "epoch": 0.336,
+ "grad_norm": 1.3114937543869019,
+ "learning_rate": 9.86962962962963e-07,
+ "logits/chosen": 1.5342715978622437,
+ "logits/rejected": 1.5572118759155273,
+ "logps/chosen": -152.65264892578125,
+ "logps/rejected": -168.03384399414062,
+ "loss": 0.6972955703735352,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.08869597315788269,
+ "rewards/margins": -0.007436127867549658,
+ "rewards/rejected": 0.09613210707902908,
+ "step": 420
+ },
+ {
+ "epoch": 0.344,
+ "grad_norm": 1.0040781497955322,
+ "learning_rate": 9.84e-07,
+ "logits/chosen": 1.5979712009429932,
+ "logits/rejected": 1.6713664531707764,
+ "logps/chosen": -164.2566375732422,
+ "logps/rejected": -152.1428680419922,
+ "loss": 0.6897010326385498,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.08101107180118561,
+ "rewards/margins": 0.007684691343456507,
+ "rewards/rejected": 0.07332637906074524,
+ "step": 430
+ },
+ {
+ "epoch": 0.352,
+ "grad_norm": 1.0801892280578613,
+ "learning_rate": 9.81037037037037e-07,
+ "logits/chosen": 1.439635992050171,
+ "logits/rejected": 1.3282365798950195,
+ "logps/chosen": -157.23403930664062,
+ "logps/rejected": -146.54806518554688,
+ "loss": 0.6881664752960205,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.08112544566392899,
+ "rewards/margins": 0.010971959680318832,
+ "rewards/rejected": 0.07015348225831985,
+ "step": 440
+ },
+ {
+ "epoch": 0.36,
+ "grad_norm": 1.1139949560165405,
+ "learning_rate": 9.78074074074074e-07,
+ "logits/chosen": 1.6345123052597046,
+ "logits/rejected": 1.5876491069793701,
+ "logps/chosen": -157.6160125732422,
+ "logps/rejected": -160.02040100097656,
+ "loss": 0.6868602275848389,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.07638200372457504,
+ "rewards/margins": 0.013464884832501411,
+ "rewards/rejected": 0.06291711330413818,
+ "step": 450
+ },
+ {
+ "epoch": 0.368,
+ "grad_norm": 1.158035397529602,
+ "learning_rate": 9.751111111111112e-07,
+ "logits/chosen": 1.5430586338043213,
+ "logits/rejected": 1.6022913455963135,
+ "logps/chosen": -151.98919677734375,
+ "logps/rejected": -164.90565490722656,
+ "loss": 0.6891930580139161,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06889279186725616,
+ "rewards/margins": 0.008791089057922363,
+ "rewards/rejected": 0.0601017065346241,
+ "step": 460
+ },
+ {
+ "epoch": 0.376,
+ "grad_norm": 1.0903704166412354,
+ "learning_rate": 9.721481481481481e-07,
+ "logits/chosen": 1.313881754875183,
+ "logits/rejected": 1.3182194232940674,
+ "logps/chosen": -131.5592041015625,
+ "logps/rejected": -123.84442138671875,
+ "loss": 0.6889429569244385,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.08150745928287506,
+ "rewards/margins": 0.009235726669430733,
+ "rewards/rejected": 0.07227173447608948,
+ "step": 470
+ },
+ {
+ "epoch": 0.384,
+ "grad_norm": 1.0766969919204712,
+ "learning_rate": 9.69185185185185e-07,
+ "logits/chosen": 1.425756812095642,
+ "logits/rejected": 1.476854920387268,
+ "logps/chosen": -148.72279357910156,
+ "logps/rejected": -164.40951538085938,
+ "loss": 0.6854116439819335,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1009502187371254,
+ "rewards/margins": 0.0166789498180151,
+ "rewards/rejected": 0.08427127450704575,
+ "step": 480
+ },
+ {
+ "epoch": 0.392,
+ "grad_norm": 1.2078129053115845,
+ "learning_rate": 9.662222222222222e-07,
+ "logits/chosen": 1.5511282682418823,
+ "logits/rejected": 1.611383080482483,
+ "logps/chosen": -143.22006225585938,
+ "logps/rejected": -159.70176696777344,
+ "loss": 0.6968960762023926,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.08779363334178925,
+ "rewards/margins": -0.00598606513813138,
+ "rewards/rejected": 0.09377971291542053,
+ "step": 490
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.9106306433677673,
+ "learning_rate": 9.632592592592593e-07,
+ "logits/chosen": 1.4217239618301392,
+ "logits/rejected": 1.505446195602417,
+ "logps/chosen": -144.34109497070312,
+ "logps/rejected": -135.52430725097656,
+ "loss": 0.691196870803833,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06902458518743515,
+ "rewards/margins": 0.005059319548308849,
+ "rewards/rejected": 0.06396526843309402,
+ "step": 500
+ },
+ {
+ "epoch": 0.4,
+ "eval_logits/chosen": 1.5185325145721436,
+ "eval_logits/rejected": 1.5510345697402954,
+ "eval_logps/chosen": -152.7921905517578,
+ "eval_logps/rejected": -147.3463134765625,
+ "eval_loss": 0.6898502111434937,
+ "eval_rewards/accuracies": 0.5099999904632568,
+ "eval_rewards/chosen": 0.07566512376070023,
+ "eval_rewards/margins": 0.008083080872893333,
+ "eval_rewards/rejected": 0.06758204847574234,
+ "eval_runtime": 90.8294,
+ "eval_samples_per_second": 5.505,
+ "eval_steps_per_second": 2.752,
+ "step": 500
+ },
+ {
+ "epoch": 0.408,
+ "grad_norm": 1.1637762784957886,
+ "learning_rate": 9.602962962962962e-07,
+ "logits/chosen": 1.5965330600738525,
+ "logits/rejected": 1.5338464975357056,
+ "logps/chosen": -131.49871826171875,
+ "logps/rejected": -140.05995178222656,
+ "loss": 0.6868171215057373,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.08385131508111954,
+ "rewards/margins": 0.013950209133327007,
+ "rewards/rejected": 0.06990110129117966,
+ "step": 510
+ },
+ {
+ "epoch": 0.416,
+ "grad_norm": 1.229719638824463,
+ "learning_rate": 9.573333333333333e-07,
+ "logits/chosen": 1.7057275772094727,
+ "logits/rejected": 1.5898948907852173,
+ "logps/chosen": -166.4267120361328,
+ "logps/rejected": -172.22337341308594,
+ "loss": 0.6912094116210937,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.07427279651165009,
+ "rewards/margins": 0.004661654122173786,
+ "rewards/rejected": 0.06961113959550858,
+ "step": 520
+ },
+ {
+ "epoch": 0.424,
+ "grad_norm": 1.136383295059204,
+ "learning_rate": 9.543703703703705e-07,
+ "logits/chosen": 1.6207077503204346,
+ "logits/rejected": 1.5417520999908447,
+ "logps/chosen": -166.81509399414062,
+ "logps/rejected": -168.5408172607422,
+ "loss": 0.6772748470306397,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.0953373983502388,
+ "rewards/margins": 0.033697597682476044,
+ "rewards/rejected": 0.06163979694247246,
+ "step": 530
+ },
+ {
+ "epoch": 0.432,
+ "grad_norm": 1.0640541315078735,
+ "learning_rate": 9.514074074074074e-07,
+ "logits/chosen": 1.6825854778289795,
+ "logits/rejected": 1.6327041387557983,
+ "logps/chosen": -181.39956665039062,
+ "logps/rejected": -158.49647521972656,
+ "loss": 0.6883615493774414,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.10187923908233643,
+ "rewards/margins": 0.010873330757021904,
+ "rewards/rejected": 0.09100590646266937,
+ "step": 540
+ },
+ {
+ "epoch": 0.44,
+ "grad_norm": 1.477909803390503,
+ "learning_rate": 9.484444444444444e-07,
+ "logits/chosen": 1.5402686595916748,
+ "logits/rejected": 1.6487388610839844,
+ "logps/chosen": -151.53814697265625,
+ "logps/rejected": -167.8573455810547,
+ "loss": 0.6915213584899902,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09472338110208511,
+ "rewards/margins": 0.005475065670907497,
+ "rewards/rejected": 0.08924831449985504,
+ "step": 550
+ },
+ {
+ "epoch": 0.448,
+ "grad_norm": 1.0138317346572876,
+ "learning_rate": 9.454814814814814e-07,
+ "logits/chosen": 1.6286503076553345,
+ "logits/rejected": 1.5223101377487183,
+ "logps/chosen": -154.72079467773438,
+ "logps/rejected": -139.2535858154297,
+ "loss": 0.6835216522216797,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.09194488823413849,
+ "rewards/margins": 0.02140027843415737,
+ "rewards/rejected": 0.07054460793733597,
+ "step": 560
+ },
+ {
+ "epoch": 0.456,
+ "grad_norm": 0.896689772605896,
+ "learning_rate": 9.425185185185184e-07,
+ "logits/chosen": 1.4857146739959717,
+ "logits/rejected": 1.4369819164276123,
+ "logps/chosen": -158.31809997558594,
+ "logps/rejected": -152.33676147460938,
+ "loss": 0.6863360404968262,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.07895292341709137,
+ "rewards/margins": 0.016260787844657898,
+ "rewards/rejected": 0.06269213557243347,
+ "step": 570
+ },
+ {
+ "epoch": 0.464,
+ "grad_norm": 1.4772330522537231,
+ "learning_rate": 9.395555555555556e-07,
+ "logits/chosen": 1.315298318862915,
+ "logits/rejected": 1.4757534265518188,
+ "logps/chosen": -136.4202117919922,
+ "logps/rejected": -154.99362182617188,
+ "loss": 0.6916167259216308,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.04677204787731171,
+ "rewards/margins": 0.005649724509567022,
+ "rewards/rejected": 0.041122324764728546,
+ "step": 580
+ },
+ {
+ "epoch": 0.472,
+ "grad_norm": 0.9806157350540161,
+ "learning_rate": 9.365925925925926e-07,
+ "logits/chosen": 1.465427279472351,
+ "logits/rejected": 1.459496259689331,
+ "logps/chosen": -145.07470703125,
+ "logps/rejected": -131.71974182128906,
+ "loss": 0.6857181549072265,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.07979898899793625,
+ "rewards/margins": 0.016965944319963455,
+ "rewards/rejected": 0.0628330409526825,
+ "step": 590
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 1.4313396215438843,
+ "learning_rate": 9.336296296296295e-07,
+ "logits/chosen": 1.5190367698669434,
+ "logits/rejected": 1.563504934310913,
+ "logps/chosen": -132.0201416015625,
+ "logps/rejected": -158.89865112304688,
+ "loss": 0.6964848041534424,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.06544725596904755,
+ "rewards/margins": -0.004280097782611847,
+ "rewards/rejected": 0.0697273463010788,
+ "step": 600
+ },
+ {
+ "epoch": 0.48,
+ "eval_logits/chosen": 1.5031698942184448,
+ "eval_logits/rejected": 1.5348584651947021,
+ "eval_logps/chosen": -152.76596069335938,
+ "eval_logps/rejected": -147.3241729736328,
+ "eval_loss": 0.6903045773506165,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": 0.07828804850578308,
+ "eval_rewards/margins": 0.008492168970406055,
+ "eval_rewards/rejected": 0.0697958841919899,
+ "eval_runtime": 90.9881,
+ "eval_samples_per_second": 5.495,
+ "eval_steps_per_second": 2.748,
+ "step": 600
+ },
+ {
+ "epoch": 0.488,
+ "grad_norm": 0.9467533230781555,
+ "learning_rate": 9.306666666666666e-07,
+ "logits/chosen": 1.5242283344268799,
+ "logits/rejected": 1.5798314809799194,
+ "logps/chosen": -131.90048217773438,
+ "logps/rejected": -149.93222045898438,
+ "loss": 0.6899854183197022,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.08419154584407806,
+ "rewards/margins": 0.00830511748790741,
+ "rewards/rejected": 0.07588644325733185,
+ "step": 610
+ },
+ {
+ "epoch": 0.496,
+ "grad_norm": 1.8463492393493652,
+ "learning_rate": 9.277037037037037e-07,
+ "logits/chosen": 1.4965155124664307,
+ "logits/rejected": 1.6336272954940796,
+ "logps/chosen": -156.1863555908203,
+ "logps/rejected": -167.74075317382812,
+ "loss": 0.6977802276611328,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.05518193915486336,
+ "rewards/margins": -0.006808985956013203,
+ "rewards/rejected": 0.061990927904844284,
+ "step": 620
+ },
+ {
+ "epoch": 0.504,
+ "grad_norm": 1.278316617012024,
+ "learning_rate": 9.247407407407407e-07,
+ "logits/chosen": 1.5074065923690796,
+ "logits/rejected": 1.535696029663086,
+ "logps/chosen": -167.56777954101562,
+ "logps/rejected": -147.04922485351562,
+ "loss": 0.6806219100952149,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.0745314508676529,
+ "rewards/margins": 0.027104835957288742,
+ "rewards/rejected": 0.047426607459783554,
+ "step": 630
+ },
+ {
+ "epoch": 0.512,
+ "grad_norm": 1.0424679517745972,
+ "learning_rate": 9.217777777777778e-07,
+ "logits/chosen": 1.4299286603927612,
+ "logits/rejected": 1.4484076499938965,
+ "logps/chosen": -168.305908203125,
+ "logps/rejected": -152.61854553222656,
+ "loss": 0.6871783256530761,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07751411944627762,
+ "rewards/margins": 0.017045259475708008,
+ "rewards/rejected": 0.06046885997056961,
+ "step": 640
+ },
+ {
+ "epoch": 0.52,
+ "grad_norm": 1.1143999099731445,
+ "learning_rate": 9.188148148148148e-07,
+ "logits/chosen": 1.5520436763763428,
+ "logits/rejected": 1.531718134880066,
+ "logps/chosen": -154.92115783691406,
+ "logps/rejected": -126.78369140625,
+ "loss": 0.6889286518096924,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.0945608839392662,
+ "rewards/margins": 0.01080345083028078,
+ "rewards/rejected": 0.0837574377655983,
+ "step": 650
+ },
+ {
+ "epoch": 0.528,
+ "grad_norm": 1.2762326002120972,
+ "learning_rate": 9.158518518518517e-07,
+ "logits/chosen": 1.5747522115707397,
+ "logits/rejected": 1.4972833395004272,
+ "logps/chosen": -151.13914489746094,
+ "logps/rejected": -132.013916015625,
+ "loss": 0.6900454998016358,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.07303085923194885,
+ "rewards/margins": 0.008073865436017513,
+ "rewards/rejected": 0.06495700031518936,
+ "step": 660
+ },
+ {
+ "epoch": 0.536,
+ "grad_norm": 1.5207557678222656,
+ "learning_rate": 9.128888888888888e-07,
+ "logits/chosen": 1.6594665050506592,
+ "logits/rejected": 1.468492031097412,
+ "logps/chosen": -166.0108184814453,
+ "logps/rejected": -151.38555908203125,
+ "loss": 0.6934059143066407,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.09294265508651733,
+ "rewards/margins": 0.0021180796902626753,
+ "rewards/rejected": 0.09082455933094025,
+ "step": 670
+ },
+ {
+ "epoch": 0.544,
+ "grad_norm": 1.1073335409164429,
+ "learning_rate": 9.099259259259259e-07,
+ "logits/chosen": 1.5321934223175049,
+ "logits/rejected": 1.5313541889190674,
+ "logps/chosen": -144.5046844482422,
+ "logps/rejected": -133.05470275878906,
+ "loss": 0.6824594974517822,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.10782947391271591,
+ "rewards/margins": 0.0230915155261755,
+ "rewards/rejected": 0.08473796397447586,
+ "step": 680
+ },
+ {
+ "epoch": 0.552,
+ "grad_norm": 1.096970558166504,
+ "learning_rate": 9.069629629629629e-07,
+ "logits/chosen": 1.438305139541626,
+ "logits/rejected": 1.4338308572769165,
+ "logps/chosen": -173.1280517578125,
+ "logps/rejected": -160.20950317382812,
+ "loss": 0.6878445625305176,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09615986794233322,
+ "rewards/margins": 0.012350928969681263,
+ "rewards/rejected": 0.08380892872810364,
+ "step": 690
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 1.3664878606796265,
+ "learning_rate": 9.039999999999999e-07,
+ "logits/chosen": 1.530369758605957,
+ "logits/rejected": 1.5520622730255127,
+ "logps/chosen": -159.60740661621094,
+ "logps/rejected": -137.77352905273438,
+ "loss": 0.6991429805755616,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07249583303928375,
+ "rewards/margins": -0.009333941154181957,
+ "rewards/rejected": 0.08182977139949799,
+ "step": 700
+ },
+ {
+ "epoch": 0.56,
+ "eval_logits/chosen": 1.5136796236038208,
+ "eval_logits/rejected": 1.5456002950668335,
+ "eval_logps/chosen": -152.5748748779297,
+ "eval_logps/rejected": -147.17152404785156,
+ "eval_loss": 0.688511848449707,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.09739598631858826,
+ "eval_rewards/margins": 0.012335418723523617,
+ "eval_rewards/rejected": 0.08506056666374207,
+ "eval_runtime": 90.8734,
+ "eval_samples_per_second": 5.502,
+ "eval_steps_per_second": 2.751,
+ "step": 700
+ },
+ {
+ "epoch": 0.568,
+ "grad_norm": 1.224107265472412,
+ "learning_rate": 9.010370370370371e-07,
+ "logits/chosen": 1.6023967266082764,
+ "logits/rejected": 1.558224081993103,
+ "logps/chosen": -148.47120666503906,
+ "logps/rejected": -163.5648651123047,
+ "loss": 0.6865221500396729,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.12494877725839615,
+ "rewards/margins": 0.01545787788927555,
+ "rewards/rejected": 0.10949089378118515,
+ "step": 710
+ },
+ {
+ "epoch": 0.576,
+ "grad_norm": 1.2135164737701416,
+ "learning_rate": 8.98074074074074e-07,
+ "logits/chosen": 1.5902750492095947,
+ "logits/rejected": 1.7012746334075928,
+ "logps/chosen": -139.7012939453125,
+ "logps/rejected": -135.42225646972656,
+ "loss": 0.7014039039611817,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.09102506935596466,
+ "rewards/margins": -0.014695463702082634,
+ "rewards/rejected": 0.10572052001953125,
+ "step": 720
+ },
+ {
+ "epoch": 0.584,
+ "grad_norm": 1.1953649520874023,
+ "learning_rate": 8.95111111111111e-07,
+ "logits/chosen": 1.6540731191635132,
+ "logits/rejected": 1.6386349201202393,
+ "logps/chosen": -153.12847900390625,
+ "logps/rejected": -149.66319274902344,
+ "loss": 0.6952126502990723,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.1124584898352623,
+ "rewards/margins": -0.0012063594767823815,
+ "rewards/rejected": 0.11366486549377441,
+ "step": 730
+ },
+ {
+ "epoch": 0.592,
+ "grad_norm": 1.147875428199768,
+ "learning_rate": 8.921481481481481e-07,
+ "logits/chosen": 1.5578330755233765,
+ "logits/rejected": 1.4689290523529053,
+ "logps/chosen": -156.93788146972656,
+ "logps/rejected": -134.1832733154297,
+ "loss": 0.6919830799102783,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.09214536100625992,
+ "rewards/margins": 0.004291228950023651,
+ "rewards/rejected": 0.08785412460565567,
+ "step": 740
+ },
+ {
+ "epoch": 0.6,
+ "grad_norm": 1.21685791015625,
+ "learning_rate": 8.891851851851852e-07,
+ "logits/chosen": 1.4241983890533447,
+ "logits/rejected": 1.4537718296051025,
+ "logps/chosen": -136.9163055419922,
+ "logps/rejected": -158.44491577148438,
+ "loss": 0.691422986984253,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.09173480421304703,
+ "rewards/margins": 0.00533579895272851,
+ "rewards/rejected": 0.08639900386333466,
+ "step": 750
+ },
+ {
+ "epoch": 0.608,
+ "grad_norm": 1.3629519939422607,
+ "learning_rate": 8.862222222222222e-07,
+ "logits/chosen": 1.565716028213501,
+ "logits/rejected": 1.3824570178985596,
+ "logps/chosen": -138.8521728515625,
+ "logps/rejected": -129.51028442382812,
+ "loss": 0.6931031227111817,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0763697624206543,
+ "rewards/margins": 0.0021251302678138018,
+ "rewards/rejected": 0.07424463331699371,
+ "step": 760
+ },
+ {
+ "epoch": 0.616,
+ "grad_norm": 1.1030747890472412,
+ "learning_rate": 8.832592592592593e-07,
+ "logits/chosen": 1.5781702995300293,
+ "logits/rejected": 1.5428775548934937,
+ "logps/chosen": -141.01480102539062,
+ "logps/rejected": -136.47189331054688,
+ "loss": 0.6777657032012939,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.10366280376911163,
+ "rewards/margins": 0.0333605632185936,
+ "rewards/rejected": 0.07030224055051804,
+ "step": 770
+ },
+ {
+ "epoch": 0.624,
+ "grad_norm": 1.3228758573532104,
+ "learning_rate": 8.802962962962962e-07,
+ "logits/chosen": 1.451596736907959,
+ "logits/rejected": 1.5489904880523682,
+ "logps/chosen": -153.74618530273438,
+ "logps/rejected": -159.26333618164062,
+ "loss": 0.6870471000671386,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.10403706133365631,
+ "rewards/margins": 0.014494165778160095,
+ "rewards/rejected": 0.08954289555549622,
+ "step": 780
+ },
+ {
+ "epoch": 0.632,
+ "grad_norm": 1.3267385959625244,
+ "learning_rate": 8.773333333333332e-07,
+ "logits/chosen": 1.4153831005096436,
+ "logits/rejected": 1.4497296810150146,
+ "logps/chosen": -148.05368041992188,
+ "logps/rejected": -151.3020477294922,
+ "loss": 0.6877872943878174,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.07300253212451935,
+ "rewards/margins": 0.013981563039124012,
+ "rewards/rejected": 0.05902096629142761,
+ "step": 790
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 1.2227483987808228,
+ "learning_rate": 8.743703703703703e-07,
+ "logits/chosen": 1.6773914098739624,
+ "logits/rejected": 1.6918518543243408,
+ "logps/chosen": -180.06863403320312,
+ "logps/rejected": -194.30752563476562,
+ "loss": 0.7015917778015137,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.07480724155902863,
+ "rewards/margins": -0.013628068380057812,
+ "rewards/rejected": 0.08843531459569931,
+ "step": 800
+ },
+ {
+ "epoch": 0.64,
+ "eval_logits/chosen": 1.5051140785217285,
+ "eval_logits/rejected": 1.5367870330810547,
+ "eval_logps/chosen": -152.8026885986328,
+ "eval_logps/rejected": -147.39437866210938,
+ "eval_loss": 0.688633918762207,
+ "eval_rewards/accuracies": 0.5220000147819519,
+ "eval_rewards/chosen": 0.07461711764335632,
+ "eval_rewards/margins": 0.011839868500828743,
+ "eval_rewards/rejected": 0.06277725100517273,
+ "eval_runtime": 90.9206,
+ "eval_samples_per_second": 5.499,
+ "eval_steps_per_second": 2.75,
+ "step": 800
+ },
+ {
+ "epoch": 0.648,
+ "grad_norm": 1.209770679473877,
+ "learning_rate": 8.714074074074074e-07,
+ "logits/chosen": 1.3785041570663452,
+ "logits/rejected": 1.60085928440094,
+ "logps/chosen": -114.5325927734375,
+ "logps/rejected": -151.88900756835938,
+ "loss": 0.6890413761138916,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0640551745891571,
+ "rewards/margins": 0.010126596316695213,
+ "rewards/rejected": 0.05392857640981674,
+ "step": 810
+ },
+ {
+ "epoch": 0.656,
+ "grad_norm": 1.0250163078308105,
+ "learning_rate": 8.684444444444444e-07,
+ "logits/chosen": 1.5827982425689697,
+ "logits/rejected": 1.6239618062973022,
+ "logps/chosen": -149.01904296875,
+ "logps/rejected": -141.9372100830078,
+ "loss": 0.6728014469146728,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.0830039232969284,
+ "rewards/margins": 0.043090131133794785,
+ "rewards/rejected": 0.03991378843784332,
+ "step": 820
+ },
+ {
+ "epoch": 0.664,
+ "grad_norm": 1.0627655982971191,
+ "learning_rate": 8.654814814814814e-07,
+ "logits/chosen": 1.501873254776001,
+ "logits/rejected": 1.537386417388916,
+ "logps/chosen": -138.75169372558594,
+ "logps/rejected": -134.00970458984375,
+ "loss": 0.6801187992095947,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0640982910990715,
+ "rewards/margins": 0.02855527400970459,
+ "rewards/rejected": 0.03554301708936691,
+ "step": 830
+ },
+ {
+ "epoch": 0.672,
+ "grad_norm": 1.2453433275222778,
+ "learning_rate": 8.625185185185186e-07,
+ "logits/chosen": 1.4766005277633667,
+ "logits/rejected": 1.5185787677764893,
+ "logps/chosen": -148.41250610351562,
+ "logps/rejected": -157.73989868164062,
+ "loss": 0.6788079738616943,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.06551454961299896,
+ "rewards/margins": 0.03301671892404556,
+ "rewards/rejected": 0.0324978306889534,
+ "step": 840
+ },
+ {
+ "epoch": 0.68,
+ "grad_norm": 1.191698670387268,
+ "learning_rate": 8.595555555555555e-07,
+ "logits/chosen": 1.555551528930664,
+ "logits/rejected": 1.5438499450683594,
+ "logps/chosen": -160.77488708496094,
+ "logps/rejected": -141.9876251220703,
+ "loss": 0.6801597595214843,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.053498052060604095,
+ "rewards/margins": 0.028470590710639954,
+ "rewards/rejected": 0.025027472525835037,
+ "step": 850
+ },
+ {
+ "epoch": 0.688,
+ "grad_norm": 0.9382092356681824,
+ "learning_rate": 8.565925925925925e-07,
+ "logits/chosen": 1.5482738018035889,
+ "logits/rejected": 1.5833700895309448,
+ "logps/chosen": -139.68118286132812,
+ "logps/rejected": -130.05947875976562,
+ "loss": 0.6825921535491943,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.06382572650909424,
+ "rewards/margins": 0.023111844435334206,
+ "rewards/rejected": 0.04071388021111488,
+ "step": 860
+ },
+ {
+ "epoch": 0.696,
+ "grad_norm": 1.2248382568359375,
+ "learning_rate": 8.536296296296296e-07,
+ "logits/chosen": 1.4529926776885986,
+ "logits/rejected": 1.4423902034759521,
+ "logps/chosen": -154.4588165283203,
+ "logps/rejected": -153.44949340820312,
+ "loss": 0.6915592193603516,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.03207828477025032,
+ "rewards/margins": 0.006785523146390915,
+ "rewards/rejected": 0.025292763486504555,
+ "step": 870
+ },
+ {
+ "epoch": 0.704,
+ "grad_norm": 1.2174893617630005,
+ "learning_rate": 8.506666666666667e-07,
+ "logits/chosen": 1.4073113203048706,
+ "logits/rejected": 1.668140172958374,
+ "logps/chosen": -145.18460083007812,
+ "logps/rejected": -157.0777587890625,
+ "loss": 0.6922356605529785,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.049450717866420746,
+ "rewards/margins": 0.004049716051667929,
+ "rewards/rejected": 0.04540099948644638,
+ "step": 880
+ },
+ {
+ "epoch": 0.712,
+ "grad_norm": 1.3186850547790527,
+ "learning_rate": 8.477037037037037e-07,
+ "logits/chosen": 1.2923892736434937,
+ "logits/rejected": 1.491051435470581,
+ "logps/chosen": -149.03018188476562,
+ "logps/rejected": -161.61715698242188,
+ "loss": 0.6841150760650635,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.038961172103881836,
+ "rewards/margins": 0.020759906619787216,
+ "rewards/rejected": 0.01820126175880432,
+ "step": 890
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 1.1123124361038208,
+ "learning_rate": 8.447407407407407e-07,
+ "logits/chosen": 1.546547293663025,
+ "logits/rejected": 1.682992696762085,
+ "logps/chosen": -166.2056884765625,
+ "logps/rejected": -180.73330688476562,
+ "loss": 0.6826215744018554,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.07279330492019653,
+ "rewards/margins": 0.025264525786042213,
+ "rewards/rejected": 0.04752878472208977,
+ "step": 900
+ },
+ {
+ "epoch": 0.72,
+ "eval_logits/chosen": 1.4819762706756592,
+ "eval_logits/rejected": 1.513027548789978,
+ "eval_logps/chosen": -153.1695556640625,
+ "eval_logps/rejected": -147.7560577392578,
+ "eval_loss": 0.689373254776001,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": 0.03792775049805641,
+ "eval_rewards/margins": 0.011320074088871479,
+ "eval_rewards/rejected": 0.026607677340507507,
+ "eval_runtime": 90.9603,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 900
+ },
+ {
+ "epoch": 0.728,
+ "grad_norm": 1.1507582664489746,
+ "learning_rate": 8.417777777777777e-07,
+ "logits/chosen": 1.5831308364868164,
+ "logits/rejected": 1.3737132549285889,
+ "logps/chosen": -140.00253295898438,
+ "logps/rejected": -138.45155334472656,
+ "loss": 0.6873391628265381,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.05066479369997978,
+ "rewards/margins": 0.015466553159058094,
+ "rewards/rejected": 0.03519824147224426,
+ "step": 910
+ },
+ {
+ "epoch": 0.736,
+ "grad_norm": 1.415456771850586,
+ "learning_rate": 8.388148148148147e-07,
+ "logits/chosen": 1.3000686168670654,
+ "logits/rejected": 1.2642180919647217,
+ "logps/chosen": -160.14027404785156,
+ "logps/rejected": -163.32827758789062,
+ "loss": 0.687142276763916,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.06618930399417877,
+ "rewards/margins": 0.014807088300585747,
+ "rewards/rejected": 0.05138222128152847,
+ "step": 920
+ },
+ {
+ "epoch": 0.744,
+ "grad_norm": 1.3228634595870972,
+ "learning_rate": 8.358518518518518e-07,
+ "logits/chosen": 1.5204170942306519,
+ "logits/rejected": 1.6129217147827148,
+ "logps/chosen": -157.99832153320312,
+ "logps/rejected": -162.09567260742188,
+ "loss": 0.6790886878967285,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.05940447002649307,
+ "rewards/margins": 0.03161252290010452,
+ "rewards/rejected": 0.027791941538453102,
+ "step": 930
+ },
+ {
+ "epoch": 0.752,
+ "grad_norm": 1.048064947128296,
+ "learning_rate": 8.328888888888889e-07,
+ "logits/chosen": 1.5515320301055908,
+ "logits/rejected": 1.676857352256775,
+ "logps/chosen": -148.4853515625,
+ "logps/rejected": -174.4320526123047,
+ "loss": 0.6935453414916992,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.010225490666925907,
+ "rewards/margins": 0.0021693313028663397,
+ "rewards/rejected": 0.008056161925196648,
+ "step": 940
+ },
+ {
+ "epoch": 0.76,
+ "grad_norm": 1.247799038887024,
+ "learning_rate": 8.299259259259259e-07,
+ "logits/chosen": 1.384231686592102,
+ "logits/rejected": 1.4611561298370361,
+ "logps/chosen": -140.81443786621094,
+ "logps/rejected": -153.81326293945312,
+ "loss": 0.6805226325988769,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.053223587572574615,
+ "rewards/margins": 0.02933613583445549,
+ "rewards/rejected": 0.023887457326054573,
+ "step": 950
+ },
+ {
+ "epoch": 0.768,
+ "grad_norm": 1.1611264944076538,
+ "learning_rate": 8.269629629629629e-07,
+ "logits/chosen": 1.4987528324127197,
+ "logits/rejected": 1.3684321641921997,
+ "logps/chosen": -135.81056213378906,
+ "logps/rejected": -119.05133056640625,
+ "loss": 0.6954316616058349,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.0197703056037426,
+ "rewards/margins": -0.0012637012405321002,
+ "rewards/rejected": 0.021034007892012596,
+ "step": 960
+ },
+ {
+ "epoch": 0.776,
+ "grad_norm": 0.964948832988739,
+ "learning_rate": 8.24e-07,
+ "logits/chosen": 1.612053632736206,
+ "logits/rejected": 1.7114235162734985,
+ "logps/chosen": -153.56137084960938,
+ "logps/rejected": -149.17323303222656,
+ "loss": 0.6916662693023682,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.05082285404205322,
+ "rewards/margins": 0.0056606316938996315,
+ "rewards/rejected": 0.04516221210360527,
+ "step": 970
+ },
+ {
+ "epoch": 0.784,
+ "grad_norm": 1.409449815750122,
+ "learning_rate": 8.21037037037037e-07,
+ "logits/chosen": 1.384060025215149,
+ "logits/rejected": 1.316361904144287,
+ "logps/chosen": -168.75360107421875,
+ "logps/rejected": -141.79537963867188,
+ "loss": 0.6845529556274415,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.0886571854352951,
+ "rewards/margins": 0.02084263041615486,
+ "rewards/rejected": 0.06781454384326935,
+ "step": 980
+ },
+ {
+ "epoch": 0.792,
+ "grad_norm": 1.0246562957763672,
+ "learning_rate": 8.18074074074074e-07,
+ "logits/chosen": 1.4986507892608643,
+ "logits/rejected": 1.571103811264038,
+ "logps/chosen": -165.22254943847656,
+ "logps/rejected": -168.63540649414062,
+ "loss": 0.6790967464447022,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.07517905533313751,
+ "rewards/margins": 0.03272219002246857,
+ "rewards/rejected": 0.04245685786008835,
+ "step": 990
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 1.014426827430725,
+ "learning_rate": 8.15111111111111e-07,
+ "logits/chosen": 1.4708755016326904,
+ "logits/rejected": 1.4550955295562744,
+ "logps/chosen": -149.49874877929688,
+ "logps/rejected": -125.0966567993164,
+ "loss": 0.6892098903656005,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.07825140655040741,
+ "rewards/margins": 0.01007872074842453,
+ "rewards/rejected": 0.06817268580198288,
+ "step": 1000
+ },
+ {
+ "epoch": 0.8,
+ "eval_logits/chosen": 1.4972777366638184,
+ "eval_logits/rejected": 1.5283576250076294,
+ "eval_logps/chosen": -152.8240966796875,
+ "eval_logps/rejected": -147.44175720214844,
+ "eval_loss": 0.6880542039871216,
+ "eval_rewards/accuracies": 0.5419999957084656,
+ "eval_rewards/chosen": 0.07247376441955566,
+ "eval_rewards/margins": 0.01443479023873806,
+ "eval_rewards/rejected": 0.058038972318172455,
+ "eval_runtime": 90.827,
+ "eval_samples_per_second": 5.505,
+ "eval_steps_per_second": 2.752,
+ "step": 1000
+ },
+ {
+ "epoch": 0.808,
+ "grad_norm": 1.1155726909637451,
+ "learning_rate": 8.121481481481482e-07,
+ "logits/chosen": 1.547481894493103,
+ "logits/rejected": 1.5461527109146118,
+ "logps/chosen": -154.89791870117188,
+ "logps/rejected": -142.05111694335938,
+ "loss": 0.6809016704559326,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.10093016922473907,
+ "rewards/margins": 0.027038773521780968,
+ "rewards/rejected": 0.07389138638973236,
+ "step": 1010
+ },
+ {
+ "epoch": 0.816,
+ "grad_norm": 1.6716786623001099,
+ "learning_rate": 8.091851851851852e-07,
+ "logits/chosen": 1.3954919576644897,
+ "logits/rejected": 1.5088036060333252,
+ "logps/chosen": -143.11846923828125,
+ "logps/rejected": -146.41482543945312,
+ "loss": 0.6849985122680664,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.05970728397369385,
+ "rewards/margins": 0.021999523043632507,
+ "rewards/rejected": 0.03770776465535164,
+ "step": 1020
+ },
+ {
+ "epoch": 0.824,
+ "grad_norm": 1.308182716369629,
+ "learning_rate": 8.062222222222221e-07,
+ "logits/chosen": 1.532488465309143,
+ "logits/rejected": 1.3316718339920044,
+ "logps/chosen": -163.06858825683594,
+ "logps/rejected": -136.57989501953125,
+ "loss": 0.6789252281188964,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.11086118221282959,
+ "rewards/margins": 0.033117204904556274,
+ "rewards/rejected": 0.07774396985769272,
+ "step": 1030
+ },
+ {
+ "epoch": 0.832,
+ "grad_norm": 1.1019353866577148,
+ "learning_rate": 8.032592592592592e-07,
+ "logits/chosen": 1.4798121452331543,
+ "logits/rejected": 1.5203083753585815,
+ "logps/chosen": -140.3281707763672,
+ "logps/rejected": -141.06216430664062,
+ "loss": 0.6724937915802002,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.1202714815735817,
+ "rewards/margins": 0.046712182462215424,
+ "rewards/rejected": 0.07355931401252747,
+ "step": 1040
+ },
+ {
+ "epoch": 0.84,
+ "grad_norm": 1.2966418266296387,
+ "learning_rate": 8.002962962962963e-07,
+ "logits/chosen": 1.3034439086914062,
+ "logits/rejected": 1.4363505840301514,
+ "logps/chosen": -142.1992645263672,
+ "logps/rejected": -132.0046844482422,
+ "loss": 0.6848338603973388,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.11126589775085449,
+ "rewards/margins": 0.019870441406965256,
+ "rewards/rejected": 0.09139545261859894,
+ "step": 1050
+ },
+ {
+ "epoch": 0.848,
+ "grad_norm": 1.0321130752563477,
+ "learning_rate": 7.973333333333333e-07,
+ "logits/chosen": 1.3257265090942383,
+ "logits/rejected": 1.4649587869644165,
+ "logps/chosen": -112.98075103759766,
+ "logps/rejected": -127.4059829711914,
+ "loss": 0.6679783344268799,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.17311830818653107,
+ "rewards/margins": 0.05631983280181885,
+ "rewards/rejected": 0.11679847538471222,
+ "step": 1060
+ },
+ {
+ "epoch": 0.856,
+ "grad_norm": 1.2037222385406494,
+ "learning_rate": 7.943703703703704e-07,
+ "logits/chosen": 1.3766987323760986,
+ "logits/rejected": 1.2802103757858276,
+ "logps/chosen": -154.8142547607422,
+ "logps/rejected": -148.680908203125,
+ "loss": 0.668931531906128,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1449051946401596,
+ "rewards/margins": 0.05721765011548996,
+ "rewards/rejected": 0.08768755197525024,
+ "step": 1070
+ },
+ {
+ "epoch": 0.864,
+ "grad_norm": 0.9230766296386719,
+ "learning_rate": 7.914074074074074e-07,
+ "logits/chosen": 1.447328805923462,
+ "logits/rejected": 1.5235106945037842,
+ "logps/chosen": -150.21812438964844,
+ "logps/rejected": -123.29923248291016,
+ "loss": 0.6716424465179444,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.16066041588783264,
+ "rewards/margins": 0.04824484512209892,
+ "rewards/rejected": 0.11241557449102402,
+ "step": 1080
+ },
+ {
+ "epoch": 0.872,
+ "grad_norm": 1.0423877239227295,
+ "learning_rate": 7.884444444444443e-07,
+ "logits/chosen": 1.6920299530029297,
+ "logits/rejected": 1.6123542785644531,
+ "logps/chosen": -175.1676025390625,
+ "logps/rejected": -185.07223510742188,
+ "loss": 0.6723968505859375,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.16490009427070618,
+ "rewards/margins": 0.046577490866184235,
+ "rewards/rejected": 0.11832261085510254,
+ "step": 1090
+ },
+ {
+ "epoch": 0.88,
+ "grad_norm": 1.4142779111862183,
+ "learning_rate": 7.854814814814814e-07,
+ "logits/chosen": 1.453111171722412,
+ "logits/rejected": 1.4981731176376343,
+ "logps/chosen": -139.9765167236328,
+ "logps/rejected": -153.63058471679688,
+ "loss": 0.6872457027435303,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.13096000254154205,
+ "rewards/margins": 0.01702108606696129,
+ "rewards/rejected": 0.11393891274929047,
+ "step": 1100
+ },
+ {
+ "epoch": 0.88,
+ "eval_logits/chosen": 1.5076195001602173,
+ "eval_logits/rejected": 1.5373249053955078,
+ "eval_logps/chosen": -152.1697998046875,
+ "eval_logps/rejected": -146.83502197265625,
+ "eval_loss": 0.6873304843902588,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.13790392875671387,
+ "eval_rewards/margins": 0.019191108644008636,
+ "eval_rewards/rejected": 0.11871281266212463,
+ "eval_runtime": 91.0157,
+ "eval_samples_per_second": 5.494,
+ "eval_steps_per_second": 2.747,
+ "step": 1100
+ },
+ {
+ "epoch": 0.888,
+ "grad_norm": 1.413996696472168,
+ "learning_rate": 7.825185185185185e-07,
+ "logits/chosen": 1.5183905363082886,
+ "logits/rejected": 1.4942539930343628,
+ "logps/chosen": -156.383544921875,
+ "logps/rejected": -145.61398315429688,
+ "loss": 0.6810199737548828,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.14243564009666443,
+ "rewards/margins": 0.03136260434985161,
+ "rewards/rejected": 0.11107305437326431,
+ "step": 1110
+ },
+ {
+ "epoch": 0.896,
+ "grad_norm": 1.0046707391738892,
+ "learning_rate": 7.795555555555555e-07,
+ "logits/chosen": 1.4982213973999023,
+ "logits/rejected": 1.5465660095214844,
+ "logps/chosen": -156.79293823242188,
+ "logps/rejected": -150.77853393554688,
+ "loss": 0.6808318614959716,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1551930010318756,
+ "rewards/margins": 0.028187204152345657,
+ "rewards/rejected": 0.12700578570365906,
+ "step": 1120
+ },
+ {
+ "epoch": 0.904,
+ "grad_norm": 1.125625729560852,
+ "learning_rate": 7.765925925925925e-07,
+ "logits/chosen": 1.6053575277328491,
+ "logits/rejected": 1.47335946559906,
+ "logps/chosen": -148.05690002441406,
+ "logps/rejected": -135.36598205566406,
+ "loss": 0.6889908790588379,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.15128108859062195,
+ "rewards/margins": 0.015811312943696976,
+ "rewards/rejected": 0.13546979427337646,
+ "step": 1130
+ },
+ {
+ "epoch": 0.912,
+ "grad_norm": 1.0933659076690674,
+ "learning_rate": 7.736296296296297e-07,
+ "logits/chosen": 1.5862042903900146,
+ "logits/rejected": 1.5827430486679077,
+ "logps/chosen": -156.10145568847656,
+ "logps/rejected": -142.94851684570312,
+ "loss": 0.6753206253051758,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1547432243824005,
+ "rewards/margins": 0.04229574277997017,
+ "rewards/rejected": 0.11244747787714005,
+ "step": 1140
+ },
+ {
+ "epoch": 0.92,
+ "grad_norm": 1.0774686336517334,
+ "learning_rate": 7.706666666666667e-07,
+ "logits/chosen": 1.2369152307510376,
+ "logits/rejected": 1.2727023363113403,
+ "logps/chosen": -139.23147583007812,
+ "logps/rejected": -130.53038024902344,
+ "loss": 0.6967329025268555,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": 0.10801626741886139,
+ "rewards/margins": 0.004670143127441406,
+ "rewards/rejected": 0.10334610939025879,
+ "step": 1150
+ },
+ {
+ "epoch": 0.928,
+ "grad_norm": 1.1111780405044556,
+ "learning_rate": 7.677037037037036e-07,
+ "logits/chosen": 1.501250982284546,
+ "logits/rejected": 1.4272868633270264,
+ "logps/chosen": -127.09858703613281,
+ "logps/rejected": -114.9041748046875,
+ "loss": 0.6853152275085449,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.1366150826215744,
+ "rewards/margins": 0.021945688873529434,
+ "rewards/rejected": 0.11466939747333527,
+ "step": 1160
+ },
+ {
+ "epoch": 0.936,
+ "grad_norm": 1.5593854188919067,
+ "learning_rate": 7.647407407407407e-07,
+ "logits/chosen": 1.727927803993225,
+ "logits/rejected": 1.6510727405548096,
+ "logps/chosen": -156.70993041992188,
+ "logps/rejected": -167.07266235351562,
+ "loss": 0.6884279251098633,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.14902853965759277,
+ "rewards/margins": 0.01685408689081669,
+ "rewards/rejected": 0.13217447698116302,
+ "step": 1170
+ },
+ {
+ "epoch": 0.944,
+ "grad_norm": 2.0064918994903564,
+ "learning_rate": 7.617777777777778e-07,
+ "logits/chosen": 1.6262975931167603,
+ "logits/rejected": 1.58951997756958,
+ "logps/chosen": -186.59210205078125,
+ "logps/rejected": -151.2965850830078,
+ "loss": 0.6906405448913574,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.14898104965686798,
+ "rewards/margins": 0.013303476385772228,
+ "rewards/rejected": 0.13567757606506348,
+ "step": 1180
+ },
+ {
+ "epoch": 0.952,
+ "grad_norm": 1.209291696548462,
+ "learning_rate": 7.588148148148148e-07,
+ "logits/chosen": 1.639369249343872,
+ "logits/rejected": 1.6400636434555054,
+ "logps/chosen": -166.24684143066406,
+ "logps/rejected": -196.35305786132812,
+ "loss": 0.6789956092834473,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1348457634449005,
+ "rewards/margins": 0.03475186973810196,
+ "rewards/rejected": 0.10009388625621796,
+ "step": 1190
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 1.209230899810791,
+ "learning_rate": 7.558518518518519e-07,
+ "logits/chosen": 1.542942762374878,
+ "logits/rejected": 1.5128916501998901,
+ "logps/chosen": -138.994384765625,
+ "logps/rejected": -145.42880249023438,
+ "loss": 0.6936234951019287,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.13895569741725922,
+ "rewards/margins": 0.005582497920840979,
+ "rewards/rejected": 0.1333732008934021,
+ "step": 1200
+ },
+ {
+ "epoch": 0.96,
+ "eval_logits/chosen": 1.5055651664733887,
+ "eval_logits/rejected": 1.5351229906082153,
+ "eval_logps/chosen": -152.2256622314453,
+ "eval_logps/rejected": -146.909912109375,
+ "eval_loss": 0.6866763830184937,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.13231882452964783,
+ "eval_rewards/margins": 0.02109396643936634,
+ "eval_rewards/rejected": 0.11122485250234604,
+ "eval_runtime": 90.8141,
+ "eval_samples_per_second": 5.506,
+ "eval_steps_per_second": 2.753,
+ "step": 1200
+ },
+ {
+ "epoch": 0.968,
+ "grad_norm": 1.2058950662612915,
+ "learning_rate": 7.528888888888889e-07,
+ "logits/chosen": 1.3591210842132568,
+ "logits/rejected": 1.3811699151992798,
+ "logps/chosen": -155.1759033203125,
+ "logps/rejected": -141.97772216796875,
+ "loss": 0.6902606964111329,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.153532013297081,
+ "rewards/margins": 0.011359233409166336,
+ "rewards/rejected": 0.14217278361320496,
+ "step": 1210
+ },
+ {
+ "epoch": 0.976,
+ "grad_norm": 1.366702914237976,
+ "learning_rate": 7.499259259259258e-07,
+ "logits/chosen": 1.3841370344161987,
+ "logits/rejected": 1.3791372776031494,
+ "logps/chosen": -145.47613525390625,
+ "logps/rejected": -123.9173583984375,
+ "loss": 0.6971709251403808,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.10521620512008667,
+ "rewards/margins": -0.0030749079305678606,
+ "rewards/rejected": 0.1082911267876625,
+ "step": 1220
+ },
+ {
+ "epoch": 0.984,
+ "grad_norm": 1.477455496788025,
+ "learning_rate": 7.469629629629629e-07,
+ "logits/chosen": 1.4447228908538818,
+ "logits/rejected": 1.6022417545318604,
+ "logps/chosen": -137.9462127685547,
+ "logps/rejected": -150.25559997558594,
+ "loss": 0.6915127277374268,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.15108394622802734,
+ "rewards/margins": 0.009388584643602371,
+ "rewards/rejected": 0.14169538021087646,
+ "step": 1230
+ },
+ {
+ "epoch": 0.992,
+ "grad_norm": 1.4407854080200195,
+ "learning_rate": 7.44e-07,
+ "logits/chosen": 1.3998825550079346,
+ "logits/rejected": 1.5924925804138184,
+ "logps/chosen": -144.93775939941406,
+ "logps/rejected": -148.02542114257812,
+ "loss": 0.6699621677398682,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.146137997508049,
+ "rewards/margins": 0.05326627567410469,
+ "rewards/rejected": 0.09287171810865402,
+ "step": 1240
+ },
+ {
+ "epoch": 1.0,
+ "grad_norm": 1.183438777923584,
+ "learning_rate": 7.41037037037037e-07,
+ "logits/chosen": 1.5393705368041992,
+ "logits/rejected": 1.6312555074691772,
+ "logps/chosen": -145.61672973632812,
+ "logps/rejected": -158.9256591796875,
+ "loss": 0.6811168670654297,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.1304798424243927,
+ "rewards/margins": 0.03360626846551895,
+ "rewards/rejected": 0.09687358886003494,
+ "step": 1250
+ },
+ {
+ "epoch": 1.008,
+ "grad_norm": 1.2436553239822388,
+ "learning_rate": 7.38074074074074e-07,
+ "logits/chosen": 1.4758602380752563,
+ "logits/rejected": 1.534609079360962,
+ "logps/chosen": -124.26841735839844,
+ "logps/rejected": -132.41867065429688,
+ "loss": 0.688177490234375,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.12570872902870178,
+ "rewards/margins": 0.014875045046210289,
+ "rewards/rejected": 0.11083368957042694,
+ "step": 1260
+ },
+ {
+ "epoch": 1.016,
+ "grad_norm": 1.3853868246078491,
+ "learning_rate": 7.351111111111112e-07,
+ "logits/chosen": 1.7195091247558594,
+ "logits/rejected": 1.632851004600525,
+ "logps/chosen": -160.8590850830078,
+ "logps/rejected": -153.91671752929688,
+ "loss": 0.6669504165649414,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.181864932179451,
+ "rewards/margins": 0.06175640970468521,
+ "rewards/rejected": 0.12010850757360458,
+ "step": 1270
+ },
+ {
+ "epoch": 1.024,
+ "grad_norm": 1.2765302658081055,
+ "learning_rate": 7.321481481481481e-07,
+ "logits/chosen": 1.5782036781311035,
+ "logits/rejected": 1.5713088512420654,
+ "logps/chosen": -154.3319549560547,
+ "logps/rejected": -141.43345642089844,
+ "loss": 0.6734190464019776,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.12361389398574829,
+ "rewards/margins": 0.04626934230327606,
+ "rewards/rejected": 0.07734455168247223,
+ "step": 1280
+ },
+ {
+ "epoch": 1.032,
+ "grad_norm": 1.0421397686004639,
+ "learning_rate": 7.291851851851851e-07,
+ "logits/chosen": 1.5199967622756958,
+ "logits/rejected": 1.5965670347213745,
+ "logps/chosen": -145.28416442871094,
+ "logps/rejected": -133.72256469726562,
+ "loss": 0.6804514408111573,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.11794980615377426,
+ "rewards/margins": 0.030916064977645874,
+ "rewards/rejected": 0.08703374862670898,
+ "step": 1290
+ },
+ {
+ "epoch": 1.04,
+ "grad_norm": 1.032158613204956,
+ "learning_rate": 7.262222222222222e-07,
+ "logits/chosen": 1.6042639017105103,
+ "logits/rejected": 1.4255346059799194,
+ "logps/chosen": -150.4803009033203,
+ "logps/rejected": -130.2232208251953,
+ "loss": 0.6676507472991944,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.15466037392616272,
+ "rewards/margins": 0.05808692425489426,
+ "rewards/rejected": 0.09657344967126846,
+ "step": 1300
+ },
+ {
+ "epoch": 1.04,
+ "eval_logits/chosen": 1.5118356943130493,
+ "eval_logits/rejected": 1.5418950319290161,
+ "eval_logps/chosen": -152.23300170898438,
+ "eval_logps/rejected": -146.9257049560547,
+ "eval_loss": 0.6859617233276367,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.13158410787582397,
+ "eval_rewards/margins": 0.021939631551504135,
+ "eval_rewards/rejected": 0.10964448750019073,
+ "eval_runtime": 90.9624,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 1300
+ },
+ {
+ "epoch": 1.048,
+ "grad_norm": 1.4942574501037598,
+ "learning_rate": 7.232592592592593e-07,
+ "logits/chosen": 1.689648985862732,
+ "logits/rejected": 1.691888451576233,
+ "logps/chosen": -166.02809143066406,
+ "logps/rejected": -160.30601501464844,
+ "loss": 0.6770285606384278,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.14377735555171967,
+ "rewards/margins": 0.03825496509671211,
+ "rewards/rejected": 0.10552239418029785,
+ "step": 1310
+ },
+ {
+ "epoch": 1.056,
+ "grad_norm": 1.3656011819839478,
+ "learning_rate": 7.202962962962963e-07,
+ "logits/chosen": 1.3065154552459717,
+ "logits/rejected": 1.5681203603744507,
+ "logps/chosen": -129.1146240234375,
+ "logps/rejected": -151.2584991455078,
+ "loss": 0.6844655036926269,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.12841743230819702,
+ "rewards/margins": 0.02178655005991459,
+ "rewards/rejected": 0.10663086175918579,
+ "step": 1320
+ },
+ {
+ "epoch": 1.064,
+ "grad_norm": 1.1723405122756958,
+ "learning_rate": 7.173333333333333e-07,
+ "logits/chosen": 1.6219158172607422,
+ "logits/rejected": 1.6032555103302002,
+ "logps/chosen": -156.4185028076172,
+ "logps/rejected": -146.28067016601562,
+ "loss": 0.6650479316711426,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1538492739200592,
+ "rewards/margins": 0.06132146716117859,
+ "rewards/rejected": 0.09252782166004181,
+ "step": 1330
+ },
+ {
+ "epoch": 1.072,
+ "grad_norm": 1.2311245203018188,
+ "learning_rate": 7.143703703703703e-07,
+ "logits/chosen": 1.4625260829925537,
+ "logits/rejected": 1.392937421798706,
+ "logps/chosen": -153.05941772460938,
+ "logps/rejected": -145.37841796875,
+ "loss": 0.6570021152496338,
+ "rewards/accuracies": 0.762499988079071,
+ "rewards/chosen": 0.16768500208854675,
+ "rewards/margins": 0.0787494108080864,
+ "rewards/rejected": 0.08893559873104095,
+ "step": 1340
+ },
+ {
+ "epoch": 1.08,
+ "grad_norm": 1.314666748046875,
+ "learning_rate": 7.114074074074073e-07,
+ "logits/chosen": 1.4812809228897095,
+ "logits/rejected": 1.3796050548553467,
+ "logps/chosen": -165.70205688476562,
+ "logps/rejected": -140.13150024414062,
+ "loss": 0.6535200595855712,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.20691323280334473,
+ "rewards/margins": 0.0911526307463646,
+ "rewards/rejected": 0.11576057970523834,
+ "step": 1350
+ },
+ {
+ "epoch": 1.088,
+ "grad_norm": 0.9556720852851868,
+ "learning_rate": 7.084444444444444e-07,
+ "logits/chosen": 1.6368863582611084,
+ "logits/rejected": 1.678134560585022,
+ "logps/chosen": -166.56253051757812,
+ "logps/rejected": -164.36416625976562,
+ "loss": 0.6742628574371338,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.15145304799079895,
+ "rewards/margins": 0.04319921135902405,
+ "rewards/rejected": 0.1082538366317749,
+ "step": 1360
+ },
+ {
+ "epoch": 1.096,
+ "grad_norm": 0.945633053779602,
+ "learning_rate": 7.054814814814815e-07,
+ "logits/chosen": 1.4285938739776611,
+ "logits/rejected": 1.5576260089874268,
+ "logps/chosen": -131.5234832763672,
+ "logps/rejected": -128.02340698242188,
+ "loss": 0.6670140743255615,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.16399362683296204,
+ "rewards/margins": 0.05922812223434448,
+ "rewards/rejected": 0.10476551204919815,
+ "step": 1370
+ },
+ {
+ "epoch": 1.104,
+ "grad_norm": 1.1189475059509277,
+ "learning_rate": 7.025185185185185e-07,
+ "logits/chosen": 1.5019981861114502,
+ "logits/rejected": 1.667614221572876,
+ "logps/chosen": -161.61854553222656,
+ "logps/rejected": -166.39996337890625,
+ "loss": 0.6742690086364747,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.15786050260066986,
+ "rewards/margins": 0.044500257819890976,
+ "rewards/rejected": 0.11336024850606918,
+ "step": 1380
+ },
+ {
+ "epoch": 1.112,
+ "grad_norm": 1.3382912874221802,
+ "learning_rate": 6.995555555555555e-07,
+ "logits/chosen": 1.4972480535507202,
+ "logits/rejected": 1.4946129322052002,
+ "logps/chosen": -166.43682861328125,
+ "logps/rejected": -162.3004913330078,
+ "loss": 0.6557814121246338,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.18964914977550507,
+ "rewards/margins": 0.08500295877456665,
+ "rewards/rejected": 0.10464620590209961,
+ "step": 1390
+ },
+ {
+ "epoch": 1.12,
+ "grad_norm": 1.2123515605926514,
+ "learning_rate": 6.965925925925926e-07,
+ "logits/chosen": 1.456635594367981,
+ "logits/rejected": 1.5964475870132446,
+ "logps/chosen": -152.08261108398438,
+ "logps/rejected": -161.51023864746094,
+ "loss": 0.6708066940307618,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.216823011636734,
+ "rewards/margins": 0.050363689661026,
+ "rewards/rejected": 0.166459321975708,
+ "step": 1400
+ },
+ {
+ "epoch": 1.12,
+ "eval_logits/chosen": 1.521518588066101,
+ "eval_logits/rejected": 1.5510718822479248,
+ "eval_logps/chosen": -151.858154296875,
+ "eval_logps/rejected": -146.5819549560547,
+ "eval_loss": 0.6853657364845276,
+ "eval_rewards/accuracies": 0.5440000295639038,
+ "eval_rewards/chosen": 0.16906805336475372,
+ "eval_rewards/margins": 0.025049500167369843,
+ "eval_rewards/rejected": 0.14401854574680328,
+ "eval_runtime": 90.8779,
+ "eval_samples_per_second": 5.502,
+ "eval_steps_per_second": 2.751,
+ "step": 1400
+ },
+ {
+ "epoch": 1.1280000000000001,
+ "grad_norm": 1.3488291501998901,
+ "learning_rate": 6.936296296296296e-07,
+ "logits/chosen": 1.6009830236434937,
+ "logits/rejected": 1.6035544872283936,
+ "logps/chosen": -145.44662475585938,
+ "logps/rejected": -149.9284210205078,
+ "loss": 0.679435396194458,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.1647307425737381,
+ "rewards/margins": 0.03348377346992493,
+ "rewards/rejected": 0.13124695420265198,
+ "step": 1410
+ },
+ {
+ "epoch": 1.1360000000000001,
+ "grad_norm": 1.6453970670700073,
+ "learning_rate": 6.906666666666666e-07,
+ "logits/chosen": 1.593658447265625,
+ "logits/rejected": 1.648276925086975,
+ "logps/chosen": -135.2926788330078,
+ "logps/rejected": -150.9585723876953,
+ "loss": 0.6875734329223633,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.15669891238212585,
+ "rewards/margins": 0.01690087839961052,
+ "rewards/rejected": 0.13979804515838623,
+ "step": 1420
+ },
+ {
+ "epoch": 1.144,
+ "grad_norm": 1.6117807626724243,
+ "learning_rate": 6.877037037037036e-07,
+ "logits/chosen": 1.4510356187820435,
+ "logits/rejected": 1.2334567308425903,
+ "logps/chosen": -153.94854736328125,
+ "logps/rejected": -147.78079223632812,
+ "loss": 0.6689413070678711,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1819056123495102,
+ "rewards/margins": 0.058564942330121994,
+ "rewards/rejected": 0.1233406662940979,
+ "step": 1430
+ },
+ {
+ "epoch": 1.152,
+ "grad_norm": 1.5533809661865234,
+ "learning_rate": 6.847407407407408e-07,
+ "logits/chosen": 1.4535889625549316,
+ "logits/rejected": 1.4273865222930908,
+ "logps/chosen": -170.42904663085938,
+ "logps/rejected": -151.9979705810547,
+ "loss": 0.6845246315002441,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.14223966002464294,
+ "rewards/margins": 0.022389881312847137,
+ "rewards/rejected": 0.1198497861623764,
+ "step": 1440
+ },
+ {
+ "epoch": 1.16,
+ "grad_norm": 1.5696005821228027,
+ "learning_rate": 6.817777777777778e-07,
+ "logits/chosen": 1.4418725967407227,
+ "logits/rejected": 1.4354021549224854,
+ "logps/chosen": -153.962158203125,
+ "logps/rejected": -148.40512084960938,
+ "loss": 0.6761108875274658,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.15529735386371613,
+ "rewards/margins": 0.040999218821525574,
+ "rewards/rejected": 0.11429812759160995,
+ "step": 1450
+ },
+ {
+ "epoch": 1.168,
+ "grad_norm": 1.0711781978607178,
+ "learning_rate": 6.788148148148147e-07,
+ "logits/chosen": 1.5659481287002563,
+ "logits/rejected": 1.5640677213668823,
+ "logps/chosen": -149.816162109375,
+ "logps/rejected": -174.68643188476562,
+ "loss": 0.677489185333252,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1510745733976364,
+ "rewards/margins": 0.03914733976125717,
+ "rewards/rejected": 0.11192724853754044,
+ "step": 1460
+ },
+ {
+ "epoch": 1.176,
+ "grad_norm": 1.2336673736572266,
+ "learning_rate": 6.758518518518518e-07,
+ "logits/chosen": 1.4275394678115845,
+ "logits/rejected": 1.6394407749176025,
+ "logps/chosen": -137.93788146972656,
+ "logps/rejected": -154.97195434570312,
+ "loss": 0.6785802364349365,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10557998716831207,
+ "rewards/margins": 0.0352485254406929,
+ "rewards/rejected": 0.07033145427703857,
+ "step": 1470
+ },
+ {
+ "epoch": 1.184,
+ "grad_norm": 1.0030723810195923,
+ "learning_rate": 6.728888888888888e-07,
+ "logits/chosen": 1.4652725458145142,
+ "logits/rejected": 1.4992272853851318,
+ "logps/chosen": -143.87374877929688,
+ "logps/rejected": -139.5693817138672,
+ "loss": 0.6700259208679199,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.13436684012413025,
+ "rewards/margins": 0.05286343768239021,
+ "rewards/rejected": 0.08150340616703033,
+ "step": 1480
+ },
+ {
+ "epoch": 1.192,
+ "grad_norm": 1.0897104740142822,
+ "learning_rate": 6.699259259259259e-07,
+ "logits/chosen": 1.7033601999282837,
+ "logits/rejected": 1.617489218711853,
+ "logps/chosen": -140.43667602539062,
+ "logps/rejected": -146.59683227539062,
+ "loss": 0.6685672283172608,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.11724893748760223,
+ "rewards/margins": 0.055138181895017624,
+ "rewards/rejected": 0.06211075931787491,
+ "step": 1490
+ },
+ {
+ "epoch": 1.2,
+ "grad_norm": 0.8573256731033325,
+ "learning_rate": 6.66962962962963e-07,
+ "logits/chosen": 1.5181013345718384,
+ "logits/rejected": 1.5965152978897095,
+ "logps/chosen": -147.78506469726562,
+ "logps/rejected": -142.18930053710938,
+ "loss": 0.6564288139343262,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.14890792965888977,
+ "rewards/margins": 0.08058460801839828,
+ "rewards/rejected": 0.06832331418991089,
+ "step": 1500
+ },
+ {
+ "epoch": 1.2,
+ "eval_logits/chosen": 1.488415002822876,
+ "eval_logits/rejected": 1.5179662704467773,
+ "eval_logps/chosen": -152.60232543945312,
+ "eval_logps/rejected": -147.2954864501953,
+ "eval_loss": 0.6861968040466309,
+ "eval_rewards/accuracies": 0.5400000214576721,
+ "eval_rewards/chosen": 0.09465231001377106,
+ "eval_rewards/margins": 0.021986663341522217,
+ "eval_rewards/rejected": 0.07266565412282944,
+ "eval_runtime": 90.8951,
+ "eval_samples_per_second": 5.501,
+ "eval_steps_per_second": 2.75,
+ "step": 1500
+ },
+ {
+ "epoch": 1.208,
+ "grad_norm": 1.29599928855896,
+ "learning_rate": 6.64e-07,
+ "logits/chosen": 1.4851064682006836,
+ "logits/rejected": 1.4355719089508057,
+ "logps/chosen": -140.07186889648438,
+ "logps/rejected": -155.5626983642578,
+ "loss": 0.6695255756378173,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08214708417654037,
+ "rewards/margins": 0.05618969723582268,
+ "rewards/rejected": 0.025957385078072548,
+ "step": 1510
+ },
+ {
+ "epoch": 1.216,
+ "grad_norm": 1.3030505180358887,
+ "learning_rate": 6.61037037037037e-07,
+ "logits/chosen": 1.3785858154296875,
+ "logits/rejected": 1.3867576122283936,
+ "logps/chosen": -143.06137084960938,
+ "logps/rejected": -152.8805694580078,
+ "loss": 0.6704819202423096,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.08582031726837158,
+ "rewards/margins": 0.052868831902742386,
+ "rewards/rejected": 0.0329514816403389,
+ "step": 1520
+ },
+ {
+ "epoch": 1.224,
+ "grad_norm": 1.2935470342636108,
+ "learning_rate": 6.58074074074074e-07,
+ "logits/chosen": 1.4631284475326538,
+ "logits/rejected": 1.4908760786056519,
+ "logps/chosen": -181.2951202392578,
+ "logps/rejected": -173.83523559570312,
+ "loss": 0.6680389881134033,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.11432291567325592,
+ "rewards/margins": 0.05878107622265816,
+ "rewards/rejected": 0.05554184317588806,
+ "step": 1530
+ },
+ {
+ "epoch": 1.232,
+ "grad_norm": 1.4426783323287964,
+ "learning_rate": 6.551111111111111e-07,
+ "logits/chosen": 1.4765878915786743,
+ "logits/rejected": 1.4511891603469849,
+ "logps/chosen": -167.4827117919922,
+ "logps/rejected": -153.88833618164062,
+ "loss": 0.6684586524963378,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.08437791466712952,
+ "rewards/margins": 0.05823448300361633,
+ "rewards/rejected": 0.026143427938222885,
+ "step": 1540
+ },
+ {
+ "epoch": 1.24,
+ "grad_norm": 1.3253812789916992,
+ "learning_rate": 6.521481481481481e-07,
+ "logits/chosen": 1.2496798038482666,
+ "logits/rejected": 1.5193806886672974,
+ "logps/chosen": -130.33685302734375,
+ "logps/rejected": -132.5361328125,
+ "loss": 0.6695978164672851,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.10806921869516373,
+ "rewards/margins": 0.05488727241754532,
+ "rewards/rejected": 0.05318193882703781,
+ "step": 1550
+ },
+ {
+ "epoch": 1.248,
+ "grad_norm": 1.1890907287597656,
+ "learning_rate": 6.491851851851851e-07,
+ "logits/chosen": 1.2053601741790771,
+ "logits/rejected": 1.2588948011398315,
+ "logps/chosen": -132.61026000976562,
+ "logps/rejected": -150.66477966308594,
+ "loss": 0.673579216003418,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.062341660261154175,
+ "rewards/margins": 0.04531678557395935,
+ "rewards/rejected": 0.017024870961904526,
+ "step": 1560
+ },
+ {
+ "epoch": 1.256,
+ "grad_norm": 1.2076733112335205,
+ "learning_rate": 6.462222222222223e-07,
+ "logits/chosen": 1.5180461406707764,
+ "logits/rejected": 1.4118196964263916,
+ "logps/chosen": -120.61619567871094,
+ "logps/rejected": -146.42056274414062,
+ "loss": 0.6747167587280274,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.0764162540435791,
+ "rewards/margins": 0.04275941476225853,
+ "rewards/rejected": 0.03365683928132057,
+ "step": 1570
+ },
+ {
+ "epoch": 1.264,
+ "grad_norm": 1.3394429683685303,
+ "learning_rate": 6.432592592592593e-07,
+ "logits/chosen": 1.2879914045333862,
+ "logits/rejected": 1.3957126140594482,
+ "logps/chosen": -150.98355102539062,
+ "logps/rejected": -172.91090393066406,
+ "loss": 0.6560922622680664,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.09853875637054443,
+ "rewards/margins": 0.08982758224010468,
+ "rewards/rejected": 0.008711177855730057,
+ "step": 1580
+ },
+ {
+ "epoch": 1.272,
+ "grad_norm": 1.303663730621338,
+ "learning_rate": 6.402962962962962e-07,
+ "logits/chosen": 1.5617700815200806,
+ "logits/rejected": 1.6307615041732788,
+ "logps/chosen": -157.00338745117188,
+ "logps/rejected": -159.2900848388672,
+ "loss": 0.6639400959014893,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.07477572560310364,
+ "rewards/margins": 0.06561199575662613,
+ "rewards/rejected": 0.009163733571767807,
+ "step": 1590
+ },
+ {
+ "epoch": 1.28,
+ "grad_norm": 1.4180800914764404,
+ "learning_rate": 6.373333333333333e-07,
+ "logits/chosen": 1.3017181158065796,
+ "logits/rejected": 1.3270524740219116,
+ "logps/chosen": -134.52029418945312,
+ "logps/rejected": -135.45938110351562,
+ "loss": 0.6564930438995361,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.08265326917171478,
+ "rewards/margins": 0.09048662334680557,
+ "rewards/rejected": -0.007833337411284447,
+ "step": 1600
+ },
+ {
+ "epoch": 1.28,
+ "eval_logits/chosen": 1.4600619077682495,
+ "eval_logits/rejected": 1.489198923110962,
+ "eval_logps/chosen": -152.86001586914062,
+ "eval_logps/rejected": -147.54771423339844,
+ "eval_loss": 0.6871278882026672,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": 0.0688827633857727,
+ "eval_rewards/margins": 0.021439960226416588,
+ "eval_rewards/rejected": 0.04744280129671097,
+ "eval_runtime": 91.0555,
+ "eval_samples_per_second": 5.491,
+ "eval_steps_per_second": 2.746,
+ "step": 1600
+ },
+ {
+ "epoch": 1.288,
+ "grad_norm": 1.1791326999664307,
+ "learning_rate": 6.343703703703703e-07,
+ "logits/chosen": 1.4442191123962402,
+ "logits/rejected": 1.5532598495483398,
+ "logps/chosen": -141.93701171875,
+ "logps/rejected": -147.68682861328125,
+ "loss": 0.6539246082305908,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.09121527522802353,
+ "rewards/margins": 0.08849794417619705,
+ "rewards/rejected": 0.002717340365052223,
+ "step": 1610
+ },
+ {
+ "epoch": 1.296,
+ "grad_norm": 1.1131889820098877,
+ "learning_rate": 6.314074074074074e-07,
+ "logits/chosen": 1.3450567722320557,
+ "logits/rejected": 1.4408035278320312,
+ "logps/chosen": -157.0798797607422,
+ "logps/rejected": -161.1500701904297,
+ "loss": 0.6505428314208984,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.11321337521076202,
+ "rewards/margins": 0.09579396992921829,
+ "rewards/rejected": 0.017419395968317986,
+ "step": 1620
+ },
+ {
+ "epoch": 1.304,
+ "grad_norm": 1.6172596216201782,
+ "learning_rate": 6.284444444444445e-07,
+ "logits/chosen": 1.478847622871399,
+ "logits/rejected": 1.3894659280776978,
+ "logps/chosen": -140.17453002929688,
+ "logps/rejected": -169.06289672851562,
+ "loss": 0.6720306396484375,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.08244314789772034,
+ "rewards/margins": 0.062018074095249176,
+ "rewards/rejected": 0.020425086840987206,
+ "step": 1630
+ },
+ {
+ "epoch": 1.312,
+ "grad_norm": 1.1743669509887695,
+ "learning_rate": 6.254814814814815e-07,
+ "logits/chosen": 1.4197287559509277,
+ "logits/rejected": 1.3319528102874756,
+ "logps/chosen": -157.23072814941406,
+ "logps/rejected": -139.50074768066406,
+ "loss": 0.6712700843811035,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.04650931805372238,
+ "rewards/margins": 0.057532649487257004,
+ "rewards/rejected": -0.0110233249142766,
+ "step": 1640
+ },
+ {
+ "epoch": 1.32,
+ "grad_norm": 1.3225634098052979,
+ "learning_rate": 6.225185185185184e-07,
+ "logits/chosen": 1.35457444190979,
+ "logits/rejected": 1.3751428127288818,
+ "logps/chosen": -137.4834747314453,
+ "logps/rejected": -134.4203643798828,
+ "loss": 0.6679590225219727,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.07574301958084106,
+ "rewards/margins": 0.057132184505462646,
+ "rewards/rejected": 0.01861082948744297,
+ "step": 1650
+ },
+ {
+ "epoch": 1.328,
+ "grad_norm": 1.5913130044937134,
+ "learning_rate": 6.195555555555555e-07,
+ "logits/chosen": 1.5328034162521362,
+ "logits/rejected": 1.3161242008209229,
+ "logps/chosen": -147.09475708007812,
+ "logps/rejected": -135.3501434326172,
+ "loss": 0.6575756072998047,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.1077059879899025,
+ "rewards/margins": 0.08081348240375519,
+ "rewards/rejected": 0.026892507448792458,
+ "step": 1660
+ },
+ {
+ "epoch": 1.336,
+ "grad_norm": 0.9880369901657104,
+ "learning_rate": 6.165925925925926e-07,
+ "logits/chosen": 1.3911128044128418,
+ "logits/rejected": 1.3688428401947021,
+ "logps/chosen": -152.17483520507812,
+ "logps/rejected": -149.79383850097656,
+ "loss": 0.6656455516815185,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.10711929947137833,
+ "rewards/margins": 0.0669747143983841,
+ "rewards/rejected": 0.04014457389712334,
+ "step": 1670
+ },
+ {
+ "epoch": 1.3439999999999999,
+ "grad_norm": 1.125959873199463,
+ "learning_rate": 6.136296296296296e-07,
+ "logits/chosen": 1.4232629537582397,
+ "logits/rejected": 1.5579107999801636,
+ "logps/chosen": -160.2217559814453,
+ "logps/rejected": -148.1455535888672,
+ "loss": 0.6710956573486329,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10255889594554901,
+ "rewards/margins": 0.05178038403391838,
+ "rewards/rejected": 0.05077851936221123,
+ "step": 1680
+ },
+ {
+ "epoch": 1.3519999999999999,
+ "grad_norm": 1.3627047538757324,
+ "learning_rate": 6.106666666666666e-07,
+ "logits/chosen": 1.4259792566299438,
+ "logits/rejected": 1.4750150442123413,
+ "logps/chosen": -152.88381958007812,
+ "logps/rejected": -153.72708129882812,
+ "loss": 0.6512804508209229,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.14147944748401642,
+ "rewards/margins": 0.09307968616485596,
+ "rewards/rejected": 0.04839975759387016,
+ "step": 1690
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "grad_norm": 1.0653239488601685,
+ "learning_rate": 6.077037037037038e-07,
+ "logits/chosen": 1.340463399887085,
+ "logits/rejected": 1.3752565383911133,
+ "logps/chosen": -150.36248779296875,
+ "logps/rejected": -151.90725708007812,
+ "loss": 0.6768710613250732,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.08575484901666641,
+ "rewards/margins": 0.043545305728912354,
+ "rewards/rejected": 0.04220953956246376,
+ "step": 1700
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "eval_logits/chosen": 1.4521433115005493,
+ "eval_logits/rejected": 1.4804331064224243,
+ "eval_logps/chosen": -152.77755737304688,
+ "eval_logps/rejected": -147.48025512695312,
+ "eval_loss": 0.6873273849487305,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": 0.07712872326374054,
+ "eval_rewards/margins": 0.022939365357160568,
+ "eval_rewards/rejected": 0.05418936163187027,
+ "eval_runtime": 90.962,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 1700
+ },
+ {
+ "epoch": 1.3679999999999999,
+ "grad_norm": 1.3996928930282593,
+ "learning_rate": 6.047407407407407e-07,
+ "logits/chosen": 1.41148042678833,
+ "logits/rejected": 1.4520132541656494,
+ "logps/chosen": -143.23252868652344,
+ "logps/rejected": -180.05331420898438,
+ "loss": 0.655043363571167,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10279543697834015,
+ "rewards/margins": 0.0895618349313736,
+ "rewards/rejected": 0.013233604840934277,
+ "step": 1710
+ },
+ {
+ "epoch": 1.376,
+ "grad_norm": 1.4253696203231812,
+ "learning_rate": 6.017777777777777e-07,
+ "logits/chosen": 1.3833668231964111,
+ "logits/rejected": 1.409301519393921,
+ "logps/chosen": -149.3744659423828,
+ "logps/rejected": -151.1966552734375,
+ "loss": 0.6865286827087402,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.06293162703514099,
+ "rewards/margins": 0.02295227348804474,
+ "rewards/rejected": 0.03997935354709625,
+ "step": 1720
+ },
+ {
+ "epoch": 1.384,
+ "grad_norm": 1.185731053352356,
+ "learning_rate": 5.988148148148148e-07,
+ "logits/chosen": 1.528174877166748,
+ "logits/rejected": 1.6338379383087158,
+ "logps/chosen": -160.7788543701172,
+ "logps/rejected": -159.85354614257812,
+ "loss": 0.6458437442779541,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.12789444625377655,
+ "rewards/margins": 0.1114773377776146,
+ "rewards/rejected": 0.016417117789387703,
+ "step": 1730
+ },
+ {
+ "epoch": 1.392,
+ "grad_norm": 1.2963365316390991,
+ "learning_rate": 5.958518518518518e-07,
+ "logits/chosen": 1.4939125776290894,
+ "logits/rejected": 1.4197361469268799,
+ "logps/chosen": -162.34744262695312,
+ "logps/rejected": -177.93087768554688,
+ "loss": 0.6620248317718506,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.11072772741317749,
+ "rewards/margins": 0.07320726662874222,
+ "rewards/rejected": 0.03752044588327408,
+ "step": 1740
+ },
+ {
+ "epoch": 1.4,
+ "grad_norm": 1.3535394668579102,
+ "learning_rate": 5.928888888888889e-07,
+ "logits/chosen": 1.3506572246551514,
+ "logits/rejected": 1.424142837524414,
+ "logps/chosen": -139.69985961914062,
+ "logps/rejected": -144.9062042236328,
+ "loss": 0.6815268516540527,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.07917657494544983,
+ "rewards/margins": 0.03688482195138931,
+ "rewards/rejected": 0.04229176044464111,
+ "step": 1750
+ },
+ {
+ "epoch": 1.408,
+ "grad_norm": 1.471380352973938,
+ "learning_rate": 5.899259259259259e-07,
+ "logits/chosen": 1.5949152708053589,
+ "logits/rejected": 1.710013747215271,
+ "logps/chosen": -170.3776092529297,
+ "logps/rejected": -177.1616973876953,
+ "loss": 0.6533854961395263,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.11633266508579254,
+ "rewards/margins": 0.09301649034023285,
+ "rewards/rejected": 0.023316185921430588,
+ "step": 1760
+ },
+ {
+ "epoch": 1.416,
+ "grad_norm": 1.3164308071136475,
+ "learning_rate": 5.869629629629629e-07,
+ "logits/chosen": 1.5159170627593994,
+ "logits/rejected": 1.4434025287628174,
+ "logps/chosen": -161.05157470703125,
+ "logps/rejected": -140.06948852539062,
+ "loss": 0.660222864151001,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.10675680637359619,
+ "rewards/margins": 0.07750208675861359,
+ "rewards/rejected": 0.029254715889692307,
+ "step": 1770
+ },
+ {
+ "epoch": 1.424,
+ "grad_norm": 1.102005124092102,
+ "learning_rate": 5.839999999999999e-07,
+ "logits/chosen": 1.349440336227417,
+ "logits/rejected": 1.1565381288528442,
+ "logps/chosen": -150.61328125,
+ "logps/rejected": -151.36001586914062,
+ "loss": 0.660042142868042,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.09433183073997498,
+ "rewards/margins": 0.07778269052505493,
+ "rewards/rejected": 0.016549136489629745,
+ "step": 1780
+ },
+ {
+ "epoch": 1.432,
+ "grad_norm": 1.2305164337158203,
+ "learning_rate": 5.81037037037037e-07,
+ "logits/chosen": 1.4631009101867676,
+ "logits/rejected": 1.497757911682129,
+ "logps/chosen": -156.7335968017578,
+ "logps/rejected": -150.91065979003906,
+ "loss": 0.6564284324645996,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.13712921738624573,
+ "rewards/margins": 0.09247532486915588,
+ "rewards/rejected": 0.04465390369296074,
+ "step": 1790
+ },
+ {
+ "epoch": 1.44,
+ "grad_norm": 1.3459731340408325,
+ "learning_rate": 5.780740740740741e-07,
+ "logits/chosen": 1.5344784259796143,
+ "logits/rejected": 1.5709785223007202,
+ "logps/chosen": -166.15830993652344,
+ "logps/rejected": -158.678955078125,
+ "loss": 0.6511439800262451,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1227751225233078,
+ "rewards/margins": 0.0968184620141983,
+ "rewards/rejected": 0.025956671684980392,
+ "step": 1800
+ },
+ {
+ "epoch": 1.44,
+ "eval_logits/chosen": 1.4476159811019897,
+ "eval_logits/rejected": 1.4754574298858643,
+ "eval_logps/chosen": -152.70677185058594,
+ "eval_logps/rejected": -147.41787719726562,
+ "eval_loss": 0.6876539587974548,
+ "eval_rewards/accuracies": 0.5199999809265137,
+ "eval_rewards/chosen": 0.08420863002538681,
+ "eval_rewards/margins": 0.023782728239893913,
+ "eval_rewards/rejected": 0.06042589247226715,
+ "eval_runtime": 91.0223,
+ "eval_samples_per_second": 5.493,
+ "eval_steps_per_second": 2.747,
+ "step": 1800
+ },
+ {
+ "epoch": 1.448,
+ "grad_norm": 1.3158259391784668,
+ "learning_rate": 5.751111111111111e-07,
+ "logits/chosen": 1.425363302230835,
+ "logits/rejected": 1.350799798965454,
+ "logps/chosen": -151.4562530517578,
+ "logps/rejected": -134.9906005859375,
+ "loss": 0.6759338855743409,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1467444747686386,
+ "rewards/margins": 0.047227732837200165,
+ "rewards/rejected": 0.09951673448085785,
+ "step": 1810
+ },
+ {
+ "epoch": 1.456,
+ "grad_norm": 1.179150938987732,
+ "learning_rate": 5.721481481481481e-07,
+ "logits/chosen": 1.6323440074920654,
+ "logits/rejected": 1.4893152713775635,
+ "logps/chosen": -149.57675170898438,
+ "logps/rejected": -146.96237182617188,
+ "loss": 0.657456636428833,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1445317566394806,
+ "rewards/margins": 0.0896807312965393,
+ "rewards/rejected": 0.05485100671648979,
+ "step": 1820
+ },
+ {
+ "epoch": 1.464,
+ "grad_norm": 1.0987558364868164,
+ "learning_rate": 5.691851851851853e-07,
+ "logits/chosen": 1.3851706981658936,
+ "logits/rejected": 1.381049633026123,
+ "logps/chosen": -163.92648315429688,
+ "logps/rejected": -142.801025390625,
+ "loss": 0.6631375312805176,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.14093783497810364,
+ "rewards/margins": 0.07281022518873215,
+ "rewards/rejected": 0.06812760978937149,
+ "step": 1830
+ },
+ {
+ "epoch": 1.472,
+ "grad_norm": 1.1079164743423462,
+ "learning_rate": 5.662222222222222e-07,
+ "logits/chosen": 1.4228973388671875,
+ "logits/rejected": 1.3453642129898071,
+ "logps/chosen": -148.5561981201172,
+ "logps/rejected": -141.28009033203125,
+ "loss": 0.6612800121307373,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.10088491439819336,
+ "rewards/margins": 0.08228114247322083,
+ "rewards/rejected": 0.018603766337037086,
+ "step": 1840
+ },
+ {
+ "epoch": 1.48,
+ "grad_norm": 1.0415616035461426,
+ "learning_rate": 5.632592592592592e-07,
+ "logits/chosen": 1.4923865795135498,
+ "logits/rejected": 1.536971092224121,
+ "logps/chosen": -142.55166625976562,
+ "logps/rejected": -149.22906494140625,
+ "loss": 0.6690784931182862,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.10785870254039764,
+ "rewards/margins": 0.05975862219929695,
+ "rewards/rejected": 0.04810008406639099,
+ "step": 1850
+ },
+ {
+ "epoch": 1.488,
+ "grad_norm": 1.4004729986190796,
+ "learning_rate": 5.602962962962962e-07,
+ "logits/chosen": 1.4783391952514648,
+ "logits/rejected": 1.5112550258636475,
+ "logps/chosen": -153.54034423828125,
+ "logps/rejected": -118.77142333984375,
+ "loss": 0.6666333675384521,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.12789246439933777,
+ "rewards/margins": 0.06238863617181778,
+ "rewards/rejected": 0.06550382077693939,
+ "step": 1860
+ },
+ {
+ "epoch": 1.496,
+ "grad_norm": 1.1908437013626099,
+ "learning_rate": 5.573333333333333e-07,
+ "logits/chosen": 1.5748203992843628,
+ "logits/rejected": 1.4827988147735596,
+ "logps/chosen": -162.0931854248047,
+ "logps/rejected": -156.00779724121094,
+ "loss": 0.6626818656921387,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.11168892681598663,
+ "rewards/margins": 0.07420919090509415,
+ "rewards/rejected": 0.03747973591089249,
+ "step": 1870
+ },
+ {
+ "epoch": 1.504,
+ "grad_norm": 1.288218379020691,
+ "learning_rate": 5.543703703703704e-07,
+ "logits/chosen": 1.5260225534439087,
+ "logits/rejected": 1.5843703746795654,
+ "logps/chosen": -148.2488555908203,
+ "logps/rejected": -161.31570434570312,
+ "loss": 0.6677760124206543,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.12626001238822937,
+ "rewards/margins": 0.06881437450647354,
+ "rewards/rejected": 0.05744565278291702,
+ "step": 1880
+ },
+ {
+ "epoch": 1.512,
+ "grad_norm": 1.2899844646453857,
+ "learning_rate": 5.514074074074074e-07,
+ "logits/chosen": 1.417141318321228,
+ "logits/rejected": 1.389639973640442,
+ "logps/chosen": -160.5532989501953,
+ "logps/rejected": -158.82923889160156,
+ "loss": 0.6510573863983155,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.15593740344047546,
+ "rewards/margins": 0.09999983012676239,
+ "rewards/rejected": 0.05593757703900337,
+ "step": 1890
+ },
+ {
+ "epoch": 1.52,
+ "grad_norm": 1.3312058448791504,
+ "learning_rate": 5.484444444444444e-07,
+ "logits/chosen": 1.4996871948242188,
+ "logits/rejected": 1.6273930072784424,
+ "logps/chosen": -154.84347534179688,
+ "logps/rejected": -130.4560089111328,
+ "loss": 0.6844823360443115,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.13181835412979126,
+ "rewards/margins": 0.029756689444184303,
+ "rewards/rejected": 0.10206165164709091,
+ "step": 1900
+ },
+ {
+ "epoch": 1.52,
+ "eval_logits/chosen": 1.4582936763763428,
+ "eval_logits/rejected": 1.4853557348251343,
+ "eval_logps/chosen": -152.36231994628906,
+ "eval_logps/rejected": -147.1087188720703,
+ "eval_loss": 0.6870284676551819,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": 0.1186550110578537,
+ "eval_rewards/margins": 0.027311701327562332,
+ "eval_rewards/rejected": 0.09134332090616226,
+ "eval_runtime": 90.913,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 1900
+ },
+ {
+ "epoch": 1.528,
+ "grad_norm": 1.2257205247879028,
+ "learning_rate": 5.454814814814814e-07,
+ "logits/chosen": 1.4909894466400146,
+ "logits/rejected": 1.4579002857208252,
+ "logps/chosen": -129.2428741455078,
+ "logps/rejected": -149.32220458984375,
+ "loss": 0.6677265644073487,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.13237449526786804,
+ "rewards/margins": 0.059117116034030914,
+ "rewards/rejected": 0.07325738668441772,
+ "step": 1910
+ },
+ {
+ "epoch": 1.536,
+ "grad_norm": 1.3048031330108643,
+ "learning_rate": 5.425185185185185e-07,
+ "logits/chosen": 1.5237693786621094,
+ "logits/rejected": 1.4846075773239136,
+ "logps/chosen": -159.10055541992188,
+ "logps/rejected": -155.69888305664062,
+ "loss": 0.6709336280822754,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.1088198646903038,
+ "rewards/margins": 0.05951980873942375,
+ "rewards/rejected": 0.04930006340146065,
+ "step": 1920
+ },
+ {
+ "epoch": 1.544,
+ "grad_norm": 1.3463319540023804,
+ "learning_rate": 5.395555555555556e-07,
+ "logits/chosen": 1.6476523876190186,
+ "logits/rejected": 1.6236053705215454,
+ "logps/chosen": -171.18319702148438,
+ "logps/rejected": -172.63796997070312,
+ "loss": 0.6670461177825928,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.13160961866378784,
+ "rewards/margins": 0.06587684154510498,
+ "rewards/rejected": 0.06573276221752167,
+ "step": 1930
+ },
+ {
+ "epoch": 1.552,
+ "grad_norm": 1.2326266765594482,
+ "learning_rate": 5.365925925925926e-07,
+ "logits/chosen": 1.4379268884658813,
+ "logits/rejected": 1.5206880569458008,
+ "logps/chosen": -157.2953643798828,
+ "logps/rejected": -153.21218872070312,
+ "loss": 0.6558516502380372,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.17594724893569946,
+ "rewards/margins": 0.09524654597043991,
+ "rewards/rejected": 0.08070070296525955,
+ "step": 1940
+ },
+ {
+ "epoch": 1.56,
+ "grad_norm": 1.1362297534942627,
+ "learning_rate": 5.336296296296296e-07,
+ "logits/chosen": 1.524285078048706,
+ "logits/rejected": 1.3964688777923584,
+ "logps/chosen": -161.221435546875,
+ "logps/rejected": -154.8270263671875,
+ "loss": 0.6746312141418457,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.12519574165344238,
+ "rewards/margins": 0.05550859495997429,
+ "rewards/rejected": 0.06968715041875839,
+ "step": 1950
+ },
+ {
+ "epoch": 1.568,
+ "grad_norm": 1.5411877632141113,
+ "learning_rate": 5.306666666666665e-07,
+ "logits/chosen": 1.2426170110702515,
+ "logits/rejected": 1.3256272077560425,
+ "logps/chosen": -137.97537231445312,
+ "logps/rejected": -159.09580993652344,
+ "loss": 0.6492344856262207,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.16253654658794403,
+ "rewards/margins": 0.12099182605743408,
+ "rewards/rejected": 0.04154472425580025,
+ "step": 1960
+ },
+ {
+ "epoch": 1.576,
+ "grad_norm": 1.2111555337905884,
+ "learning_rate": 5.277037037037037e-07,
+ "logits/chosen": 1.3337433338165283,
+ "logits/rejected": 1.479209065437317,
+ "logps/chosen": -149.0251007080078,
+ "logps/rejected": -140.0357208251953,
+ "loss": 0.6559633731842041,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.13673515617847443,
+ "rewards/margins": 0.0912124291062355,
+ "rewards/rejected": 0.04552273079752922,
+ "step": 1970
+ },
+ {
+ "epoch": 1.584,
+ "grad_norm": 1.1228832006454468,
+ "learning_rate": 5.247407407407407e-07,
+ "logits/chosen": 1.4376949071884155,
+ "logits/rejected": 1.5033822059631348,
+ "logps/chosen": -144.91751098632812,
+ "logps/rejected": -147.37057495117188,
+ "loss": 0.6593158721923829,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.1573731154203415,
+ "rewards/margins": 0.08546318858861923,
+ "rewards/rejected": 0.07190994173288345,
+ "step": 1980
+ },
+ {
+ "epoch": 1.592,
+ "grad_norm": 0.9601779580116272,
+ "learning_rate": 5.217777777777777e-07,
+ "logits/chosen": 1.2674219608306885,
+ "logits/rejected": 1.3877508640289307,
+ "logps/chosen": -116.023193359375,
+ "logps/rejected": -122.81617736816406,
+ "loss": 0.6842892646789551,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.13847123086452484,
+ "rewards/margins": 0.026916390284895897,
+ "rewards/rejected": 0.11155486106872559,
+ "step": 1990
+ },
+ {
+ "epoch": 1.6,
+ "grad_norm": 1.2849960327148438,
+ "learning_rate": 5.188148148148148e-07,
+ "logits/chosen": 1.6982529163360596,
+ "logits/rejected": 1.6600332260131836,
+ "logps/chosen": -143.352294921875,
+ "logps/rejected": -150.0862274169922,
+ "loss": 0.6666803359985352,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.13707581162452698,
+ "rewards/margins": 0.06178014352917671,
+ "rewards/rejected": 0.07529564201831818,
+ "step": 2000
+ },
+ {
+ "epoch": 1.6,
+ "eval_logits/chosen": 1.4610096216201782,
+ "eval_logits/rejected": 1.4881619215011597,
+ "eval_logps/chosen": -152.36614990234375,
+ "eval_logps/rejected": -147.1319580078125,
+ "eval_loss": 0.6861760020256042,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": 0.11826877295970917,
+ "eval_rewards/margins": 0.02924923785030842,
+ "eval_rewards/rejected": 0.0890195295214653,
+ "eval_runtime": 91.0197,
+ "eval_samples_per_second": 5.493,
+ "eval_steps_per_second": 2.747,
+ "step": 2000
+ },
+ {
+ "epoch": 1.608,
+ "grad_norm": 1.731399655342102,
+ "learning_rate": 5.158518518518519e-07,
+ "logits/chosen": 1.4297409057617188,
+ "logits/rejected": 1.5017236471176147,
+ "logps/chosen": -164.93292236328125,
+ "logps/rejected": -164.80677795410156,
+ "loss": 0.6650710582733155,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.20246899127960205,
+ "rewards/margins": 0.0675087422132492,
+ "rewards/rejected": 0.13496026396751404,
+ "step": 2010
+ },
+ {
+ "epoch": 1.616,
+ "grad_norm": 1.2728053331375122,
+ "learning_rate": 5.128888888888888e-07,
+ "logits/chosen": 1.305499792098999,
+ "logits/rejected": 1.436187744140625,
+ "logps/chosen": -143.90879821777344,
+ "logps/rejected": -158.8233184814453,
+ "loss": 0.6744787693023682,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.17535606026649475,
+ "rewards/margins": 0.04918169230222702,
+ "rewards/rejected": 0.12617434561252594,
+ "step": 2020
+ },
+ {
+ "epoch": 1.624,
+ "grad_norm": 1.4291980266571045,
+ "learning_rate": 5.099259259259259e-07,
+ "logits/chosen": 1.4624104499816895,
+ "logits/rejected": 1.4936870336532593,
+ "logps/chosen": -151.03884887695312,
+ "logps/rejected": -127.40641784667969,
+ "loss": 0.6721909046173096,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.1530950516462326,
+ "rewards/margins": 0.0540502592921257,
+ "rewards/rejected": 0.0990448072552681,
+ "step": 2030
+ },
+ {
+ "epoch": 1.6320000000000001,
+ "grad_norm": 1.5909459590911865,
+ "learning_rate": 5.069629629629629e-07,
+ "logits/chosen": 1.468285322189331,
+ "logits/rejected": 1.3470795154571533,
+ "logps/chosen": -158.12046813964844,
+ "logps/rejected": -160.971923828125,
+ "loss": 0.6752189636230469,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.12029504776000977,
+ "rewards/margins": 0.05528820678591728,
+ "rewards/rejected": 0.06500682979822159,
+ "step": 2040
+ },
+ {
+ "epoch": 1.6400000000000001,
+ "grad_norm": 1.2923390865325928,
+ "learning_rate": 5.04e-07,
+ "logits/chosen": 1.530002236366272,
+ "logits/rejected": 1.5821081399917603,
+ "logps/chosen": -157.1008758544922,
+ "logps/rejected": -158.16775512695312,
+ "loss": 0.6889143943786621,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.14681917428970337,
+ "rewards/margins": 0.01898271217942238,
+ "rewards/rejected": 0.1278364509344101,
+ "step": 2050
+ },
+ {
+ "epoch": 1.6480000000000001,
+ "grad_norm": 1.2025870084762573,
+ "learning_rate": 5.010370370370371e-07,
+ "logits/chosen": 1.4749194383621216,
+ "logits/rejected": 1.3301353454589844,
+ "logps/chosen": -119.29649353027344,
+ "logps/rejected": -145.0949249267578,
+ "loss": 0.6332630634307861,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.1944357454776764,
+ "rewards/margins": 0.14449670910835266,
+ "rewards/rejected": 0.049939028918743134,
+ "step": 2060
+ },
+ {
+ "epoch": 1.6560000000000001,
+ "grad_norm": 1.3136606216430664,
+ "learning_rate": 4.98074074074074e-07,
+ "logits/chosen": 1.5436251163482666,
+ "logits/rejected": 1.4338414669036865,
+ "logps/chosen": -157.49942016601562,
+ "logps/rejected": -140.4504852294922,
+ "loss": 0.6923964977264404,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.13830263912677765,
+ "rewards/margins": 0.010729723609983921,
+ "rewards/rejected": 0.1275729238986969,
+ "step": 2070
+ },
+ {
+ "epoch": 1.6640000000000001,
+ "grad_norm": 1.2378884553909302,
+ "learning_rate": 4.951111111111111e-07,
+ "logits/chosen": 1.5672242641448975,
+ "logits/rejected": 1.4682090282440186,
+ "logps/chosen": -158.99301147460938,
+ "logps/rejected": -149.21221923828125,
+ "loss": 0.6473859786987305,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.1993202269077301,
+ "rewards/margins": 0.10525582730770111,
+ "rewards/rejected": 0.0940643846988678,
+ "step": 2080
+ },
+ {
+ "epoch": 1.6720000000000002,
+ "grad_norm": 1.1976169347763062,
+ "learning_rate": 4.921481481481482e-07,
+ "logits/chosen": 1.4160411357879639,
+ "logits/rejected": 1.3142106533050537,
+ "logps/chosen": -153.40621948242188,
+ "logps/rejected": -154.57308959960938,
+ "loss": 0.6821109771728515,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.14698973298072815,
+ "rewards/margins": 0.034348614513874054,
+ "rewards/rejected": 0.1126411110162735,
+ "step": 2090
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "grad_norm": 1.3028976917266846,
+ "learning_rate": 4.891851851851852e-07,
+ "logits/chosen": 1.61053466796875,
+ "logits/rejected": 1.526831030845642,
+ "logps/chosen": -164.3324737548828,
+ "logps/rejected": -149.382080078125,
+ "loss": 0.6684560298919677,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.22455796599388123,
+ "rewards/margins": 0.06641162931919098,
+ "rewards/rejected": 0.15814633667469025,
+ "step": 2100
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "eval_logits/chosen": 1.4865608215332031,
+ "eval_logits/rejected": 1.5141921043395996,
+ "eval_logps/chosen": -151.979248046875,
+ "eval_logps/rejected": -146.76809692382812,
+ "eval_loss": 0.6849560141563416,
+ "eval_rewards/accuracies": 0.5379999876022339,
+ "eval_rewards/chosen": 0.15696033835411072,
+ "eval_rewards/margins": 0.0315566249191761,
+ "eval_rewards/rejected": 0.1254037320613861,
+ "eval_runtime": 90.9172,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 2100
+ },
+ {
+ "epoch": 1.688,
+ "grad_norm": 1.3767266273498535,
+ "learning_rate": 4.862222222222222e-07,
+ "logits/chosen": 1.458231806755066,
+ "logits/rejected": 1.3821945190429688,
+ "logps/chosen": -166.67698669433594,
+ "logps/rejected": -156.74313354492188,
+ "loss": 0.6658836364746094,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.17039546370506287,
+ "rewards/margins": 0.07431827485561371,
+ "rewards/rejected": 0.09607716649770737,
+ "step": 2110
+ },
+ {
+ "epoch": 1.696,
+ "grad_norm": 1.4396629333496094,
+ "learning_rate": 4.832592592592592e-07,
+ "logits/chosen": 1.5568702220916748,
+ "logits/rejected": 1.5459610223770142,
+ "logps/chosen": -133.0255889892578,
+ "logps/rejected": -138.2047882080078,
+ "loss": 0.6704818725585937,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.173842191696167,
+ "rewards/margins": 0.06081608682870865,
+ "rewards/rejected": 0.11302611976861954,
+ "step": 2120
+ },
+ {
+ "epoch": 1.704,
+ "grad_norm": 2.3225553035736084,
+ "learning_rate": 4.802962962962962e-07,
+ "logits/chosen": 1.5432292222976685,
+ "logits/rejected": 1.521958589553833,
+ "logps/chosen": -165.04318237304688,
+ "logps/rejected": -163.95855712890625,
+ "loss": 0.6567724704742431,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.20300237834453583,
+ "rewards/margins": 0.09381169825792313,
+ "rewards/rejected": 0.1091906875371933,
+ "step": 2130
+ },
+ {
+ "epoch": 1.712,
+ "grad_norm": 1.3185398578643799,
+ "learning_rate": 4.773333333333333e-07,
+ "logits/chosen": 1.2340335845947266,
+ "logits/rejected": 1.3610973358154297,
+ "logps/chosen": -139.11878967285156,
+ "logps/rejected": -146.57260131835938,
+ "loss": 0.671626091003418,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.17043188214302063,
+ "rewards/margins": 0.054147154092788696,
+ "rewards/rejected": 0.11628471314907074,
+ "step": 2140
+ },
+ {
+ "epoch": 1.72,
+ "grad_norm": 1.618586540222168,
+ "learning_rate": 4.7437037037037034e-07,
+ "logits/chosen": 1.5895780324935913,
+ "logits/rejected": 1.613523244857788,
+ "logps/chosen": -156.53970336914062,
+ "logps/rejected": -154.35116577148438,
+ "loss": 0.6798102855682373,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.20041461288928986,
+ "rewards/margins": 0.03897023946046829,
+ "rewards/rejected": 0.16144435107707977,
+ "step": 2150
+ },
+ {
+ "epoch": 1.728,
+ "grad_norm": 0.8500939607620239,
+ "learning_rate": 4.7140740740740736e-07,
+ "logits/chosen": 1.3736517429351807,
+ "logits/rejected": 1.361998438835144,
+ "logps/chosen": -137.701904296875,
+ "logps/rejected": -139.91567993164062,
+ "loss": 0.6568773269653321,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.16317322850227356,
+ "rewards/margins": 0.08119454979896545,
+ "rewards/rejected": 0.0819786787033081,
+ "step": 2160
+ },
+ {
+ "epoch": 1.736,
+ "grad_norm": 1.2291405200958252,
+ "learning_rate": 4.6844444444444444e-07,
+ "logits/chosen": 1.474157452583313,
+ "logits/rejected": 1.4488928318023682,
+ "logps/chosen": -152.36732482910156,
+ "logps/rejected": -149.70980834960938,
+ "loss": 0.6551193714141845,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.2117660939693451,
+ "rewards/margins": 0.0950237363576889,
+ "rewards/rejected": 0.11674235761165619,
+ "step": 2170
+ },
+ {
+ "epoch": 1.744,
+ "grad_norm": 1.2946466207504272,
+ "learning_rate": 4.6548148148148146e-07,
+ "logits/chosen": 1.257048487663269,
+ "logits/rejected": 1.3120336532592773,
+ "logps/chosen": -143.43148803710938,
+ "logps/rejected": -169.9266815185547,
+ "loss": 0.6712408542633057,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.1496409922838211,
+ "rewards/margins": 0.05779259279370308,
+ "rewards/rejected": 0.09184839576482773,
+ "step": 2180
+ },
+ {
+ "epoch": 1.752,
+ "grad_norm": 1.0697646141052246,
+ "learning_rate": 4.625185185185185e-07,
+ "logits/chosen": 1.5544202327728271,
+ "logits/rejected": 1.5186070203781128,
+ "logps/chosen": -170.5447235107422,
+ "logps/rejected": -165.55209350585938,
+ "loss": 0.6674113750457764,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.12943358719348907,
+ "rewards/margins": 0.07018069922924042,
+ "rewards/rejected": 0.05925289914011955,
+ "step": 2190
+ },
+ {
+ "epoch": 1.76,
+ "grad_norm": 1.228698968887329,
+ "learning_rate": 4.595555555555555e-07,
+ "logits/chosen": 1.46980881690979,
+ "logits/rejected": 1.2382839918136597,
+ "logps/chosen": -133.5864715576172,
+ "logps/rejected": -135.65968322753906,
+ "loss": 0.6756285667419434,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.18240585923194885,
+ "rewards/margins": 0.052959512919187546,
+ "rewards/rejected": 0.1294463574886322,
+ "step": 2200
+ },
+ {
+ "epoch": 1.76,
+ "eval_logits/chosen": 1.4826997518539429,
+ "eval_logits/rejected": 1.5107221603393555,
+ "eval_logps/chosen": -152.19204711914062,
+ "eval_logps/rejected": -146.98675537109375,
+ "eval_loss": 0.6843143105506897,
+ "eval_rewards/accuracies": 0.5379999876022339,
+ "eval_rewards/chosen": 0.13567964732646942,
+ "eval_rewards/margins": 0.03213911131024361,
+ "eval_rewards/rejected": 0.10354051738977432,
+ "eval_runtime": 91.0116,
+ "eval_samples_per_second": 5.494,
+ "eval_steps_per_second": 2.747,
+ "step": 2200
+ },
+ {
+ "epoch": 1.768,
+ "grad_norm": 1.4145433902740479,
+ "learning_rate": 4.565925925925926e-07,
+ "logits/chosen": 1.49526047706604,
+ "logits/rejected": 1.519241213798523,
+ "logps/chosen": -152.7605438232422,
+ "logps/rejected": -140.98629760742188,
+ "loss": 0.6503778457641601,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.16126050055027008,
+ "rewards/margins": 0.0973939448595047,
+ "rewards/rejected": 0.06386655569076538,
+ "step": 2210
+ },
+ {
+ "epoch": 1.776,
+ "grad_norm": 1.4429577589035034,
+ "learning_rate": 4.536296296296296e-07,
+ "logits/chosen": 1.4706772565841675,
+ "logits/rejected": 1.3714580535888672,
+ "logps/chosen": -135.95150756835938,
+ "logps/rejected": -165.23587036132812,
+ "loss": 0.6639570713043212,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.18487155437469482,
+ "rewards/margins": 0.07484755665063858,
+ "rewards/rejected": 0.11002399027347565,
+ "step": 2220
+ },
+ {
+ "epoch": 1.784,
+ "grad_norm": 1.0622409582138062,
+ "learning_rate": 4.506666666666666e-07,
+ "logits/chosen": 1.2506721019744873,
+ "logits/rejected": 1.4802286624908447,
+ "logps/chosen": -129.645751953125,
+ "logps/rejected": -131.7567596435547,
+ "loss": 0.6788074493408203,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.12520259618759155,
+ "rewards/margins": 0.03735826909542084,
+ "rewards/rejected": 0.08784433454275131,
+ "step": 2230
+ },
+ {
+ "epoch": 1.792,
+ "grad_norm": 1.0001399517059326,
+ "learning_rate": 4.477037037037037e-07,
+ "logits/chosen": 1.4206364154815674,
+ "logits/rejected": 1.5338770151138306,
+ "logps/chosen": -143.95074462890625,
+ "logps/rejected": -160.1469268798828,
+ "loss": 0.6695474147796631,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.13798081874847412,
+ "rewards/margins": 0.05759159475564957,
+ "rewards/rejected": 0.08038921654224396,
+ "step": 2240
+ },
+ {
+ "epoch": 1.8,
+ "grad_norm": 1.3531230688095093,
+ "learning_rate": 4.4474074074074067e-07,
+ "logits/chosen": 1.5709412097930908,
+ "logits/rejected": 1.5301862955093384,
+ "logps/chosen": -161.30087280273438,
+ "logps/rejected": -150.31236267089844,
+ "loss": 0.6771373271942138,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.1488293707370758,
+ "rewards/margins": 0.04127617925405502,
+ "rewards/rejected": 0.10755319893360138,
+ "step": 2250
+ },
+ {
+ "epoch": 1.808,
+ "grad_norm": 1.37367582321167,
+ "learning_rate": 4.4177777777777775e-07,
+ "logits/chosen": 1.36393141746521,
+ "logits/rejected": 1.484881043434143,
+ "logps/chosen": -125.18086242675781,
+ "logps/rejected": -148.78546142578125,
+ "loss": 0.6596842288970948,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.15397249162197113,
+ "rewards/margins": 0.07893262803554535,
+ "rewards/rejected": 0.07503987848758698,
+ "step": 2260
+ },
+ {
+ "epoch": 1.8159999999999998,
+ "grad_norm": 1.4511909484863281,
+ "learning_rate": 4.388148148148148e-07,
+ "logits/chosen": 1.3140859603881836,
+ "logits/rejected": 1.3879334926605225,
+ "logps/chosen": -131.48226928710938,
+ "logps/rejected": -123.94035339355469,
+ "loss": 0.6698450088500977,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.12767434120178223,
+ "rewards/margins": 0.06716184318065643,
+ "rewards/rejected": 0.060512494295835495,
+ "step": 2270
+ },
+ {
+ "epoch": 1.8239999999999998,
+ "grad_norm": 1.3375639915466309,
+ "learning_rate": 4.3585185185185184e-07,
+ "logits/chosen": 1.5821285247802734,
+ "logits/rejected": 1.43180251121521,
+ "logps/chosen": -167.2334747314453,
+ "logps/rejected": -145.28871154785156,
+ "loss": 0.658758020401001,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.12011411041021347,
+ "rewards/margins": 0.08019578456878662,
+ "rewards/rejected": 0.039918314665555954,
+ "step": 2280
+ },
+ {
+ "epoch": 1.8319999999999999,
+ "grad_norm": 1.2121729850769043,
+ "learning_rate": 4.3288888888888887e-07,
+ "logits/chosen": 1.3520320653915405,
+ "logits/rejected": 1.18943452835083,
+ "logps/chosen": -147.15621948242188,
+ "logps/rejected": -132.015625,
+ "loss": 0.6655458450317383,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08098682016134262,
+ "rewards/margins": 0.07385818660259247,
+ "rewards/rejected": 0.007128633558750153,
+ "step": 2290
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "grad_norm": 1.1618680953979492,
+ "learning_rate": 4.2992592592592594e-07,
+ "logits/chosen": 1.4779785871505737,
+ "logits/rejected": 1.448462724685669,
+ "logps/chosen": -157.75070190429688,
+ "logps/rejected": -141.3148956298828,
+ "loss": 0.6813467979431153,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.12400279194116592,
+ "rewards/margins": 0.034347161650657654,
+ "rewards/rejected": 0.08965563774108887,
+ "step": 2300
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "eval_logits/chosen": 1.4821760654449463,
+ "eval_logits/rejected": 1.5101500749588013,
+ "eval_logps/chosen": -152.2089080810547,
+ "eval_logps/rejected": -147.0087432861328,
+ "eval_loss": 0.6842693090438843,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.13399286568164825,
+ "eval_rewards/margins": 0.032653141766786575,
+ "eval_rewards/rejected": 0.10133973509073257,
+ "eval_runtime": 90.8665,
+ "eval_samples_per_second": 5.503,
+ "eval_steps_per_second": 2.751,
+ "step": 2300
+ },
+ {
+ "epoch": 1.8479999999999999,
+ "grad_norm": 1.1095342636108398,
+ "learning_rate": 4.2696296296296296e-07,
+ "logits/chosen": 1.6475988626480103,
+ "logits/rejected": 1.6654495000839233,
+ "logps/chosen": -165.965087890625,
+ "logps/rejected": -153.98214721679688,
+ "loss": 0.6605272769927979,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.2209581881761551,
+ "rewards/margins": 0.07579347491264343,
+ "rewards/rejected": 0.14516471326351166,
+ "step": 2310
+ },
+ {
+ "epoch": 1.8559999999999999,
+ "grad_norm": 1.1993674039840698,
+ "learning_rate": 4.24e-07,
+ "logits/chosen": 1.5783694982528687,
+ "logits/rejected": 1.406640648841858,
+ "logps/chosen": -139.3554229736328,
+ "logps/rejected": -126.39006042480469,
+ "loss": 0.6817263126373291,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1467253863811493,
+ "rewards/margins": 0.03637007623910904,
+ "rewards/rejected": 0.11035531759262085,
+ "step": 2320
+ },
+ {
+ "epoch": 1.8639999999999999,
+ "grad_norm": 1.4875127077102661,
+ "learning_rate": 4.21037037037037e-07,
+ "logits/chosen": 1.4171388149261475,
+ "logits/rejected": 1.4270119667053223,
+ "logps/chosen": -145.1273651123047,
+ "logps/rejected": -157.85816955566406,
+ "loss": 0.6678358554840088,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.14816439151763916,
+ "rewards/margins": 0.06963086873292923,
+ "rewards/rejected": 0.07853354513645172,
+ "step": 2330
+ },
+ {
+ "epoch": 1.8719999999999999,
+ "grad_norm": 1.2899141311645508,
+ "learning_rate": 4.180740740740741e-07,
+ "logits/chosen": 1.3682881593704224,
+ "logits/rejected": 1.3383492231369019,
+ "logps/chosen": -167.96450805664062,
+ "logps/rejected": -158.2183837890625,
+ "loss": 0.6754446029663086,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.1453542709350586,
+ "rewards/margins": 0.047066912055015564,
+ "rewards/rejected": 0.09828735888004303,
+ "step": 2340
+ },
+ {
+ "epoch": 1.88,
+ "grad_norm": 1.4864925146102905,
+ "learning_rate": 4.151111111111111e-07,
+ "logits/chosen": 1.5181632041931152,
+ "logits/rejected": 1.6898539066314697,
+ "logps/chosen": -146.9915771484375,
+ "logps/rejected": -156.66282653808594,
+ "loss": 0.6776726245880127,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.1270596832036972,
+ "rewards/margins": 0.04481229558587074,
+ "rewards/rejected": 0.08224737644195557,
+ "step": 2350
+ },
+ {
+ "epoch": 1.888,
+ "grad_norm": 1.3912899494171143,
+ "learning_rate": 4.1214814814814813e-07,
+ "logits/chosen": 1.279984474182129,
+ "logits/rejected": 1.258424997329712,
+ "logps/chosen": -162.26925659179688,
+ "logps/rejected": -153.519287109375,
+ "loss": 0.680011796951294,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1301378756761551,
+ "rewards/margins": 0.03849628567695618,
+ "rewards/rejected": 0.09164158999919891,
+ "step": 2360
+ },
+ {
+ "epoch": 1.896,
+ "grad_norm": 1.2498691082000732,
+ "learning_rate": 4.091851851851852e-07,
+ "logits/chosen": 1.5644304752349854,
+ "logits/rejected": 1.5302040576934814,
+ "logps/chosen": -181.20506286621094,
+ "logps/rejected": -164.98670959472656,
+ "loss": 0.6632030963897705,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.16487805545330048,
+ "rewards/margins": 0.08321762084960938,
+ "rewards/rejected": 0.0816604346036911,
+ "step": 2370
+ },
+ {
+ "epoch": 1.904,
+ "grad_norm": 1.3421261310577393,
+ "learning_rate": 4.0622222222222217e-07,
+ "logits/chosen": 1.5388444662094116,
+ "logits/rejected": 1.5939340591430664,
+ "logps/chosen": -151.34434509277344,
+ "logps/rejected": -149.8802490234375,
+ "loss": 0.691883659362793,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.11643080413341522,
+ "rewards/margins": 0.017300304025411606,
+ "rewards/rejected": 0.09913049638271332,
+ "step": 2380
+ },
+ {
+ "epoch": 1.912,
+ "grad_norm": 1.4378565549850464,
+ "learning_rate": 4.0325925925925925e-07,
+ "logits/chosen": 1.4750030040740967,
+ "logits/rejected": 1.385709524154663,
+ "logps/chosen": -159.98593139648438,
+ "logps/rejected": -144.0281219482422,
+ "loss": 0.6825335502624512,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.1235724687576294,
+ "rewards/margins": 0.03918968886137009,
+ "rewards/rejected": 0.0843827873468399,
+ "step": 2390
+ },
+ {
+ "epoch": 1.92,
+ "grad_norm": 1.1941649913787842,
+ "learning_rate": 4.002962962962963e-07,
+ "logits/chosen": 1.3726251125335693,
+ "logits/rejected": 1.5375208854675293,
+ "logps/chosen": -149.9639129638672,
+ "logps/rejected": -142.49295043945312,
+ "loss": 0.6461881160736084,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.12773704528808594,
+ "rewards/margins": 0.1104915589094162,
+ "rewards/rejected": 0.017245473340153694,
+ "step": 2400
+ },
+ {
+ "epoch": 1.92,
+ "eval_logits/chosen": 1.4780151844024658,
+ "eval_logits/rejected": 1.506591558456421,
+ "eval_logps/chosen": -152.50015258789062,
+ "eval_logps/rejected": -147.30506896972656,
+ "eval_loss": 0.6834847331047058,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": 0.10486816614866257,
+ "eval_rewards/margins": 0.03315889462828636,
+ "eval_rewards/rejected": 0.0717092752456665,
+ "eval_runtime": 90.9835,
+ "eval_samples_per_second": 5.496,
+ "eval_steps_per_second": 2.748,
+ "step": 2400
+ },
+ {
+ "epoch": 1.928,
+ "grad_norm": 1.1473901271820068,
+ "learning_rate": 3.973333333333333e-07,
+ "logits/chosen": 1.3499658107757568,
+ "logits/rejected": 1.479353666305542,
+ "logps/chosen": -153.15272521972656,
+ "logps/rejected": -170.32418823242188,
+ "loss": 0.6756062507629395,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.09217077493667603,
+ "rewards/margins": 0.05041539669036865,
+ "rewards/rejected": 0.041755370795726776,
+ "step": 2410
+ },
+ {
+ "epoch": 1.936,
+ "grad_norm": 1.4407395124435425,
+ "learning_rate": 3.9437037037037037e-07,
+ "logits/chosen": 1.5876386165618896,
+ "logits/rejected": 1.5701191425323486,
+ "logps/chosen": -144.0182342529297,
+ "logps/rejected": -130.75460815429688,
+ "loss": 0.6733745574951172,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.1180434376001358,
+ "rewards/margins": 0.050890080630779266,
+ "rewards/rejected": 0.06715335696935654,
+ "step": 2420
+ },
+ {
+ "epoch": 1.944,
+ "grad_norm": 1.46868097782135,
+ "learning_rate": 3.914074074074074e-07,
+ "logits/chosen": 1.4809507131576538,
+ "logits/rejected": 1.4793384075164795,
+ "logps/chosen": -144.4136199951172,
+ "logps/rejected": -140.72645568847656,
+ "loss": 0.65852952003479,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.13238464295864105,
+ "rewards/margins": 0.08860992640256882,
+ "rewards/rejected": 0.043774720281362534,
+ "step": 2430
+ },
+ {
+ "epoch": 1.952,
+ "grad_norm": 1.2951419353485107,
+ "learning_rate": 3.884444444444444e-07,
+ "logits/chosen": 1.4519541263580322,
+ "logits/rejected": 1.6060655117034912,
+ "logps/chosen": -150.94955444335938,
+ "logps/rejected": -146.84732055664062,
+ "loss": 0.650700569152832,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.14693069458007812,
+ "rewards/margins": 0.09734389185905457,
+ "rewards/rejected": 0.04958680272102356,
+ "step": 2440
+ },
+ {
+ "epoch": 1.96,
+ "grad_norm": 1.2181923389434814,
+ "learning_rate": 3.854814814814815e-07,
+ "logits/chosen": 1.5463917255401611,
+ "logits/rejected": 1.4773080348968506,
+ "logps/chosen": -150.07005310058594,
+ "logps/rejected": -163.2088165283203,
+ "loss": 0.6521890163421631,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.15517377853393555,
+ "rewards/margins": 0.09567751735448837,
+ "rewards/rejected": 0.05949625372886658,
+ "step": 2450
+ },
+ {
+ "epoch": 1.968,
+ "grad_norm": 1.2294185161590576,
+ "learning_rate": 3.825185185185185e-07,
+ "logits/chosen": 1.4533751010894775,
+ "logits/rejected": 1.4323598146438599,
+ "logps/chosen": -156.0291290283203,
+ "logps/rejected": -161.64663696289062,
+ "loss": 0.6698451042175293,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.07660157233476639,
+ "rewards/margins": 0.05659238249063492,
+ "rewards/rejected": 0.020009174942970276,
+ "step": 2460
+ },
+ {
+ "epoch": 1.976,
+ "grad_norm": 1.0025874376296997,
+ "learning_rate": 3.7955555555555553e-07,
+ "logits/chosen": 1.6077829599380493,
+ "logits/rejected": 1.502532720565796,
+ "logps/chosen": -168.87271118164062,
+ "logps/rejected": -160.9680633544922,
+ "loss": 0.6690214157104493,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.11292171478271484,
+ "rewards/margins": 0.06120762228965759,
+ "rewards/rejected": 0.05171409994363785,
+ "step": 2470
+ },
+ {
+ "epoch": 1.984,
+ "grad_norm": 1.1663116216659546,
+ "learning_rate": 3.7659259259259255e-07,
+ "logits/chosen": 1.6099679470062256,
+ "logits/rejected": 1.6698023080825806,
+ "logps/chosen": -147.36190795898438,
+ "logps/rejected": -145.37986755371094,
+ "loss": 0.666702127456665,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1266266405582428,
+ "rewards/margins": 0.0683312863111496,
+ "rewards/rejected": 0.0582953505218029,
+ "step": 2480
+ },
+ {
+ "epoch": 1.992,
+ "grad_norm": 1.1562272310256958,
+ "learning_rate": 3.7362962962962963e-07,
+ "logits/chosen": 1.5222467184066772,
+ "logits/rejected": 1.592881441116333,
+ "logps/chosen": -145.1186065673828,
+ "logps/rejected": -158.30718994140625,
+ "loss": 0.6763694763183594,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.09335663169622421,
+ "rewards/margins": 0.04241436719894409,
+ "rewards/rejected": 0.05094226449728012,
+ "step": 2490
+ },
+ {
+ "epoch": 2.0,
+ "grad_norm": 1.5228629112243652,
+ "learning_rate": 3.7066666666666665e-07,
+ "logits/chosen": 1.5318562984466553,
+ "logits/rejected": 1.5440677404403687,
+ "logps/chosen": -178.16098022460938,
+ "logps/rejected": -143.88893127441406,
+ "loss": 0.6654858112335205,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0874411091208458,
+ "rewards/margins": 0.07056570053100586,
+ "rewards/rejected": 0.01687539741396904,
+ "step": 2500
+ },
+ {
+ "epoch": 2.0,
+ "eval_logits/chosen": 1.4583442211151123,
+ "eval_logits/rejected": 1.486719012260437,
+ "eval_logps/chosen": -152.8921661376953,
+ "eval_logps/rejected": -147.6856231689453,
+ "eval_loss": 0.6842741370201111,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.06566743552684784,
+ "eval_rewards/margins": 0.032015930861234665,
+ "eval_rewards/rejected": 0.03365150839090347,
+ "eval_runtime": 90.8716,
+ "eval_samples_per_second": 5.502,
+ "eval_steps_per_second": 2.751,
+ "step": 2500
+ },
+ {
+ "epoch": 2.008,
+ "grad_norm": 1.2679792642593384,
+ "learning_rate": 3.6770370370370367e-07,
+ "logits/chosen": 1.542234182357788,
+ "logits/rejected": 1.6291179656982422,
+ "logps/chosen": -142.90040588378906,
+ "logps/rejected": -161.39028930664062,
+ "loss": 0.6518040657043457,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.12705746293067932,
+ "rewards/margins": 0.09586121141910553,
+ "rewards/rejected": 0.031196262687444687,
+ "step": 2510
+ },
+ {
+ "epoch": 2.016,
+ "grad_norm": 1.2428960800170898,
+ "learning_rate": 3.6474074074074075e-07,
+ "logits/chosen": 1.5037320852279663,
+ "logits/rejected": 1.5008131265640259,
+ "logps/chosen": -154.43722534179688,
+ "logps/rejected": -179.9853515625,
+ "loss": 0.6436567306518555,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.09548825025558472,
+ "rewards/margins": 0.11602531373500824,
+ "rewards/rejected": -0.020537082105875015,
+ "step": 2520
+ },
+ {
+ "epoch": 2.024,
+ "grad_norm": 1.3352172374725342,
+ "learning_rate": 3.617777777777777e-07,
+ "logits/chosen": 1.5730235576629639,
+ "logits/rejected": 1.5479223728179932,
+ "logps/chosen": -143.82000732421875,
+ "logps/rejected": -154.56626892089844,
+ "loss": 0.6623670578002929,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.11983202397823334,
+ "rewards/margins": 0.07249654829502106,
+ "rewards/rejected": 0.04733546823263168,
+ "step": 2530
+ },
+ {
+ "epoch": 2.032,
+ "grad_norm": 1.2124547958374023,
+ "learning_rate": 3.588148148148148e-07,
+ "logits/chosen": 1.2967132329940796,
+ "logits/rejected": 1.198378324508667,
+ "logps/chosen": -134.57449340820312,
+ "logps/rejected": -124.48946380615234,
+ "loss": 0.6602365016937256,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.10377311706542969,
+ "rewards/margins": 0.07978598773479462,
+ "rewards/rejected": 0.023987144231796265,
+ "step": 2540
+ },
+ {
+ "epoch": 2.04,
+ "grad_norm": 1.7321484088897705,
+ "learning_rate": 3.5585185185185187e-07,
+ "logits/chosen": 1.4171000719070435,
+ "logits/rejected": 1.3644115924835205,
+ "logps/chosen": -153.2901611328125,
+ "logps/rejected": -143.07582092285156,
+ "loss": 0.6586461067199707,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.10289964824914932,
+ "rewards/margins": 0.08438605815172195,
+ "rewards/rejected": 0.018513580784201622,
+ "step": 2550
+ },
+ {
+ "epoch": 2.048,
+ "grad_norm": 1.2162495851516724,
+ "learning_rate": 3.5288888888888884e-07,
+ "logits/chosen": 1.5142430067062378,
+ "logits/rejected": 1.4998128414154053,
+ "logps/chosen": -156.86972045898438,
+ "logps/rejected": -155.01321411132812,
+ "loss": 0.633359432220459,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.12783676385879517,
+ "rewards/margins": 0.13910353183746338,
+ "rewards/rejected": -0.011266755871474743,
+ "step": 2560
+ },
+ {
+ "epoch": 2.056,
+ "grad_norm": 1.7551093101501465,
+ "learning_rate": 3.499259259259259e-07,
+ "logits/chosen": 1.3865793943405151,
+ "logits/rejected": 1.4954140186309814,
+ "logps/chosen": -156.48683166503906,
+ "logps/rejected": -148.37991333007812,
+ "loss": 0.6553828716278076,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.09534349292516708,
+ "rewards/margins": 0.08968523144721985,
+ "rewards/rejected": 0.005658247508108616,
+ "step": 2570
+ },
+ {
+ "epoch": 2.064,
+ "grad_norm": 1.282270908355713,
+ "learning_rate": 3.4696296296296293e-07,
+ "logits/chosen": 1.3754762411117554,
+ "logits/rejected": 1.3920589685440063,
+ "logps/chosen": -130.45675659179688,
+ "logps/rejected": -128.36073303222656,
+ "loss": 0.6361326217651367,
+ "rewards/accuracies": 0.8125,
+ "rewards/chosen": 0.16388167440891266,
+ "rewards/margins": 0.13015803694725037,
+ "rewards/rejected": 0.0337236113846302,
+ "step": 2580
+ },
+ {
+ "epoch": 2.072,
+ "grad_norm": 1.2549991607666016,
+ "learning_rate": 3.4399999999999996e-07,
+ "logits/chosen": 1.4334301948547363,
+ "logits/rejected": 1.4073057174682617,
+ "logps/chosen": -142.28848266601562,
+ "logps/rejected": -143.86415100097656,
+ "loss": 0.6604367733001709,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.10965392738580704,
+ "rewards/margins": 0.08058565109968185,
+ "rewards/rejected": 0.02906828001141548,
+ "step": 2590
+ },
+ {
+ "epoch": 2.08,
+ "grad_norm": 1.1193031072616577,
+ "learning_rate": 3.4103703703703703e-07,
+ "logits/chosen": 1.4255090951919556,
+ "logits/rejected": 1.3589593172073364,
+ "logps/chosen": -145.86782836914062,
+ "logps/rejected": -134.277099609375,
+ "loss": 0.6415833950042724,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.15703223645687103,
+ "rewards/margins": 0.11860913038253784,
+ "rewards/rejected": 0.03842310234904289,
+ "step": 2600
+ },
+ {
+ "epoch": 2.08,
+ "eval_logits/chosen": 1.4633173942565918,
+ "eval_logits/rejected": 1.4914942979812622,
+ "eval_logps/chosen": -152.7378387451172,
+ "eval_logps/rejected": -147.54226684570312,
+ "eval_loss": 0.684092104434967,
+ "eval_rewards/accuracies": 0.5559999942779541,
+ "eval_rewards/chosen": 0.08109957724809647,
+ "eval_rewards/margins": 0.033111557364463806,
+ "eval_rewards/rejected": 0.047988031059503555,
+ "eval_runtime": 90.9652,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 2600
+ },
+ {
+ "epoch": 2.088,
+ "grad_norm": 1.138805627822876,
+ "learning_rate": 3.3807407407407405e-07,
+ "logits/chosen": 1.6175915002822876,
+ "logits/rejected": 1.7044296264648438,
+ "logps/chosen": -161.02072143554688,
+ "logps/rejected": -150.9885711669922,
+ "loss": 0.6464156627655029,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.1512492299079895,
+ "rewards/margins": 0.11052403599023819,
+ "rewards/rejected": 0.040725212544202805,
+ "step": 2610
+ },
+ {
+ "epoch": 2.096,
+ "grad_norm": 1.2945690155029297,
+ "learning_rate": 3.3511111111111113e-07,
+ "logits/chosen": 1.7169849872589111,
+ "logits/rejected": 1.5235084295272827,
+ "logps/chosen": -178.69578552246094,
+ "logps/rejected": -147.8590850830078,
+ "loss": 0.6443929195404052,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.1698969006538391,
+ "rewards/margins": 0.11732057482004166,
+ "rewards/rejected": 0.052576325833797455,
+ "step": 2620
+ },
+ {
+ "epoch": 2.104,
+ "grad_norm": 1.4025486707687378,
+ "learning_rate": 3.321481481481481e-07,
+ "logits/chosen": 1.3389716148376465,
+ "logits/rejected": 1.4465675354003906,
+ "logps/chosen": -154.75857543945312,
+ "logps/rejected": -151.66146850585938,
+ "loss": 0.651912260055542,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.06493876874446869,
+ "rewards/margins": 0.09662672132253647,
+ "rewards/rejected": -0.03168795630335808,
+ "step": 2630
+ },
+ {
+ "epoch": 2.112,
+ "grad_norm": 1.5229434967041016,
+ "learning_rate": 3.291851851851852e-07,
+ "logits/chosen": 1.733494758605957,
+ "logits/rejected": 1.6284431219100952,
+ "logps/chosen": -159.22984313964844,
+ "logps/rejected": -148.73074340820312,
+ "loss": 0.6609207153320312,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.12649953365325928,
+ "rewards/margins": 0.07567869126796722,
+ "rewards/rejected": 0.050820838660001755,
+ "step": 2640
+ },
+ {
+ "epoch": 2.12,
+ "grad_norm": 1.4460108280181885,
+ "learning_rate": 3.2622222222222225e-07,
+ "logits/chosen": 1.4306278228759766,
+ "logits/rejected": 1.5417077541351318,
+ "logps/chosen": -158.87351989746094,
+ "logps/rejected": -174.2522430419922,
+ "loss": 0.6391911983489991,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.13457423448562622,
+ "rewards/margins": 0.12862522900104523,
+ "rewards/rejected": 0.005949012935161591,
+ "step": 2650
+ },
+ {
+ "epoch": 2.128,
+ "grad_norm": 1.3597654104232788,
+ "learning_rate": 3.232592592592592e-07,
+ "logits/chosen": 1.329627513885498,
+ "logits/rejected": 1.3987334966659546,
+ "logps/chosen": -165.3042449951172,
+ "logps/rejected": -153.50428771972656,
+ "loss": 0.6513238906860351,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.12750771641731262,
+ "rewards/margins": 0.10093003511428833,
+ "rewards/rejected": 0.026577681303024292,
+ "step": 2660
+ },
+ {
+ "epoch": 2.136,
+ "grad_norm": 1.2453831434249878,
+ "learning_rate": 3.202962962962963e-07,
+ "logits/chosen": 1.5347964763641357,
+ "logits/rejected": 1.5230233669281006,
+ "logps/chosen": -154.97242736816406,
+ "logps/rejected": -130.9337158203125,
+ "loss": 0.6872330665588379,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.08273214101791382,
+ "rewards/margins": 0.03720199316740036,
+ "rewards/rejected": 0.04553014412522316,
+ "step": 2670
+ },
+ {
+ "epoch": 2.144,
+ "grad_norm": 1.9659223556518555,
+ "learning_rate": 3.173333333333333e-07,
+ "logits/chosen": 1.5074447393417358,
+ "logits/rejected": 1.4809236526489258,
+ "logps/chosen": -146.32305908203125,
+ "logps/rejected": -143.98345947265625,
+ "loss": 0.6821617603302002,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.08092048764228821,
+ "rewards/margins": 0.03855252265930176,
+ "rewards/rejected": 0.04236796498298645,
+ "step": 2680
+ },
+ {
+ "epoch": 2.152,
+ "grad_norm": 1.2072023153305054,
+ "learning_rate": 3.1437037037037034e-07,
+ "logits/chosen": 1.3799779415130615,
+ "logits/rejected": 1.3887022733688354,
+ "logps/chosen": -143.9578094482422,
+ "logps/rejected": -159.37698364257812,
+ "loss": 0.656886100769043,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0652204304933548,
+ "rewards/margins": 0.08883848786354065,
+ "rewards/rejected": -0.02361806109547615,
+ "step": 2690
+ },
+ {
+ "epoch": 2.16,
+ "grad_norm": 1.1380783319473267,
+ "learning_rate": 3.114074074074074e-07,
+ "logits/chosen": 1.5064178705215454,
+ "logits/rejected": 1.6023409366607666,
+ "logps/chosen": -132.29075622558594,
+ "logps/rejected": -128.1918487548828,
+ "loss": 0.6357467651367188,
+ "rewards/accuracies": 0.762499988079071,
+ "rewards/chosen": 0.15728428959846497,
+ "rewards/margins": 0.13087645173072815,
+ "rewards/rejected": 0.02640785276889801,
+ "step": 2700
+ },
+ {
+ "epoch": 2.16,
+ "eval_logits/chosen": 1.4565616846084595,
+ "eval_logits/rejected": 1.4844369888305664,
+ "eval_logps/chosen": -152.74916076660156,
+ "eval_logps/rejected": -147.5604248046875,
+ "eval_loss": 0.6842489242553711,
+ "eval_rewards/accuracies": 0.5580000281333923,
+ "eval_rewards/chosen": 0.0799703598022461,
+ "eval_rewards/margins": 0.03379753977060318,
+ "eval_rewards/rejected": 0.046172816306352615,
+ "eval_runtime": 90.8529,
+ "eval_samples_per_second": 5.503,
+ "eval_steps_per_second": 2.752,
+ "step": 2700
+ },
+ {
+ "epoch": 2.168,
+ "grad_norm": 1.435728669166565,
+ "learning_rate": 3.0844444444444444e-07,
+ "logits/chosen": 1.4452109336853027,
+ "logits/rejected": 1.4124813079833984,
+ "logps/chosen": -151.26815795898438,
+ "logps/rejected": -143.44091796875,
+ "loss": 0.6585721969604492,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.0672367662191391,
+ "rewards/margins": 0.07900270074605942,
+ "rewards/rejected": -0.011765943840146065,
+ "step": 2710
+ },
+ {
+ "epoch": 2.176,
+ "grad_norm": 1.3085405826568604,
+ "learning_rate": 3.0548148148148146e-07,
+ "logits/chosen": 1.2030339241027832,
+ "logits/rejected": 1.2627733945846558,
+ "logps/chosen": -138.23812866210938,
+ "logps/rejected": -134.25238037109375,
+ "loss": 0.6320697784423828,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.10239285230636597,
+ "rewards/margins": 0.14901991188526154,
+ "rewards/rejected": -0.04662706330418587,
+ "step": 2720
+ },
+ {
+ "epoch": 2.184,
+ "grad_norm": 1.5403406620025635,
+ "learning_rate": 3.025185185185185e-07,
+ "logits/chosen": 1.4745285511016846,
+ "logits/rejected": 1.4968340396881104,
+ "logps/chosen": -153.49771118164062,
+ "logps/rejected": -148.9380645751953,
+ "loss": 0.6523497104644775,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.09372548013925552,
+ "rewards/margins": 0.09992895275354385,
+ "rewards/rejected": -0.006203479133546352,
+ "step": 2730
+ },
+ {
+ "epoch": 2.192,
+ "grad_norm": 1.5256744623184204,
+ "learning_rate": 2.9955555555555556e-07,
+ "logits/chosen": 1.39058518409729,
+ "logits/rejected": 1.3656699657440186,
+ "logps/chosen": -148.22085571289062,
+ "logps/rejected": -149.59144592285156,
+ "loss": 0.6467072010040283,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.09406543523073196,
+ "rewards/margins": 0.10628072172403336,
+ "rewards/rejected": -0.012215293012559414,
+ "step": 2740
+ },
+ {
+ "epoch": 2.2,
+ "grad_norm": 1.128711462020874,
+ "learning_rate": 2.965925925925926e-07,
+ "logits/chosen": 1.557297945022583,
+ "logits/rejected": 1.5189030170440674,
+ "logps/chosen": -160.21929931640625,
+ "logps/rejected": -151.35910034179688,
+ "loss": 0.6474220275878906,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.13500377535820007,
+ "rewards/margins": 0.10226242244243622,
+ "rewards/rejected": 0.03274135664105415,
+ "step": 2750
+ },
+ {
+ "epoch": 2.208,
+ "grad_norm": 1.3706861734390259,
+ "learning_rate": 2.936296296296296e-07,
+ "logits/chosen": 1.431715965270996,
+ "logits/rejected": 1.4564027786254883,
+ "logps/chosen": -142.4249267578125,
+ "logps/rejected": -143.3875732421875,
+ "loss": 0.6685743808746338,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.09871350228786469,
+ "rewards/margins": 0.06824100762605667,
+ "rewards/rejected": 0.03047250770032406,
+ "step": 2760
+ },
+ {
+ "epoch": 2.216,
+ "grad_norm": 1.3409324884414673,
+ "learning_rate": 2.906666666666667e-07,
+ "logits/chosen": 1.332448124885559,
+ "logits/rejected": 1.3167566061019897,
+ "logps/chosen": -146.07406616210938,
+ "logps/rejected": -138.0575408935547,
+ "loss": 0.6424362182617187,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.05286739021539688,
+ "rewards/margins": 0.12220163643360138,
+ "rewards/rejected": -0.0693342313170433,
+ "step": 2770
+ },
+ {
+ "epoch": 2.224,
+ "grad_norm": 1.5424753427505493,
+ "learning_rate": 2.8770370370370364e-07,
+ "logits/chosen": 1.7004855871200562,
+ "logits/rejected": 1.5205724239349365,
+ "logps/chosen": -180.87466430664062,
+ "logps/rejected": -163.62646484375,
+ "loss": 0.6519207000732422,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.12406478077173233,
+ "rewards/margins": 0.10040261596441269,
+ "rewards/rejected": 0.023662149906158447,
+ "step": 2780
+ },
+ {
+ "epoch": 2.232,
+ "grad_norm": 1.2184964418411255,
+ "learning_rate": 2.847407407407407e-07,
+ "logits/chosen": 1.4738783836364746,
+ "logits/rejected": 1.4357870817184448,
+ "logps/chosen": -164.53448486328125,
+ "logps/rejected": -142.24685668945312,
+ "loss": 0.6661330223083496,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.09590204805135727,
+ "rewards/margins": 0.06964156031608582,
+ "rewards/rejected": 0.026260489597916603,
+ "step": 2790
+ },
+ {
+ "epoch": 2.24,
+ "grad_norm": 1.2697800397872925,
+ "learning_rate": 2.817777777777778e-07,
+ "logits/chosen": 1.4889880418777466,
+ "logits/rejected": 1.5381649732589722,
+ "logps/chosen": -181.35275268554688,
+ "logps/rejected": -159.2748565673828,
+ "loss": 0.6470918655395508,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.10390053689479828,
+ "rewards/margins": 0.10561826080083847,
+ "rewards/rejected": -0.00171771424356848,
+ "step": 2800
+ },
+ {
+ "epoch": 2.24,
+ "eval_logits/chosen": 1.450643539428711,
+ "eval_logits/rejected": 1.478151559829712,
+ "eval_logps/chosen": -152.76634216308594,
+ "eval_logps/rejected": -147.58448791503906,
+ "eval_loss": 0.6843582987785339,
+ "eval_rewards/accuracies": 0.5619999766349792,
+ "eval_rewards/chosen": 0.07825153321027756,
+ "eval_rewards/margins": 0.03448594734072685,
+ "eval_rewards/rejected": 0.04376557841897011,
+ "eval_runtime": 90.9417,
+ "eval_samples_per_second": 5.498,
+ "eval_steps_per_second": 2.749,
+ "step": 2800
+ },
+ {
+ "epoch": 2.248,
+ "grad_norm": 1.161729097366333,
+ "learning_rate": 2.7881481481481476e-07,
+ "logits/chosen": 1.4012717008590698,
+ "logits/rejected": 1.365496277809143,
+ "logps/chosen": -142.15725708007812,
+ "logps/rejected": -131.11892700195312,
+ "loss": 0.6921159744262695,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.048748843371868134,
+ "rewards/margins": 0.018229911103844643,
+ "rewards/rejected": 0.030518928542733192,
+ "step": 2810
+ },
+ {
+ "epoch": 2.2560000000000002,
+ "grad_norm": 1.403849720954895,
+ "learning_rate": 2.7585185185185184e-07,
+ "logits/chosen": 1.6802637577056885,
+ "logits/rejected": 1.6305253505706787,
+ "logps/chosen": -162.37313842773438,
+ "logps/rejected": -165.4166717529297,
+ "loss": 0.6506373405456543,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.14859917759895325,
+ "rewards/margins": 0.09807908535003662,
+ "rewards/rejected": 0.05052009969949722,
+ "step": 2820
+ },
+ {
+ "epoch": 2.2640000000000002,
+ "grad_norm": 1.3189504146575928,
+ "learning_rate": 2.728888888888889e-07,
+ "logits/chosen": 1.4633572101593018,
+ "logits/rejected": 1.4499682188034058,
+ "logps/chosen": -156.68209838867188,
+ "logps/rejected": -155.54751586914062,
+ "loss": 0.6692873954772949,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.13907714188098907,
+ "rewards/margins": 0.06127101182937622,
+ "rewards/rejected": 0.07780613750219345,
+ "step": 2830
+ },
+ {
+ "epoch": 2.2720000000000002,
+ "grad_norm": 1.5040212869644165,
+ "learning_rate": 2.699259259259259e-07,
+ "logits/chosen": 1.4402498006820679,
+ "logits/rejected": 1.4848177433013916,
+ "logps/chosen": -145.59292602539062,
+ "logps/rejected": -142.07449340820312,
+ "loss": 0.6696759223937988,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1008831262588501,
+ "rewards/margins": 0.06455080956220627,
+ "rewards/rejected": 0.03633231669664383,
+ "step": 2840
+ },
+ {
+ "epoch": 2.2800000000000002,
+ "grad_norm": 1.4957029819488525,
+ "learning_rate": 2.6696296296296296e-07,
+ "logits/chosen": 1.309990406036377,
+ "logits/rejected": 1.2410783767700195,
+ "logps/chosen": -145.6426239013672,
+ "logps/rejected": -155.6902313232422,
+ "loss": 0.6614094257354737,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.08516388386487961,
+ "rewards/margins": 0.0843389481306076,
+ "rewards/rejected": 0.0008249226957559586,
+ "step": 2850
+ },
+ {
+ "epoch": 2.288,
+ "grad_norm": 1.1865259408950806,
+ "learning_rate": 2.64e-07,
+ "logits/chosen": 1.4638307094573975,
+ "logits/rejected": 1.5667332410812378,
+ "logps/chosen": -128.8794708251953,
+ "logps/rejected": -121.29227447509766,
+ "loss": 0.6763367652893066,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.11934898048639297,
+ "rewards/margins": 0.05003399774432182,
+ "rewards/rejected": 0.06931495666503906,
+ "step": 2860
+ },
+ {
+ "epoch": 2.296,
+ "grad_norm": 2.0794501304626465,
+ "learning_rate": 2.61037037037037e-07,
+ "logits/chosen": 1.1402711868286133,
+ "logits/rejected": 1.4019138813018799,
+ "logps/chosen": -147.0840301513672,
+ "logps/rejected": -157.21463012695312,
+ "loss": 0.6764614105224609,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.09646089375019073,
+ "rewards/margins": 0.05578025057911873,
+ "rewards/rejected": 0.04068063572049141,
+ "step": 2870
+ },
+ {
+ "epoch": 2.304,
+ "grad_norm": 1.318089485168457,
+ "learning_rate": 2.580740740740741e-07,
+ "logits/chosen": 1.3814247846603394,
+ "logits/rejected": 1.2060496807098389,
+ "logps/chosen": -131.8695068359375,
+ "logps/rejected": -124.2463607788086,
+ "loss": 0.6537458896636963,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.0847344920039177,
+ "rewards/margins": 0.10118331015110016,
+ "rewards/rejected": -0.016448814421892166,
+ "step": 2880
+ },
+ {
+ "epoch": 2.312,
+ "grad_norm": 1.1566070318222046,
+ "learning_rate": 2.551111111111111e-07,
+ "logits/chosen": 1.449655294418335,
+ "logits/rejected": 1.4626872539520264,
+ "logps/chosen": -148.62901306152344,
+ "logps/rejected": -162.71693420410156,
+ "loss": 0.6510843753814697,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.1346159428358078,
+ "rewards/margins": 0.09821879863739014,
+ "rewards/rejected": 0.03639713674783707,
+ "step": 2890
+ },
+ {
+ "epoch": 2.32,
+ "grad_norm": 1.3676846027374268,
+ "learning_rate": 2.521481481481481e-07,
+ "logits/chosen": 1.5381054878234863,
+ "logits/rejected": 1.4480576515197754,
+ "logps/chosen": -159.05850219726562,
+ "logps/rejected": -143.50048828125,
+ "loss": 0.6565275669097901,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.09088721126317978,
+ "rewards/margins": 0.08976241946220398,
+ "rewards/rejected": 0.0011247784132137895,
+ "step": 2900
+ },
+ {
+ "epoch": 2.32,
+ "eval_logits/chosen": 1.4630789756774902,
+ "eval_logits/rejected": 1.4909931421279907,
+ "eval_logps/chosen": -152.56454467773438,
+ "eval_logps/rejected": -147.39541625976562,
+ "eval_loss": 0.6835448145866394,
+ "eval_rewards/accuracies": 0.5540000200271606,
+ "eval_rewards/chosen": 0.09842933714389801,
+ "eval_rewards/margins": 0.03575750067830086,
+ "eval_rewards/rejected": 0.06267183274030685,
+ "eval_runtime": 90.9368,
+ "eval_samples_per_second": 5.498,
+ "eval_steps_per_second": 2.749,
+ "step": 2900
+ },
+ {
+ "epoch": 2.328,
+ "grad_norm": 1.2419706583023071,
+ "learning_rate": 2.4918518518518515e-07,
+ "logits/chosen": 1.3978856801986694,
+ "logits/rejected": 1.4488260746002197,
+ "logps/chosen": -131.99398803710938,
+ "logps/rejected": -155.72616577148438,
+ "loss": 0.6545028209686279,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.1015889048576355,
+ "rewards/margins": 0.0939277708530426,
+ "rewards/rejected": 0.007661131210625172,
+ "step": 2910
+ },
+ {
+ "epoch": 2.336,
+ "grad_norm": 0.8262892961502075,
+ "learning_rate": 2.462222222222222e-07,
+ "logits/chosen": 1.260616660118103,
+ "logits/rejected": 1.298372507095337,
+ "logps/chosen": -131.44737243652344,
+ "logps/rejected": -132.9801483154297,
+ "loss": 0.6633480072021485,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.11483611911535263,
+ "rewards/margins": 0.08126294612884521,
+ "rewards/rejected": 0.033573172986507416,
+ "step": 2920
+ },
+ {
+ "epoch": 2.344,
+ "grad_norm": 1.0795193910598755,
+ "learning_rate": 2.4325925925925924e-07,
+ "logits/chosen": 1.6006931066513062,
+ "logits/rejected": 1.5708739757537842,
+ "logps/chosen": -136.45252990722656,
+ "logps/rejected": -145.27377319335938,
+ "loss": 0.6421307563781739,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.1252741515636444,
+ "rewards/margins": 0.12073083221912384,
+ "rewards/rejected": 0.004543319344520569,
+ "step": 2930
+ },
+ {
+ "epoch": 2.352,
+ "grad_norm": 1.2647486925125122,
+ "learning_rate": 2.4029629629629627e-07,
+ "logits/chosen": 1.386439561843872,
+ "logits/rejected": 1.5220506191253662,
+ "logps/chosen": -160.830078125,
+ "logps/rejected": -166.8371124267578,
+ "loss": 0.6581830024719239,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.08102302998304367,
+ "rewards/margins": 0.08525596559047699,
+ "rewards/rejected": -0.004232937004417181,
+ "step": 2940
+ },
+ {
+ "epoch": 2.36,
+ "grad_norm": 1.1739935874938965,
+ "learning_rate": 2.3733333333333334e-07,
+ "logits/chosen": 1.4723713397979736,
+ "logits/rejected": 1.313079595565796,
+ "logps/chosen": -145.6066436767578,
+ "logps/rejected": -132.0267791748047,
+ "loss": 0.6555693626403809,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10856165736913681,
+ "rewards/margins": 0.10550457239151001,
+ "rewards/rejected": 0.003057088004425168,
+ "step": 2950
+ },
+ {
+ "epoch": 2.368,
+ "grad_norm": 1.1559282541275024,
+ "learning_rate": 2.3437037037037036e-07,
+ "logits/chosen": 1.6269900798797607,
+ "logits/rejected": 1.4614148139953613,
+ "logps/chosen": -166.4721221923828,
+ "logps/rejected": -142.86746215820312,
+ "loss": 0.6763554573059082,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.10653790086507797,
+ "rewards/margins": 0.04229436814785004,
+ "rewards/rejected": 0.06424354016780853,
+ "step": 2960
+ },
+ {
+ "epoch": 2.376,
+ "grad_norm": 1.315238118171692,
+ "learning_rate": 2.3140740740740739e-07,
+ "logits/chosen": 1.5281031131744385,
+ "logits/rejected": 1.6713653802871704,
+ "logps/chosen": -155.27975463867188,
+ "logps/rejected": -140.60409545898438,
+ "loss": 0.6465449333190918,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1229790598154068,
+ "rewards/margins": 0.10460098087787628,
+ "rewards/rejected": 0.018378086388111115,
+ "step": 2970
+ },
+ {
+ "epoch": 2.384,
+ "grad_norm": 1.5117779970169067,
+ "learning_rate": 2.2844444444444443e-07,
+ "logits/chosen": 1.4258384704589844,
+ "logits/rejected": 1.3107579946517944,
+ "logps/chosen": -153.65931701660156,
+ "logps/rejected": -136.7742156982422,
+ "loss": 0.6342881679534912,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.09818007051944733,
+ "rewards/margins": 0.1422421634197235,
+ "rewards/rejected": -0.04406210035085678,
+ "step": 2980
+ },
+ {
+ "epoch": 2.392,
+ "grad_norm": 1.543182134628296,
+ "learning_rate": 2.2548148148148146e-07,
+ "logits/chosen": 1.374324917793274,
+ "logits/rejected": 1.5730586051940918,
+ "logps/chosen": -137.39154052734375,
+ "logps/rejected": -156.12655639648438,
+ "loss": 0.660968017578125,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.1029653325676918,
+ "rewards/margins": 0.0822048932313919,
+ "rewards/rejected": 0.020760437473654747,
+ "step": 2990
+ },
+ {
+ "epoch": 2.4,
+ "grad_norm": 1.5507569313049316,
+ "learning_rate": 2.2251851851851853e-07,
+ "logits/chosen": 1.232704520225525,
+ "logits/rejected": 1.3583004474639893,
+ "logps/chosen": -153.07070922851562,
+ "logps/rejected": -152.47064208984375,
+ "loss": 0.6697997093200684,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.03783693164587021,
+ "rewards/margins": 0.06472334265708923,
+ "rewards/rejected": -0.02688641846179962,
+ "step": 3000
+ },
+ {
+ "epoch": 2.4,
+ "eval_logits/chosen": 1.4536765813827515,
+ "eval_logits/rejected": 1.4812932014465332,
+ "eval_logps/chosen": -152.67945861816406,
+ "eval_logps/rejected": -147.50315856933594,
+ "eval_loss": 0.6841781139373779,
+ "eval_rewards/accuracies": 0.5619999766349792,
+ "eval_rewards/chosen": 0.08694048970937729,
+ "eval_rewards/margins": 0.035040292888879776,
+ "eval_rewards/rejected": 0.05190018564462662,
+ "eval_runtime": 90.957,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.749,
+ "step": 3000
+ },
+ {
+ "epoch": 2.408,
+ "grad_norm": 1.299101710319519,
+ "learning_rate": 2.1955555555555555e-07,
+ "logits/chosen": 1.300927996635437,
+ "logits/rejected": 1.4192984104156494,
+ "logps/chosen": -140.5328826904297,
+ "logps/rejected": -157.88247680664062,
+ "loss": 0.6539476871490478,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.04641692712903023,
+ "rewards/margins": 0.1018022894859314,
+ "rewards/rejected": -0.055385369807481766,
+ "step": 3010
+ },
+ {
+ "epoch": 2.416,
+ "grad_norm": 1.597330093383789,
+ "learning_rate": 2.1659259259259258e-07,
+ "logits/chosen": 1.410930871963501,
+ "logits/rejected": 1.4487859010696411,
+ "logps/chosen": -147.2603759765625,
+ "logps/rejected": -141.80209350585938,
+ "loss": 0.6582518577575683,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.05355765298008919,
+ "rewards/margins": 0.08682949841022491,
+ "rewards/rejected": -0.033271849155426025,
+ "step": 3020
+ },
+ {
+ "epoch": 2.424,
+ "grad_norm": 1.2904349565505981,
+ "learning_rate": 2.1362962962962962e-07,
+ "logits/chosen": 1.431049108505249,
+ "logits/rejected": 1.4038130044937134,
+ "logps/chosen": -134.06564331054688,
+ "logps/rejected": -139.49826049804688,
+ "loss": 0.6637657642364502,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.12606436014175415,
+ "rewards/margins": 0.07909566909074783,
+ "rewards/rejected": 0.04696869105100632,
+ "step": 3030
+ },
+ {
+ "epoch": 2.432,
+ "grad_norm": 1.3195490837097168,
+ "learning_rate": 2.1066666666666665e-07,
+ "logits/chosen": 1.5150330066680908,
+ "logits/rejected": 1.3475278615951538,
+ "logps/chosen": -157.96343994140625,
+ "logps/rejected": -160.56509399414062,
+ "loss": 0.6553669452667237,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.10793514549732208,
+ "rewards/margins": 0.09139807522296906,
+ "rewards/rejected": 0.01653706096112728,
+ "step": 3040
+ },
+ {
+ "epoch": 2.44,
+ "grad_norm": 1.022117018699646,
+ "learning_rate": 2.077037037037037e-07,
+ "logits/chosen": 1.411431074142456,
+ "logits/rejected": 1.5498731136322021,
+ "logps/chosen": -142.24293518066406,
+ "logps/rejected": -152.7814483642578,
+ "loss": 0.6480430603027344,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.13466320931911469,
+ "rewards/margins": 0.11154146492481232,
+ "rewards/rejected": 0.023121753707528114,
+ "step": 3050
+ },
+ {
+ "epoch": 2.448,
+ "grad_norm": 1.3372952938079834,
+ "learning_rate": 2.0474074074074074e-07,
+ "logits/chosen": 1.3349260091781616,
+ "logits/rejected": 1.36862051486969,
+ "logps/chosen": -141.4913787841797,
+ "logps/rejected": -153.81234741210938,
+ "loss": 0.6436941623687744,
+ "rewards/accuracies": 0.7749999761581421,
+ "rewards/chosen": 0.09609787166118622,
+ "rewards/margins": 0.12482483685016632,
+ "rewards/rejected": -0.02872694469988346,
+ "step": 3060
+ },
+ {
+ "epoch": 2.456,
+ "grad_norm": 1.2636305093765259,
+ "learning_rate": 2.0177777777777777e-07,
+ "logits/chosen": 1.5101096630096436,
+ "logits/rejected": 1.4085323810577393,
+ "logps/chosen": -158.89871215820312,
+ "logps/rejected": -157.28538513183594,
+ "loss": 0.6665299892425537,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.13253602385520935,
+ "rewards/margins": 0.06450225412845612,
+ "rewards/rejected": 0.06803376972675323,
+ "step": 3070
+ },
+ {
+ "epoch": 2.464,
+ "grad_norm": 1.3486571311950684,
+ "learning_rate": 1.988148148148148e-07,
+ "logits/chosen": 1.327289342880249,
+ "logits/rejected": 1.3173015117645264,
+ "logps/chosen": -160.33554077148438,
+ "logps/rejected": -153.06150817871094,
+ "loss": 0.6544498443603516,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10490100085735321,
+ "rewards/margins": 0.10563824325799942,
+ "rewards/rejected": -0.0007372468826361,
+ "step": 3080
+ },
+ {
+ "epoch": 2.472,
+ "grad_norm": 1.1544814109802246,
+ "learning_rate": 1.9585185185185186e-07,
+ "logits/chosen": 1.208981990814209,
+ "logits/rejected": 1.221260905265808,
+ "logps/chosen": -133.82061767578125,
+ "logps/rejected": -134.70327758789062,
+ "loss": 0.6506445407867432,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.07232929766178131,
+ "rewards/margins": 0.1016290932893753,
+ "rewards/rejected": -0.029299795627593994,
+ "step": 3090
+ },
+ {
+ "epoch": 2.48,
+ "grad_norm": 1.2893116474151611,
+ "learning_rate": 1.9288888888888889e-07,
+ "logits/chosen": 1.3482762575149536,
+ "logits/rejected": 1.3961074352264404,
+ "logps/chosen": -145.61500549316406,
+ "logps/rejected": -135.99945068359375,
+ "loss": 0.6463598728179931,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.16556979715824127,
+ "rewards/margins": 0.12888674437999725,
+ "rewards/rejected": 0.03668304160237312,
+ "step": 3100
+ },
+ {
+ "epoch": 2.48,
+ "eval_logits/chosen": 1.4538236856460571,
+ "eval_logits/rejected": 1.4815431833267212,
+ "eval_logps/chosen": -152.71058654785156,
+ "eval_logps/rejected": -147.53634643554688,
+ "eval_loss": 0.6841146349906921,
+ "eval_rewards/accuracies": 0.5619999766349792,
+ "eval_rewards/chosen": 0.08382504433393478,
+ "eval_rewards/margins": 0.03524418920278549,
+ "eval_rewards/rejected": 0.048580847680568695,
+ "eval_runtime": 90.8524,
+ "eval_samples_per_second": 5.503,
+ "eval_steps_per_second": 2.752,
+ "step": 3100
+ },
+ {
+ "epoch": 2.488,
+ "grad_norm": 1.2138270139694214,
+ "learning_rate": 1.899259259259259e-07,
+ "logits/chosen": 1.4320173263549805,
+ "logits/rejected": 1.379774570465088,
+ "logps/chosen": -140.01589965820312,
+ "logps/rejected": -145.36776733398438,
+ "loss": 0.6643844604492187,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.09917240589857101,
+ "rewards/margins": 0.07538153976202011,
+ "rewards/rejected": 0.023790866136550903,
+ "step": 3110
+ },
+ {
+ "epoch": 2.496,
+ "grad_norm": 1.2045173645019531,
+ "learning_rate": 1.8696296296296296e-07,
+ "logits/chosen": 1.2402794361114502,
+ "logits/rejected": 1.4080727100372314,
+ "logps/chosen": -119.64544677734375,
+ "logps/rejected": -134.0382843017578,
+ "loss": 0.6308258056640625,
+ "rewards/accuracies": 0.7749999761581421,
+ "rewards/chosen": 0.06476704776287079,
+ "rewards/margins": 0.15264371037483215,
+ "rewards/rejected": -0.08787664026021957,
+ "step": 3120
+ },
+ {
+ "epoch": 2.504,
+ "grad_norm": 1.2934551239013672,
+ "learning_rate": 1.8399999999999998e-07,
+ "logits/chosen": 1.521606206893921,
+ "logits/rejected": 1.4204734563827515,
+ "logps/chosen": -136.7379608154297,
+ "logps/rejected": -141.2797088623047,
+ "loss": 0.6567333221435547,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.14066217839717865,
+ "rewards/margins": 0.09437786787748337,
+ "rewards/rejected": 0.04628429934382439,
+ "step": 3130
+ },
+ {
+ "epoch": 2.512,
+ "grad_norm": 1.4545506238937378,
+ "learning_rate": 1.8103703703703703e-07,
+ "logits/chosen": 1.5223419666290283,
+ "logits/rejected": 1.369768738746643,
+ "logps/chosen": -157.4479217529297,
+ "logps/rejected": -146.4864501953125,
+ "loss": 0.6391398906707764,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.12953564524650574,
+ "rewards/margins": 0.13037580251693726,
+ "rewards/rejected": -0.0008401572704315186,
+ "step": 3140
+ },
+ {
+ "epoch": 2.52,
+ "grad_norm": 1.2595700025558472,
+ "learning_rate": 1.7807407407407408e-07,
+ "logits/chosen": 1.4436731338500977,
+ "logits/rejected": 1.5720274448394775,
+ "logps/chosen": -166.989013671875,
+ "logps/rejected": -162.21975708007812,
+ "loss": 0.6528693199157715,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.09743344783782959,
+ "rewards/margins": 0.09220697730779648,
+ "rewards/rejected": 0.005226473324000835,
+ "step": 3150
+ },
+ {
+ "epoch": 2.528,
+ "grad_norm": 1.2463562488555908,
+ "learning_rate": 1.751111111111111e-07,
+ "logits/chosen": 1.4497953653335571,
+ "logits/rejected": 1.4076560735702515,
+ "logps/chosen": -150.78482055664062,
+ "logps/rejected": -155.48902893066406,
+ "loss": 0.6709191799163818,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.13361872732639313,
+ "rewards/margins": 0.06404821574687958,
+ "rewards/rejected": 0.06957050412893295,
+ "step": 3160
+ },
+ {
+ "epoch": 2.536,
+ "grad_norm": 1.1280947923660278,
+ "learning_rate": 1.7214814814814815e-07,
+ "logits/chosen": 1.5186564922332764,
+ "logits/rejected": 1.5018190145492554,
+ "logps/chosen": -155.66006469726562,
+ "logps/rejected": -159.46339416503906,
+ "loss": 0.6649936676025391,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.11657847464084625,
+ "rewards/margins": 0.07113558799028397,
+ "rewards/rejected": 0.045442890375852585,
+ "step": 3170
+ },
+ {
+ "epoch": 2.544,
+ "grad_norm": 1.5582311153411865,
+ "learning_rate": 1.6918518518518517e-07,
+ "logits/chosen": 1.5405746698379517,
+ "logits/rejected": 1.5165503025054932,
+ "logps/chosen": -164.27452087402344,
+ "logps/rejected": -171.57701110839844,
+ "loss": 0.6631642818450928,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.07613547891378403,
+ "rewards/margins": 0.07466691732406616,
+ "rewards/rejected": 0.0014685706701129675,
+ "step": 3180
+ },
+ {
+ "epoch": 2.552,
+ "grad_norm": 1.3221781253814697,
+ "learning_rate": 1.6622222222222222e-07,
+ "logits/chosen": 1.59696364402771,
+ "logits/rejected": 1.3756712675094604,
+ "logps/chosen": -175.3300018310547,
+ "logps/rejected": -160.0670623779297,
+ "loss": 0.6846892833709717,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.09736991673707962,
+ "rewards/margins": 0.028879499062895775,
+ "rewards/rejected": 0.0684904232621193,
+ "step": 3190
+ },
+ {
+ "epoch": 2.56,
+ "grad_norm": 1.2958283424377441,
+ "learning_rate": 1.6325925925925927e-07,
+ "logits/chosen": 1.5975316762924194,
+ "logits/rejected": 1.3776990175247192,
+ "logps/chosen": -167.61512756347656,
+ "logps/rejected": -151.68203735351562,
+ "loss": 0.6497097969055176,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.13339915871620178,
+ "rewards/margins": 0.10488065332174301,
+ "rewards/rejected": 0.02851850353181362,
+ "step": 3200
+ },
+ {
+ "epoch": 2.56,
+ "eval_logits/chosen": 1.4546222686767578,
+ "eval_logits/rejected": 1.4822379350662231,
+ "eval_logps/chosen": -152.66015625,
+ "eval_logps/rejected": -147.49147033691406,
+ "eval_loss": 0.6839378476142883,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.08886973559856415,
+ "eval_rewards/margins": 0.03580106422305107,
+ "eval_rewards/rejected": 0.053068675100803375,
+ "eval_runtime": 90.9562,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.749,
+ "step": 3200
+ },
+ {
+ "epoch": 2.568,
+ "grad_norm": 2.3782362937927246,
+ "learning_rate": 1.602962962962963e-07,
+ "logits/chosen": 1.4312902688980103,
+ "logits/rejected": 1.6004657745361328,
+ "logps/chosen": -166.62925720214844,
+ "logps/rejected": -158.98095703125,
+ "loss": 0.6500686645507813,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.14471536874771118,
+ "rewards/margins": 0.10505994409322739,
+ "rewards/rejected": 0.039655424654483795,
+ "step": 3210
+ },
+ {
+ "epoch": 2.576,
+ "grad_norm": 1.392526388168335,
+ "learning_rate": 1.573333333333333e-07,
+ "logits/chosen": 1.453791856765747,
+ "logits/rejected": 1.5730758905410767,
+ "logps/chosen": -161.05404663085938,
+ "logps/rejected": -164.30911254882812,
+ "loss": 0.6707005977630616,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.1165541261434555,
+ "rewards/margins": 0.05886579677462578,
+ "rewards/rejected": 0.05768834426999092,
+ "step": 3220
+ },
+ {
+ "epoch": 2.584,
+ "grad_norm": 1.4355558156967163,
+ "learning_rate": 1.5437037037037036e-07,
+ "logits/chosen": 1.3674612045288086,
+ "logits/rejected": 1.5156943798065186,
+ "logps/chosen": -147.24728393554688,
+ "logps/rejected": -156.7361297607422,
+ "loss": 0.6483449935913086,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.12978991866111755,
+ "rewards/margins": 0.10252753645181656,
+ "rewards/rejected": 0.02726239338517189,
+ "step": 3230
+ },
+ {
+ "epoch": 2.592,
+ "grad_norm": 1.3843603134155273,
+ "learning_rate": 1.514074074074074e-07,
+ "logits/chosen": 1.5766382217407227,
+ "logits/rejected": 1.5126440525054932,
+ "logps/chosen": -171.20852661132812,
+ "logps/rejected": -143.60977172851562,
+ "loss": 0.6324634075164794,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.19362562894821167,
+ "rewards/margins": 0.14052848517894745,
+ "rewards/rejected": 0.05309714749455452,
+ "step": 3240
+ },
+ {
+ "epoch": 2.6,
+ "grad_norm": 1.089949369430542,
+ "learning_rate": 1.4844444444444443e-07,
+ "logits/chosen": 1.3081635236740112,
+ "logits/rejected": 1.4264672994613647,
+ "logps/chosen": -149.9308319091797,
+ "logps/rejected": -153.34234619140625,
+ "loss": 0.6382611751556396,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.1245744600892067,
+ "rewards/margins": 0.12832382321357727,
+ "rewards/rejected": -0.0037493593990802765,
+ "step": 3250
+ },
+ {
+ "epoch": 2.608,
+ "grad_norm": 1.4414907693862915,
+ "learning_rate": 1.4548148148148148e-07,
+ "logits/chosen": 1.4160754680633545,
+ "logits/rejected": 1.4785319566726685,
+ "logps/chosen": -147.95248413085938,
+ "logps/rejected": -152.06893920898438,
+ "loss": 0.6700046539306641,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.05685792490839958,
+ "rewards/margins": 0.056677937507629395,
+ "rewards/rejected": 0.00017998255498241633,
+ "step": 3260
+ },
+ {
+ "epoch": 2.616,
+ "grad_norm": 1.3908545970916748,
+ "learning_rate": 1.425185185185185e-07,
+ "logits/chosen": 1.4107372760772705,
+ "logits/rejected": 1.442772626876831,
+ "logps/chosen": -163.569091796875,
+ "logps/rejected": -153.9599609375,
+ "loss": 0.6501737117767334,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.07901959121227264,
+ "rewards/margins": 0.105490542948246,
+ "rewards/rejected": -0.02647094801068306,
+ "step": 3270
+ },
+ {
+ "epoch": 2.624,
+ "grad_norm": 1.3507336378097534,
+ "learning_rate": 1.3955555555555553e-07,
+ "logits/chosen": 1.5124311447143555,
+ "logits/rejected": 1.405656337738037,
+ "logps/chosen": -141.2902374267578,
+ "logps/rejected": -172.03335571289062,
+ "loss": 0.6489161014556885,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.13159234821796417,
+ "rewards/margins": 0.10707925260066986,
+ "rewards/rejected": 0.024513086304068565,
+ "step": 3280
+ },
+ {
+ "epoch": 2.632,
+ "grad_norm": 1.4356945753097534,
+ "learning_rate": 1.365925925925926e-07,
+ "logits/chosen": 1.4829243421554565,
+ "logits/rejected": 1.5267254114151,
+ "logps/chosen": -156.767578125,
+ "logps/rejected": -166.2552032470703,
+ "loss": 0.661461877822876,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.1381731629371643,
+ "rewards/margins": 0.08749156445264816,
+ "rewards/rejected": 0.050681598484516144,
+ "step": 3290
+ },
+ {
+ "epoch": 2.64,
+ "grad_norm": 1.2448052167892456,
+ "learning_rate": 1.3362962962962962e-07,
+ "logits/chosen": 1.4656591415405273,
+ "logits/rejected": 1.5108649730682373,
+ "logps/chosen": -151.31231689453125,
+ "logps/rejected": -135.82418823242188,
+ "loss": 0.6609601974487305,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.12814496457576752,
+ "rewards/margins": 0.0794161856174469,
+ "rewards/rejected": 0.048728786408901215,
+ "step": 3300
+ },
+ {
+ "epoch": 2.64,
+ "eval_logits/chosen": 1.4599801301956177,
+ "eval_logits/rejected": 1.4877806901931763,
+ "eval_logps/chosen": -152.58131408691406,
+ "eval_logps/rejected": -147.42189025878906,
+ "eval_loss": 0.6834326386451721,
+ "eval_rewards/accuracies": 0.5580000281333923,
+ "eval_rewards/chosen": 0.0967537984251976,
+ "eval_rewards/margins": 0.03672795370221138,
+ "eval_rewards/rejected": 0.06002584472298622,
+ "eval_runtime": 90.8651,
+ "eval_samples_per_second": 5.503,
+ "eval_steps_per_second": 2.751,
+ "step": 3300
+ },
+ {
+ "epoch": 2.648,
+ "grad_norm": 1.4104338884353638,
+ "learning_rate": 1.3066666666666665e-07,
+ "logits/chosen": 1.3793938159942627,
+ "logits/rejected": 1.4125133752822876,
+ "logps/chosen": -146.07693481445312,
+ "logps/rejected": -162.7715606689453,
+ "loss": 0.660995626449585,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.09182589501142502,
+ "rewards/margins": 0.08458633720874786,
+ "rewards/rejected": 0.00723955687135458,
+ "step": 3310
+ },
+ {
+ "epoch": 2.656,
+ "grad_norm": 1.6272552013397217,
+ "learning_rate": 1.277037037037037e-07,
+ "logits/chosen": 1.5128822326660156,
+ "logits/rejected": 1.472041368484497,
+ "logps/chosen": -168.37808227539062,
+ "logps/rejected": -138.04177856445312,
+ "loss": 0.6664434909820557,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.12670311331748962,
+ "rewards/margins": 0.06635840237140656,
+ "rewards/rejected": 0.06034470722079277,
+ "step": 3320
+ },
+ {
+ "epoch": 2.664,
+ "grad_norm": 1.6957371234893799,
+ "learning_rate": 1.2474074074074072e-07,
+ "logits/chosen": 1.6430788040161133,
+ "logits/rejected": 1.701340675354004,
+ "logps/chosen": -175.08358764648438,
+ "logps/rejected": -186.39157104492188,
+ "loss": 0.6366147041320801,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.19551560282707214,
+ "rewards/margins": 0.1353006660938263,
+ "rewards/rejected": 0.06021494418382645,
+ "step": 3330
+ },
+ {
+ "epoch": 2.672,
+ "grad_norm": 1.248181700706482,
+ "learning_rate": 1.2177777777777777e-07,
+ "logits/chosen": 1.5288277864456177,
+ "logits/rejected": 1.5558831691741943,
+ "logps/chosen": -160.19473266601562,
+ "logps/rejected": -172.4878387451172,
+ "loss": 0.6668370246887207,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.11863049119710922,
+ "rewards/margins": 0.06773928552865982,
+ "rewards/rejected": 0.05089121311903,
+ "step": 3340
+ },
+ {
+ "epoch": 2.68,
+ "grad_norm": 1.4140652418136597,
+ "learning_rate": 1.1881481481481481e-07,
+ "logits/chosen": 1.5105751752853394,
+ "logits/rejected": 1.4798310995101929,
+ "logps/chosen": -148.14773559570312,
+ "logps/rejected": -137.95657348632812,
+ "loss": 0.6380892753601074,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.15043237805366516,
+ "rewards/margins": 0.12980502843856812,
+ "rewards/rejected": 0.020627355203032494,
+ "step": 3350
+ },
+ {
+ "epoch": 2.6879999999999997,
+ "grad_norm": 1.2615612745285034,
+ "learning_rate": 1.1585185185185185e-07,
+ "logits/chosen": 1.464707374572754,
+ "logits/rejected": 1.4781959056854248,
+ "logps/chosen": -140.28860473632812,
+ "logps/rejected": -141.41860961914062,
+ "loss": 0.6501630306243896,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.13670575618743896,
+ "rewards/margins": 0.09955373406410217,
+ "rewards/rejected": 0.037152040749788284,
+ "step": 3360
+ },
+ {
+ "epoch": 2.6959999999999997,
+ "grad_norm": 1.385250210762024,
+ "learning_rate": 1.1288888888888889e-07,
+ "logits/chosen": 1.5589094161987305,
+ "logits/rejected": 1.447178840637207,
+ "logps/chosen": -148.5657958984375,
+ "logps/rejected": -169.35621643066406,
+ "loss": 0.6499518871307373,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.1626952886581421,
+ "rewards/margins": 0.10765236616134644,
+ "rewards/rejected": 0.055042944848537445,
+ "step": 3370
+ },
+ {
+ "epoch": 2.7039999999999997,
+ "grad_norm": 1.4454525709152222,
+ "learning_rate": 1.0992592592592592e-07,
+ "logits/chosen": 1.400112271308899,
+ "logits/rejected": 1.4862735271453857,
+ "logps/chosen": -150.48345947265625,
+ "logps/rejected": -168.9849395751953,
+ "loss": 0.6562737464904785,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.10018215328454971,
+ "rewards/margins": 0.08878926187753677,
+ "rewards/rejected": 0.011392900720238686,
+ "step": 3380
+ },
+ {
+ "epoch": 2.7119999999999997,
+ "grad_norm": 1.9209951162338257,
+ "learning_rate": 1.0696296296296297e-07,
+ "logits/chosen": 1.3544957637786865,
+ "logits/rejected": 1.3679192066192627,
+ "logps/chosen": -179.68997192382812,
+ "logps/rejected": -175.43161010742188,
+ "loss": 0.662507963180542,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.08724933862686157,
+ "rewards/margins": 0.08174990117549896,
+ "rewards/rejected": 0.005499442107975483,
+ "step": 3390
+ },
+ {
+ "epoch": 2.7199999999999998,
+ "grad_norm": 1.295901894569397,
+ "learning_rate": 1.0399999999999999e-07,
+ "logits/chosen": 1.2417107820510864,
+ "logits/rejected": 1.2812005281448364,
+ "logps/chosen": -149.10450744628906,
+ "logps/rejected": -141.39944458007812,
+ "loss": 0.6393757820129394,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.12547731399536133,
+ "rewards/margins": 0.12965723872184753,
+ "rewards/rejected": -0.004179902374744415,
+ "step": 3400
+ },
+ {
+ "epoch": 2.7199999999999998,
+ "eval_logits/chosen": 1.457220435142517,
+ "eval_logits/rejected": 1.4849778413772583,
+ "eval_logps/chosen": -152.62132263183594,
+ "eval_logps/rejected": -147.45970153808594,
+ "eval_loss": 0.6836552023887634,
+ "eval_rewards/accuracies": 0.5640000104904175,
+ "eval_rewards/chosen": 0.09275158494710922,
+ "eval_rewards/margins": 0.03650682047009468,
+ "eval_rewards/rejected": 0.05624476447701454,
+ "eval_runtime": 90.9464,
+ "eval_samples_per_second": 5.498,
+ "eval_steps_per_second": 2.749,
+ "step": 3400
+ },
+ {
+ "epoch": 2.7279999999999998,
+ "grad_norm": 1.3785442113876343,
+ "learning_rate": 1.0103703703703703e-07,
+ "logits/chosen": 1.502685785293579,
+ "logits/rejected": 1.5352259874343872,
+ "logps/chosen": -165.4739532470703,
+ "logps/rejected": -150.7890167236328,
+ "loss": 0.6662552833557129,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.1556030958890915,
+ "rewards/margins": 0.06861518323421478,
+ "rewards/rejected": 0.0869879275560379,
+ "step": 3410
+ },
+ {
+ "epoch": 2.7359999999999998,
+ "grad_norm": 1.2958600521087646,
+ "learning_rate": 9.807407407407408e-08,
+ "logits/chosen": 1.536089301109314,
+ "logits/rejected": 1.5964341163635254,
+ "logps/chosen": -169.01927185058594,
+ "logps/rejected": -164.82464599609375,
+ "loss": 0.6383768081665039,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.1638927459716797,
+ "rewards/margins": 0.1290481686592102,
+ "rewards/rejected": 0.03484458476305008,
+ "step": 3420
+ },
+ {
+ "epoch": 2.7439999999999998,
+ "grad_norm": 1.3208509683609009,
+ "learning_rate": 9.51111111111111e-08,
+ "logits/chosen": 1.4485461711883545,
+ "logits/rejected": 1.3840972185134888,
+ "logps/chosen": -127.16844177246094,
+ "logps/rejected": -139.8473358154297,
+ "loss": 0.637468671798706,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.13635967671871185,
+ "rewards/margins": 0.12859989702701569,
+ "rewards/rejected": 0.007759793661534786,
+ "step": 3430
+ },
+ {
+ "epoch": 2.752,
+ "grad_norm": 1.6750396490097046,
+ "learning_rate": 9.214814814814815e-08,
+ "logits/chosen": 1.3725965023040771,
+ "logits/rejected": 1.2694950103759766,
+ "logps/chosen": -160.5334930419922,
+ "logps/rejected": -151.38027954101562,
+ "loss": 0.6750851631164551,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.1054903045296669,
+ "rewards/margins": 0.053377915173769,
+ "rewards/rejected": 0.0521123930811882,
+ "step": 3440
+ },
+ {
+ "epoch": 2.76,
+ "grad_norm": 1.6593612432479858,
+ "learning_rate": 8.918518518518518e-08,
+ "logits/chosen": 1.3901779651641846,
+ "logits/rejected": 1.5386005640029907,
+ "logps/chosen": -153.39671325683594,
+ "logps/rejected": -146.6219482421875,
+ "loss": 0.6775777339935303,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.08287060260772705,
+ "rewards/margins": 0.0443020761013031,
+ "rewards/rejected": 0.03856852278113365,
+ "step": 3450
+ },
+ {
+ "epoch": 2.768,
+ "grad_norm": 1.1425881385803223,
+ "learning_rate": 8.622222222222222e-08,
+ "logits/chosen": 1.6038954257965088,
+ "logits/rejected": 1.6097686290740967,
+ "logps/chosen": -157.6035614013672,
+ "logps/rejected": -149.43051147460938,
+ "loss": 0.6633115768432617,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1399986296892166,
+ "rewards/margins": 0.0743202343583107,
+ "rewards/rejected": 0.06567840278148651,
+ "step": 3460
+ },
+ {
+ "epoch": 2.776,
+ "grad_norm": 1.5609575510025024,
+ "learning_rate": 8.325925925925925e-08,
+ "logits/chosen": 1.36880624294281,
+ "logits/rejected": 1.1539722681045532,
+ "logps/chosen": -149.6429901123047,
+ "logps/rejected": -151.99984741210938,
+ "loss": 0.6565720081329346,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.09346573799848557,
+ "rewards/margins": 0.09520730376243591,
+ "rewards/rejected": -0.0017415669281035662,
+ "step": 3470
+ },
+ {
+ "epoch": 2.784,
+ "grad_norm": 1.1983734369277954,
+ "learning_rate": 8.029629629629629e-08,
+ "logits/chosen": 1.5328073501586914,
+ "logits/rejected": 1.3884016275405884,
+ "logps/chosen": -150.6219482421875,
+ "logps/rejected": -154.71389770507812,
+ "loss": 0.6380839347839355,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1845501959323883,
+ "rewards/margins": 0.13372965157032013,
+ "rewards/rejected": 0.05082052946090698,
+ "step": 3480
+ },
+ {
+ "epoch": 2.792,
+ "grad_norm": 1.2442429065704346,
+ "learning_rate": 7.733333333333334e-08,
+ "logits/chosen": 1.4287585020065308,
+ "logits/rejected": 1.6142936944961548,
+ "logps/chosen": -176.40530395507812,
+ "logps/rejected": -187.75030517578125,
+ "loss": 0.6455347537994385,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.17798057198524475,
+ "rewards/margins": 0.11338325589895248,
+ "rewards/rejected": 0.06459730863571167,
+ "step": 3490
+ },
+ {
+ "epoch": 2.8,
+ "grad_norm": 1.9635435342788696,
+ "learning_rate": 7.437037037037036e-08,
+ "logits/chosen": 1.2022366523742676,
+ "logits/rejected": 1.2814809083938599,
+ "logps/chosen": -132.84152221679688,
+ "logps/rejected": -153.19830322265625,
+ "loss": 0.6350316047668457,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.08858552575111389,
+ "rewards/margins": 0.1404990553855896,
+ "rewards/rejected": -0.05191350728273392,
+ "step": 3500
+ },
+ {
+ "epoch": 2.8,
+ "eval_logits/chosen": 1.4583321809768677,
+ "eval_logits/rejected": 1.4860731363296509,
+ "eval_logps/chosen": -152.62744140625,
+ "eval_logps/rejected": -147.46978759765625,
+ "eval_loss": 0.683444619178772,
+ "eval_rewards/accuracies": 0.5619999766349792,
+ "eval_rewards/chosen": 0.09214015305042267,
+ "eval_rewards/margins": 0.03690274432301521,
+ "eval_rewards/rejected": 0.055237412452697754,
+ "eval_runtime": 90.9644,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.748,
+ "step": 3500
+ },
+ {
+ "epoch": 2.808,
+ "grad_norm": 1.1622490882873535,
+ "learning_rate": 7.140740740740741e-08,
+ "logits/chosen": 1.4402045011520386,
+ "logits/rejected": 1.5743377208709717,
+ "logps/chosen": -123.78458404541016,
+ "logps/rejected": -148.7012939453125,
+ "loss": 0.6563457012176513,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.11645811796188354,
+ "rewards/margins": 0.09171206504106522,
+ "rewards/rejected": 0.02474605105817318,
+ "step": 3510
+ },
+ {
+ "epoch": 2.816,
+ "grad_norm": 1.3605060577392578,
+ "learning_rate": 6.844444444444444e-08,
+ "logits/chosen": 1.3449777364730835,
+ "logits/rejected": 1.3641579151153564,
+ "logps/chosen": -137.8799285888672,
+ "logps/rejected": -141.6542510986328,
+ "loss": 0.6603388786315918,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.15797407925128937,
+ "rewards/margins": 0.08372689038515091,
+ "rewards/rejected": 0.07424716651439667,
+ "step": 3520
+ },
+ {
+ "epoch": 2.824,
+ "grad_norm": 1.3372809886932373,
+ "learning_rate": 6.548148148148148e-08,
+ "logits/chosen": 1.599825143814087,
+ "logits/rejected": 1.6121799945831299,
+ "logps/chosen": -167.2816162109375,
+ "logps/rejected": -162.6077423095703,
+ "loss": 0.6652153015136719,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1697283685207367,
+ "rewards/margins": 0.06826295703649521,
+ "rewards/rejected": 0.10146540403366089,
+ "step": 3530
+ },
+ {
+ "epoch": 2.832,
+ "grad_norm": 1.2685668468475342,
+ "learning_rate": 6.251851851851852e-08,
+ "logits/chosen": 1.5327260494232178,
+ "logits/rejected": 1.5467902421951294,
+ "logps/chosen": -153.3846893310547,
+ "logps/rejected": -166.84727478027344,
+ "loss": 0.6487001895904541,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.17645052075386047,
+ "rewards/margins": 0.10123439878225327,
+ "rewards/rejected": 0.07521611452102661,
+ "step": 3540
+ },
+ {
+ "epoch": 2.84,
+ "grad_norm": 1.197698712348938,
+ "learning_rate": 5.955555555555555e-08,
+ "logits/chosen": 1.5396482944488525,
+ "logits/rejected": 1.4628225564956665,
+ "logps/chosen": -159.66380310058594,
+ "logps/rejected": -145.74610900878906,
+ "loss": 0.6531730651855469,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.11850614845752716,
+ "rewards/margins": 0.09591890871524811,
+ "rewards/rejected": 0.022587241604924202,
+ "step": 3550
+ },
+ {
+ "epoch": 2.848,
+ "grad_norm": 1.5673083066940308,
+ "learning_rate": 5.6592592592592586e-08,
+ "logits/chosen": 1.4155744314193726,
+ "logits/rejected": 1.350127935409546,
+ "logps/chosen": -142.23983764648438,
+ "logps/rejected": -155.3253173828125,
+ "loss": 0.6413177490234375,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.10945747047662735,
+ "rewards/margins": 0.13470008969306946,
+ "rewards/rejected": -0.025242626667022705,
+ "step": 3560
+ },
+ {
+ "epoch": 2.856,
+ "grad_norm": 1.3042010068893433,
+ "learning_rate": 5.362962962962963e-08,
+ "logits/chosen": 1.4582467079162598,
+ "logits/rejected": 1.4870424270629883,
+ "logps/chosen": -148.69073486328125,
+ "logps/rejected": -151.39779663085938,
+ "loss": 0.6614701271057128,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.10823602974414825,
+ "rewards/margins": 0.08666074275970459,
+ "rewards/rejected": 0.021575292572379112,
+ "step": 3570
+ },
+ {
+ "epoch": 2.864,
+ "grad_norm": 1.1495215892791748,
+ "learning_rate": 5.0666666666666664e-08,
+ "logits/chosen": 1.52294921875,
+ "logits/rejected": 1.5980182886123657,
+ "logps/chosen": -141.8367919921875,
+ "logps/rejected": -177.9903564453125,
+ "loss": 0.6496347427368164,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.13001732528209686,
+ "rewards/margins": 0.10187549889087677,
+ "rewards/rejected": 0.028141820803284645,
+ "step": 3580
+ },
+ {
+ "epoch": 2.872,
+ "grad_norm": 0.9840258955955505,
+ "learning_rate": 4.7703703703703706e-08,
+ "logits/chosen": 1.5534389019012451,
+ "logits/rejected": 1.5350894927978516,
+ "logps/chosen": -145.63079833984375,
+ "logps/rejected": -133.99974060058594,
+ "loss": 0.6615052223205566,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.10790165513753891,
+ "rewards/margins": 0.07739261537790298,
+ "rewards/rejected": 0.03050905093550682,
+ "step": 3590
+ },
+ {
+ "epoch": 2.88,
+ "grad_norm": 1.5131075382232666,
+ "learning_rate": 4.474074074074074e-08,
+ "logits/chosen": 1.564985752105713,
+ "logits/rejected": 1.5553464889526367,
+ "logps/chosen": -149.2957000732422,
+ "logps/rejected": -159.87326049804688,
+ "loss": 0.6592265605926514,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.11044289916753769,
+ "rewards/margins": 0.08047507703304291,
+ "rewards/rejected": 0.02996782958507538,
+ "step": 3600
+ },
+ {
+ "epoch": 2.88,
+ "eval_logits/chosen": 1.4542793035507202,
+ "eval_logits/rejected": 1.4820283651351929,
+ "eval_logps/chosen": -152.6907196044922,
+ "eval_logps/rejected": -147.53453063964844,
+ "eval_loss": 0.6834748983383179,
+ "eval_rewards/accuracies": 0.5619999766349792,
+ "eval_rewards/chosen": 0.08581218123435974,
+ "eval_rewards/margins": 0.03705006465315819,
+ "eval_rewards/rejected": 0.04876211658120155,
+ "eval_runtime": 90.9457,
+ "eval_samples_per_second": 5.498,
+ "eval_steps_per_second": 2.749,
+ "step": 3600
+ },
+ {
+ "epoch": 2.888,
+ "grad_norm": 1.344807505607605,
+ "learning_rate": 4.177777777777777e-08,
+ "logits/chosen": 1.4743510484695435,
+ "logits/rejected": 1.5635039806365967,
+ "logps/chosen": -156.87158203125,
+ "logps/rejected": -154.7716522216797,
+ "loss": 0.6525380134582519,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.13008974492549896,
+ "rewards/margins": 0.09934906661510468,
+ "rewards/rejected": 0.030740663409233093,
+ "step": 3610
+ },
+ {
+ "epoch": 2.896,
+ "grad_norm": 1.357977271080017,
+ "learning_rate": 3.881481481481481e-08,
+ "logits/chosen": 1.4685356616973877,
+ "logits/rejected": 1.4745405912399292,
+ "logps/chosen": -157.85101318359375,
+ "logps/rejected": -145.76034545898438,
+ "loss": 0.6684325695037842,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.12483832985162735,
+ "rewards/margins": 0.062453120946884155,
+ "rewards/rejected": 0.06238521263003349,
+ "step": 3620
+ },
+ {
+ "epoch": 2.904,
+ "grad_norm": 1.2610769271850586,
+ "learning_rate": 3.585185185185185e-08,
+ "logits/chosen": 1.5312964916229248,
+ "logits/rejected": 1.5217597484588623,
+ "logps/chosen": -148.142578125,
+ "logps/rejected": -138.1629638671875,
+ "loss": 0.6344244003295898,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.1255989372730255,
+ "rewards/margins": 0.13656888902187347,
+ "rewards/rejected": -0.010969944298267365,
+ "step": 3630
+ },
+ {
+ "epoch": 2.912,
+ "grad_norm": 1.4132224321365356,
+ "learning_rate": 3.288888888888889e-08,
+ "logits/chosen": 1.4116708040237427,
+ "logits/rejected": 1.4336539506912231,
+ "logps/chosen": -122.974609375,
+ "logps/rejected": -147.6199493408203,
+ "loss": 0.6176250457763672,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.15114453434944153,
+ "rewards/margins": 0.17734341323375702,
+ "rewards/rejected": -0.026198897510766983,
+ "step": 3640
+ },
+ {
+ "epoch": 2.92,
+ "grad_norm": 1.253814935684204,
+ "learning_rate": 2.9925925925925926e-08,
+ "logits/chosen": 1.436292052268982,
+ "logits/rejected": 1.394976019859314,
+ "logps/chosen": -141.23611450195312,
+ "logps/rejected": -140.11265563964844,
+ "loss": 0.6578958988189697,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.12222155183553696,
+ "rewards/margins": 0.10659031569957733,
+ "rewards/rejected": 0.015631232410669327,
+ "step": 3650
+ },
+ {
+ "epoch": 2.928,
+ "grad_norm": 1.2387349605560303,
+ "learning_rate": 2.696296296296296e-08,
+ "logits/chosen": 1.5451767444610596,
+ "logits/rejected": 1.5703151226043701,
+ "logps/chosen": -169.91392517089844,
+ "logps/rejected": -166.9705047607422,
+ "loss": 0.670601749420166,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.08693405240774155,
+ "rewards/margins": 0.05981532484292984,
+ "rewards/rejected": 0.027118723839521408,
+ "step": 3660
+ },
+ {
+ "epoch": 2.936,
+ "grad_norm": 1.3434607982635498,
+ "learning_rate": 2.4e-08,
+ "logits/chosen": 1.509053111076355,
+ "logits/rejected": 1.5241844654083252,
+ "logps/chosen": -145.90908813476562,
+ "logps/rejected": -135.3159637451172,
+ "loss": 0.672095775604248,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.12775234878063202,
+ "rewards/margins": 0.05578482151031494,
+ "rewards/rejected": 0.07196752727031708,
+ "step": 3670
+ },
+ {
+ "epoch": 2.944,
+ "grad_norm": 1.3283329010009766,
+ "learning_rate": 2.103703703703704e-08,
+ "logits/chosen": 1.4883911609649658,
+ "logits/rejected": 1.3725128173828125,
+ "logps/chosen": -143.17108154296875,
+ "logps/rejected": -137.8993377685547,
+ "loss": 0.6566094398498535,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.11801548302173615,
+ "rewards/margins": 0.08855273574590683,
+ "rewards/rejected": 0.029462745413184166,
+ "step": 3680
+ },
+ {
+ "epoch": 2.952,
+ "grad_norm": 1.1680418252944946,
+ "learning_rate": 1.8074074074074074e-08,
+ "logits/chosen": 1.5205543041229248,
+ "logits/rejected": 1.564670205116272,
+ "logps/chosen": -142.9960174560547,
+ "logps/rejected": -158.27052307128906,
+ "loss": 0.6416749954223633,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.09179408848285675,
+ "rewards/margins": 0.11523264646530151,
+ "rewards/rejected": -0.02343853935599327,
+ "step": 3690
+ },
+ {
+ "epoch": 2.96,
+ "grad_norm": 1.400667667388916,
+ "learning_rate": 1.511111111111111e-08,
+ "logits/chosen": 1.4704561233520508,
+ "logits/rejected": 1.4768496751785278,
+ "logps/chosen": -153.42848205566406,
+ "logps/rejected": -143.26663208007812,
+ "loss": 0.6594854831695557,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.14385171234607697,
+ "rewards/margins": 0.07918992638587952,
+ "rewards/rejected": 0.06466178596019745,
+ "step": 3700
+ },
+ {
+ "epoch": 2.96,
+ "eval_logits/chosen": 1.4535651206970215,
+ "eval_logits/rejected": 1.4812530279159546,
+ "eval_logps/chosen": -152.7117156982422,
+ "eval_logps/rejected": -147.5548095703125,
+ "eval_loss": 0.6834741234779358,
+ "eval_rewards/accuracies": 0.5619999766349792,
+ "eval_rewards/chosen": 0.08371291309595108,
+ "eval_rewards/margins": 0.03697838634252548,
+ "eval_rewards/rejected": 0.046734534204006195,
+ "eval_runtime": 90.9514,
+ "eval_samples_per_second": 5.497,
+ "eval_steps_per_second": 2.749,
+ "step": 3700
+ },
+ {
+ "epoch": 2.968,
+ "grad_norm": 1.3115962743759155,
+ "learning_rate": 1.2148148148148147e-08,
+ "logits/chosen": 1.461692214012146,
+ "logits/rejected": 1.54447603225708,
+ "logps/chosen": -172.43798828125,
+ "logps/rejected": -157.37420654296875,
+ "loss": 0.6359637260437012,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.14317496120929718,
+ "rewards/margins": 0.13744616508483887,
+ "rewards/rejected": 0.005728775169700384,
+ "step": 3710
+ },
+ {
+ "epoch": 2.976,
+ "grad_norm": 1.3789044618606567,
+ "learning_rate": 9.185185185185184e-09,
+ "logits/chosen": 1.378190517425537,
+ "logits/rejected": 1.5256309509277344,
+ "logps/chosen": -161.46340942382812,
+ "logps/rejected": -170.3937530517578,
+ "loss": 0.6403630256652832,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.1301015168428421,
+ "rewards/margins": 0.12454215437173843,
+ "rewards/rejected": 0.005559341516345739,
+ "step": 3720
+ },
+ {
+ "epoch": 2.984,
+ "grad_norm": 1.077886700630188,
+ "learning_rate": 6.222222222222222e-09,
+ "logits/chosen": 1.210209608078003,
+ "logits/rejected": 1.3575526475906372,
+ "logps/chosen": -125.64872741699219,
+ "logps/rejected": -134.8231201171875,
+ "loss": 0.6446949958801269,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.1091424822807312,
+ "rewards/margins": 0.11449601501226425,
+ "rewards/rejected": -0.005353519693017006,
+ "step": 3730
+ },
+ {
+ "epoch": 2.992,
+ "grad_norm": 1.4235198497772217,
+ "learning_rate": 3.2592592592592587e-09,
+ "logits/chosen": 1.415931224822998,
+ "logits/rejected": 1.4967601299285889,
+ "logps/chosen": -148.02452087402344,
+ "logps/rejected": -145.53929138183594,
+ "loss": 0.6488478660583497,
+ "rewards/accuracies": 0.762499988079071,
+ "rewards/chosen": 0.13844045996665955,
+ "rewards/margins": 0.10229086875915527,
+ "rewards/rejected": 0.03614959865808487,
+ "step": 3740
+ },
+ {
+ "epoch": 3.0,
+ "grad_norm": 1.141985535621643,
+ "learning_rate": 2.962962962962963e-10,
+ "logits/chosen": 1.5848835706710815,
+ "logits/rejected": 1.6290578842163086,
+ "logps/chosen": -147.43020629882812,
+ "logps/rejected": -154.41482543945312,
+ "loss": 0.6445570468902588,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.10755322873592377,
+ "rewards/margins": 0.11339461803436279,
+ "rewards/rejected": -0.005841395817697048,
+ "step": 3750
+ }
+ ],
+ "logging_steps": 10,
+ "max_steps": 3750,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": true
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-3750/training_args.bin b/v3/DPO/DPO_10k/lora/checkpoint-3750/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..b3fc3d76093802087869ab1fee5e37b9c830fe77
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-3750/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4f90e4a7053b7fb440d68cb64a3d198396af09db942d08839c5f4f0f0b8a0c8
+size 6097
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-500/README.md b/v3/DPO/DPO_10k/lora/checkpoint-500/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..250500f798b4682b2cd2c35cd1fc366677c215dd
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-500/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-500/adapter_config.json b/v3/DPO/DPO_10k/lora/checkpoint-500/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..039869f8d79ca6c48a793a29a79267da6d8f81c2
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-500/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "v_proj",
+ "gate_proj",
+ "down_proj",
+ "k_proj",
+ "o_proj",
+ "up_proj",
+ "q_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-500/adapter_model.safetensors b/v3/DPO/DPO_10k/lora/checkpoint-500/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a22e6a99e83f6319f72c2800232fb33efd6a34b7
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-500/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92b041e23f06a4b3f927901cfcc40617b711398a009814c27e08dae4222353fb
+size 180385008
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-500/chat_template.jinja b/v3/DPO/DPO_10k/lora/checkpoint-500/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-500/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-500/optimizer.pt b/v3/DPO/DPO_10k/lora/checkpoint-500/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..92c1f773b393914f6ee22b7bb3e8083d11b22e29
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-500/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:588a0ba3dc1790967ab2ee685203a511f1fe7ab4ef93fe1500a33c1664dc59d4
+size 360902475
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-500/rng_state.pth b/v3/DPO/DPO_10k/lora/checkpoint-500/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..3ab68364cdd28cbfdc34c4fcc2e4416efba5ff2a
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-500/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01f9a0f7843a37be87edd23f4e88aa93b38b95cc2c07503eeb1cf2e4632453a2
+size 14645
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-500/scaler.pt b/v3/DPO/DPO_10k/lora/checkpoint-500/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..86c857e8514e5db52a765434b135695dac4c9c36
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-500/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f77569c2e850b04af982cc8c1389f1430851448915c593b69e5da36ce05b71d7
+size 1383
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-500/scheduler.pt b/v3/DPO/DPO_10k/lora/checkpoint-500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..68722c4b4d89453dcaebcb870f779336255e072e
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70fa187cfdd1aa8bef9fb840ac38c6f8c85452b1b2d1b2488d3e6311801ce053
+size 1465
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-500/tokenizer.json b/v3/DPO/DPO_10k/lora/checkpoint-500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-500/tokenizer_config.json b/v3/DPO/DPO_10k/lora/checkpoint-500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-500/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-500/trainer_state.json b/v3/DPO/DPO_10k/lora/checkpoint-500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..469e1452052efca0b8dec25fb4e1787666904d42
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-500/trainer_state.json
@@ -0,0 +1,864 @@
+{
+ "best_global_step": 500,
+ "best_metric": 0.6898502111434937,
+ "best_model_checkpoint": "output/lora/checkpoint-500",
+ "epoch": 0.4,
+ "eval_steps": 100,
+ "global_step": 500,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.008,
+ "grad_norm": 1.1179773807525635,
+ "learning_rate": 2.4e-08,
+ "logits/chosen": 1.4545083045959473,
+ "logits/rejected": 1.425490379333496,
+ "logps/chosen": -132.46189880371094,
+ "logps/rejected": -148.20260620117188,
+ "loss": 0.6930626392364502,
+ "rewards/accuracies": 0.21250000596046448,
+ "rewards/chosen": -6.165739614516497e-05,
+ "rewards/margins": 0.00017333509458694607,
+ "rewards/rejected": -0.00023499250528402627,
+ "step": 10
+ },
+ {
+ "epoch": 0.016,
+ "grad_norm": 1.3502839803695679,
+ "learning_rate": 5.0666666666666664e-08,
+ "logits/chosen": 1.5668801069259644,
+ "logits/rejected": 1.5191389322280884,
+ "logps/chosen": -157.63624572753906,
+ "logps/rejected": -144.2716827392578,
+ "loss": 0.6930380821228027,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.0009244013344869018,
+ "rewards/margins": 0.00023505210992880166,
+ "rewards/rejected": 0.0006893490790389478,
+ "step": 20
+ },
+ {
+ "epoch": 0.024,
+ "grad_norm": 1.0219553709030151,
+ "learning_rate": 7.733333333333334e-08,
+ "logits/chosen": 1.8921797275543213,
+ "logits/rejected": 1.7342685461044312,
+ "logps/chosen": -164.55799865722656,
+ "logps/rejected": -158.92752075195312,
+ "loss": 0.69290189743042,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0009391927160322666,
+ "rewards/margins": 0.0005032324115745723,
+ "rewards/rejected": 0.0004359603044576943,
+ "step": 30
+ },
+ {
+ "epoch": 0.032,
+ "grad_norm": 1.2056113481521606,
+ "learning_rate": 1.0399999999999999e-07,
+ "logits/chosen": 1.4950047731399536,
+ "logits/rejected": 1.6163270473480225,
+ "logps/chosen": -145.54183959960938,
+ "logps/rejected": -134.19979858398438,
+ "loss": 0.6935500621795654,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.0006421065190806985,
+ "rewards/margins": -0.0007943272357806563,
+ "rewards/rejected": 0.0014364338712766767,
+ "step": 40
+ },
+ {
+ "epoch": 0.04,
+ "grad_norm": 1.1029725074768066,
+ "learning_rate": 1.3066666666666665e-07,
+ "logits/chosen": 1.4076533317565918,
+ "logits/rejected": 1.4791977405548096,
+ "logps/chosen": -133.23324584960938,
+ "logps/rejected": -138.3593292236328,
+ "loss": 0.6925833702087403,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.0012372838100418448,
+ "rewards/margins": 0.001139522879384458,
+ "rewards/rejected": 9.776116348803043e-05,
+ "step": 50
+ },
+ {
+ "epoch": 0.048,
+ "grad_norm": 0.9960266351699829,
+ "learning_rate": 1.573333333333333e-07,
+ "logits/chosen": 1.407166600227356,
+ "logits/rejected": 1.376042127609253,
+ "logps/chosen": -137.8921661376953,
+ "logps/rejected": -127.97320556640625,
+ "loss": 0.693614149093628,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.00021316049969755113,
+ "rewards/margins": -0.0009249996510334313,
+ "rewards/rejected": 0.0007118391804397106,
+ "step": 60
+ },
+ {
+ "epoch": 0.056,
+ "grad_norm": 1.0946824550628662,
+ "learning_rate": 1.8399999999999998e-07,
+ "logits/chosen": 1.5944501161575317,
+ "logits/rejected": 1.7248146533966064,
+ "logps/chosen": -153.5526123046875,
+ "logps/rejected": -157.19735717773438,
+ "loss": 0.6928449153900147,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0026370862033218145,
+ "rewards/margins": 0.0006150150438770652,
+ "rewards/rejected": 0.0020220710430294275,
+ "step": 70
+ },
+ {
+ "epoch": 0.064,
+ "grad_norm": 1.0322489738464355,
+ "learning_rate": 2.1066666666666665e-07,
+ "logits/chosen": 1.4586353302001953,
+ "logits/rejected": 1.4842995405197144,
+ "logps/chosen": -149.03665161132812,
+ "logps/rejected": -136.24081420898438,
+ "loss": 0.6932015895843506,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.004061593674123287,
+ "rewards/margins": -9.99594121822156e-05,
+ "rewards/rejected": 0.004161553457379341,
+ "step": 80
+ },
+ {
+ "epoch": 0.072,
+ "grad_norm": 1.0012447834014893,
+ "learning_rate": 2.3733333333333334e-07,
+ "logits/chosen": 1.5075902938842773,
+ "logits/rejected": 1.5986944437026978,
+ "logps/chosen": -159.3433074951172,
+ "logps/rejected": -151.85072326660156,
+ "loss": 0.6928757190704345,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.006978762801736593,
+ "rewards/margins": 0.0005627512582577765,
+ "rewards/rejected": 0.006416010670363903,
+ "step": 90
+ },
+ {
+ "epoch": 0.08,
+ "grad_norm": 1.280081033706665,
+ "learning_rate": 2.64e-07,
+ "logits/chosen": 1.448286533355713,
+ "logits/rejected": 1.5480695962905884,
+ "logps/chosen": -151.12625122070312,
+ "logps/rejected": -171.24545288085938,
+ "loss": 0.6931126117706299,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.00693341251462698,
+ "rewards/margins": 9.057526767719537e-05,
+ "rewards/rejected": 0.006842837668955326,
+ "step": 100
+ },
+ {
+ "epoch": 0.08,
+ "eval_logits/chosen": 1.518183946609497,
+ "eval_logits/rejected": 1.5525447130203247,
+ "eval_logps/chosen": -153.46630859375,
+ "eval_logps/rejected": -147.95005798339844,
+ "eval_loss": 0.6926330327987671,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.008253541775047779,
+ "eval_rewards/margins": 0.0010449704714119434,
+ "eval_rewards/rejected": 0.007208569906651974,
+ "eval_runtime": 91.0812,
+ "eval_samples_per_second": 5.49,
+ "eval_steps_per_second": 2.745,
+ "step": 100
+ },
+ {
+ "epoch": 0.088,
+ "grad_norm": 1.3912373781204224,
+ "learning_rate": 2.906666666666667e-07,
+ "logits/chosen": 1.6896222829818726,
+ "logits/rejected": 1.6516921520233154,
+ "logps/chosen": -173.70030212402344,
+ "logps/rejected": -155.5503692626953,
+ "loss": 0.6926521778106689,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.010179834440350533,
+ "rewards/margins": 0.0010137532372027636,
+ "rewards/rejected": 0.009166081435978413,
+ "step": 110
+ },
+ {
+ "epoch": 0.096,
+ "grad_norm": 1.1518888473510742,
+ "learning_rate": 3.173333333333333e-07,
+ "logits/chosen": 1.5166860818862915,
+ "logits/rejected": 1.5136979818344116,
+ "logps/chosen": -136.39906311035156,
+ "logps/rejected": -140.86561584472656,
+ "loss": 0.6932918548583984,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": 0.00907944981008768,
+ "rewards/margins": -0.00026998287648893893,
+ "rewards/rejected": 0.009349432773888111,
+ "step": 120
+ },
+ {
+ "epoch": 0.104,
+ "grad_norm": 1.3213376998901367,
+ "learning_rate": 3.4399999999999996e-07,
+ "logits/chosen": 1.4598973989486694,
+ "logits/rejected": 1.6739647388458252,
+ "logps/chosen": -159.12461853027344,
+ "logps/rejected": -163.39346313476562,
+ "loss": 0.693086051940918,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.010477779433131218,
+ "rewards/margins": 0.00014539004769176245,
+ "rewards/rejected": 0.010332388803362846,
+ "step": 130
+ },
+ {
+ "epoch": 0.112,
+ "grad_norm": 0.8833863735198975,
+ "learning_rate": 3.7066666666666665e-07,
+ "logits/chosen": 1.3829412460327148,
+ "logits/rejected": 1.4100173711776733,
+ "logps/chosen": -132.7149658203125,
+ "logps/rejected": -126.85478210449219,
+ "loss": 0.6932382583618164,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": 0.010262183845043182,
+ "rewards/margins": -0.0001672578218858689,
+ "rewards/rejected": 0.010429441928863525,
+ "step": 140
+ },
+ {
+ "epoch": 0.12,
+ "grad_norm": 1.1472455263137817,
+ "learning_rate": 3.973333333333333e-07,
+ "logits/chosen": 1.4954869747161865,
+ "logits/rejected": 1.5857340097427368,
+ "logps/chosen": -165.10568237304688,
+ "logps/rejected": -158.97433471679688,
+ "loss": 0.6916103363037109,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.014177520759403706,
+ "rewards/margins": 0.003108317730948329,
+ "rewards/rejected": 0.011069201864302158,
+ "step": 150
+ },
+ {
+ "epoch": 0.128,
+ "grad_norm": 1.0628798007965088,
+ "learning_rate": 4.24e-07,
+ "logits/chosen": 1.6717243194580078,
+ "logits/rejected": 1.705092191696167,
+ "logps/chosen": -159.332275390625,
+ "logps/rejected": -147.21646118164062,
+ "loss": 0.6935629844665527,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": 0.016994481906294823,
+ "rewards/margins": -0.0007874655420891941,
+ "rewards/rejected": 0.017781946808099747,
+ "step": 160
+ },
+ {
+ "epoch": 0.136,
+ "grad_norm": 1.390810251235962,
+ "learning_rate": 4.506666666666666e-07,
+ "logits/chosen": 1.4719234704971313,
+ "logits/rejected": 1.499526023864746,
+ "logps/chosen": -148.67251586914062,
+ "logps/rejected": -161.58021545410156,
+ "loss": 0.6926599979400635,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.021239640191197395,
+ "rewards/margins": 0.0010346340714022517,
+ "rewards/rejected": 0.020205006003379822,
+ "step": 170
+ },
+ {
+ "epoch": 0.144,
+ "grad_norm": 1.139707326889038,
+ "learning_rate": 4.773333333333333e-07,
+ "logits/chosen": 1.4833905696868896,
+ "logits/rejected": 1.637584924697876,
+ "logps/chosen": -158.96731567382812,
+ "logps/rejected": -161.9830322265625,
+ "loss": 0.6919547557830811,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.019416773691773415,
+ "rewards/margins": 0.0024296878837049007,
+ "rewards/rejected": 0.016987085342407227,
+ "step": 180
+ },
+ {
+ "epoch": 0.152,
+ "grad_norm": 1.2029796838760376,
+ "learning_rate": 5.04e-07,
+ "logits/chosen": 1.5398027896881104,
+ "logits/rejected": 1.6678705215454102,
+ "logps/chosen": -143.00283813476562,
+ "logps/rejected": -144.89378356933594,
+ "loss": 0.692262840270996,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.018960200250148773,
+ "rewards/margins": 0.0018175147706642747,
+ "rewards/rejected": 0.017142686992883682,
+ "step": 190
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 1.0527437925338745,
+ "learning_rate": 5.306666666666665e-07,
+ "logits/chosen": 1.6087623834609985,
+ "logits/rejected": 1.5181543827056885,
+ "logps/chosen": -159.35073852539062,
+ "logps/rejected": -133.59890747070312,
+ "loss": 0.6924069881439209,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.01926913857460022,
+ "rewards/margins": 0.0015339398523792624,
+ "rewards/rejected": 0.0177351962774992,
+ "step": 200
+ },
+ {
+ "epoch": 0.16,
+ "eval_logits/chosen": 1.5193672180175781,
+ "eval_logits/rejected": 1.5534639358520508,
+ "eval_logps/chosen": -153.34857177734375,
+ "eval_logps/rejected": -147.8425750732422,
+ "eval_loss": 0.6921458840370178,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.020027177408337593,
+ "eval_rewards/margins": 0.0020714527927339077,
+ "eval_rewards/rejected": 0.017955724149942398,
+ "eval_runtime": 90.9762,
+ "eval_samples_per_second": 5.496,
+ "eval_steps_per_second": 2.748,
+ "step": 200
+ },
+ {
+ "epoch": 0.168,
+ "grad_norm": 1.1897239685058594,
+ "learning_rate": 5.573333333333333e-07,
+ "logits/chosen": 1.6267732381820679,
+ "logits/rejected": 1.5775268077850342,
+ "logps/chosen": -159.51437377929688,
+ "logps/rejected": -170.17762756347656,
+ "loss": 0.6925770759582519,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.02078000269830227,
+ "rewards/margins": 0.0012053751852363348,
+ "rewards/rejected": 0.01957462914288044,
+ "step": 210
+ },
+ {
+ "epoch": 0.176,
+ "grad_norm": 0.9995236396789551,
+ "learning_rate": 5.839999999999999e-07,
+ "logits/chosen": 1.5781934261322021,
+ "logits/rejected": 1.642251968383789,
+ "logps/chosen": -159.95309448242188,
+ "logps/rejected": -144.36058044433594,
+ "loss": 0.6931475639343262,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.023322442546486855,
+ "rewards/margins": 6.392716750269756e-05,
+ "rewards/rejected": 0.02325851283967495,
+ "step": 220
+ },
+ {
+ "epoch": 0.184,
+ "grad_norm": 0.8819260597229004,
+ "learning_rate": 6.106666666666666e-07,
+ "logits/chosen": 1.459991693496704,
+ "logits/rejected": 1.6060739755630493,
+ "logps/chosen": -162.04745483398438,
+ "logps/rejected": -139.5411834716797,
+ "loss": 0.6919463634490967,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.026052182540297508,
+ "rewards/margins": 0.002481849165633321,
+ "rewards/rejected": 0.023570330813527107,
+ "step": 230
+ },
+ {
+ "epoch": 0.192,
+ "grad_norm": 1.1055208444595337,
+ "learning_rate": 6.373333333333333e-07,
+ "logits/chosen": 1.558739185333252,
+ "logits/rejected": 1.5965116024017334,
+ "logps/chosen": -171.9166259765625,
+ "logps/rejected": -145.66433715820312,
+ "loss": 0.6912176132202148,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.033868782222270966,
+ "rewards/margins": 0.004017929546535015,
+ "rewards/rejected": 0.029850851744413376,
+ "step": 240
+ },
+ {
+ "epoch": 0.2,
+ "grad_norm": 1.1372733116149902,
+ "learning_rate": 6.64e-07,
+ "logits/chosen": 1.5322860479354858,
+ "logits/rejected": 1.4695795774459839,
+ "logps/chosen": -161.7803955078125,
+ "logps/rejected": -148.91949462890625,
+ "loss": 0.6940969944000244,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.034065864980220795,
+ "rewards/margins": -0.0016761379083618522,
+ "rewards/rejected": 0.035742007195949554,
+ "step": 250
+ },
+ {
+ "epoch": 0.208,
+ "grad_norm": 1.0917680263519287,
+ "learning_rate": 6.906666666666666e-07,
+ "logits/chosen": 1.4094547033309937,
+ "logits/rejected": 1.50186026096344,
+ "logps/chosen": -132.91455078125,
+ "logps/rejected": -146.79714965820312,
+ "loss": 0.6930203914642334,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.03089061938226223,
+ "rewards/margins": 0.00040971505222842097,
+ "rewards/rejected": 0.030480902642011642,
+ "step": 260
+ },
+ {
+ "epoch": 0.216,
+ "grad_norm": 1.0280259847640991,
+ "learning_rate": 7.173333333333333e-07,
+ "logits/chosen": 1.531285285949707,
+ "logits/rejected": 1.6163132190704346,
+ "logps/chosen": -146.902587890625,
+ "logps/rejected": -147.8347625732422,
+ "loss": 0.6944749355316162,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.028514739125967026,
+ "rewards/margins": -0.0025452517438679934,
+ "rewards/rejected": 0.0310599897056818,
+ "step": 270
+ },
+ {
+ "epoch": 0.224,
+ "grad_norm": 1.398708701133728,
+ "learning_rate": 7.44e-07,
+ "logits/chosen": 1.563852071762085,
+ "logits/rejected": 1.396703839302063,
+ "logps/chosen": -172.68153381347656,
+ "logps/rejected": -155.80215454101562,
+ "loss": 0.689476490020752,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03224216774106026,
+ "rewards/margins": 0.007473687641322613,
+ "rewards/rejected": 0.02476847730576992,
+ "step": 280
+ },
+ {
+ "epoch": 0.232,
+ "grad_norm": 1.0394141674041748,
+ "learning_rate": 7.706666666666667e-07,
+ "logits/chosen": 1.5581729412078857,
+ "logits/rejected": 1.5819021463394165,
+ "logps/chosen": -144.41465759277344,
+ "logps/rejected": -153.55406188964844,
+ "loss": 0.6927877902984619,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.03272664546966553,
+ "rewards/margins": 0.0008557910332456231,
+ "rewards/rejected": 0.031870849430561066,
+ "step": 290
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 1.2557185888290405,
+ "learning_rate": 7.973333333333333e-07,
+ "logits/chosen": 1.5062446594238281,
+ "logits/rejected": 1.5786281824111938,
+ "logps/chosen": -153.63558959960938,
+ "logps/rejected": -165.46617126464844,
+ "loss": 0.6897892951965332,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03914078325033188,
+ "rewards/margins": 0.006873926613479853,
+ "rewards/rejected": 0.032266855239868164,
+ "step": 300
+ },
+ {
+ "epoch": 0.24,
+ "eval_logits/chosen": 1.5237834453582764,
+ "eval_logits/rejected": 1.557741641998291,
+ "eval_logps/chosen": -153.19883728027344,
+ "eval_logps/rejected": -147.7100830078125,
+ "eval_loss": 0.6913501024246216,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.03499976545572281,
+ "eval_rewards/margins": 0.0037925743963569403,
+ "eval_rewards/rejected": 0.031207194551825523,
+ "eval_runtime": 90.9133,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 300
+ },
+ {
+ "epoch": 0.248,
+ "grad_norm": 1.1200144290924072,
+ "learning_rate": 8.24e-07,
+ "logits/chosen": 1.618949294090271,
+ "logits/rejected": 1.6625702381134033,
+ "logps/chosen": -141.8953857421875,
+ "logps/rejected": -156.6927032470703,
+ "loss": 0.694425106048584,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.03162423521280289,
+ "rewards/margins": -0.0023833750747144222,
+ "rewards/rejected": 0.034007612615823746,
+ "step": 310
+ },
+ {
+ "epoch": 0.256,
+ "grad_norm": 1.0823743343353271,
+ "learning_rate": 8.506666666666667e-07,
+ "logits/chosen": 1.5974153280258179,
+ "logits/rejected": 1.6237560510635376,
+ "logps/chosen": -139.2996368408203,
+ "logps/rejected": -141.94407653808594,
+ "loss": 0.6914949893951416,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.029445698484778404,
+ "rewards/margins": 0.0034329916816204786,
+ "rewards/rejected": 0.026012707501649857,
+ "step": 320
+ },
+ {
+ "epoch": 0.264,
+ "grad_norm": 1.1765722036361694,
+ "learning_rate": 8.773333333333332e-07,
+ "logits/chosen": 1.465215802192688,
+ "logits/rejected": 1.4882014989852905,
+ "logps/chosen": -158.4329833984375,
+ "logps/rejected": -183.94259643554688,
+ "loss": 0.6925934791564942,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.026302698999643326,
+ "rewards/margins": 0.0012350418837741017,
+ "rewards/rejected": 0.025067657232284546,
+ "step": 330
+ },
+ {
+ "epoch": 0.272,
+ "grad_norm": 1.0667294263839722,
+ "learning_rate": 9.039999999999999e-07,
+ "logits/chosen": 1.7538875341415405,
+ "logits/rejected": 1.6171789169311523,
+ "logps/chosen": -159.63790893554688,
+ "logps/rejected": -151.1880340576172,
+ "loss": 0.6922041416168213,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.026619985699653625,
+ "rewards/margins": 0.002031774492934346,
+ "rewards/rejected": 0.024588212370872498,
+ "step": 340
+ },
+ {
+ "epoch": 0.28,
+ "grad_norm": 1.2210006713867188,
+ "learning_rate": 9.306666666666666e-07,
+ "logits/chosen": 1.5815479755401611,
+ "logits/rejected": 1.710217833518982,
+ "logps/chosen": -167.21743774414062,
+ "logps/rejected": -196.77989196777344,
+ "loss": 0.6899692058563233,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.03636014088988304,
+ "rewards/margins": 0.006536121014505625,
+ "rewards/rejected": 0.029824022203683853,
+ "step": 350
+ },
+ {
+ "epoch": 0.288,
+ "grad_norm": 0.9623845219612122,
+ "learning_rate": 9.573333333333333e-07,
+ "logits/chosen": 1.4416790008544922,
+ "logits/rejected": 1.3618693351745605,
+ "logps/chosen": -163.27401733398438,
+ "logps/rejected": -169.3351287841797,
+ "loss": 0.6910821437835694,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.037200987339019775,
+ "rewards/margins": 0.004338678438216448,
+ "rewards/rejected": 0.032862309366464615,
+ "step": 360
+ },
+ {
+ "epoch": 0.296,
+ "grad_norm": 1.1005645990371704,
+ "learning_rate": 9.84e-07,
+ "logits/chosen": 1.6214616298675537,
+ "logits/rejected": 1.6572606563568115,
+ "logps/chosen": -162.95504760742188,
+ "logps/rejected": -154.91807556152344,
+ "loss": 0.6895030021667481,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.047198958694934845,
+ "rewards/margins": 0.0075958347879350185,
+ "rewards/rejected": 0.03960312157869339,
+ "step": 370
+ },
+ {
+ "epoch": 0.304,
+ "grad_norm": 1.1126338243484497,
+ "learning_rate": 9.988148148148148e-07,
+ "logits/chosen": 1.4842126369476318,
+ "logits/rejected": 1.410930871963501,
+ "logps/chosen": -143.09182739257812,
+ "logps/rejected": -153.48135375976562,
+ "loss": 0.6885370254516602,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.05975018069148064,
+ "rewards/margins": 0.00981281790882349,
+ "rewards/rejected": 0.04993735998868942,
+ "step": 380
+ },
+ {
+ "epoch": 0.312,
+ "grad_norm": 1.1146304607391357,
+ "learning_rate": 9.95851851851852e-07,
+ "logits/chosen": 1.608332633972168,
+ "logits/rejected": 1.5848472118377686,
+ "logps/chosen": -152.25250244140625,
+ "logps/rejected": -138.348876953125,
+ "loss": 0.6851807117462159,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.07630724459886551,
+ "rewards/margins": 0.016460279002785683,
+ "rewards/rejected": 0.059846967458724976,
+ "step": 390
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 1.2686601877212524,
+ "learning_rate": 9.928888888888889e-07,
+ "logits/chosen": 1.5840070247650146,
+ "logits/rejected": 1.5679785013198853,
+ "logps/chosen": -155.90219116210938,
+ "logps/rejected": -148.89549255371094,
+ "loss": 0.6891039371490478,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.08129279315471649,
+ "rewards/margins": 0.008997146971523762,
+ "rewards/rejected": 0.072295643389225,
+ "step": 400
+ },
+ {
+ "epoch": 0.32,
+ "eval_logits/chosen": 1.5358340740203857,
+ "eval_logits/rejected": 1.5689741373062134,
+ "eval_logps/chosen": -152.6895294189453,
+ "eval_logps/rejected": -147.23422241210938,
+ "eval_loss": 0.6900544166564941,
+ "eval_rewards/accuracies": 0.5419999957084656,
+ "eval_rewards/chosen": 0.08593228459358215,
+ "eval_rewards/margins": 0.0071397931315004826,
+ "eval_rewards/rejected": 0.0787924975156784,
+ "eval_runtime": 90.9072,
+ "eval_samples_per_second": 5.5,
+ "eval_steps_per_second": 2.75,
+ "step": 400
+ },
+ {
+ "epoch": 0.328,
+ "grad_norm": 1.1653108596801758,
+ "learning_rate": 9.899259259259258e-07,
+ "logits/chosen": 1.5450490713119507,
+ "logits/rejected": 1.4428151845932007,
+ "logps/chosen": -148.61581420898438,
+ "logps/rejected": -188.90919494628906,
+ "loss": 0.6918989181518554,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0946170911192894,
+ "rewards/margins": 0.003271549940109253,
+ "rewards/rejected": 0.09134554117918015,
+ "step": 410
+ },
+ {
+ "epoch": 0.336,
+ "grad_norm": 1.3114937543869019,
+ "learning_rate": 9.86962962962963e-07,
+ "logits/chosen": 1.5342715978622437,
+ "logits/rejected": 1.5572118759155273,
+ "logps/chosen": -152.65264892578125,
+ "logps/rejected": -168.03384399414062,
+ "loss": 0.6972955703735352,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": 0.08869597315788269,
+ "rewards/margins": -0.007436127867549658,
+ "rewards/rejected": 0.09613210707902908,
+ "step": 420
+ },
+ {
+ "epoch": 0.344,
+ "grad_norm": 1.0040781497955322,
+ "learning_rate": 9.84e-07,
+ "logits/chosen": 1.5979712009429932,
+ "logits/rejected": 1.6713664531707764,
+ "logps/chosen": -164.2566375732422,
+ "logps/rejected": -152.1428680419922,
+ "loss": 0.6897010326385498,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.08101107180118561,
+ "rewards/margins": 0.007684691343456507,
+ "rewards/rejected": 0.07332637906074524,
+ "step": 430
+ },
+ {
+ "epoch": 0.352,
+ "grad_norm": 1.0801892280578613,
+ "learning_rate": 9.81037037037037e-07,
+ "logits/chosen": 1.439635992050171,
+ "logits/rejected": 1.3282365798950195,
+ "logps/chosen": -157.23403930664062,
+ "logps/rejected": -146.54806518554688,
+ "loss": 0.6881664752960205,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.08112544566392899,
+ "rewards/margins": 0.010971959680318832,
+ "rewards/rejected": 0.07015348225831985,
+ "step": 440
+ },
+ {
+ "epoch": 0.36,
+ "grad_norm": 1.1139949560165405,
+ "learning_rate": 9.78074074074074e-07,
+ "logits/chosen": 1.6345123052597046,
+ "logits/rejected": 1.5876491069793701,
+ "logps/chosen": -157.6160125732422,
+ "logps/rejected": -160.02040100097656,
+ "loss": 0.6868602275848389,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.07638200372457504,
+ "rewards/margins": 0.013464884832501411,
+ "rewards/rejected": 0.06291711330413818,
+ "step": 450
+ },
+ {
+ "epoch": 0.368,
+ "grad_norm": 1.158035397529602,
+ "learning_rate": 9.751111111111112e-07,
+ "logits/chosen": 1.5430586338043213,
+ "logits/rejected": 1.6022913455963135,
+ "logps/chosen": -151.98919677734375,
+ "logps/rejected": -164.90565490722656,
+ "loss": 0.6891930580139161,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06889279186725616,
+ "rewards/margins": 0.008791089057922363,
+ "rewards/rejected": 0.0601017065346241,
+ "step": 460
+ },
+ {
+ "epoch": 0.376,
+ "grad_norm": 1.0903704166412354,
+ "learning_rate": 9.721481481481481e-07,
+ "logits/chosen": 1.313881754875183,
+ "logits/rejected": 1.3182194232940674,
+ "logps/chosen": -131.5592041015625,
+ "logps/rejected": -123.84442138671875,
+ "loss": 0.6889429569244385,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.08150745928287506,
+ "rewards/margins": 0.009235726669430733,
+ "rewards/rejected": 0.07227173447608948,
+ "step": 470
+ },
+ {
+ "epoch": 0.384,
+ "grad_norm": 1.0766969919204712,
+ "learning_rate": 9.69185185185185e-07,
+ "logits/chosen": 1.425756812095642,
+ "logits/rejected": 1.476854920387268,
+ "logps/chosen": -148.72279357910156,
+ "logps/rejected": -164.40951538085938,
+ "loss": 0.6854116439819335,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.1009502187371254,
+ "rewards/margins": 0.0166789498180151,
+ "rewards/rejected": 0.08427127450704575,
+ "step": 480
+ },
+ {
+ "epoch": 0.392,
+ "grad_norm": 1.2078129053115845,
+ "learning_rate": 9.662222222222222e-07,
+ "logits/chosen": 1.5511282682418823,
+ "logits/rejected": 1.611383080482483,
+ "logps/chosen": -143.22006225585938,
+ "logps/rejected": -159.70176696777344,
+ "loss": 0.6968960762023926,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.08779363334178925,
+ "rewards/margins": -0.00598606513813138,
+ "rewards/rejected": 0.09377971291542053,
+ "step": 490
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.9106306433677673,
+ "learning_rate": 9.632592592592593e-07,
+ "logits/chosen": 1.4217239618301392,
+ "logits/rejected": 1.505446195602417,
+ "logps/chosen": -144.34109497070312,
+ "logps/rejected": -135.52430725097656,
+ "loss": 0.691196870803833,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06902458518743515,
+ "rewards/margins": 0.005059319548308849,
+ "rewards/rejected": 0.06396526843309402,
+ "step": 500
+ },
+ {
+ "epoch": 0.4,
+ "eval_logits/chosen": 1.5185325145721436,
+ "eval_logits/rejected": 1.5510345697402954,
+ "eval_logps/chosen": -152.7921905517578,
+ "eval_logps/rejected": -147.3463134765625,
+ "eval_loss": 0.6898502111434937,
+ "eval_rewards/accuracies": 0.5099999904632568,
+ "eval_rewards/chosen": 0.07566512376070023,
+ "eval_rewards/margins": 0.008083080872893333,
+ "eval_rewards/rejected": 0.06758204847574234,
+ "eval_runtime": 90.8294,
+ "eval_samples_per_second": 5.505,
+ "eval_steps_per_second": 2.752,
+ "step": 500
+ }
+ ],
+ "logging_steps": 10,
+ "max_steps": 3750,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/v3/DPO/DPO_10k/lora/checkpoint-500/training_args.bin b/v3/DPO/DPO_10k/lora/checkpoint-500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..b3fc3d76093802087869ab1fee5e37b9c830fe77
--- /dev/null
+++ b/v3/DPO/DPO_10k/lora/checkpoint-500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4f90e4a7053b7fb440d68cb64a3d198396af09db942d08839c5f4f0f0b8a0c8
+size 6097
diff --git a/v3/DPO/DPO_1k/DPO_1k/README.md b/v3/DPO/DPO_1k/DPO_1k/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..250500f798b4682b2cd2c35cd1fc366677c215dd
--- /dev/null
+++ b/v3/DPO/DPO_1k/DPO_1k/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/DPO/DPO_1k/DPO_1k/adapter_config.json b/v3/DPO/DPO_1k/DPO_1k/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..402e6dd6fb2e3433da281c60aa4532d3bab802d4
--- /dev/null
+++ b/v3/DPO/DPO_1k/DPO_1k/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "up_proj",
+ "q_proj",
+ "o_proj",
+ "k_proj",
+ "gate_proj",
+ "v_proj",
+ "down_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/DPO/DPO_1k/DPO_1k/adapter_model.safetensors b/v3/DPO/DPO_1k/DPO_1k/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5430ecd212c15d822d7787e530b216ed7b71aaa9
--- /dev/null
+++ b/v3/DPO/DPO_1k/DPO_1k/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acfba8673c6a71f789168973f38679b9b62058dfae764ad42b20f3312999008e
+size 180385008
diff --git a/v3/DPO/DPO_1k/MDPO_1k/chat_template.jinja b/v3/DPO/DPO_1k/MDPO_1k/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/DPO/DPO_1k/MDPO_1k/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/DPO/DPO_1k/MDPO_1k/config.json b/v3/DPO/DPO_1k/MDPO_1k/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e9ec18bc1ae18922052a57480e28401f3b9c6b84
--- /dev/null
+++ b/v3/DPO/DPO_1k/MDPO_1k/config.json
@@ -0,0 +1,40 @@
+{
+ "architectures": [
+ "LlamaForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 128000,
+ "dtype": "float16",
+ "eos_token_id": [
+ 128001,
+ 128008,
+ 128009
+ ],
+ "head_dim": 64,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 8192,
+ "max_position_embeddings": 131072,
+ "mlp_bias": false,
+ "model_type": "llama",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 16,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "pretraining_tp": 1,
+ "rms_norm_eps": 1e-05,
+ "rope_parameters": {
+ "factor": 32.0,
+ "high_freq_factor": 4.0,
+ "low_freq_factor": 1.0,
+ "original_max_position_embeddings": 8192,
+ "rope_theta": 500000.0,
+ "rope_type": "llama3"
+ },
+ "tie_word_embeddings": true,
+ "transformers_version": "5.2.0",
+ "use_cache": true,
+ "vocab_size": 128256
+}
diff --git a/v3/DPO/DPO_1k/MDPO_1k/generation_config.json b/v3/DPO/DPO_1k/MDPO_1k/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..898a2e748a41a4b7a931b754f8abbdac02039fae
--- /dev/null
+++ b/v3/DPO/DPO_1k/MDPO_1k/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "bos_token_id": 128000,
+ "do_sample": true,
+ "eos_token_id": [
+ 128001,
+ 128008,
+ 128009
+ ],
+ "temperature": 0.6,
+ "top_p": 0.9,
+ "transformers_version": "5.2.0"
+}
diff --git a/v3/DPO/DPO_1k/MDPO_1k/model.safetensors b/v3/DPO/DPO_1k/MDPO_1k/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d7bc637526f7afc49970be9585096532653ea37f
--- /dev/null
+++ b/v3/DPO/DPO_1k/MDPO_1k/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d748019d871b4afbd7b97295b2e21034a8a5ccff73f09a42257ddfc86893637
+size 2471645464
diff --git a/v3/DPO/DPO_1k/MDPO_1k/tokenizer.json b/v3/DPO/DPO_1k/MDPO_1k/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/DPO/DPO_1k/MDPO_1k/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/DPO/DPO_1k/MDPO_1k/tokenizer_config.json b/v3/DPO/DPO_1k/MDPO_1k/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/DPO/DPO_1k/MDPO_1k/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/DPO/DPO_1k/lora/README.md b/v3/DPO/DPO_1k/lora/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..a8b56a79d7d600f309aa8e332891f4d78e1b487a
--- /dev/null
+++ b/v3/DPO/DPO_1k/lora/README.md
@@ -0,0 +1,68 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: transformers
+model_name: lora
+tags:
+- generated_from_trainer
+- dpo
+- trl
+licence: license
+---
+
+# Model Card for lora
+
+This model is a fine-tuned version of [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[](https://wandb.ai/sea-rod/huggingface/runs/lilqm6x4)
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- TRL: 0.28.0
+- Transformers: 5.2.0
+- Pytorch: 2.10.0
+- Datasets: 4.5.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+ title = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+ author = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+ year = 2023,
+ booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+ url = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+ editor = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+ title = {{TRL: Transformers Reinforcement Learning}},
+ author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+ license = {Apache-2.0},
+ url = {https://github.com/huggingface/trl},
+ year = {2020}
+}
+```
\ No newline at end of file
diff --git a/v3/DPO/DPO_1k/lora/checkpoint-375/README.md b/v3/DPO/DPO_1k/lora/checkpoint-375/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..250500f798b4682b2cd2c35cd1fc366677c215dd
--- /dev/null
+++ b/v3/DPO/DPO_1k/lora/checkpoint-375/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/DPO/DPO_1k/lora/checkpoint-375/adapter_config.json b/v3/DPO/DPO_1k/lora/checkpoint-375/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..402e6dd6fb2e3433da281c60aa4532d3bab802d4
--- /dev/null
+++ b/v3/DPO/DPO_1k/lora/checkpoint-375/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "up_proj",
+ "q_proj",
+ "o_proj",
+ "k_proj",
+ "gate_proj",
+ "v_proj",
+ "down_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/DPO/DPO_1k/lora/checkpoint-375/adapter_model.safetensors b/v3/DPO/DPO_1k/lora/checkpoint-375/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5430ecd212c15d822d7787e530b216ed7b71aaa9
--- /dev/null
+++ b/v3/DPO/DPO_1k/lora/checkpoint-375/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acfba8673c6a71f789168973f38679b9b62058dfae764ad42b20f3312999008e
+size 180385008
diff --git a/v3/DPO/DPO_1k/lora/checkpoint-375/chat_template.jinja b/v3/DPO/DPO_1k/lora/checkpoint-375/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/DPO/DPO_1k/lora/checkpoint-375/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/DPO/DPO_1k/lora/checkpoint-375/optimizer.pt b/v3/DPO/DPO_1k/lora/checkpoint-375/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..79ff966c5c4f2543c15b6120cd310b5d2a3cac3f
--- /dev/null
+++ b/v3/DPO/DPO_1k/lora/checkpoint-375/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:275c2b1bbe8daf5bdbaeb6e3d945061086a1db36372cd532217c3821c3747cc5
+size 360902475
diff --git a/v3/DPO/DPO_1k/lora/checkpoint-375/rng_state.pth b/v3/DPO/DPO_1k/lora/checkpoint-375/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..7859360e6e5ad44f3ed47e7c17f4cafb2aebd41a
--- /dev/null
+++ b/v3/DPO/DPO_1k/lora/checkpoint-375/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50dd579ef1d2d932c3d7726ef0f6ec72d0e25b023ae2fd8b67112ac75e1041b9
+size 14645
diff --git a/v3/DPO/DPO_1k/lora/checkpoint-375/scaler.pt b/v3/DPO/DPO_1k/lora/checkpoint-375/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..096ec41fd7ece1a6761c13f49e4aeb268a4a2e4a
--- /dev/null
+++ b/v3/DPO/DPO_1k/lora/checkpoint-375/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac58ef7454dc327cdd1497d560e3c1cadc5881183c106e69fda6701af10248e8
+size 1383
diff --git a/v3/DPO/DPO_1k/lora/checkpoint-375/scheduler.pt b/v3/DPO/DPO_1k/lora/checkpoint-375/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b268469a2ea82f64f2bda71e30957ac1a335f742
--- /dev/null
+++ b/v3/DPO/DPO_1k/lora/checkpoint-375/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953de1507cdebcb1a23d3c9e90f38af5aa0829e9bca1c3deb0662aa977177d27
+size 1465
diff --git a/v3/DPO/DPO_1k/lora/checkpoint-375/tokenizer.json b/v3/DPO/DPO_1k/lora/checkpoint-375/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/DPO/DPO_1k/lora/checkpoint-375/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/DPO/DPO_1k/lora/checkpoint-375/tokenizer_config.json b/v3/DPO/DPO_1k/lora/checkpoint-375/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/DPO/DPO_1k/lora/checkpoint-375/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/DPO/DPO_1k/lora/checkpoint-375/trainer_state.json b/v3/DPO/DPO_1k/lora/checkpoint-375/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..aba5818cb6c0f66d828b269e17800ad1a8a4df23
--- /dev/null
+++ b/v3/DPO/DPO_1k/lora/checkpoint-375/trainer_state.json
@@ -0,0 +1,1181 @@
+{
+ "best_global_step": 360,
+ "best_metric": 0.691532552242279,
+ "best_model_checkpoint": null,
+ "epoch": 3.0,
+ "eval_steps": 10,
+ "global_step": 375,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.08,
+ "grad_norm": 1.1099787950515747,
+ "learning_rate": 2.3684210526315787e-07,
+ "logits/chosen": 1.6321640014648438,
+ "logits/rejected": 1.4893718957901,
+ "logps/chosen": -139.11293029785156,
+ "logps/rejected": -131.92221069335938,
+ "loss": 0.6938377380371094,
+ "rewards/accuracies": 0.32499998807907104,
+ "rewards/chosen": -0.0003625393146649003,
+ "rewards/margins": -0.0013720847200602293,
+ "rewards/rejected": 0.001009545405395329,
+ "step": 10
+ },
+ {
+ "epoch": 0.08,
+ "eval_logits/chosen": 1.5145372152328491,
+ "eval_logits/rejected": 1.549047827720642,
+ "eval_logps/chosen": -153.54446411132812,
+ "eval_logps/rejected": -148.01809692382812,
+ "eval_loss": 0.6931373476982117,
+ "eval_rewards/accuracies": 0.49000000953674316,
+ "eval_rewards/chosen": 0.0004369899688754231,
+ "eval_rewards/margins": 3.07468326354865e-05,
+ "eval_rewards/rejected": 0.00040624308167025447,
+ "eval_runtime": 92.0087,
+ "eval_samples_per_second": 5.434,
+ "eval_steps_per_second": 2.717,
+ "step": 10
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 1.0536019802093506,
+ "learning_rate": 5e-07,
+ "logits/chosen": 1.5109329223632812,
+ "logits/rejected": 1.6265140771865845,
+ "logps/chosen": -146.97775268554688,
+ "logps/rejected": -146.86245727539062,
+ "loss": 0.693108081817627,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.0023965430445969105,
+ "rewards/margins": 8.569951023673639e-05,
+ "rewards/rejected": 0.0023108436726033688,
+ "step": 20
+ },
+ {
+ "epoch": 0.16,
+ "eval_logits/chosen": 1.5149668455123901,
+ "eval_logits/rejected": 1.5492897033691406,
+ "eval_logps/chosen": -153.52540588378906,
+ "eval_logps/rejected": -148.00369262695312,
+ "eval_loss": 0.6929041743278503,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": 0.0023428802378475666,
+ "eval_rewards/margins": 0.0004977969801984727,
+ "eval_rewards/rejected": 0.001845083199441433,
+ "eval_runtime": 91.9011,
+ "eval_samples_per_second": 5.441,
+ "eval_steps_per_second": 2.72,
+ "step": 20
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 0.9590275883674622,
+ "learning_rate": 7.631578947368421e-07,
+ "logits/chosen": 1.5057928562164307,
+ "logits/rejected": 1.660310983657837,
+ "logps/chosen": -145.98333740234375,
+ "logps/rejected": -158.2825927734375,
+ "loss": 0.6925349235534668,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.005280056037008762,
+ "rewards/margins": 0.0012419605627655983,
+ "rewards/rejected": 0.004038095474243164,
+ "step": 30
+ },
+ {
+ "epoch": 0.24,
+ "eval_logits/chosen": 1.516362190246582,
+ "eval_logits/rejected": 1.5507477521896362,
+ "eval_logps/chosen": -153.49749755859375,
+ "eval_logps/rejected": -147.97317504882812,
+ "eval_loss": 0.6930363774299622,
+ "eval_rewards/accuracies": 0.5040000081062317,
+ "eval_rewards/chosen": 0.005134128034114838,
+ "eval_rewards/margins": 0.00023609661730006337,
+ "eval_rewards/rejected": 0.004898030776530504,
+ "eval_runtime": 91.6912,
+ "eval_samples_per_second": 5.453,
+ "eval_steps_per_second": 2.727,
+ "step": 30
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 1.2454571723937988,
+ "learning_rate": 9.970326409495548e-07,
+ "logits/chosen": 1.470884919166565,
+ "logits/rejected": 1.3818800449371338,
+ "logps/chosen": -138.23550415039062,
+ "logps/rejected": -146.94664001464844,
+ "loss": 0.6922728538513183,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.009227929636836052,
+ "rewards/margins": 0.0017647624481469393,
+ "rewards/rejected": 0.007463166955858469,
+ "step": 40
+ },
+ {
+ "epoch": 0.32,
+ "eval_logits/chosen": 1.5191068649291992,
+ "eval_logits/rejected": 1.553483486175537,
+ "eval_logps/chosen": -153.4440460205078,
+ "eval_logps/rejected": -147.92205810546875,
+ "eval_loss": 0.6929237842559814,
+ "eval_rewards/accuracies": 0.49000000953674316,
+ "eval_rewards/chosen": 0.010480178520083427,
+ "eval_rewards/margins": 0.00047138563240878284,
+ "eval_rewards/rejected": 0.010008794255554676,
+ "eval_runtime": 91.6588,
+ "eval_samples_per_second": 5.455,
+ "eval_steps_per_second": 2.728,
+ "step": 40
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 1.2237528562545776,
+ "learning_rate": 9.67359050445104e-07,
+ "logits/chosen": 1.4704697132110596,
+ "logits/rejected": 1.7169246673583984,
+ "logps/chosen": -162.7308807373047,
+ "logps/rejected": -149.7134552001953,
+ "loss": 0.6919724941253662,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.016776496544480324,
+ "rewards/margins": 0.0023813582956790924,
+ "rewards/rejected": 0.01439514011144638,
+ "step": 50
+ },
+ {
+ "epoch": 0.4,
+ "eval_logits/chosen": 1.5223050117492676,
+ "eval_logits/rejected": 1.5566374063491821,
+ "eval_logps/chosen": -153.3477325439453,
+ "eval_logps/rejected": -147.83493041992188,
+ "eval_loss": 0.6924771666526794,
+ "eval_rewards/accuracies": 0.5080000162124634,
+ "eval_rewards/chosen": 0.020111385732889175,
+ "eval_rewards/margins": 0.001389803597703576,
+ "eval_rewards/rejected": 0.018721580505371094,
+ "eval_runtime": 91.7228,
+ "eval_samples_per_second": 5.451,
+ "eval_steps_per_second": 2.726,
+ "step": 50
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 1.204073190689087,
+ "learning_rate": 9.376854599406528e-07,
+ "logits/chosen": 1.541168451309204,
+ "logits/rejected": 1.6895427703857422,
+ "logps/chosen": -150.58872985839844,
+ "logps/rejected": -174.12901306152344,
+ "loss": 0.6933607578277587,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.025453919544816017,
+ "rewards/margins": -0.0003413918020669371,
+ "rewards/rejected": 0.025795307010412216,
+ "step": 60
+ },
+ {
+ "epoch": 0.48,
+ "eval_logits/chosen": 1.5224460363388062,
+ "eval_logits/rejected": 1.5566353797912598,
+ "eval_logps/chosen": -153.314453125,
+ "eval_logps/rejected": -147.8090362548828,
+ "eval_loss": 0.6921172738075256,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": 0.023439867421984673,
+ "eval_rewards/margins": 0.0021286739502102137,
+ "eval_rewards/rejected": 0.021311195567250252,
+ "eval_runtime": 91.8819,
+ "eval_samples_per_second": 5.442,
+ "eval_steps_per_second": 2.721,
+ "step": 60
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 1.2116458415985107,
+ "learning_rate": 9.080118694362017e-07,
+ "logits/chosen": 1.6586517095565796,
+ "logits/rejected": 1.6767410039901733,
+ "logps/chosen": -157.19911193847656,
+ "logps/rejected": -158.79193115234375,
+ "loss": 0.6931434154510498,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": 0.025548268109560013,
+ "rewards/margins": 8.960017294157296e-05,
+ "rewards/rejected": 0.02545866370201111,
+ "step": 70
+ },
+ {
+ "epoch": 0.56,
+ "eval_logits/chosen": 1.521816372871399,
+ "eval_logits/rejected": 1.5559982061386108,
+ "eval_logps/chosen": -153.3275146484375,
+ "eval_logps/rejected": -147.81773376464844,
+ "eval_loss": 0.6923356652259827,
+ "eval_rewards/accuracies": 0.5260000228881836,
+ "eval_rewards/chosen": 0.02213224396109581,
+ "eval_rewards/margins": 0.0016910441918298602,
+ "eval_rewards/rejected": 0.020441202446818352,
+ "eval_runtime": 91.6721,
+ "eval_samples_per_second": 5.454,
+ "eval_steps_per_second": 2.727,
+ "step": 70
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 1.3703562021255493,
+ "learning_rate": 8.783382789317507e-07,
+ "logits/chosen": 1.4678456783294678,
+ "logits/rejected": 1.5785505771636963,
+ "logps/chosen": -140.6632843017578,
+ "logps/rejected": -142.6456298828125,
+ "loss": 0.6928178787231445,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.021536555141210556,
+ "rewards/margins": 0.0007222103886306286,
+ "rewards/rejected": 0.02081434614956379,
+ "step": 80
+ },
+ {
+ "epoch": 0.64,
+ "eval_logits/chosen": 1.519076943397522,
+ "eval_logits/rejected": 1.55327570438385,
+ "eval_logps/chosen": -153.37380981445312,
+ "eval_logps/rejected": -147.86260986328125,
+ "eval_loss": 0.6924052834510803,
+ "eval_rewards/accuracies": 0.515999972820282,
+ "eval_rewards/chosen": 0.01750360056757927,
+ "eval_rewards/margins": 0.0015494764083996415,
+ "eval_rewards/rejected": 0.015954123809933662,
+ "eval_runtime": 91.6149,
+ "eval_samples_per_second": 5.458,
+ "eval_steps_per_second": 2.729,
+ "step": 80
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 1.2956594228744507,
+ "learning_rate": 8.486646884272997e-07,
+ "logits/chosen": 1.5918622016906738,
+ "logits/rejected": 1.5741856098175049,
+ "logps/chosen": -152.06668090820312,
+ "logps/rejected": -157.22036743164062,
+ "loss": 0.6927751541137696,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.015795515850186348,
+ "rewards/margins": 0.0007950471481308341,
+ "rewards/rejected": 0.01500046718865633,
+ "step": 90
+ },
+ {
+ "epoch": 0.72,
+ "eval_logits/chosen": 1.5141069889068604,
+ "eval_logits/rejected": 1.548289179801941,
+ "eval_logps/chosen": -153.44723510742188,
+ "eval_logps/rejected": -147.93753051757812,
+ "eval_loss": 0.6923331022262573,
+ "eval_rewards/accuracies": 0.5559999942779541,
+ "eval_rewards/chosen": 0.010160998441278934,
+ "eval_rewards/margins": 0.0016996588092297316,
+ "eval_rewards/rejected": 0.008461340330541134,
+ "eval_runtime": 91.5644,
+ "eval_samples_per_second": 5.461,
+ "eval_steps_per_second": 2.73,
+ "step": 90
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 1.1156457662582397,
+ "learning_rate": 8.189910979228485e-07,
+ "logits/chosen": 1.4542659521102905,
+ "logits/rejected": 1.5833475589752197,
+ "logps/chosen": -152.64093017578125,
+ "logps/rejected": -158.0126495361328,
+ "loss": 0.6919864654541016,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.007315542548894882,
+ "rewards/margins": 0.002385933417826891,
+ "rewards/rejected": 0.004929609131067991,
+ "step": 100
+ },
+ {
+ "epoch": 0.8,
+ "eval_logits/chosen": 1.5120500326156616,
+ "eval_logits/rejected": 1.5461506843566895,
+ "eval_logps/chosen": -153.46383666992188,
+ "eval_logps/rejected": -147.94837951660156,
+ "eval_loss": 0.6926317811012268,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.008500359952449799,
+ "eval_rewards/margins": 0.0011226902715861797,
+ "eval_rewards/rejected": 0.007377668283879757,
+ "eval_runtime": 91.6164,
+ "eval_samples_per_second": 5.458,
+ "eval_steps_per_second": 2.729,
+ "step": 100
+ },
+ {
+ "epoch": 0.88,
+ "grad_norm": 1.361391544342041,
+ "learning_rate": 7.893175074183976e-07,
+ "logits/chosen": 1.3737733364105225,
+ "logits/rejected": 1.3109161853790283,
+ "logps/chosen": -136.71749877929688,
+ "logps/rejected": -129.01361083984375,
+ "loss": 0.6945322990417481,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": 0.003221578197553754,
+ "rewards/margins": -0.002658813027665019,
+ "rewards/rejected": 0.0058803921565413475,
+ "step": 110
+ },
+ {
+ "epoch": 0.88,
+ "eval_logits/chosen": 1.5104182958602905,
+ "eval_logits/rejected": 1.5443729162216187,
+ "eval_logps/chosen": -153.4846649169922,
+ "eval_logps/rejected": -147.97154235839844,
+ "eval_loss": 0.6925206780433655,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": 0.006417811382561922,
+ "eval_rewards/margins": 0.0013568454887717962,
+ "eval_rewards/rejected": 0.0050609661266207695,
+ "eval_runtime": 91.5931,
+ "eval_samples_per_second": 5.459,
+ "eval_steps_per_second": 2.729,
+ "step": 110
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 1.190990686416626,
+ "learning_rate": 7.596439169139466e-07,
+ "logits/chosen": 1.546119213104248,
+ "logits/rejected": 1.6396499872207642,
+ "logps/chosen": -141.16188049316406,
+ "logps/rejected": -135.41415405273438,
+ "loss": 0.6925000190734864,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.01047755777835846,
+ "rewards/margins": 0.0013591502793133259,
+ "rewards/rejected": 0.009118407033383846,
+ "step": 120
+ },
+ {
+ "epoch": 0.96,
+ "eval_logits/chosen": 1.51317298412323,
+ "eval_logits/rejected": 1.5472123622894287,
+ "eval_logps/chosen": -153.44869995117188,
+ "eval_logps/rejected": -147.93692016601562,
+ "eval_loss": 0.6924490928649902,
+ "eval_rewards/accuracies": 0.5199999809265137,
+ "eval_rewards/chosen": 0.010014238767325878,
+ "eval_rewards/margins": 0.0014920332469046116,
+ "eval_rewards/rejected": 0.00852220505475998,
+ "eval_runtime": 91.5004,
+ "eval_samples_per_second": 5.464,
+ "eval_steps_per_second": 2.732,
+ "step": 120
+ },
+ {
+ "epoch": 1.04,
+ "grad_norm": 0.9366094470024109,
+ "learning_rate": 7.299703264094955e-07,
+ "logits/chosen": 1.536211371421814,
+ "logits/rejected": 1.5287096500396729,
+ "logps/chosen": -140.06198120117188,
+ "logps/rejected": -125.5794677734375,
+ "loss": 0.6875796794891358,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.019292745739221573,
+ "rewards/margins": 0.011286661960184574,
+ "rewards/rejected": 0.008006083779036999,
+ "step": 130
+ },
+ {
+ "epoch": 1.04,
+ "eval_logits/chosen": 1.5151466131210327,
+ "eval_logits/rejected": 1.5491855144500732,
+ "eval_logps/chosen": -153.4193572998047,
+ "eval_logps/rejected": -147.90896606445312,
+ "eval_loss": 0.692375659942627,
+ "eval_rewards/accuracies": 0.5419999957084656,
+ "eval_rewards/chosen": 0.012948009185492992,
+ "eval_rewards/margins": 0.001630293671041727,
+ "eval_rewards/rejected": 0.011317715980112553,
+ "eval_runtime": 91.4948,
+ "eval_samples_per_second": 5.465,
+ "eval_steps_per_second": 2.732,
+ "step": 130
+ },
+ {
+ "epoch": 1.12,
+ "grad_norm": 0.9696089029312134,
+ "learning_rate": 7.002967359050444e-07,
+ "logits/chosen": 1.6676204204559326,
+ "logits/rejected": 1.7247302532196045,
+ "logps/chosen": -140.10838317871094,
+ "logps/rejected": -132.8035888671875,
+ "loss": 0.6826080799102783,
+ "rewards/accuracies": 0.887499988079071,
+ "rewards/chosen": 0.02765839174389839,
+ "rewards/margins": 0.02128567174077034,
+ "rewards/rejected": 0.006372719071805477,
+ "step": 140
+ },
+ {
+ "epoch": 1.12,
+ "eval_logits/chosen": 1.5169621706008911,
+ "eval_logits/rejected": 1.5509902238845825,
+ "eval_logps/chosen": -153.3729248046875,
+ "eval_logps/rejected": -147.8663330078125,
+ "eval_loss": 0.6921952962875366,
+ "eval_rewards/accuracies": 0.5379999876022339,
+ "eval_rewards/chosen": 0.017592232674360275,
+ "eval_rewards/margins": 0.0020089212339371443,
+ "eval_rewards/rejected": 0.015583310276269913,
+ "eval_runtime": 91.4751,
+ "eval_samples_per_second": 5.466,
+ "eval_steps_per_second": 2.733,
+ "step": 140
+ },
+ {
+ "epoch": 1.2,
+ "grad_norm": 1.1575056314468384,
+ "learning_rate": 6.706231454005934e-07,
+ "logits/chosen": 1.617916464805603,
+ "logits/rejected": 1.5512969493865967,
+ "logps/chosen": -142.42713928222656,
+ "logps/rejected": -151.55618286132812,
+ "loss": 0.6822887420654297,
+ "rewards/accuracies": 0.875,
+ "rewards/chosen": 0.03180321678519249,
+ "rewards/margins": 0.021957319229841232,
+ "rewards/rejected": 0.009845896624028683,
+ "step": 150
+ },
+ {
+ "epoch": 1.2,
+ "eval_logits/chosen": 1.5179307460784912,
+ "eval_logits/rejected": 1.5517919063568115,
+ "eval_logps/chosen": -153.3365020751953,
+ "eval_logps/rejected": -147.83668518066406,
+ "eval_loss": 0.6918668150901794,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.02123303711414337,
+ "eval_rewards/margins": 0.002686727326363325,
+ "eval_rewards/rejected": 0.01854631118476391,
+ "eval_runtime": 91.5031,
+ "eval_samples_per_second": 5.464,
+ "eval_steps_per_second": 2.732,
+ "step": 150
+ },
+ {
+ "epoch": 1.28,
+ "grad_norm": 1.131994366645813,
+ "learning_rate": 6.409495548961425e-07,
+ "logits/chosen": 1.3928228616714478,
+ "logits/rejected": 1.543765664100647,
+ "logps/chosen": -138.6504364013672,
+ "logps/rejected": -157.1339569091797,
+ "loss": 0.6779372692108154,
+ "rewards/accuracies": 0.949999988079071,
+ "rewards/chosen": 0.04010936990380287,
+ "rewards/margins": 0.030827566981315613,
+ "rewards/rejected": 0.009281802922487259,
+ "step": 160
+ },
+ {
+ "epoch": 1.28,
+ "eval_logits/chosen": 1.518864393234253,
+ "eval_logits/rejected": 1.552842617034912,
+ "eval_logps/chosen": -153.30166625976562,
+ "eval_logps/rejected": -147.79891967773438,
+ "eval_loss": 0.6920233368873596,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": 0.024718036875128746,
+ "eval_rewards/margins": 0.002396443858742714,
+ "eval_rewards/rejected": 0.022321591153740883,
+ "eval_runtime": 91.7707,
+ "eval_samples_per_second": 5.448,
+ "eval_steps_per_second": 2.724,
+ "step": 160
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "grad_norm": 1.1497802734375,
+ "learning_rate": 6.112759643916914e-07,
+ "logits/chosen": 1.4033597707748413,
+ "logits/rejected": 1.5339505672454834,
+ "logps/chosen": -124.35432434082031,
+ "logps/rejected": -161.1474609375,
+ "loss": 0.680607795715332,
+ "rewards/accuracies": 0.824999988079071,
+ "rewards/chosen": 0.03419633209705353,
+ "rewards/margins": 0.02547808550298214,
+ "rewards/rejected": 0.008718245662748814,
+ "step": 170
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "eval_logits/chosen": 1.5178680419921875,
+ "eval_logits/rejected": 1.5517812967300415,
+ "eval_logps/chosen": -153.30496215820312,
+ "eval_logps/rejected": -147.80471801757812,
+ "eval_loss": 0.6919159889221191,
+ "eval_rewards/accuracies": 0.5260000228881836,
+ "eval_rewards/chosen": 0.02438722550868988,
+ "eval_rewards/margins": 0.002643684623762965,
+ "eval_rewards/rejected": 0.021743541583418846,
+ "eval_runtime": 91.5892,
+ "eval_samples_per_second": 5.459,
+ "eval_steps_per_second": 2.73,
+ "step": 170
+ },
+ {
+ "epoch": 1.44,
+ "grad_norm": 1.0237778425216675,
+ "learning_rate": 5.816023738872403e-07,
+ "logits/chosen": 1.5374921560287476,
+ "logits/rejected": 1.6105371713638306,
+ "logps/chosen": -155.44931030273438,
+ "logps/rejected": -144.53395080566406,
+ "loss": 0.68257737159729,
+ "rewards/accuracies": 0.7875000238418579,
+ "rewards/chosen": 0.03792569041252136,
+ "rewards/margins": 0.021481072530150414,
+ "rewards/rejected": 0.01644461788237095,
+ "step": 180
+ },
+ {
+ "epoch": 1.44,
+ "eval_logits/chosen": 1.5193660259246826,
+ "eval_logits/rejected": 1.5532726049423218,
+ "eval_logps/chosen": -153.29214477539062,
+ "eval_logps/rejected": -147.7940673828125,
+ "eval_loss": 0.6918110251426697,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": 0.02567088045179844,
+ "eval_rewards/margins": 0.002862172434106469,
+ "eval_rewards/rejected": 0.022808710113167763,
+ "eval_runtime": 91.6386,
+ "eval_samples_per_second": 5.456,
+ "eval_steps_per_second": 2.728,
+ "step": 180
+ },
+ {
+ "epoch": 1.52,
+ "grad_norm": 1.4238712787628174,
+ "learning_rate": 5.519287833827893e-07,
+ "logits/chosen": 1.551783800125122,
+ "logits/rejected": 1.5306293964385986,
+ "logps/chosen": -154.03009033203125,
+ "logps/rejected": -148.2330780029297,
+ "loss": 0.6788228988647461,
+ "rewards/accuracies": 0.875,
+ "rewards/chosen": 0.034665219485759735,
+ "rewards/margins": 0.0292512234300375,
+ "rewards/rejected": 0.005413994658738375,
+ "step": 190
+ },
+ {
+ "epoch": 1.52,
+ "eval_logits/chosen": 1.5185673236846924,
+ "eval_logits/rejected": 1.5524872541427612,
+ "eval_logps/chosen": -153.31253051757812,
+ "eval_logps/rejected": -147.81149291992188,
+ "eval_loss": 0.6919614672660828,
+ "eval_rewards/accuracies": 0.5180000066757202,
+ "eval_rewards/chosen": 0.023632025346159935,
+ "eval_rewards/margins": 0.0025663499254733324,
+ "eval_rewards/rejected": 0.02106567472219467,
+ "eval_runtime": 91.6118,
+ "eval_samples_per_second": 5.458,
+ "eval_steps_per_second": 2.729,
+ "step": 190
+ },
+ {
+ "epoch": 1.6,
+ "grad_norm": 0.965166449546814,
+ "learning_rate": 5.222551928783383e-07,
+ "logits/chosen": 1.5729683637619019,
+ "logits/rejected": 1.6736443042755127,
+ "logps/chosen": -175.38226318359375,
+ "logps/rejected": -152.61105346679688,
+ "loss": 0.6791690826416016,
+ "rewards/accuracies": 0.800000011920929,
+ "rewards/chosen": 0.040378399193286896,
+ "rewards/margins": 0.028407778590917587,
+ "rewards/rejected": 0.011970620602369308,
+ "step": 200
+ },
+ {
+ "epoch": 1.6,
+ "eval_logits/chosen": 1.5189275741577148,
+ "eval_logits/rejected": 1.5527902841567993,
+ "eval_logps/chosen": -153.29649353027344,
+ "eval_logps/rejected": -147.7986602783203,
+ "eval_loss": 0.691815972328186,
+ "eval_rewards/accuracies": 0.5199999809265137,
+ "eval_rewards/chosen": 0.025234410539269447,
+ "eval_rewards/margins": 0.0028851027600467205,
+ "eval_rewards/rejected": 0.02234930545091629,
+ "eval_runtime": 91.6894,
+ "eval_samples_per_second": 5.453,
+ "eval_steps_per_second": 2.727,
+ "step": 200
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "grad_norm": 1.3004869222640991,
+ "learning_rate": 4.925816023738872e-07,
+ "logits/chosen": 1.4969855546951294,
+ "logits/rejected": 1.536221981048584,
+ "logps/chosen": -137.86892700195312,
+ "logps/rejected": -136.7208251953125,
+ "loss": 0.6805938720703125,
+ "rewards/accuracies": 0.800000011920929,
+ "rewards/chosen": 0.04161720722913742,
+ "rewards/margins": 0.025478878989815712,
+ "rewards/rejected": 0.016138331964612007,
+ "step": 210
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "eval_logits/chosen": 1.519534707069397,
+ "eval_logits/rejected": 1.5532875061035156,
+ "eval_logps/chosen": -153.25706481933594,
+ "eval_logps/rejected": -147.76138305664062,
+ "eval_loss": 0.6917269229888916,
+ "eval_rewards/accuracies": 0.5239999890327454,
+ "eval_rewards/chosen": 0.02917887456715107,
+ "eval_rewards/margins": 0.0031047509983181953,
+ "eval_rewards/rejected": 0.02607412077486515,
+ "eval_runtime": 91.4998,
+ "eval_samples_per_second": 5.464,
+ "eval_steps_per_second": 2.732,
+ "step": 210
+ },
+ {
+ "epoch": 1.76,
+ "grad_norm": 0.9438315033912659,
+ "learning_rate": 4.6290801186943617e-07,
+ "logits/chosen": 1.672853708267212,
+ "logits/rejected": 1.6711323261260986,
+ "logps/chosen": -154.64735412597656,
+ "logps/rejected": -158.96665954589844,
+ "loss": 0.6800187587738037,
+ "rewards/accuracies": 0.8500000238418579,
+ "rewards/chosen": 0.043148353695869446,
+ "rewards/margins": 0.02666427567601204,
+ "rewards/rejected": 0.016484078019857407,
+ "step": 220
+ },
+ {
+ "epoch": 1.76,
+ "eval_logits/chosen": 1.5207358598709106,
+ "eval_logits/rejected": 1.554581880569458,
+ "eval_logps/chosen": -153.2335968017578,
+ "eval_logps/rejected": -147.73776245117188,
+ "eval_loss": 0.6917403340339661,
+ "eval_rewards/accuracies": 0.5199999809265137,
+ "eval_rewards/chosen": 0.031523481011390686,
+ "eval_rewards/margins": 0.0030857159290462732,
+ "eval_rewards/rejected": 0.02843775972723961,
+ "eval_runtime": 91.6029,
+ "eval_samples_per_second": 5.458,
+ "eval_steps_per_second": 2.729,
+ "step": 220
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "grad_norm": 0.9313515424728394,
+ "learning_rate": 4.332344213649852e-07,
+ "logits/chosen": 1.412987470626831,
+ "logits/rejected": 1.318050503730774,
+ "logps/chosen": -138.3455352783203,
+ "logps/rejected": -132.15701293945312,
+ "loss": 0.6811746120452881,
+ "rewards/accuracies": 0.800000011920929,
+ "rewards/chosen": 0.04524313658475876,
+ "rewards/margins": 0.02441518008708954,
+ "rewards/rejected": 0.02082795649766922,
+ "step": 230
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "eval_logits/chosen": 1.5209001302719116,
+ "eval_logits/rejected": 1.554688572883606,
+ "eval_logps/chosen": -153.22760009765625,
+ "eval_logps/rejected": -147.7340087890625,
+ "eval_loss": 0.6916303038597107,
+ "eval_rewards/accuracies": 0.5239999890327454,
+ "eval_rewards/chosen": 0.03212640434503555,
+ "eval_rewards/margins": 0.003312844317406416,
+ "eval_rewards/rejected": 0.02881355956196785,
+ "eval_runtime": 91.5879,
+ "eval_samples_per_second": 5.459,
+ "eval_steps_per_second": 2.73,
+ "step": 230
+ },
+ {
+ "epoch": 1.92,
+ "grad_norm": 0.9393594861030579,
+ "learning_rate": 4.035608308605341e-07,
+ "logits/chosen": 1.6411993503570557,
+ "logits/rejected": 1.7736005783081055,
+ "logps/chosen": -143.26681518554688,
+ "logps/rejected": -150.54849243164062,
+ "loss": 0.6799068450927734,
+ "rewards/accuracies": 0.8500000238418579,
+ "rewards/chosen": 0.047449741512537,
+ "rewards/margins": 0.026947852224111557,
+ "rewards/rejected": 0.020501883700489998,
+ "step": 240
+ },
+ {
+ "epoch": 1.92,
+ "eval_logits/chosen": 1.5195804834365845,
+ "eval_logits/rejected": 1.553446650505066,
+ "eval_logps/chosen": -153.2545166015625,
+ "eval_logps/rejected": -147.75694274902344,
+ "eval_loss": 0.6918333172798157,
+ "eval_rewards/accuracies": 0.5220000147819519,
+ "eval_rewards/chosen": 0.02943290024995804,
+ "eval_rewards/margins": 0.002911545103415847,
+ "eval_rewards/rejected": 0.026521356776356697,
+ "eval_runtime": 91.751,
+ "eval_samples_per_second": 5.45,
+ "eval_steps_per_second": 2.725,
+ "step": 240
+ },
+ {
+ "epoch": 2.0,
+ "grad_norm": 0.8810976147651672,
+ "learning_rate": 3.7388724035608303e-07,
+ "logits/chosen": 1.4585398435592651,
+ "logits/rejected": 1.6135761737823486,
+ "logps/chosen": -144.0837860107422,
+ "logps/rejected": -164.10047912597656,
+ "loss": 0.6806133270263672,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.0476423017680645,
+ "rewards/margins": 0.025543373078107834,
+ "rewards/rejected": 0.022098926827311516,
+ "step": 250
+ },
+ {
+ "epoch": 2.0,
+ "eval_logits/chosen": 1.518336534500122,
+ "eval_logits/rejected": 1.5521376132965088,
+ "eval_logps/chosen": -153.27587890625,
+ "eval_logps/rejected": -147.77859497070312,
+ "eval_loss": 0.6918182373046875,
+ "eval_rewards/accuracies": 0.5080000162124634,
+ "eval_rewards/chosen": 0.02729707397520542,
+ "eval_rewards/margins": 0.0029408105183392763,
+ "eval_rewards/rejected": 0.024356262758374214,
+ "eval_runtime": 91.638,
+ "eval_samples_per_second": 5.456,
+ "eval_steps_per_second": 2.728,
+ "step": 250
+ },
+ {
+ "epoch": 2.08,
+ "grad_norm": 1.381959319114685,
+ "learning_rate": 3.4421364985163206e-07,
+ "logits/chosen": 1.5461595058441162,
+ "logits/rejected": 1.5189964771270752,
+ "logps/chosen": -147.18014526367188,
+ "logps/rejected": -140.32498168945312,
+ "loss": 0.6733437061309815,
+ "rewards/accuracies": 0.9125000238418579,
+ "rewards/chosen": 0.04400390386581421,
+ "rewards/margins": 0.040534622967243195,
+ "rewards/rejected": 0.003469283226877451,
+ "step": 260
+ },
+ {
+ "epoch": 2.08,
+ "eval_logits/chosen": 1.5175292491912842,
+ "eval_logits/rejected": 1.5512447357177734,
+ "eval_logps/chosen": -153.28150939941406,
+ "eval_logps/rejected": -147.7823486328125,
+ "eval_loss": 0.6919183135032654,
+ "eval_rewards/accuracies": 0.5199999809265137,
+ "eval_rewards/chosen": 0.02673187479376793,
+ "eval_rewards/margins": 0.002751472406089306,
+ "eval_rewards/rejected": 0.0239803995937109,
+ "eval_runtime": 91.6634,
+ "eval_samples_per_second": 5.455,
+ "eval_steps_per_second": 2.727,
+ "step": 260
+ },
+ {
+ "epoch": 2.16,
+ "grad_norm": 1.2985988855361938,
+ "learning_rate": 3.14540059347181e-07,
+ "logits/chosen": 1.4951808452606201,
+ "logits/rejected": 1.5748175382614136,
+ "logps/chosen": -143.20095825195312,
+ "logps/rejected": -141.10385131835938,
+ "loss": 0.6718563079833985,
+ "rewards/accuracies": 0.887499988079071,
+ "rewards/chosen": 0.04751860350370407,
+ "rewards/margins": 0.04354388266801834,
+ "rewards/rejected": 0.003974726889282465,
+ "step": 270
+ },
+ {
+ "epoch": 2.16,
+ "eval_logits/chosen": 1.517343521118164,
+ "eval_logits/rejected": 1.551146149635315,
+ "eval_logps/chosen": -153.27561950683594,
+ "eval_logps/rejected": -147.78273010253906,
+ "eval_loss": 0.6916126608848572,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.027322430163621902,
+ "eval_rewards/margins": 0.003382232040166855,
+ "eval_rewards/rejected": 0.023940199986100197,
+ "eval_runtime": 91.6025,
+ "eval_samples_per_second": 5.458,
+ "eval_steps_per_second": 2.729,
+ "step": 270
+ },
+ {
+ "epoch": 2.24,
+ "grad_norm": 1.1583564281463623,
+ "learning_rate": 2.8486646884272995e-07,
+ "logits/chosen": 1.511849284172058,
+ "logits/rejected": 1.44902765750885,
+ "logps/chosen": -145.66714477539062,
+ "logps/rejected": -134.85720825195312,
+ "loss": 0.6712348461151123,
+ "rewards/accuracies": 0.925000011920929,
+ "rewards/chosen": 0.052734870463609695,
+ "rewards/margins": 0.044918566942214966,
+ "rewards/rejected": 0.007816305384039879,
+ "step": 280
+ },
+ {
+ "epoch": 2.24,
+ "eval_logits/chosen": 1.516547441482544,
+ "eval_logits/rejected": 1.5503180027008057,
+ "eval_logps/chosen": -153.27293395996094,
+ "eval_logps/rejected": -147.77932739257812,
+ "eval_loss": 0.6916624903678894,
+ "eval_rewards/accuracies": 0.5260000228881836,
+ "eval_rewards/chosen": 0.027590708807110786,
+ "eval_rewards/margins": 0.0033070938661694527,
+ "eval_rewards/rejected": 0.02428361587226391,
+ "eval_runtime": 91.7361,
+ "eval_samples_per_second": 5.45,
+ "eval_steps_per_second": 2.725,
+ "step": 280
+ },
+ {
+ "epoch": 2.32,
+ "grad_norm": 1.1054120063781738,
+ "learning_rate": 2.551928783382789e-07,
+ "logits/chosen": 1.4237782955169678,
+ "logits/rejected": 1.5945169925689697,
+ "logps/chosen": -137.7429962158203,
+ "logps/rejected": -148.33139038085938,
+ "loss": 0.671449613571167,
+ "rewards/accuracies": 0.925000011920929,
+ "rewards/chosen": 0.05219203978776932,
+ "rewards/margins": 0.04431880638003349,
+ "rewards/rejected": 0.0078732306137681,
+ "step": 290
+ },
+ {
+ "epoch": 2.32,
+ "eval_logits/chosen": 1.5170159339904785,
+ "eval_logits/rejected": 1.550805687904358,
+ "eval_logps/chosen": -153.26280212402344,
+ "eval_logps/rejected": -147.76779174804688,
+ "eval_loss": 0.6917348504066467,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.02860422618687153,
+ "eval_rewards/margins": 0.0031696020159870386,
+ "eval_rewards/rejected": 0.025434626266360283,
+ "eval_runtime": 91.5841,
+ "eval_samples_per_second": 5.459,
+ "eval_steps_per_second": 2.73,
+ "step": 290
+ },
+ {
+ "epoch": 2.4,
+ "grad_norm": 1.07664954662323,
+ "learning_rate": 2.2551928783382787e-07,
+ "logits/chosen": 1.437131643295288,
+ "logits/rejected": 1.5557130575180054,
+ "logps/chosen": -136.46463012695312,
+ "logps/rejected": -133.51284790039062,
+ "loss": 0.6737070560455323,
+ "rewards/accuracies": 0.8500000238418579,
+ "rewards/chosen": 0.052551962435245514,
+ "rewards/margins": 0.039616115391254425,
+ "rewards/rejected": 0.012935848906636238,
+ "step": 300
+ },
+ {
+ "epoch": 2.4,
+ "eval_logits/chosen": 1.517410159111023,
+ "eval_logits/rejected": 1.5511317253112793,
+ "eval_logps/chosen": -153.2540283203125,
+ "eval_logps/rejected": -147.76036071777344,
+ "eval_loss": 0.6916749477386475,
+ "eval_rewards/accuracies": 0.5059999823570251,
+ "eval_rewards/chosen": 0.029480623081326485,
+ "eval_rewards/margins": 0.003301949007436633,
+ "eval_rewards/rejected": 0.026178676635026932,
+ "eval_runtime": 91.4824,
+ "eval_samples_per_second": 5.466,
+ "eval_steps_per_second": 2.733,
+ "step": 300
+ },
+ {
+ "epoch": 2.48,
+ "grad_norm": 1.215026617050171,
+ "learning_rate": 1.9584569732937684e-07,
+ "logits/chosen": 1.551770806312561,
+ "logits/rejected": 1.6192102432250977,
+ "logps/chosen": -143.90609741210938,
+ "logps/rejected": -143.22312927246094,
+ "loss": 0.6664128303527832,
+ "rewards/accuracies": 0.925000011920929,
+ "rewards/chosen": 0.05427182838320732,
+ "rewards/margins": 0.054780732840299606,
+ "rewards/rejected": -0.0005089022452011704,
+ "step": 310
+ },
+ {
+ "epoch": 2.48,
+ "eval_logits/chosen": 1.5172474384307861,
+ "eval_logits/rejected": 1.55105459690094,
+ "eval_logps/chosen": -153.2525177001953,
+ "eval_logps/rejected": -147.757080078125,
+ "eval_loss": 0.691773533821106,
+ "eval_rewards/accuracies": 0.5139999985694885,
+ "eval_rewards/chosen": 0.02963165193796158,
+ "eval_rewards/margins": 0.003123925533145666,
+ "eval_rewards/rejected": 0.026507727801799774,
+ "eval_runtime": 91.7435,
+ "eval_samples_per_second": 5.45,
+ "eval_steps_per_second": 2.725,
+ "step": 310
+ },
+ {
+ "epoch": 2.56,
+ "grad_norm": 1.1716984510421753,
+ "learning_rate": 1.661721068249258e-07,
+ "logits/chosen": 1.5159642696380615,
+ "logits/rejected": 1.5621572732925415,
+ "logps/chosen": -171.1765899658203,
+ "logps/rejected": -169.19384765625,
+ "loss": 0.6687184810638428,
+ "rewards/accuracies": 0.862500011920929,
+ "rewards/chosen": 0.05426288768649101,
+ "rewards/margins": 0.04987890645861626,
+ "rewards/rejected": 0.004383978433907032,
+ "step": 320
+ },
+ {
+ "epoch": 2.56,
+ "eval_logits/chosen": 1.516568660736084,
+ "eval_logits/rejected": 1.5502315759658813,
+ "eval_logps/chosen": -153.261474609375,
+ "eval_logps/rejected": -147.76637268066406,
+ "eval_loss": 0.6917595267295837,
+ "eval_rewards/accuracies": 0.5139999985694885,
+ "eval_rewards/chosen": 0.028737707063555717,
+ "eval_rewards/margins": 0.0031595926266163588,
+ "eval_rewards/rejected": 0.02557811327278614,
+ "eval_runtime": 91.6465,
+ "eval_samples_per_second": 5.456,
+ "eval_steps_per_second": 2.728,
+ "step": 320
+ },
+ {
+ "epoch": 2.64,
+ "grad_norm": 0.9244529008865356,
+ "learning_rate": 1.3649851632047476e-07,
+ "logits/chosen": 1.4917646646499634,
+ "logits/rejected": 1.6086304187774658,
+ "logps/chosen": -145.4997100830078,
+ "logps/rejected": -149.67369079589844,
+ "loss": 0.6736223697662354,
+ "rewards/accuracies": 0.8500000238418579,
+ "rewards/chosen": 0.04983464628458023,
+ "rewards/margins": 0.039811473339796066,
+ "rewards/rejected": 0.01002317015081644,
+ "step": 330
+ },
+ {
+ "epoch": 2.64,
+ "eval_logits/chosen": 1.515918493270874,
+ "eval_logits/rejected": 1.5496324300765991,
+ "eval_logps/chosen": -153.2665252685547,
+ "eval_logps/rejected": -147.77523803710938,
+ "eval_loss": 0.691569447517395,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": 0.028231728821992874,
+ "eval_rewards/margins": 0.0035420504864305258,
+ "eval_rewards/rejected": 0.024689678102731705,
+ "eval_runtime": 91.6598,
+ "eval_samples_per_second": 5.455,
+ "eval_steps_per_second": 2.727,
+ "step": 330
+ },
+ {
+ "epoch": 2.7199999999999998,
+ "grad_norm": 1.502729892730713,
+ "learning_rate": 1.0682492581602373e-07,
+ "logits/chosen": 1.6879287958145142,
+ "logits/rejected": 1.6182794570922852,
+ "logps/chosen": -146.68263244628906,
+ "logps/rejected": -165.4949188232422,
+ "loss": 0.6722164630889893,
+ "rewards/accuracies": 0.8500000238418579,
+ "rewards/chosen": 0.05252299830317497,
+ "rewards/margins": 0.0430002398788929,
+ "rewards/rejected": 0.009522760286927223,
+ "step": 340
+ },
+ {
+ "epoch": 2.7199999999999998,
+ "eval_logits/chosen": 1.5161068439483643,
+ "eval_logits/rejected": 1.5497750043869019,
+ "eval_logps/chosen": -153.26605224609375,
+ "eval_logps/rejected": -147.77371215820312,
+ "eval_loss": 0.6916317343711853,
+ "eval_rewards/accuracies": 0.5440000295639038,
+ "eval_rewards/chosen": 0.028278229758143425,
+ "eval_rewards/margins": 0.003435892518609762,
+ "eval_rewards/rejected": 0.024842336773872375,
+ "eval_runtime": 91.6857,
+ "eval_samples_per_second": 5.453,
+ "eval_steps_per_second": 2.727,
+ "step": 340
+ },
+ {
+ "epoch": 2.8,
+ "grad_norm": 1.1309890747070312,
+ "learning_rate": 7.715133531157269e-08,
+ "logits/chosen": 1.5167756080627441,
+ "logits/rejected": 1.5120360851287842,
+ "logps/chosen": -132.57003784179688,
+ "logps/rejected": -134.81997680664062,
+ "loss": 0.6760876178741455,
+ "rewards/accuracies": 0.8999999761581421,
+ "rewards/chosen": 0.04849396273493767,
+ "rewards/margins": 0.03476421907544136,
+ "rewards/rejected": 0.013729743659496307,
+ "step": 350
+ },
+ {
+ "epoch": 2.8,
+ "eval_logits/chosen": 1.515918493270874,
+ "eval_logits/rejected": 1.5495837926864624,
+ "eval_logps/chosen": -153.2740020751953,
+ "eval_logps/rejected": -147.77586364746094,
+ "eval_loss": 0.6919155120849609,
+ "eval_rewards/accuracies": 0.5099999904632568,
+ "eval_rewards/chosen": 0.02748536318540573,
+ "eval_rewards/margins": 0.0028558254707604647,
+ "eval_rewards/rejected": 0.024629538878798485,
+ "eval_runtime": 91.6859,
+ "eval_samples_per_second": 5.453,
+ "eval_steps_per_second": 2.727,
+ "step": 350
+ },
+ {
+ "epoch": 2.88,
+ "grad_norm": 1.3074413537979126,
+ "learning_rate": 4.747774480712166e-08,
+ "logits/chosen": 1.581860899925232,
+ "logits/rejected": 1.565870761871338,
+ "logps/chosen": -166.4083709716797,
+ "logps/rejected": -159.31626892089844,
+ "loss": 0.6719220161437989,
+ "rewards/accuracies": 0.862500011920929,
+ "rewards/chosen": 0.05766367167234421,
+ "rewards/margins": 0.04334929957985878,
+ "rewards/rejected": 0.014314373955130577,
+ "step": 360
+ },
+ {
+ "epoch": 2.88,
+ "eval_logits/chosen": 1.516121506690979,
+ "eval_logits/rejected": 1.5497403144836426,
+ "eval_logps/chosen": -153.26768493652344,
+ "eval_logps/rejected": -147.7772216796875,
+ "eval_loss": 0.691532552242279,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": 0.02811509743332863,
+ "eval_rewards/margins": 0.0036220289766788483,
+ "eval_rewards/rejected": 0.02449306845664978,
+ "eval_runtime": 91.72,
+ "eval_samples_per_second": 5.451,
+ "eval_steps_per_second": 2.726,
+ "step": 360
+ },
+ {
+ "epoch": 2.96,
+ "grad_norm": 1.0647271871566772,
+ "learning_rate": 1.7804154302670622e-08,
+ "logits/chosen": 1.4320623874664307,
+ "logits/rejected": 1.60173761844635,
+ "logps/chosen": -124.555908203125,
+ "logps/rejected": -156.7512664794922,
+ "loss": 0.6746182918548584,
+ "rewards/accuracies": 0.862500011920929,
+ "rewards/chosen": 0.0526120588183403,
+ "rewards/margins": 0.03778594732284546,
+ "rewards/rejected": 0.01482611894607544,
+ "step": 370
+ },
+ {
+ "epoch": 2.96,
+ "eval_logits/chosen": 1.5159584283828735,
+ "eval_logits/rejected": 1.5496013164520264,
+ "eval_logps/chosen": -153.26437377929688,
+ "eval_logps/rejected": -147.7732696533203,
+ "eval_loss": 0.6915634274482727,
+ "eval_rewards/accuracies": 0.5120000243186951,
+ "eval_rewards/chosen": 0.028447875753045082,
+ "eval_rewards/margins": 0.0035595870576798916,
+ "eval_rewards/rejected": 0.024888288229703903,
+ "eval_runtime": 91.7173,
+ "eval_samples_per_second": 5.452,
+ "eval_steps_per_second": 2.726,
+ "step": 370
+ }
+ ],
+ "logging_steps": 10,
+ "max_steps": 375,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": true
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/v3/DPO/DPO_1k/lora/checkpoint-375/training_args.bin b/v3/DPO/DPO_1k/lora/checkpoint-375/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..3e886d2019fd0b48fb6fe8daca6682f7c33cd328
--- /dev/null
+++ b/v3/DPO/DPO_1k/lora/checkpoint-375/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae6fe36d54feeb74309543d4eb00c98d1bbcfd3aca732d59918c68b1d6774432
+size 6097
diff --git a/v3/DPO/DPO_5k/DPO_5k/README.md b/v3/DPO/DPO_5k/DPO_5k/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..250500f798b4682b2cd2c35cd1fc366677c215dd
--- /dev/null
+++ b/v3/DPO/DPO_5k/DPO_5k/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/DPO/DPO_5k/DPO_5k/adapter_config.json b/v3/DPO/DPO_5k/DPO_5k/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..695e5e16d4a5436a17790fe2e31fc2a9e1631d03
--- /dev/null
+++ b/v3/DPO/DPO_5k/DPO_5k/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "gate_proj",
+ "k_proj",
+ "q_proj",
+ "down_proj",
+ "o_proj",
+ "v_proj",
+ "up_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/DPO/DPO_5k/DPO_5k/adapter_model.safetensors b/v3/DPO/DPO_5k/DPO_5k/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6635f4cb505655e646a63ebae1b9d26e21c5f9d2
--- /dev/null
+++ b/v3/DPO/DPO_5k/DPO_5k/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:504da931dd735d5a7d203b6528f0ef0550f7e365d2cd61c78dfefdcd3360562b
+size 180385008
diff --git a/v3/DPO/DPO_5k/MDPO_5k/chat_template.jinja b/v3/DPO/DPO_5k/MDPO_5k/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/DPO/DPO_5k/MDPO_5k/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/DPO/DPO_5k/MDPO_5k/config.json b/v3/DPO/DPO_5k/MDPO_5k/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e9ec18bc1ae18922052a57480e28401f3b9c6b84
--- /dev/null
+++ b/v3/DPO/DPO_5k/MDPO_5k/config.json
@@ -0,0 +1,40 @@
+{
+ "architectures": [
+ "LlamaForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 128000,
+ "dtype": "float16",
+ "eos_token_id": [
+ 128001,
+ 128008,
+ 128009
+ ],
+ "head_dim": 64,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 8192,
+ "max_position_embeddings": 131072,
+ "mlp_bias": false,
+ "model_type": "llama",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 16,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "pretraining_tp": 1,
+ "rms_norm_eps": 1e-05,
+ "rope_parameters": {
+ "factor": 32.0,
+ "high_freq_factor": 4.0,
+ "low_freq_factor": 1.0,
+ "original_max_position_embeddings": 8192,
+ "rope_theta": 500000.0,
+ "rope_type": "llama3"
+ },
+ "tie_word_embeddings": true,
+ "transformers_version": "5.2.0",
+ "use_cache": true,
+ "vocab_size": 128256
+}
diff --git a/v3/DPO/DPO_5k/MDPO_5k/generation_config.json b/v3/DPO/DPO_5k/MDPO_5k/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..898a2e748a41a4b7a931b754f8abbdac02039fae
--- /dev/null
+++ b/v3/DPO/DPO_5k/MDPO_5k/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "bos_token_id": 128000,
+ "do_sample": true,
+ "eos_token_id": [
+ 128001,
+ 128008,
+ 128009
+ ],
+ "temperature": 0.6,
+ "top_p": 0.9,
+ "transformers_version": "5.2.0"
+}
diff --git a/v3/DPO/DPO_5k/MDPO_5k/model.safetensors b/v3/DPO/DPO_5k/MDPO_5k/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ba5a1da22aca34718857020484120166a9ddafd4
--- /dev/null
+++ b/v3/DPO/DPO_5k/MDPO_5k/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4d61713ee447d0f3c5d031cbcf2b789791bdf40b4b4f9b8ea1431ecad09acfa
+size 2471645464
diff --git a/v3/DPO/DPO_5k/MDPO_5k/tokenizer.json b/v3/DPO/DPO_5k/MDPO_5k/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/DPO/DPO_5k/MDPO_5k/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/DPO/DPO_5k/MDPO_5k/tokenizer_config.json b/v3/DPO/DPO_5k/MDPO_5k/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/DPO/DPO_5k/MDPO_5k/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/DPO/DPO_5k/lora/README.md b/v3/DPO/DPO_5k/lora/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..a2ceb15fc2ca4b7e97704cf2e5d1f4c7e6be5d52
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/README.md
@@ -0,0 +1,68 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: transformers
+model_name: lora
+tags:
+- generated_from_trainer
+- trl
+- dpo
+licence: license
+---
+
+# Model Card for lora
+
+This model is a fine-tuned version of [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[](https://wandb.ai/sea-rod/huggingface/runs/0uweoi9o)
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- TRL: 0.28.0
+- Transformers: 5.2.0
+- Pytorch: 2.10.0
+- Datasets: 4.5.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+ title = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+ author = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+ year = 2023,
+ booktitle = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+ url = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+ editor = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+ title = {{TRL: Transformers Reinforcement Learning}},
+ author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+ license = {Apache-2.0},
+ url = {https://github.com/huggingface/trl},
+ year = {2020}
+}
+```
\ No newline at end of file
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1000/README.md b/v3/DPO/DPO_5k/lora/checkpoint-1000/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..250500f798b4682b2cd2c35cd1fc366677c215dd
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1000/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1000/adapter_config.json b/v3/DPO/DPO_5k/lora/checkpoint-1000/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..695e5e16d4a5436a17790fe2e31fc2a9e1631d03
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1000/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "gate_proj",
+ "k_proj",
+ "q_proj",
+ "down_proj",
+ "o_proj",
+ "v_proj",
+ "up_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1000/adapter_model.safetensors b/v3/DPO/DPO_5k/lora/checkpoint-1000/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6848028414dcafea2c40c23f1bfd04814e4d8776
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1000/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca2066076212283a4bc94494e0a74b02fee0596a18f157f1cf66ee217c0e4a67
+size 180385008
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1000/chat_template.jinja b/v3/DPO/DPO_5k/lora/checkpoint-1000/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1000/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1000/optimizer.pt b/v3/DPO/DPO_5k/lora/checkpoint-1000/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9e317ec74e304196895488f77e39ea71b8d86b77
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1000/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:810105e1b58aaa451e78b1620f84afe4cd387b7cf60c71e78790abb0acf32547
+size 360902475
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1000/rng_state.pth b/v3/DPO/DPO_5k/lora/checkpoint-1000/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..45da84cb9f96139d23fe2df13e9f26310fc87471
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1000/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f27403649e37262609f871dba600d7da4f0bfc387b6571584c6bc75289b7c853
+size 14645
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1000/scaler.pt b/v3/DPO/DPO_5k/lora/checkpoint-1000/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0405b11307032f43fcf2212da19c52823cc01ac4
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1000/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14ae2a2128444abab378aa06c09a61a84665f758fcc19fc46f5789b0bc1b5665
+size 1383
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1000/scheduler.pt b/v3/DPO/DPO_5k/lora/checkpoint-1000/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ed4a9fc9e26eef6deed39366d5164afb1e640425
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1000/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f9e5bef5377b623d98b024a8221f87395f0b16a317ae92d0e96ff17473ea802
+size 1465
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1000/tokenizer.json b/v3/DPO/DPO_5k/lora/checkpoint-1000/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1000/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1000/tokenizer_config.json b/v3/DPO/DPO_5k/lora/checkpoint-1000/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1000/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1000/trainer_state.json b/v3/DPO/DPO_5k/lora/checkpoint-1000/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..45c5d048f13cf3d463459b9758c294ef19412fde
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1000/trainer_state.json
@@ -0,0 +1,1854 @@
+{
+ "best_global_step": 950,
+ "best_metric": 0.6867280602455139,
+ "best_model_checkpoint": null,
+ "epoch": 1.6,
+ "eval_steps": 50,
+ "global_step": 1000,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.016,
+ "grad_norm": 1.1517287492752075,
+ "learning_rate": 4.787234042553191e-08,
+ "logits/chosen": 1.6246497631072998,
+ "logits/rejected": 1.5396833419799805,
+ "logps/chosen": -145.91531372070312,
+ "logps/rejected": -151.4051971435547,
+ "loss": 0.693363094329834,
+ "rewards/accuracies": 0.2874999940395355,
+ "rewards/chosen": -3.5572534216044005e-06,
+ "rewards/margins": -0.0004222107236273587,
+ "rewards/rejected": 0.00041865353705361485,
+ "step": 10
+ },
+ {
+ "epoch": 0.032,
+ "grad_norm": 1.3633077144622803,
+ "learning_rate": 1.0106382978723404e-07,
+ "logits/chosen": 1.313926339149475,
+ "logits/rejected": 1.4038684368133545,
+ "logps/chosen": -120.60591888427734,
+ "logps/rejected": -134.05252075195312,
+ "loss": 0.6930661201477051,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 9.576084994478151e-05,
+ "rewards/margins": 0.0001732158416416496,
+ "rewards/rejected": -7.745502807665616e-05,
+ "step": 20
+ },
+ {
+ "epoch": 0.048,
+ "grad_norm": 1.209151029586792,
+ "learning_rate": 1.5425531914893615e-07,
+ "logits/chosen": 1.5228456258773804,
+ "logits/rejected": 1.6869430541992188,
+ "logps/chosen": -162.40280151367188,
+ "logps/rejected": -179.38192749023438,
+ "loss": 0.692751693725586,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0007011269335635006,
+ "rewards/margins": 0.0008053636993281543,
+ "rewards/rejected": -0.00010423659114167094,
+ "step": 30
+ },
+ {
+ "epoch": 0.064,
+ "grad_norm": 1.3589205741882324,
+ "learning_rate": 2.074468085106383e-07,
+ "logits/chosen": 1.544001817703247,
+ "logits/rejected": 1.4715760946273804,
+ "logps/chosen": -152.7266845703125,
+ "logps/rejected": -150.9158477783203,
+ "loss": 0.6925192832946777,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.00026422025985084474,
+ "rewards/margins": 0.0012672842713072896,
+ "rewards/rejected": -0.0010030639823526144,
+ "step": 40
+ },
+ {
+ "epoch": 0.08,
+ "grad_norm": 1.202810525894165,
+ "learning_rate": 2.6063829787234044e-07,
+ "logits/chosen": 1.6794925928115845,
+ "logits/rejected": 1.7414724826812744,
+ "logps/chosen": -169.71751403808594,
+ "logps/rejected": -156.7571258544922,
+ "loss": 0.6930821895599365,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 9.479046275373548e-05,
+ "rewards/margins": 0.00014605997421313077,
+ "rewards/rejected": -5.126944597577676e-05,
+ "step": 50
+ },
+ {
+ "epoch": 0.08,
+ "eval_logits/chosen": 1.5141410827636719,
+ "eval_logits/rejected": 1.5485105514526367,
+ "eval_logps/chosen": -153.53335571289062,
+ "eval_logps/rejected": -148.00894165039062,
+ "eval_loss": 0.6930396556854248,
+ "eval_rewards/accuracies": 0.5120000243186951,
+ "eval_rewards/chosen": 0.0015483711613342166,
+ "eval_rewards/margins": 0.00022836039715912193,
+ "eval_rewards/rejected": 0.0013200106332078576,
+ "eval_runtime": 91.1536,
+ "eval_samples_per_second": 5.485,
+ "eval_steps_per_second": 2.743,
+ "step": 50
+ },
+ {
+ "epoch": 0.096,
+ "grad_norm": 1.2080731391906738,
+ "learning_rate": 3.1382978723404253e-07,
+ "logits/chosen": 1.796142339706421,
+ "logits/rejected": 1.7786738872528076,
+ "logps/chosen": -158.88607788085938,
+ "logps/rejected": -149.43331909179688,
+ "loss": 0.6934396743774414,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.001977279083803296,
+ "rewards/margins": -0.0005727410316467285,
+ "rewards/rejected": 0.0025500201154500246,
+ "step": 60
+ },
+ {
+ "epoch": 0.112,
+ "grad_norm": 0.9988052248954773,
+ "learning_rate": 3.6702127659574467e-07,
+ "logits/chosen": 1.7978185415267944,
+ "logits/rejected": 1.7301257848739624,
+ "logps/chosen": -158.67416381835938,
+ "logps/rejected": -165.23361206054688,
+ "loss": 0.6937376976013183,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.0031797506380826235,
+ "rewards/margins": -0.0011613152455538511,
+ "rewards/rejected": 0.004341065417975187,
+ "step": 70
+ },
+ {
+ "epoch": 0.128,
+ "grad_norm": 1.2025635242462158,
+ "learning_rate": 4.202127659574468e-07,
+ "logits/chosen": 1.4933072328567505,
+ "logits/rejected": 1.4721126556396484,
+ "logps/chosen": -146.546875,
+ "logps/rejected": -138.2855224609375,
+ "loss": 0.6930079936981202,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.005923466756939888,
+ "rewards/margins": 0.000295188365271315,
+ "rewards/rejected": 0.005628278478980064,
+ "step": 80
+ },
+ {
+ "epoch": 0.144,
+ "grad_norm": 1.1192086935043335,
+ "learning_rate": 4.734042553191489e-07,
+ "logits/chosen": 1.4889073371887207,
+ "logits/rejected": 1.4786438941955566,
+ "logps/chosen": -163.24220275878906,
+ "logps/rejected": -160.1715850830078,
+ "loss": 0.6927919387817383,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.00839515682309866,
+ "rewards/margins": 0.0007321214652620256,
+ "rewards/rejected": 0.007663035299628973,
+ "step": 90
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 1.2289488315582275,
+ "learning_rate": 5.26595744680851e-07,
+ "logits/chosen": 1.5521520376205444,
+ "logits/rejected": 1.620697259902954,
+ "logps/chosen": -143.54342651367188,
+ "logps/rejected": -166.68838500976562,
+ "loss": 0.6924111843109131,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.010075005702674389,
+ "rewards/margins": 0.00150866259355098,
+ "rewards/rejected": 0.008566344156861305,
+ "step": 100
+ },
+ {
+ "epoch": 0.16,
+ "eval_logits/chosen": 1.513370156288147,
+ "eval_logits/rejected": 1.5475622415542603,
+ "eval_logps/chosen": -153.44686889648438,
+ "eval_logps/rejected": -147.92510986328125,
+ "eval_loss": 0.6929221153259277,
+ "eval_rewards/accuracies": 0.4699999988079071,
+ "eval_rewards/chosen": 0.010197031311690807,
+ "eval_rewards/margins": 0.0004922974039800465,
+ "eval_rewards/rejected": 0.00970473513007164,
+ "eval_runtime": 91.4962,
+ "eval_samples_per_second": 5.465,
+ "eval_steps_per_second": 2.732,
+ "step": 100
+ },
+ {
+ "epoch": 0.176,
+ "grad_norm": 1.0589202642440796,
+ "learning_rate": 5.797872340425531e-07,
+ "logits/chosen": 1.5165660381317139,
+ "logits/rejected": 1.5812733173370361,
+ "logps/chosen": -144.24609375,
+ "logps/rejected": -138.3242645263672,
+ "loss": 0.6928606033325195,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.011964901350438595,
+ "rewards/margins": 0.0006008840864524245,
+ "rewards/rejected": 0.011364017613232136,
+ "step": 110
+ },
+ {
+ "epoch": 0.192,
+ "grad_norm": 1.1458054780960083,
+ "learning_rate": 6.329787234042553e-07,
+ "logits/chosen": 1.5244230031967163,
+ "logits/rejected": 1.594994306564331,
+ "logps/chosen": -143.27723693847656,
+ "logps/rejected": -143.82754516601562,
+ "loss": 0.6933906078338623,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": 0.009245705790817738,
+ "rewards/margins": -0.0004399990721140057,
+ "rewards/rejected": 0.009685704484581947,
+ "step": 120
+ },
+ {
+ "epoch": 0.208,
+ "grad_norm": 1.3152341842651367,
+ "learning_rate": 6.861702127659574e-07,
+ "logits/chosen": 1.6518194675445557,
+ "logits/rejected": 1.7864271402359009,
+ "logps/chosen": -156.05361938476562,
+ "logps/rejected": -151.7067108154297,
+ "loss": 0.6933954238891602,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.004633297678083181,
+ "rewards/margins": -0.00046000973088666797,
+ "rewards/rejected": 0.005093307700008154,
+ "step": 130
+ },
+ {
+ "epoch": 0.224,
+ "grad_norm": 1.1007953882217407,
+ "learning_rate": 7.393617021276596e-07,
+ "logits/chosen": 1.4447275400161743,
+ "logits/rejected": 1.522552251815796,
+ "logps/chosen": -134.9702911376953,
+ "logps/rejected": -133.505615234375,
+ "loss": 0.6926079750061035,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0014777708565816283,
+ "rewards/margins": 0.0011204290203750134,
+ "rewards/rejected": 0.00035734166158363223,
+ "step": 140
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 0.92185378074646,
+ "learning_rate": 7.925531914893616e-07,
+ "logits/chosen": 1.7024447917938232,
+ "logits/rejected": 1.6758934259414673,
+ "logps/chosen": -142.47024536132812,
+ "logps/rejected": -141.05670166015625,
+ "loss": 0.6922515869140625,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.007132537662982941,
+ "rewards/margins": 0.0018380668479949236,
+ "rewards/rejected": 0.005294471047818661,
+ "step": 150
+ },
+ {
+ "epoch": 0.24,
+ "eval_logits/chosen": 1.5091065168380737,
+ "eval_logits/rejected": 1.5432627201080322,
+ "eval_logps/chosen": -153.50772094726562,
+ "eval_logps/rejected": -147.98837280273438,
+ "eval_loss": 0.6928078532218933,
+ "eval_rewards/accuracies": 0.5139999985694885,
+ "eval_rewards/chosen": 0.004112891852855682,
+ "eval_rewards/margins": 0.0007364260382018983,
+ "eval_rewards/rejected": 0.0033764655236154795,
+ "eval_runtime": 91.3903,
+ "eval_samples_per_second": 5.471,
+ "eval_steps_per_second": 2.736,
+ "step": 150
+ },
+ {
+ "epoch": 0.256,
+ "grad_norm": 1.2662791013717651,
+ "learning_rate": 8.457446808510637e-07,
+ "logits/chosen": 1.6849098205566406,
+ "logits/rejected": 1.620693564414978,
+ "logps/chosen": -164.6056365966797,
+ "logps/rejected": -148.01400756835938,
+ "loss": 0.6930419445037842,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": 0.009472684934735298,
+ "rewards/margins": 0.0002518748806323856,
+ "rewards/rejected": 0.009220810607075691,
+ "step": 160
+ },
+ {
+ "epoch": 0.272,
+ "grad_norm": 1.0822441577911377,
+ "learning_rate": 8.989361702127659e-07,
+ "logits/chosen": 1.4992691278457642,
+ "logits/rejected": 1.5570002794265747,
+ "logps/chosen": -141.10240173339844,
+ "logps/rejected": -125.11993408203125,
+ "loss": 0.692972469329834,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.009402485564351082,
+ "rewards/margins": 0.0004053188022226095,
+ "rewards/rejected": 0.008997166529297829,
+ "step": 170
+ },
+ {
+ "epoch": 0.288,
+ "grad_norm": 0.9809871315956116,
+ "learning_rate": 9.52127659574468e-07,
+ "logits/chosen": 1.5153322219848633,
+ "logits/rejected": 1.5223143100738525,
+ "logps/chosen": -154.1062469482422,
+ "logps/rejected": -155.04188537597656,
+ "loss": 0.6921162605285645,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.016383877024054527,
+ "rewards/margins": 0.002122154226526618,
+ "rewards/rejected": 0.014261722564697266,
+ "step": 180
+ },
+ {
+ "epoch": 0.304,
+ "grad_norm": 1.1610571146011353,
+ "learning_rate": 9.99407231772377e-07,
+ "logits/chosen": 1.6443488597869873,
+ "logits/rejected": 1.5873346328735352,
+ "logps/chosen": -157.59298706054688,
+ "logps/rejected": -155.9975128173828,
+ "loss": 0.6925239086151123,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.025818049907684326,
+ "rewards/margins": 0.0013327361084520817,
+ "rewards/rejected": 0.024485310539603233,
+ "step": 190
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 1.2108772993087769,
+ "learning_rate": 9.93479549496147e-07,
+ "logits/chosen": 1.3679401874542236,
+ "logits/rejected": 1.5478118658065796,
+ "logps/chosen": -127.5545654296875,
+ "logps/rejected": -134.0937042236328,
+ "loss": 0.6889955520629882,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.033060748130083084,
+ "rewards/margins": 0.008453840389847755,
+ "rewards/rejected": 0.02460690215229988,
+ "step": 200
+ },
+ {
+ "epoch": 0.32,
+ "eval_logits/chosen": 1.5197134017944336,
+ "eval_logits/rejected": 1.5536755323410034,
+ "eval_logps/chosen": -153.1664581298828,
+ "eval_logps/rejected": -147.66543579101562,
+ "eval_loss": 0.6919757127761841,
+ "eval_rewards/accuracies": 0.5239999890327454,
+ "eval_rewards/chosen": 0.03824040666222572,
+ "eval_rewards/margins": 0.0025693033821880817,
+ "eval_rewards/rejected": 0.03567110374569893,
+ "eval_runtime": 91.3674,
+ "eval_samples_per_second": 5.472,
+ "eval_steps_per_second": 2.736,
+ "step": 200
+ },
+ {
+ "epoch": 0.336,
+ "grad_norm": 1.4075576066970825,
+ "learning_rate": 9.87551867219917e-07,
+ "logits/chosen": 1.6747233867645264,
+ "logits/rejected": 1.557538628578186,
+ "logps/chosen": -151.46202087402344,
+ "logps/rejected": -126.70817565917969,
+ "loss": 0.6903837203979493,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.0453563891351223,
+ "rewards/margins": 0.005785389803349972,
+ "rewards/rejected": 0.03957099840044975,
+ "step": 210
+ },
+ {
+ "epoch": 0.352,
+ "grad_norm": 1.05863356590271,
+ "learning_rate": 9.81624184943687e-07,
+ "logits/chosen": 1.7199232578277588,
+ "logits/rejected": 1.6891214847564697,
+ "logps/chosen": -164.67184448242188,
+ "logps/rejected": -143.46157836914062,
+ "loss": 0.6919777870178223,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.0621187686920166,
+ "rewards/margins": 0.002725570695474744,
+ "rewards/rejected": 0.05939319729804993,
+ "step": 220
+ },
+ {
+ "epoch": 0.368,
+ "grad_norm": 0.9537659287452698,
+ "learning_rate": 9.75696502667457e-07,
+ "logits/chosen": 1.5906708240509033,
+ "logits/rejected": 1.5525639057159424,
+ "logps/chosen": -149.23095703125,
+ "logps/rejected": -154.40731811523438,
+ "loss": 0.6942371845245361,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.061252254992723465,
+ "rewards/margins": -0.0017460950184613466,
+ "rewards/rejected": 0.06299833953380585,
+ "step": 230
+ },
+ {
+ "epoch": 0.384,
+ "grad_norm": 0.9180606603622437,
+ "learning_rate": 9.69768820391227e-07,
+ "logits/chosen": 1.5654420852661133,
+ "logits/rejected": 1.6795461177825928,
+ "logps/chosen": -158.65875244140625,
+ "logps/rejected": -165.13951110839844,
+ "loss": 0.692426061630249,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.06773029267787933,
+ "rewards/margins": 0.0019084472442045808,
+ "rewards/rejected": 0.06582184135913849,
+ "step": 240
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.7486060857772827,
+ "learning_rate": 9.63841138114997e-07,
+ "logits/chosen": 1.5377140045166016,
+ "logits/rejected": 1.6011888980865479,
+ "logps/chosen": -138.14199829101562,
+ "logps/rejected": -151.1943817138672,
+ "loss": 0.6908403396606445,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.06411711871623993,
+ "rewards/margins": 0.004912695847451687,
+ "rewards/rejected": 0.05920442193746567,
+ "step": 250
+ },
+ {
+ "epoch": 0.4,
+ "eval_logits/chosen": 1.534494161605835,
+ "eval_logits/rejected": 1.5683013200759888,
+ "eval_logps/chosen": -152.8787078857422,
+ "eval_logps/rejected": -147.39991760253906,
+ "eval_loss": 0.6910172700881958,
+ "eval_rewards/accuracies": 0.5460000038146973,
+ "eval_rewards/chosen": 0.06701447814702988,
+ "eval_rewards/margins": 0.004791789222508669,
+ "eval_rewards/rejected": 0.06222269684076309,
+ "eval_runtime": 91.3518,
+ "eval_samples_per_second": 5.473,
+ "eval_steps_per_second": 2.737,
+ "step": 250
+ },
+ {
+ "epoch": 0.416,
+ "grad_norm": 1.0861977338790894,
+ "learning_rate": 9.579134558387671e-07,
+ "logits/chosen": 1.5887064933776855,
+ "logits/rejected": 1.6124156713485718,
+ "logps/chosen": -145.68692016601562,
+ "logps/rejected": -150.8065948486328,
+ "loss": 0.6914952278137207,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.07477976381778717,
+ "rewards/margins": 0.00367984501644969,
+ "rewards/rejected": 0.0710999146103859,
+ "step": 260
+ },
+ {
+ "epoch": 0.432,
+ "grad_norm": 1.2531821727752686,
+ "learning_rate": 9.519857735625369e-07,
+ "logits/chosen": 1.552918553352356,
+ "logits/rejected": 1.6837631464004517,
+ "logps/chosen": -158.31802368164062,
+ "logps/rejected": -151.29379272460938,
+ "loss": 0.691353178024292,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.07334339618682861,
+ "rewards/margins": 0.004158434458076954,
+ "rewards/rejected": 0.06918496638536453,
+ "step": 270
+ },
+ {
+ "epoch": 0.448,
+ "grad_norm": 0.8373661637306213,
+ "learning_rate": 9.460580912863069e-07,
+ "logits/chosen": 1.6604083776474,
+ "logits/rejected": 1.7040210962295532,
+ "logps/chosen": -132.56422424316406,
+ "logps/rejected": -151.414794921875,
+ "loss": 0.6947796821594239,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.07179545611143112,
+ "rewards/margins": -0.002721407450735569,
+ "rewards/rejected": 0.07451687008142471,
+ "step": 280
+ },
+ {
+ "epoch": 0.464,
+ "grad_norm": 1.0909446477890015,
+ "learning_rate": 9.40130409010077e-07,
+ "logits/chosen": 1.6112158298492432,
+ "logits/rejected": 1.6040490865707397,
+ "logps/chosen": -155.4887237548828,
+ "logps/rejected": -149.31924438476562,
+ "loss": 0.6928027629852295,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.07096447050571442,
+ "rewards/margins": 0.0013116076588630676,
+ "rewards/rejected": 0.06965287029743195,
+ "step": 290
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 0.8769014477729797,
+ "learning_rate": 9.34202726733847e-07,
+ "logits/chosen": 1.4986810684204102,
+ "logits/rejected": 1.524269700050354,
+ "logps/chosen": -138.91099548339844,
+ "logps/rejected": -141.48495483398438,
+ "loss": 0.6924718379974365,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.060377366840839386,
+ "rewards/margins": 0.0016540264477953315,
+ "rewards/rejected": 0.058723341673612595,
+ "step": 300
+ },
+ {
+ "epoch": 0.48,
+ "eval_logits/chosen": 1.5390068292617798,
+ "eval_logits/rejected": 1.573019027709961,
+ "eval_logps/chosen": -152.91844177246094,
+ "eval_logps/rejected": -147.44703674316406,
+ "eval_loss": 0.6906248331069946,
+ "eval_rewards/accuracies": 0.5680000185966492,
+ "eval_rewards/chosen": 0.06304025650024414,
+ "eval_rewards/margins": 0.005528101697564125,
+ "eval_rewards/rejected": 0.057512152940034866,
+ "eval_runtime": 91.3495,
+ "eval_samples_per_second": 5.473,
+ "eval_steps_per_second": 2.737,
+ "step": 300
+ },
+ {
+ "epoch": 0.496,
+ "grad_norm": 1.050471305847168,
+ "learning_rate": 9.28275044457617e-07,
+ "logits/chosen": 1.7194430828094482,
+ "logits/rejected": 1.807745337486267,
+ "logps/chosen": -160.05563354492188,
+ "logps/rejected": -138.70767211914062,
+ "loss": 0.6887433052062988,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0673876479268074,
+ "rewards/margins": 0.009202037006616592,
+ "rewards/rejected": 0.058185599744319916,
+ "step": 310
+ },
+ {
+ "epoch": 0.512,
+ "grad_norm": 1.1666924953460693,
+ "learning_rate": 9.223473621813871e-07,
+ "logits/chosen": 1.737310767173767,
+ "logits/rejected": 1.6316074132919312,
+ "logps/chosen": -151.58229064941406,
+ "logps/rejected": -142.8289794921875,
+ "loss": 0.6919591426849365,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.06055358797311783,
+ "rewards/margins": 0.0026850434951484203,
+ "rewards/rejected": 0.057868532836437225,
+ "step": 320
+ },
+ {
+ "epoch": 0.528,
+ "grad_norm": 1.2360730171203613,
+ "learning_rate": 9.16419679905157e-07,
+ "logits/chosen": 1.4524610042572021,
+ "logits/rejected": 1.490668535232544,
+ "logps/chosen": -136.93104553222656,
+ "logps/rejected": -155.99844360351562,
+ "loss": 0.6895017623901367,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.06680475920438766,
+ "rewards/margins": 0.007821018807590008,
+ "rewards/rejected": 0.05898374319076538,
+ "step": 330
+ },
+ {
+ "epoch": 0.544,
+ "grad_norm": 1.0067647695541382,
+ "learning_rate": 9.10491997628927e-07,
+ "logits/chosen": 1.5482432842254639,
+ "logits/rejected": 1.596333622932434,
+ "logps/chosen": -142.59603881835938,
+ "logps/rejected": -120.35069274902344,
+ "loss": 0.6895721912384033,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.0686279758810997,
+ "rewards/margins": 0.007503635250031948,
+ "rewards/rejected": 0.06112435460090637,
+ "step": 340
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 1.2567517757415771,
+ "learning_rate": 9.04564315352697e-07,
+ "logits/chosen": 1.5976614952087402,
+ "logits/rejected": 1.3774999380111694,
+ "logps/chosen": -137.83567810058594,
+ "logps/rejected": -134.2790985107422,
+ "loss": 0.6924222946166992,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06484080851078033,
+ "rewards/margins": 0.0020756719168275595,
+ "rewards/rejected": 0.06276513636112213,
+ "step": 350
+ },
+ {
+ "epoch": 0.56,
+ "eval_logits/chosen": 1.5381687879562378,
+ "eval_logits/rejected": 1.5718625783920288,
+ "eval_logps/chosen": -152.91278076171875,
+ "eval_logps/rejected": -147.4539794921875,
+ "eval_loss": 0.6900433301925659,
+ "eval_rewards/accuracies": 0.5559999942779541,
+ "eval_rewards/chosen": 0.06360553950071335,
+ "eval_rewards/margins": 0.006787541322410107,
+ "eval_rewards/rejected": 0.05681798979640007,
+ "eval_runtime": 91.4768,
+ "eval_samples_per_second": 5.466,
+ "eval_steps_per_second": 2.733,
+ "step": 350
+ },
+ {
+ "epoch": 0.576,
+ "grad_norm": 1.3803397417068481,
+ "learning_rate": 8.986366330764671e-07,
+ "logits/chosen": 1.6290550231933594,
+ "logits/rejected": 1.7498657703399658,
+ "logps/chosen": -163.21676635742188,
+ "logps/rejected": -170.33084106445312,
+ "loss": 0.6899758338928222,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06653806567192078,
+ "rewards/margins": 0.006778226234018803,
+ "rewards/rejected": 0.05975983664393425,
+ "step": 360
+ },
+ {
+ "epoch": 0.592,
+ "grad_norm": 1.1363084316253662,
+ "learning_rate": 8.927089508002371e-07,
+ "logits/chosen": 1.6613603830337524,
+ "logits/rejected": 1.7096723318099976,
+ "logps/chosen": -175.3104248046875,
+ "logps/rejected": -176.24655151367188,
+ "loss": 0.690467643737793,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.061445217579603195,
+ "rewards/margins": 0.005937105976045132,
+ "rewards/rejected": 0.05550810694694519,
+ "step": 370
+ },
+ {
+ "epoch": 0.608,
+ "grad_norm": 1.139722228050232,
+ "learning_rate": 8.86781268524007e-07,
+ "logits/chosen": 1.6426604986190796,
+ "logits/rejected": 1.7499873638153076,
+ "logps/chosen": -136.9043731689453,
+ "logps/rejected": -159.98196411132812,
+ "loss": 0.6899127960205078,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.05270732566714287,
+ "rewards/margins": 0.007067640777677298,
+ "rewards/rejected": 0.04563968628644943,
+ "step": 380
+ },
+ {
+ "epoch": 0.624,
+ "grad_norm": 1.2813146114349365,
+ "learning_rate": 8.80853586247777e-07,
+ "logits/chosen": 1.3995320796966553,
+ "logits/rejected": 1.6828539371490479,
+ "logps/chosen": -131.1273651123047,
+ "logps/rejected": -167.51840209960938,
+ "loss": 0.6943269729614258,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.05107322335243225,
+ "rewards/margins": -0.0017629960784688592,
+ "rewards/rejected": 0.05283621698617935,
+ "step": 390
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 1.1288373470306396,
+ "learning_rate": 8.749259039715471e-07,
+ "logits/chosen": 1.6386661529541016,
+ "logits/rejected": 1.6272175312042236,
+ "logps/chosen": -150.4084014892578,
+ "logps/rejected": -161.52236938476562,
+ "loss": 0.6923639297485351,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.039307981729507446,
+ "rewards/margins": 0.001983280060812831,
+ "rewards/rejected": 0.037324704229831696,
+ "step": 400
+ },
+ {
+ "epoch": 0.64,
+ "eval_logits/chosen": 1.519494652748108,
+ "eval_logits/rejected": 1.5528141260147095,
+ "eval_logps/chosen": -153.15609741210938,
+ "eval_logps/rejected": -147.69198608398438,
+ "eval_loss": 0.6903249025344849,
+ "eval_rewards/accuracies": 0.5460000038146973,
+ "eval_rewards/chosen": 0.03927500545978546,
+ "eval_rewards/margins": 0.006258465815335512,
+ "eval_rewards/rejected": 0.033016543835401535,
+ "eval_runtime": 91.4257,
+ "eval_samples_per_second": 5.469,
+ "eval_steps_per_second": 2.734,
+ "step": 400
+ },
+ {
+ "epoch": 0.656,
+ "grad_norm": 1.355100154876709,
+ "learning_rate": 8.689982216953171e-07,
+ "logits/chosen": 1.5310468673706055,
+ "logits/rejected": 1.6076139211654663,
+ "logps/chosen": -157.39776611328125,
+ "logps/rejected": -145.91275024414062,
+ "loss": 0.68691987991333,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.04244391992688179,
+ "rewards/margins": 0.01302963774651289,
+ "rewards/rejected": 0.029414281249046326,
+ "step": 410
+ },
+ {
+ "epoch": 0.672,
+ "grad_norm": 1.1569719314575195,
+ "learning_rate": 8.630705394190871e-07,
+ "logits/chosen": 1.5669739246368408,
+ "logits/rejected": 1.5956265926361084,
+ "logps/chosen": -132.22671508789062,
+ "logps/rejected": -132.46981811523438,
+ "loss": 0.6866061210632324,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.04782433062791824,
+ "rewards/margins": 0.013592484407126904,
+ "rewards/rejected": 0.034231849014759064,
+ "step": 420
+ },
+ {
+ "epoch": 0.688,
+ "grad_norm": 1.1847658157348633,
+ "learning_rate": 8.57142857142857e-07,
+ "logits/chosen": 1.6808805465698242,
+ "logits/rejected": 1.7047704458236694,
+ "logps/chosen": -140.24200439453125,
+ "logps/rejected": -140.81124877929688,
+ "loss": 0.6933694362640381,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.047805462032556534,
+ "rewards/margins": 0.00025355032994411886,
+ "rewards/rejected": 0.04755191132426262,
+ "step": 430
+ },
+ {
+ "epoch": 0.704,
+ "grad_norm": 0.9608289003372192,
+ "learning_rate": 8.512151748666271e-07,
+ "logits/chosen": 1.663816213607788,
+ "logits/rejected": 1.6789140701293945,
+ "logps/chosen": -155.818603515625,
+ "logps/rejected": -145.31468200683594,
+ "loss": 0.6929332733154296,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.042977917939424515,
+ "rewards/margins": 0.0008615514962002635,
+ "rewards/rejected": 0.04211636632680893,
+ "step": 440
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 1.3275092840194702,
+ "learning_rate": 8.452874925903971e-07,
+ "logits/chosen": 1.4646484851837158,
+ "logits/rejected": 1.6175050735473633,
+ "logps/chosen": -151.04867553710938,
+ "logps/rejected": -147.8173828125,
+ "loss": 0.6903160572052002,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.03235078603029251,
+ "rewards/margins": 0.00654552411288023,
+ "rewards/rejected": 0.025805264711380005,
+ "step": 450
+ },
+ {
+ "epoch": 0.72,
+ "eval_logits/chosen": 1.5208625793457031,
+ "eval_logits/rejected": 1.5539581775665283,
+ "eval_logps/chosen": -153.08497619628906,
+ "eval_logps/rejected": -147.63291931152344,
+ "eval_loss": 0.6898273229598999,
+ "eval_rewards/accuracies": 0.5680000185966492,
+ "eval_rewards/chosen": 0.046387214213609695,
+ "eval_rewards/margins": 0.0074648563750088215,
+ "eval_rewards/rejected": 0.038922350853681564,
+ "eval_runtime": 91.384,
+ "eval_samples_per_second": 5.471,
+ "eval_steps_per_second": 2.736,
+ "step": 450
+ },
+ {
+ "epoch": 0.736,
+ "grad_norm": 1.3754528760910034,
+ "learning_rate": 8.393598103141671e-07,
+ "logits/chosen": 1.5359246730804443,
+ "logits/rejected": 1.604922890663147,
+ "logps/chosen": -137.5425262451172,
+ "logps/rejected": -161.14151000976562,
+ "loss": 0.6894284725189209,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.03450053930282593,
+ "rewards/margins": 0.007881317287683487,
+ "rewards/rejected": 0.02661922201514244,
+ "step": 460
+ },
+ {
+ "epoch": 0.752,
+ "grad_norm": 1.045366883277893,
+ "learning_rate": 8.334321280379371e-07,
+ "logits/chosen": 1.4338958263397217,
+ "logits/rejected": 1.5732755661010742,
+ "logps/chosen": -142.7367401123047,
+ "logps/rejected": -129.0875244140625,
+ "loss": 0.687935209274292,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03967561572790146,
+ "rewards/margins": 0.01102046575397253,
+ "rewards/rejected": 0.028655150905251503,
+ "step": 470
+ },
+ {
+ "epoch": 0.768,
+ "grad_norm": 1.232444167137146,
+ "learning_rate": 8.275044457617071e-07,
+ "logits/chosen": 1.7112070322036743,
+ "logits/rejected": 1.8045260906219482,
+ "logps/chosen": -170.90280151367188,
+ "logps/rejected": -164.41964721679688,
+ "loss": 0.6896602630615234,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.04624806344509125,
+ "rewards/margins": 0.0076986560598015785,
+ "rewards/rejected": 0.038549408316612244,
+ "step": 480
+ },
+ {
+ "epoch": 0.784,
+ "grad_norm": 1.2316750288009644,
+ "learning_rate": 8.215767634854771e-07,
+ "logits/chosen": 1.660528540611267,
+ "logits/rejected": 1.5905355215072632,
+ "logps/chosen": -166.0521697998047,
+ "logps/rejected": -164.01641845703125,
+ "loss": 0.6967196464538574,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": 0.022944677621126175,
+ "rewards/margins": -0.006097245961427689,
+ "rewards/rejected": 0.029041921719908714,
+ "step": 490
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 1.0982184410095215,
+ "learning_rate": 8.156490812092471e-07,
+ "logits/chosen": 1.5519205331802368,
+ "logits/rejected": 1.6819387674331665,
+ "logps/chosen": -137.09713745117188,
+ "logps/rejected": -139.62698364257812,
+ "loss": 0.6872400283813477,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.027622828260064125,
+ "rewards/margins": 0.012441830709576607,
+ "rewards/rejected": 0.015180995687842369,
+ "step": 500
+ },
+ {
+ "epoch": 0.8,
+ "eval_logits/chosen": 1.513393521308899,
+ "eval_logits/rejected": 1.5464614629745483,
+ "eval_logps/chosen": -153.2435302734375,
+ "eval_logps/rejected": -147.79180908203125,
+ "eval_loss": 0.6898292303085327,
+ "eval_rewards/accuracies": 0.5559999942779541,
+ "eval_rewards/chosen": 0.030531544238328934,
+ "eval_rewards/margins": 0.0074988617561757565,
+ "eval_rewards/rejected": 0.02303268015384674,
+ "eval_runtime": 91.4769,
+ "eval_samples_per_second": 5.466,
+ "eval_steps_per_second": 2.733,
+ "step": 500
+ },
+ {
+ "epoch": 0.816,
+ "grad_norm": 0.9861111640930176,
+ "learning_rate": 8.097213989330171e-07,
+ "logits/chosen": 1.7147877216339111,
+ "logits/rejected": 1.6248664855957031,
+ "logps/chosen": -166.3791961669922,
+ "logps/rejected": -152.08753967285156,
+ "loss": 0.6864038944244385,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.0399431511759758,
+ "rewards/margins": 0.014103055000305176,
+ "rewards/rejected": 0.025840098038315773,
+ "step": 510
+ },
+ {
+ "epoch": 0.832,
+ "grad_norm": 1.118251919746399,
+ "learning_rate": 8.037937166567872e-07,
+ "logits/chosen": 1.6375319957733154,
+ "logits/rejected": 1.6414417028427124,
+ "logps/chosen": -150.5874786376953,
+ "logps/rejected": -179.5028076171875,
+ "loss": 0.6879529476165771,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.03941132128238678,
+ "rewards/margins": 0.011486245319247246,
+ "rewards/rejected": 0.027925079688429832,
+ "step": 520
+ },
+ {
+ "epoch": 0.848,
+ "grad_norm": 1.354157567024231,
+ "learning_rate": 7.978660343805572e-07,
+ "logits/chosen": 1.5268175601959229,
+ "logits/rejected": 1.5682529211044312,
+ "logps/chosen": -144.65402221679688,
+ "logps/rejected": -136.36709594726562,
+ "loss": 0.6815488815307618,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.03592699021100998,
+ "rewards/margins": 0.02444549836218357,
+ "rewards/rejected": 0.011481491848826408,
+ "step": 530
+ },
+ {
+ "epoch": 0.864,
+ "grad_norm": 1.0916801691055298,
+ "learning_rate": 7.919383521043271e-07,
+ "logits/chosen": 1.4136483669281006,
+ "logits/rejected": 1.5612545013427734,
+ "logps/chosen": -128.98541259765625,
+ "logps/rejected": -149.62124633789062,
+ "loss": 0.6931351661682129,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.014613496139645576,
+ "rewards/margins": 0.0007547929999418557,
+ "rewards/rejected": 0.013858701102435589,
+ "step": 540
+ },
+ {
+ "epoch": 0.88,
+ "grad_norm": 1.2422736883163452,
+ "learning_rate": 7.860106698280971e-07,
+ "logits/chosen": 1.6898448467254639,
+ "logits/rejected": 1.7594168186187744,
+ "logps/chosen": -175.46859741210938,
+ "logps/rejected": -185.50271606445312,
+ "loss": 0.6864456176757813,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.030645186081528664,
+ "rewards/margins": 0.01440943218767643,
+ "rewards/rejected": 0.016235752031207085,
+ "step": 550
+ },
+ {
+ "epoch": 0.88,
+ "eval_logits/chosen": 1.5022823810577393,
+ "eval_logits/rejected": 1.5351035594940186,
+ "eval_logps/chosen": -153.427490234375,
+ "eval_logps/rejected": -147.97894287109375,
+ "eval_loss": 0.6898054480552673,
+ "eval_rewards/accuracies": 0.5580000281333923,
+ "eval_rewards/chosen": 0.012135841883718967,
+ "eval_rewards/margins": 0.007815876975655556,
+ "eval_rewards/rejected": 0.004319964908063412,
+ "eval_runtime": 91.5476,
+ "eval_samples_per_second": 5.462,
+ "eval_steps_per_second": 2.731,
+ "step": 550
+ },
+ {
+ "epoch": 0.896,
+ "grad_norm": 1.1276408433914185,
+ "learning_rate": 7.800829875518672e-07,
+ "logits/chosen": 1.466566801071167,
+ "logits/rejected": 1.3821640014648438,
+ "logps/chosen": -145.90614318847656,
+ "logps/rejected": -152.33578491210938,
+ "loss": 0.6930467128753662,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.01578265056014061,
+ "rewards/margins": 0.001082198810763657,
+ "rewards/rejected": 0.014700451865792274,
+ "step": 560
+ },
+ {
+ "epoch": 0.912,
+ "grad_norm": 1.0809053182601929,
+ "learning_rate": 7.741553052756372e-07,
+ "logits/chosen": 1.6201200485229492,
+ "logits/rejected": 1.6171897649765015,
+ "logps/chosen": -150.1614990234375,
+ "logps/rejected": -137.40005493164062,
+ "loss": 0.6915182590484619,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.015585039742290974,
+ "rewards/margins": 0.003875966649502516,
+ "rewards/rejected": 0.01170907262712717,
+ "step": 570
+ },
+ {
+ "epoch": 0.928,
+ "grad_norm": 1.3044863939285278,
+ "learning_rate": 7.682276229994072e-07,
+ "logits/chosen": 1.688551902770996,
+ "logits/rejected": 1.6596629619598389,
+ "logps/chosen": -168.21104431152344,
+ "logps/rejected": -137.5209197998047,
+ "loss": 0.6888096332550049,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.00975596159696579,
+ "rewards/margins": 0.009401545859873295,
+ "rewards/rejected": 0.00035441629006527364,
+ "step": 580
+ },
+ {
+ "epoch": 0.944,
+ "grad_norm": 1.3138145208358765,
+ "learning_rate": 7.622999407231771e-07,
+ "logits/chosen": 1.5617314577102661,
+ "logits/rejected": 1.4952644109725952,
+ "logps/chosen": -177.6552734375,
+ "logps/rejected": -156.41143798828125,
+ "loss": 0.6811863422393799,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.023103689774870872,
+ "rewards/margins": 0.02511446550488472,
+ "rewards/rejected": -0.0020107750315219164,
+ "step": 590
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 1.284184217453003,
+ "learning_rate": 7.563722584469472e-07,
+ "logits/chosen": 1.5531961917877197,
+ "logits/rejected": 1.5904725790023804,
+ "logps/chosen": -163.69615173339844,
+ "logps/rejected": -155.56857299804688,
+ "loss": 0.6912278652191162,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.021354522556066513,
+ "rewards/margins": 0.005431803874671459,
+ "rewards/rejected": 0.01592271961271763,
+ "step": 600
+ },
+ {
+ "epoch": 0.96,
+ "eval_logits/chosen": 1.5025317668914795,
+ "eval_logits/rejected": 1.5352839231491089,
+ "eval_logps/chosen": -153.3654022216797,
+ "eval_logps/rejected": -147.92263793945312,
+ "eval_loss": 0.6896048784255981,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": 0.018345065414905548,
+ "eval_rewards/margins": 0.008394664153456688,
+ "eval_rewards/rejected": 0.009950404055416584,
+ "eval_runtime": 91.5004,
+ "eval_samples_per_second": 5.464,
+ "eval_steps_per_second": 2.732,
+ "step": 600
+ },
+ {
+ "epoch": 0.976,
+ "grad_norm": 1.136492371559143,
+ "learning_rate": 7.504445761707172e-07,
+ "logits/chosen": 1.5187523365020752,
+ "logits/rejected": 1.5115095376968384,
+ "logps/chosen": -123.2374267578125,
+ "logps/rejected": -121.64241790771484,
+ "loss": 0.6866262912750244,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.04122164472937584,
+ "rewards/margins": 0.013987530954182148,
+ "rewards/rejected": 0.027234116569161415,
+ "step": 610
+ },
+ {
+ "epoch": 0.992,
+ "grad_norm": 1.3648184537887573,
+ "learning_rate": 7.445168938944872e-07,
+ "logits/chosen": 1.2267796993255615,
+ "logits/rejected": 1.318672776222229,
+ "logps/chosen": -125.6270751953125,
+ "logps/rejected": -139.3863067626953,
+ "loss": 0.6841369152069092,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.02149296924471855,
+ "rewards/margins": 0.02048521675169468,
+ "rewards/rejected": 0.001007752143777907,
+ "step": 620
+ },
+ {
+ "epoch": 1.008,
+ "grad_norm": 0.8776348233222961,
+ "learning_rate": 7.385892116182572e-07,
+ "logits/chosen": 1.4997355937957764,
+ "logits/rejected": 1.376514196395874,
+ "logps/chosen": -152.75949096679688,
+ "logps/rejected": -142.19186401367188,
+ "loss": 0.6843707084655761,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.03699764981865883,
+ "rewards/margins": 0.01918930374085903,
+ "rewards/rejected": 0.01780834048986435,
+ "step": 630
+ },
+ {
+ "epoch": 1.024,
+ "grad_norm": 1.2698174715042114,
+ "learning_rate": 7.326615293420272e-07,
+ "logits/chosen": 1.4477511644363403,
+ "logits/rejected": 1.5352327823638916,
+ "logps/chosen": -146.83685302734375,
+ "logps/rejected": -151.5980224609375,
+ "loss": 0.6780704021453857,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.04432147741317749,
+ "rewards/margins": 0.031893618404865265,
+ "rewards/rejected": 0.012427863664925098,
+ "step": 640
+ },
+ {
+ "epoch": 1.04,
+ "grad_norm": 1.309323787689209,
+ "learning_rate": 7.267338470657972e-07,
+ "logits/chosen": 1.6456422805786133,
+ "logits/rejected": 1.7331817150115967,
+ "logps/chosen": -148.64895629882812,
+ "logps/rejected": -160.71920776367188,
+ "loss": 0.6779833793640136,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.049473900347948074,
+ "rewards/margins": 0.0323341004550457,
+ "rewards/rejected": 0.017139799892902374,
+ "step": 650
+ },
+ {
+ "epoch": 1.04,
+ "eval_logits/chosen": 1.4995282888412476,
+ "eval_logits/rejected": 1.5318833589553833,
+ "eval_logps/chosen": -153.2689208984375,
+ "eval_logps/rejected": -147.83871459960938,
+ "eval_loss": 0.689191997051239,
+ "eval_rewards/accuracies": 0.5460000038146973,
+ "eval_rewards/chosen": 0.02799258381128311,
+ "eval_rewards/margins": 0.009650195948779583,
+ "eval_rewards/rejected": 0.018342390656471252,
+ "eval_runtime": 91.5151,
+ "eval_samples_per_second": 5.464,
+ "eval_steps_per_second": 2.732,
+ "step": 650
+ },
+ {
+ "epoch": 1.056,
+ "grad_norm": 1.1198511123657227,
+ "learning_rate": 7.208061647895672e-07,
+ "logits/chosen": 1.393791675567627,
+ "logits/rejected": 1.537705659866333,
+ "logps/chosen": -141.07870483398438,
+ "logps/rejected": -151.03375244140625,
+ "loss": 0.6852189064025879,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.028253447264432907,
+ "rewards/margins": 0.01726841926574707,
+ "rewards/rejected": 0.010985029861330986,
+ "step": 660
+ },
+ {
+ "epoch": 1.072,
+ "grad_norm": 1.2956689596176147,
+ "learning_rate": 7.148784825133372e-07,
+ "logits/chosen": 1.5089902877807617,
+ "logits/rejected": 1.5116627216339111,
+ "logps/chosen": -153.54898071289062,
+ "logps/rejected": -154.49032592773438,
+ "loss": 0.6772716045379639,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.04329819604754448,
+ "rewards/margins": 0.03352857381105423,
+ "rewards/rejected": 0.0097696203738451,
+ "step": 670
+ },
+ {
+ "epoch": 1.088,
+ "grad_norm": 1.0869190692901611,
+ "learning_rate": 7.089508002371073e-07,
+ "logits/chosen": 1.5537410974502563,
+ "logits/rejected": 1.4676992893218994,
+ "logps/chosen": -128.73751831054688,
+ "logps/rejected": -131.25894165039062,
+ "loss": 0.6872283458709717,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.02748076617717743,
+ "rewards/margins": 0.013209682889282703,
+ "rewards/rejected": 0.01427108608186245,
+ "step": 680
+ },
+ {
+ "epoch": 1.104,
+ "grad_norm": 1.2243199348449707,
+ "learning_rate": 7.030231179608772e-07,
+ "logits/chosen": 1.6543573141098022,
+ "logits/rejected": 1.7242790460586548,
+ "logps/chosen": -176.51280212402344,
+ "logps/rejected": -174.8518524169922,
+ "loss": 0.675547981262207,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.05593103915452957,
+ "rewards/margins": 0.0371597483754158,
+ "rewards/rejected": 0.01877129077911377,
+ "step": 690
+ },
+ {
+ "epoch": 1.12,
+ "grad_norm": 1.3488147258758545,
+ "learning_rate": 6.970954356846472e-07,
+ "logits/chosen": 1.5460538864135742,
+ "logits/rejected": 1.5656627416610718,
+ "logps/chosen": -146.79302978515625,
+ "logps/rejected": -140.67349243164062,
+ "loss": 0.669365406036377,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.04432321712374687,
+ "rewards/margins": 0.05032513290643692,
+ "rewards/rejected": -0.006001918110996485,
+ "step": 700
+ },
+ {
+ "epoch": 1.12,
+ "eval_logits/chosen": 1.5015324354171753,
+ "eval_logits/rejected": 1.5338712930679321,
+ "eval_logps/chosen": -153.185791015625,
+ "eval_logps/rejected": -147.7559814453125,
+ "eval_loss": 0.6892233490943909,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.03630613535642624,
+ "eval_rewards/margins": 0.009690148755908012,
+ "eval_rewards/rejected": 0.026615988463163376,
+ "eval_runtime": 91.5231,
+ "eval_samples_per_second": 5.463,
+ "eval_steps_per_second": 2.732,
+ "step": 700
+ },
+ {
+ "epoch": 1.1360000000000001,
+ "grad_norm": 0.9973899722099304,
+ "learning_rate": 6.911677534084172e-07,
+ "logits/chosen": 1.6786168813705444,
+ "logits/rejected": 1.6147639751434326,
+ "logps/chosen": -175.9976806640625,
+ "logps/rejected": -145.07785034179688,
+ "loss": 0.6673734664916993,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.0721244290471077,
+ "rewards/margins": 0.054663002490997314,
+ "rewards/rejected": 0.017461419105529785,
+ "step": 710
+ },
+ {
+ "epoch": 1.152,
+ "grad_norm": 1.1944832801818848,
+ "learning_rate": 6.852400711321873e-07,
+ "logits/chosen": 1.452962040901184,
+ "logits/rejected": 1.4640306234359741,
+ "logps/chosen": -139.7812042236328,
+ "logps/rejected": -159.2677001953125,
+ "loss": 0.6787771224975586,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.07180429995059967,
+ "rewards/margins": 0.03057795763015747,
+ "rewards/rejected": 0.0412263348698616,
+ "step": 720
+ },
+ {
+ "epoch": 1.168,
+ "grad_norm": 1.4376368522644043,
+ "learning_rate": 6.793123888559573e-07,
+ "logits/chosen": 1.579025387763977,
+ "logits/rejected": 1.5297110080718994,
+ "logps/chosen": -161.11328125,
+ "logps/rejected": -164.56582641601562,
+ "loss": 0.680543327331543,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.07446161657571793,
+ "rewards/margins": 0.027710452675819397,
+ "rewards/rejected": 0.04675116389989853,
+ "step": 730
+ },
+ {
+ "epoch": 1.184,
+ "grad_norm": 1.1249836683273315,
+ "learning_rate": 6.733847065797273e-07,
+ "logits/chosen": 1.7372865676879883,
+ "logits/rejected": 1.7558908462524414,
+ "logps/chosen": -147.6786346435547,
+ "logps/rejected": -155.9397430419922,
+ "loss": 0.667045783996582,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.08958978950977325,
+ "rewards/margins": 0.05507674068212509,
+ "rewards/rejected": 0.03451304882764816,
+ "step": 740
+ },
+ {
+ "epoch": 1.2,
+ "grad_norm": 1.0353419780731201,
+ "learning_rate": 6.674570243034972e-07,
+ "logits/chosen": 1.6369037628173828,
+ "logits/rejected": 1.6020278930664062,
+ "logps/chosen": -166.5580291748047,
+ "logps/rejected": -144.23370361328125,
+ "loss": 0.6636831760406494,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.06881646066904068,
+ "rewards/margins": 0.06279782950878143,
+ "rewards/rejected": 0.0060186260379850864,
+ "step": 750
+ },
+ {
+ "epoch": 1.2,
+ "eval_logits/chosen": 1.4953525066375732,
+ "eval_logits/rejected": 1.5272784233093262,
+ "eval_logps/chosen": -153.105712890625,
+ "eval_logps/rejected": -147.6992950439453,
+ "eval_loss": 0.6883665919303894,
+ "eval_rewards/accuracies": 0.5460000038146973,
+ "eval_rewards/chosen": 0.044312380254268646,
+ "eval_rewards/margins": 0.012026800774037838,
+ "eval_rewards/rejected": 0.032285578548908234,
+ "eval_runtime": 91.4577,
+ "eval_samples_per_second": 5.467,
+ "eval_steps_per_second": 2.734,
+ "step": 750
+ },
+ {
+ "epoch": 1.216,
+ "grad_norm": 1.1240723133087158,
+ "learning_rate": 6.615293420272673e-07,
+ "logits/chosen": 1.4654918909072876,
+ "logits/rejected": 1.4545985460281372,
+ "logps/chosen": -156.47726440429688,
+ "logps/rejected": -138.5869598388672,
+ "loss": 0.6732029438018798,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.05434625223278999,
+ "rewards/margins": 0.04249870777130127,
+ "rewards/rejected": 0.011847544461488724,
+ "step": 760
+ },
+ {
+ "epoch": 1.232,
+ "grad_norm": 1.1530578136444092,
+ "learning_rate": 6.556016597510373e-07,
+ "logits/chosen": 1.5636401176452637,
+ "logits/rejected": 1.603150725364685,
+ "logps/chosen": -147.6873779296875,
+ "logps/rejected": -127.45448303222656,
+ "loss": 0.6720876216888427,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.07123544067144394,
+ "rewards/margins": 0.04455225542187691,
+ "rewards/rejected": 0.026683185249567032,
+ "step": 770
+ },
+ {
+ "epoch": 1.248,
+ "grad_norm": 1.0530054569244385,
+ "learning_rate": 6.496739774748073e-07,
+ "logits/chosen": 1.672349214553833,
+ "logits/rejected": 1.773258924484253,
+ "logps/chosen": -171.55245971679688,
+ "logps/rejected": -185.20631408691406,
+ "loss": 0.6770391464233398,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.06762111186981201,
+ "rewards/margins": 0.03522135317325592,
+ "rewards/rejected": 0.032399751245975494,
+ "step": 780
+ },
+ {
+ "epoch": 1.264,
+ "grad_norm": 0.9587673544883728,
+ "learning_rate": 6.437462951985773e-07,
+ "logits/chosen": 1.6674445867538452,
+ "logits/rejected": 1.5399541854858398,
+ "logps/chosen": -148.35202026367188,
+ "logps/rejected": -141.05218505859375,
+ "loss": 0.688765001296997,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.059925489127635956,
+ "rewards/margins": 0.011203404515981674,
+ "rewards/rejected": 0.04872208088636398,
+ "step": 790
+ },
+ {
+ "epoch": 1.28,
+ "grad_norm": 1.1872327327728271,
+ "learning_rate": 6.378186129223473e-07,
+ "logits/chosen": 1.4424043893814087,
+ "logits/rejected": 1.5345561504364014,
+ "logps/chosen": -142.57899475097656,
+ "logps/rejected": -138.53045654296875,
+ "loss": 0.6739780902862549,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.07486443221569061,
+ "rewards/margins": 0.04266515001654625,
+ "rewards/rejected": 0.032199278473854065,
+ "step": 800
+ },
+ {
+ "epoch": 1.28,
+ "eval_logits/chosen": 1.502331256866455,
+ "eval_logits/rejected": 1.5342037677764893,
+ "eval_logps/chosen": -152.94044494628906,
+ "eval_logps/rejected": -147.55020141601562,
+ "eval_loss": 0.6877647042274475,
+ "eval_rewards/accuracies": 0.5540000200271606,
+ "eval_rewards/chosen": 0.060839928686618805,
+ "eval_rewards/margins": 0.013645462691783905,
+ "eval_rewards/rejected": 0.0471944697201252,
+ "eval_runtime": 91.4749,
+ "eval_samples_per_second": 5.466,
+ "eval_steps_per_second": 2.733,
+ "step": 800
+ },
+ {
+ "epoch": 1.296,
+ "grad_norm": 1.1386040449142456,
+ "learning_rate": 6.318909306461173e-07,
+ "logits/chosen": 1.4093509912490845,
+ "logits/rejected": 1.4580585956573486,
+ "logps/chosen": -164.720947265625,
+ "logps/rejected": -142.99754333496094,
+ "loss": 0.6797928810119629,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.07539167255163193,
+ "rewards/margins": 0.0292592104524374,
+ "rewards/rejected": 0.046132463961839676,
+ "step": 810
+ },
+ {
+ "epoch": 1.312,
+ "grad_norm": 1.3657523393630981,
+ "learning_rate": 6.259632483698873e-07,
+ "logits/chosen": 1.3146798610687256,
+ "logits/rejected": 1.4988536834716797,
+ "logps/chosen": -141.5091552734375,
+ "logps/rejected": -138.624755859375,
+ "loss": 0.6713742733001709,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.06485269963741302,
+ "rewards/margins": 0.04707767441868782,
+ "rewards/rejected": 0.017775025218725204,
+ "step": 820
+ },
+ {
+ "epoch": 1.328,
+ "grad_norm": 1.207746148109436,
+ "learning_rate": 6.200355660936573e-07,
+ "logits/chosen": 1.5304832458496094,
+ "logits/rejected": 1.567846655845642,
+ "logps/chosen": -145.65390014648438,
+ "logps/rejected": -134.02752685546875,
+ "loss": 0.6716270923614502,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08788949996232986,
+ "rewards/margins": 0.04611582309007645,
+ "rewards/rejected": 0.041773684322834015,
+ "step": 830
+ },
+ {
+ "epoch": 1.3439999999999999,
+ "grad_norm": 1.3036965131759644,
+ "learning_rate": 6.141078838174274e-07,
+ "logits/chosen": 1.5840891599655151,
+ "logits/rejected": 1.500603199005127,
+ "logps/chosen": -138.60763549804688,
+ "logps/rejected": -125.70695495605469,
+ "loss": 0.6759689331054688,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10840122401714325,
+ "rewards/margins": 0.037392571568489075,
+ "rewards/rejected": 0.07100865244865417,
+ "step": 840
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "grad_norm": 1.1700375080108643,
+ "learning_rate": 6.081802015411973e-07,
+ "logits/chosen": 1.5052621364593506,
+ "logits/rejected": 1.5524170398712158,
+ "logps/chosen": -133.3436279296875,
+ "logps/rejected": -137.5284423828125,
+ "loss": 0.6731984615325928,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.09424515068531036,
+ "rewards/margins": 0.04275429993867874,
+ "rewards/rejected": 0.051490843296051025,
+ "step": 850
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "eval_logits/chosen": 1.514055609703064,
+ "eval_logits/rejected": 1.545797348022461,
+ "eval_logps/chosen": -152.71902465820312,
+ "eval_logps/rejected": -147.34742736816406,
+ "eval_loss": 0.6870437264442444,
+ "eval_rewards/accuracies": 0.5540000200271606,
+ "eval_rewards/chosen": 0.08298191428184509,
+ "eval_rewards/margins": 0.015508526936173439,
+ "eval_rewards/rejected": 0.0674733817577362,
+ "eval_runtime": 91.4891,
+ "eval_samples_per_second": 5.465,
+ "eval_steps_per_second": 2.733,
+ "step": 850
+ },
+ {
+ "epoch": 1.376,
+ "grad_norm": 1.080214023590088,
+ "learning_rate": 6.022525192649673e-07,
+ "logits/chosen": 1.4343178272247314,
+ "logits/rejected": 1.5825798511505127,
+ "logps/chosen": -148.39657592773438,
+ "logps/rejected": -129.2698516845703,
+ "loss": 0.6728739261627197,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.10437977313995361,
+ "rewards/margins": 0.04413595423102379,
+ "rewards/rejected": 0.06024380773305893,
+ "step": 860
+ },
+ {
+ "epoch": 1.392,
+ "grad_norm": 1.3011701107025146,
+ "learning_rate": 5.963248369887373e-07,
+ "logits/chosen": 1.6486666202545166,
+ "logits/rejected": 1.636071801185608,
+ "logps/chosen": -155.72866821289062,
+ "logps/rejected": -156.0375213623047,
+ "loss": 0.6663424491882324,
+ "rewards/accuracies": 0.7875000238418579,
+ "rewards/chosen": 0.0980696976184845,
+ "rewards/margins": 0.05624913424253464,
+ "rewards/rejected": 0.04182056710124016,
+ "step": 870
+ },
+ {
+ "epoch": 1.408,
+ "grad_norm": 1.1527386903762817,
+ "learning_rate": 5.903971547125074e-07,
+ "logits/chosen": 1.6375783681869507,
+ "logits/rejected": 1.5879249572753906,
+ "logps/chosen": -138.84786987304688,
+ "logps/rejected": -149.22756958007812,
+ "loss": 0.6661319255828857,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.1091323122382164,
+ "rewards/margins": 0.05697597935795784,
+ "rewards/rejected": 0.05215633660554886,
+ "step": 880
+ },
+ {
+ "epoch": 1.424,
+ "grad_norm": 1.2128634452819824,
+ "learning_rate": 5.844694724362774e-07,
+ "logits/chosen": 1.643206238746643,
+ "logits/rejected": 1.529200792312622,
+ "logps/chosen": -156.31326293945312,
+ "logps/rejected": -144.78689575195312,
+ "loss": 0.6645416736602783,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.11794288456439972,
+ "rewards/margins": 0.06102023646235466,
+ "rewards/rejected": 0.05692264437675476,
+ "step": 890
+ },
+ {
+ "epoch": 1.44,
+ "grad_norm": 0.9876027703285217,
+ "learning_rate": 5.785417901600474e-07,
+ "logits/chosen": 1.3582518100738525,
+ "logits/rejected": 1.5023400783538818,
+ "logps/chosen": -137.19717407226562,
+ "logps/rejected": -167.57350158691406,
+ "loss": 0.6799286842346192,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.09652381390333176,
+ "rewards/margins": 0.02946191094815731,
+ "rewards/rejected": 0.0670619010925293,
+ "step": 900
+ },
+ {
+ "epoch": 1.44,
+ "eval_logits/chosen": 1.5112248659133911,
+ "eval_logits/rejected": 1.542572021484375,
+ "eval_logps/chosen": -152.71963500976562,
+ "eval_logps/rejected": -147.3575897216797,
+ "eval_loss": 0.6868481636047363,
+ "eval_rewards/accuracies": 0.5720000267028809,
+ "eval_rewards/chosen": 0.08292035758495331,
+ "eval_rewards/margins": 0.016464442014694214,
+ "eval_rewards/rejected": 0.0664559081196785,
+ "eval_runtime": 91.463,
+ "eval_samples_per_second": 5.467,
+ "eval_steps_per_second": 2.733,
+ "step": 900
+ },
+ {
+ "epoch": 1.456,
+ "grad_norm": 1.0684723854064941,
+ "learning_rate": 5.726141078838173e-07,
+ "logits/chosen": 1.5465466976165771,
+ "logits/rejected": 1.500496506690979,
+ "logps/chosen": -132.5164337158203,
+ "logps/rejected": -129.40220642089844,
+ "loss": 0.6799171924591064,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08448641002178192,
+ "rewards/margins": 0.028616121038794518,
+ "rewards/rejected": 0.05587029457092285,
+ "step": 910
+ },
+ {
+ "epoch": 1.472,
+ "grad_norm": 1.0342527627944946,
+ "learning_rate": 5.666864256075874e-07,
+ "logits/chosen": 1.6495583057403564,
+ "logits/rejected": 1.7119420766830444,
+ "logps/chosen": -139.18484497070312,
+ "logps/rejected": -154.33407592773438,
+ "loss": 0.6807973861694336,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.09809015691280365,
+ "rewards/margins": 0.027513395994901657,
+ "rewards/rejected": 0.0705767497420311,
+ "step": 920
+ },
+ {
+ "epoch": 1.488,
+ "grad_norm": 1.0507510900497437,
+ "learning_rate": 5.607587433313574e-07,
+ "logits/chosen": 1.5217338800430298,
+ "logits/rejected": 1.5648618936538696,
+ "logps/chosen": -131.5649871826172,
+ "logps/rejected": -122.32954406738281,
+ "loss": 0.681494951248169,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.08206742256879807,
+ "rewards/margins": 0.026795843616127968,
+ "rewards/rejected": 0.055271584540605545,
+ "step": 930
+ },
+ {
+ "epoch": 1.504,
+ "grad_norm": 1.3102085590362549,
+ "learning_rate": 5.548310610551274e-07,
+ "logits/chosen": 1.6052824258804321,
+ "logits/rejected": 1.6236565113067627,
+ "logps/chosen": -149.2860565185547,
+ "logps/rejected": -153.01657104492188,
+ "loss": 0.6783152103424073,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.10417164862155914,
+ "rewards/margins": 0.03314245864748955,
+ "rewards/rejected": 0.0710291862487793,
+ "step": 940
+ },
+ {
+ "epoch": 1.52,
+ "grad_norm": 1.3876780271530151,
+ "learning_rate": 5.489033787788974e-07,
+ "logits/chosen": 1.689612627029419,
+ "logits/rejected": 1.6765525341033936,
+ "logps/chosen": -164.98193359375,
+ "logps/rejected": -164.33975219726562,
+ "loss": 0.6782557964324951,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.09384694695472717,
+ "rewards/margins": 0.03179098665714264,
+ "rewards/rejected": 0.062055956572294235,
+ "step": 950
+ },
+ {
+ "epoch": 1.52,
+ "eval_logits/chosen": 1.5106096267700195,
+ "eval_logits/rejected": 1.5420260429382324,
+ "eval_logps/chosen": -152.7333526611328,
+ "eval_logps/rejected": -147.37344360351562,
+ "eval_loss": 0.6867280602455139,
+ "eval_rewards/accuracies": 0.5540000200271606,
+ "eval_rewards/chosen": 0.08154840767383575,
+ "eval_rewards/margins": 0.01667776331305504,
+ "eval_rewards/rejected": 0.06487063318490982,
+ "eval_runtime": 91.5472,
+ "eval_samples_per_second": 5.462,
+ "eval_steps_per_second": 2.731,
+ "step": 950
+ },
+ {
+ "epoch": 1.536,
+ "grad_norm": 1.0544084310531616,
+ "learning_rate": 5.429756965026674e-07,
+ "logits/chosen": 1.696502685546875,
+ "logits/rejected": 1.722957968711853,
+ "logps/chosen": -177.16172790527344,
+ "logps/rejected": -160.9434051513672,
+ "loss": 0.6731346130371094,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.09499023109674454,
+ "rewards/margins": 0.04484598711133003,
+ "rewards/rejected": 0.050144243985414505,
+ "step": 960
+ },
+ {
+ "epoch": 1.552,
+ "grad_norm": 1.2123764753341675,
+ "learning_rate": 5.370480142264374e-07,
+ "logits/chosen": 1.5045363903045654,
+ "logits/rejected": 1.4909541606903076,
+ "logps/chosen": -144.6968994140625,
+ "logps/rejected": -160.55947875976562,
+ "loss": 0.6804170131683349,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.08150655776262283,
+ "rewards/margins": 0.029505986720323563,
+ "rewards/rejected": 0.05200057104229927,
+ "step": 970
+ },
+ {
+ "epoch": 1.568,
+ "grad_norm": 1.0709542036056519,
+ "learning_rate": 5.311203319502074e-07,
+ "logits/chosen": 1.5131323337554932,
+ "logits/rejected": 1.5731028318405151,
+ "logps/chosen": -157.4567108154297,
+ "logps/rejected": -149.07107543945312,
+ "loss": 0.6691867351531983,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.08909507840871811,
+ "rewards/margins": 0.05152255296707153,
+ "rewards/rejected": 0.03757254034280777,
+ "step": 980
+ },
+ {
+ "epoch": 1.584,
+ "grad_norm": 1.329915165901184,
+ "learning_rate": 5.251926496739774e-07,
+ "logits/chosen": 1.574861764907837,
+ "logits/rejected": 1.5829271078109741,
+ "logps/chosen": -143.85403442382812,
+ "logps/rejected": -140.85009765625,
+ "loss": 0.6716786384582519,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08492995798587799,
+ "rewards/margins": 0.04930455982685089,
+ "rewards/rejected": 0.0356253944337368,
+ "step": 990
+ },
+ {
+ "epoch": 1.6,
+ "grad_norm": 1.3974183797836304,
+ "learning_rate": 5.192649673977475e-07,
+ "logits/chosen": 1.5636709928512573,
+ "logits/rejected": 1.3984944820404053,
+ "logps/chosen": -148.8516845703125,
+ "logps/rejected": -143.12136840820312,
+ "loss": 0.676781702041626,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.05405520275235176,
+ "rewards/margins": 0.035432908684015274,
+ "rewards/rejected": 0.018622295930981636,
+ "step": 1000
+ },
+ {
+ "epoch": 1.6,
+ "eval_logits/chosen": 1.5051932334899902,
+ "eval_logits/rejected": 1.5366464853286743,
+ "eval_logps/chosen": -152.83253479003906,
+ "eval_logps/rejected": -147.4718475341797,
+ "eval_loss": 0.6868343353271484,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.07163143903017044,
+ "eval_rewards/margins": 0.016600240021944046,
+ "eval_rewards/rejected": 0.055031195282936096,
+ "eval_runtime": 91.6291,
+ "eval_samples_per_second": 5.457,
+ "eval_steps_per_second": 2.728,
+ "step": 1000
+ }
+ ],
+ "logging_steps": 10,
+ "max_steps": 1875,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1000/training_args.bin b/v3/DPO/DPO_5k/lora/checkpoint-1000/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..544215e69bd09b706d2c495c7bb412d0df7f83e7
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afe50d4ac4c5f2af306fafe28b341897775b898324082e319b56c66893dceabb
+size 6097
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1500/README.md b/v3/DPO/DPO_5k/lora/checkpoint-1500/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..250500f798b4682b2cd2c35cd1fc366677c215dd
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1500/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1500/adapter_config.json b/v3/DPO/DPO_5k/lora/checkpoint-1500/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..695e5e16d4a5436a17790fe2e31fc2a9e1631d03
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1500/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "gate_proj",
+ "k_proj",
+ "q_proj",
+ "down_proj",
+ "o_proj",
+ "v_proj",
+ "up_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1500/adapter_model.safetensors b/v3/DPO/DPO_5k/lora/checkpoint-1500/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7333762224a30176388291210233e42771aac414
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1500/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96d390eb1ea71496192837315af9fa60cfaa62434ae54e929c9b398444e8a4e4
+size 180385008
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1500/chat_template.jinja b/v3/DPO/DPO_5k/lora/checkpoint-1500/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1500/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1500/optimizer.pt b/v3/DPO/DPO_5k/lora/checkpoint-1500/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..280a0bd56d52f22f8e6325016c0ad7065db6b0b0
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1500/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16aaa92908b384d7f0e4860062894429343b12249e7b8eb2b4d5dc10bd834446
+size 360902475
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1500/rng_state.pth b/v3/DPO/DPO_5k/lora/checkpoint-1500/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..33ab1511c4dbfc6e61dcb7f70233792670b01e19
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1500/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:428aed83766f09a6a2d9ccfb86972eeda646225b3b0cd6accec4a90e4f900161
+size 14645
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1500/scaler.pt b/v3/DPO/DPO_5k/lora/checkpoint-1500/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3664bb212147e2acc5cde00bad4860996ad25fe2
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1500/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca372268f4fa9335030c0cb7aedb6cdba75f457da50e7a4034abb1a2d0843689
+size 1383
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1500/scheduler.pt b/v3/DPO/DPO_5k/lora/checkpoint-1500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1d517bedd00f8bf23edf0dfe316b24efd2da8a2e
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6354cbd6537a6e3911df9683caa40922381ffdceba1a5d92c0b866beb24f082
+size 1465
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1500/tokenizer.json b/v3/DPO/DPO_5k/lora/checkpoint-1500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1500/tokenizer_config.json b/v3/DPO/DPO_5k/lora/checkpoint-1500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1500/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1500/trainer_state.json b/v3/DPO/DPO_5k/lora/checkpoint-1500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..4cf6cab7c3926032cc59a75250d90713f4482ab3
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1500/trainer_state.json
@@ -0,0 +1,2764 @@
+{
+ "best_global_step": 950,
+ "best_metric": 0.6867280602455139,
+ "best_model_checkpoint": null,
+ "epoch": 2.4,
+ "eval_steps": 50,
+ "global_step": 1500,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.016,
+ "grad_norm": 1.1517287492752075,
+ "learning_rate": 4.787234042553191e-08,
+ "logits/chosen": 1.6246497631072998,
+ "logits/rejected": 1.5396833419799805,
+ "logps/chosen": -145.91531372070312,
+ "logps/rejected": -151.4051971435547,
+ "loss": 0.693363094329834,
+ "rewards/accuracies": 0.2874999940395355,
+ "rewards/chosen": -3.5572534216044005e-06,
+ "rewards/margins": -0.0004222107236273587,
+ "rewards/rejected": 0.00041865353705361485,
+ "step": 10
+ },
+ {
+ "epoch": 0.032,
+ "grad_norm": 1.3633077144622803,
+ "learning_rate": 1.0106382978723404e-07,
+ "logits/chosen": 1.313926339149475,
+ "logits/rejected": 1.4038684368133545,
+ "logps/chosen": -120.60591888427734,
+ "logps/rejected": -134.05252075195312,
+ "loss": 0.6930661201477051,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 9.576084994478151e-05,
+ "rewards/margins": 0.0001732158416416496,
+ "rewards/rejected": -7.745502807665616e-05,
+ "step": 20
+ },
+ {
+ "epoch": 0.048,
+ "grad_norm": 1.209151029586792,
+ "learning_rate": 1.5425531914893615e-07,
+ "logits/chosen": 1.5228456258773804,
+ "logits/rejected": 1.6869430541992188,
+ "logps/chosen": -162.40280151367188,
+ "logps/rejected": -179.38192749023438,
+ "loss": 0.692751693725586,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0007011269335635006,
+ "rewards/margins": 0.0008053636993281543,
+ "rewards/rejected": -0.00010423659114167094,
+ "step": 30
+ },
+ {
+ "epoch": 0.064,
+ "grad_norm": 1.3589205741882324,
+ "learning_rate": 2.074468085106383e-07,
+ "logits/chosen": 1.544001817703247,
+ "logits/rejected": 1.4715760946273804,
+ "logps/chosen": -152.7266845703125,
+ "logps/rejected": -150.9158477783203,
+ "loss": 0.6925192832946777,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.00026422025985084474,
+ "rewards/margins": 0.0012672842713072896,
+ "rewards/rejected": -0.0010030639823526144,
+ "step": 40
+ },
+ {
+ "epoch": 0.08,
+ "grad_norm": 1.202810525894165,
+ "learning_rate": 2.6063829787234044e-07,
+ "logits/chosen": 1.6794925928115845,
+ "logits/rejected": 1.7414724826812744,
+ "logps/chosen": -169.71751403808594,
+ "logps/rejected": -156.7571258544922,
+ "loss": 0.6930821895599365,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 9.479046275373548e-05,
+ "rewards/margins": 0.00014605997421313077,
+ "rewards/rejected": -5.126944597577676e-05,
+ "step": 50
+ },
+ {
+ "epoch": 0.08,
+ "eval_logits/chosen": 1.5141410827636719,
+ "eval_logits/rejected": 1.5485105514526367,
+ "eval_logps/chosen": -153.53335571289062,
+ "eval_logps/rejected": -148.00894165039062,
+ "eval_loss": 0.6930396556854248,
+ "eval_rewards/accuracies": 0.5120000243186951,
+ "eval_rewards/chosen": 0.0015483711613342166,
+ "eval_rewards/margins": 0.00022836039715912193,
+ "eval_rewards/rejected": 0.0013200106332078576,
+ "eval_runtime": 91.1536,
+ "eval_samples_per_second": 5.485,
+ "eval_steps_per_second": 2.743,
+ "step": 50
+ },
+ {
+ "epoch": 0.096,
+ "grad_norm": 1.2080731391906738,
+ "learning_rate": 3.1382978723404253e-07,
+ "logits/chosen": 1.796142339706421,
+ "logits/rejected": 1.7786738872528076,
+ "logps/chosen": -158.88607788085938,
+ "logps/rejected": -149.43331909179688,
+ "loss": 0.6934396743774414,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.001977279083803296,
+ "rewards/margins": -0.0005727410316467285,
+ "rewards/rejected": 0.0025500201154500246,
+ "step": 60
+ },
+ {
+ "epoch": 0.112,
+ "grad_norm": 0.9988052248954773,
+ "learning_rate": 3.6702127659574467e-07,
+ "logits/chosen": 1.7978185415267944,
+ "logits/rejected": 1.7301257848739624,
+ "logps/chosen": -158.67416381835938,
+ "logps/rejected": -165.23361206054688,
+ "loss": 0.6937376976013183,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.0031797506380826235,
+ "rewards/margins": -0.0011613152455538511,
+ "rewards/rejected": 0.004341065417975187,
+ "step": 70
+ },
+ {
+ "epoch": 0.128,
+ "grad_norm": 1.2025635242462158,
+ "learning_rate": 4.202127659574468e-07,
+ "logits/chosen": 1.4933072328567505,
+ "logits/rejected": 1.4721126556396484,
+ "logps/chosen": -146.546875,
+ "logps/rejected": -138.2855224609375,
+ "loss": 0.6930079936981202,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.005923466756939888,
+ "rewards/margins": 0.000295188365271315,
+ "rewards/rejected": 0.005628278478980064,
+ "step": 80
+ },
+ {
+ "epoch": 0.144,
+ "grad_norm": 1.1192086935043335,
+ "learning_rate": 4.734042553191489e-07,
+ "logits/chosen": 1.4889073371887207,
+ "logits/rejected": 1.4786438941955566,
+ "logps/chosen": -163.24220275878906,
+ "logps/rejected": -160.1715850830078,
+ "loss": 0.6927919387817383,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.00839515682309866,
+ "rewards/margins": 0.0007321214652620256,
+ "rewards/rejected": 0.007663035299628973,
+ "step": 90
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 1.2289488315582275,
+ "learning_rate": 5.26595744680851e-07,
+ "logits/chosen": 1.5521520376205444,
+ "logits/rejected": 1.620697259902954,
+ "logps/chosen": -143.54342651367188,
+ "logps/rejected": -166.68838500976562,
+ "loss": 0.6924111843109131,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.010075005702674389,
+ "rewards/margins": 0.00150866259355098,
+ "rewards/rejected": 0.008566344156861305,
+ "step": 100
+ },
+ {
+ "epoch": 0.16,
+ "eval_logits/chosen": 1.513370156288147,
+ "eval_logits/rejected": 1.5475622415542603,
+ "eval_logps/chosen": -153.44686889648438,
+ "eval_logps/rejected": -147.92510986328125,
+ "eval_loss": 0.6929221153259277,
+ "eval_rewards/accuracies": 0.4699999988079071,
+ "eval_rewards/chosen": 0.010197031311690807,
+ "eval_rewards/margins": 0.0004922974039800465,
+ "eval_rewards/rejected": 0.00970473513007164,
+ "eval_runtime": 91.4962,
+ "eval_samples_per_second": 5.465,
+ "eval_steps_per_second": 2.732,
+ "step": 100
+ },
+ {
+ "epoch": 0.176,
+ "grad_norm": 1.0589202642440796,
+ "learning_rate": 5.797872340425531e-07,
+ "logits/chosen": 1.5165660381317139,
+ "logits/rejected": 1.5812733173370361,
+ "logps/chosen": -144.24609375,
+ "logps/rejected": -138.3242645263672,
+ "loss": 0.6928606033325195,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.011964901350438595,
+ "rewards/margins": 0.0006008840864524245,
+ "rewards/rejected": 0.011364017613232136,
+ "step": 110
+ },
+ {
+ "epoch": 0.192,
+ "grad_norm": 1.1458054780960083,
+ "learning_rate": 6.329787234042553e-07,
+ "logits/chosen": 1.5244230031967163,
+ "logits/rejected": 1.594994306564331,
+ "logps/chosen": -143.27723693847656,
+ "logps/rejected": -143.82754516601562,
+ "loss": 0.6933906078338623,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": 0.009245705790817738,
+ "rewards/margins": -0.0004399990721140057,
+ "rewards/rejected": 0.009685704484581947,
+ "step": 120
+ },
+ {
+ "epoch": 0.208,
+ "grad_norm": 1.3152341842651367,
+ "learning_rate": 6.861702127659574e-07,
+ "logits/chosen": 1.6518194675445557,
+ "logits/rejected": 1.7864271402359009,
+ "logps/chosen": -156.05361938476562,
+ "logps/rejected": -151.7067108154297,
+ "loss": 0.6933954238891602,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.004633297678083181,
+ "rewards/margins": -0.00046000973088666797,
+ "rewards/rejected": 0.005093307700008154,
+ "step": 130
+ },
+ {
+ "epoch": 0.224,
+ "grad_norm": 1.1007953882217407,
+ "learning_rate": 7.393617021276596e-07,
+ "logits/chosen": 1.4447275400161743,
+ "logits/rejected": 1.522552251815796,
+ "logps/chosen": -134.9702911376953,
+ "logps/rejected": -133.505615234375,
+ "loss": 0.6926079750061035,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0014777708565816283,
+ "rewards/margins": 0.0011204290203750134,
+ "rewards/rejected": 0.00035734166158363223,
+ "step": 140
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 0.92185378074646,
+ "learning_rate": 7.925531914893616e-07,
+ "logits/chosen": 1.7024447917938232,
+ "logits/rejected": 1.6758934259414673,
+ "logps/chosen": -142.47024536132812,
+ "logps/rejected": -141.05670166015625,
+ "loss": 0.6922515869140625,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.007132537662982941,
+ "rewards/margins": 0.0018380668479949236,
+ "rewards/rejected": 0.005294471047818661,
+ "step": 150
+ },
+ {
+ "epoch": 0.24,
+ "eval_logits/chosen": 1.5091065168380737,
+ "eval_logits/rejected": 1.5432627201080322,
+ "eval_logps/chosen": -153.50772094726562,
+ "eval_logps/rejected": -147.98837280273438,
+ "eval_loss": 0.6928078532218933,
+ "eval_rewards/accuracies": 0.5139999985694885,
+ "eval_rewards/chosen": 0.004112891852855682,
+ "eval_rewards/margins": 0.0007364260382018983,
+ "eval_rewards/rejected": 0.0033764655236154795,
+ "eval_runtime": 91.3903,
+ "eval_samples_per_second": 5.471,
+ "eval_steps_per_second": 2.736,
+ "step": 150
+ },
+ {
+ "epoch": 0.256,
+ "grad_norm": 1.2662791013717651,
+ "learning_rate": 8.457446808510637e-07,
+ "logits/chosen": 1.6849098205566406,
+ "logits/rejected": 1.620693564414978,
+ "logps/chosen": -164.6056365966797,
+ "logps/rejected": -148.01400756835938,
+ "loss": 0.6930419445037842,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": 0.009472684934735298,
+ "rewards/margins": 0.0002518748806323856,
+ "rewards/rejected": 0.009220810607075691,
+ "step": 160
+ },
+ {
+ "epoch": 0.272,
+ "grad_norm": 1.0822441577911377,
+ "learning_rate": 8.989361702127659e-07,
+ "logits/chosen": 1.4992691278457642,
+ "logits/rejected": 1.5570002794265747,
+ "logps/chosen": -141.10240173339844,
+ "logps/rejected": -125.11993408203125,
+ "loss": 0.692972469329834,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.009402485564351082,
+ "rewards/margins": 0.0004053188022226095,
+ "rewards/rejected": 0.008997166529297829,
+ "step": 170
+ },
+ {
+ "epoch": 0.288,
+ "grad_norm": 0.9809871315956116,
+ "learning_rate": 9.52127659574468e-07,
+ "logits/chosen": 1.5153322219848633,
+ "logits/rejected": 1.5223143100738525,
+ "logps/chosen": -154.1062469482422,
+ "logps/rejected": -155.04188537597656,
+ "loss": 0.6921162605285645,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.016383877024054527,
+ "rewards/margins": 0.002122154226526618,
+ "rewards/rejected": 0.014261722564697266,
+ "step": 180
+ },
+ {
+ "epoch": 0.304,
+ "grad_norm": 1.1610571146011353,
+ "learning_rate": 9.99407231772377e-07,
+ "logits/chosen": 1.6443488597869873,
+ "logits/rejected": 1.5873346328735352,
+ "logps/chosen": -157.59298706054688,
+ "logps/rejected": -155.9975128173828,
+ "loss": 0.6925239086151123,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.025818049907684326,
+ "rewards/margins": 0.0013327361084520817,
+ "rewards/rejected": 0.024485310539603233,
+ "step": 190
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 1.2108772993087769,
+ "learning_rate": 9.93479549496147e-07,
+ "logits/chosen": 1.3679401874542236,
+ "logits/rejected": 1.5478118658065796,
+ "logps/chosen": -127.5545654296875,
+ "logps/rejected": -134.0937042236328,
+ "loss": 0.6889955520629882,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.033060748130083084,
+ "rewards/margins": 0.008453840389847755,
+ "rewards/rejected": 0.02460690215229988,
+ "step": 200
+ },
+ {
+ "epoch": 0.32,
+ "eval_logits/chosen": 1.5197134017944336,
+ "eval_logits/rejected": 1.5536755323410034,
+ "eval_logps/chosen": -153.1664581298828,
+ "eval_logps/rejected": -147.66543579101562,
+ "eval_loss": 0.6919757127761841,
+ "eval_rewards/accuracies": 0.5239999890327454,
+ "eval_rewards/chosen": 0.03824040666222572,
+ "eval_rewards/margins": 0.0025693033821880817,
+ "eval_rewards/rejected": 0.03567110374569893,
+ "eval_runtime": 91.3674,
+ "eval_samples_per_second": 5.472,
+ "eval_steps_per_second": 2.736,
+ "step": 200
+ },
+ {
+ "epoch": 0.336,
+ "grad_norm": 1.4075576066970825,
+ "learning_rate": 9.87551867219917e-07,
+ "logits/chosen": 1.6747233867645264,
+ "logits/rejected": 1.557538628578186,
+ "logps/chosen": -151.46202087402344,
+ "logps/rejected": -126.70817565917969,
+ "loss": 0.6903837203979493,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.0453563891351223,
+ "rewards/margins": 0.005785389803349972,
+ "rewards/rejected": 0.03957099840044975,
+ "step": 210
+ },
+ {
+ "epoch": 0.352,
+ "grad_norm": 1.05863356590271,
+ "learning_rate": 9.81624184943687e-07,
+ "logits/chosen": 1.7199232578277588,
+ "logits/rejected": 1.6891214847564697,
+ "logps/chosen": -164.67184448242188,
+ "logps/rejected": -143.46157836914062,
+ "loss": 0.6919777870178223,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.0621187686920166,
+ "rewards/margins": 0.002725570695474744,
+ "rewards/rejected": 0.05939319729804993,
+ "step": 220
+ },
+ {
+ "epoch": 0.368,
+ "grad_norm": 0.9537659287452698,
+ "learning_rate": 9.75696502667457e-07,
+ "logits/chosen": 1.5906708240509033,
+ "logits/rejected": 1.5525639057159424,
+ "logps/chosen": -149.23095703125,
+ "logps/rejected": -154.40731811523438,
+ "loss": 0.6942371845245361,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.061252254992723465,
+ "rewards/margins": -0.0017460950184613466,
+ "rewards/rejected": 0.06299833953380585,
+ "step": 230
+ },
+ {
+ "epoch": 0.384,
+ "grad_norm": 0.9180606603622437,
+ "learning_rate": 9.69768820391227e-07,
+ "logits/chosen": 1.5654420852661133,
+ "logits/rejected": 1.6795461177825928,
+ "logps/chosen": -158.65875244140625,
+ "logps/rejected": -165.13951110839844,
+ "loss": 0.692426061630249,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.06773029267787933,
+ "rewards/margins": 0.0019084472442045808,
+ "rewards/rejected": 0.06582184135913849,
+ "step": 240
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.7486060857772827,
+ "learning_rate": 9.63841138114997e-07,
+ "logits/chosen": 1.5377140045166016,
+ "logits/rejected": 1.6011888980865479,
+ "logps/chosen": -138.14199829101562,
+ "logps/rejected": -151.1943817138672,
+ "loss": 0.6908403396606445,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.06411711871623993,
+ "rewards/margins": 0.004912695847451687,
+ "rewards/rejected": 0.05920442193746567,
+ "step": 250
+ },
+ {
+ "epoch": 0.4,
+ "eval_logits/chosen": 1.534494161605835,
+ "eval_logits/rejected": 1.5683013200759888,
+ "eval_logps/chosen": -152.8787078857422,
+ "eval_logps/rejected": -147.39991760253906,
+ "eval_loss": 0.6910172700881958,
+ "eval_rewards/accuracies": 0.5460000038146973,
+ "eval_rewards/chosen": 0.06701447814702988,
+ "eval_rewards/margins": 0.004791789222508669,
+ "eval_rewards/rejected": 0.06222269684076309,
+ "eval_runtime": 91.3518,
+ "eval_samples_per_second": 5.473,
+ "eval_steps_per_second": 2.737,
+ "step": 250
+ },
+ {
+ "epoch": 0.416,
+ "grad_norm": 1.0861977338790894,
+ "learning_rate": 9.579134558387671e-07,
+ "logits/chosen": 1.5887064933776855,
+ "logits/rejected": 1.6124156713485718,
+ "logps/chosen": -145.68692016601562,
+ "logps/rejected": -150.8065948486328,
+ "loss": 0.6914952278137207,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.07477976381778717,
+ "rewards/margins": 0.00367984501644969,
+ "rewards/rejected": 0.0710999146103859,
+ "step": 260
+ },
+ {
+ "epoch": 0.432,
+ "grad_norm": 1.2531821727752686,
+ "learning_rate": 9.519857735625369e-07,
+ "logits/chosen": 1.552918553352356,
+ "logits/rejected": 1.6837631464004517,
+ "logps/chosen": -158.31802368164062,
+ "logps/rejected": -151.29379272460938,
+ "loss": 0.691353178024292,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.07334339618682861,
+ "rewards/margins": 0.004158434458076954,
+ "rewards/rejected": 0.06918496638536453,
+ "step": 270
+ },
+ {
+ "epoch": 0.448,
+ "grad_norm": 0.8373661637306213,
+ "learning_rate": 9.460580912863069e-07,
+ "logits/chosen": 1.6604083776474,
+ "logits/rejected": 1.7040210962295532,
+ "logps/chosen": -132.56422424316406,
+ "logps/rejected": -151.414794921875,
+ "loss": 0.6947796821594239,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.07179545611143112,
+ "rewards/margins": -0.002721407450735569,
+ "rewards/rejected": 0.07451687008142471,
+ "step": 280
+ },
+ {
+ "epoch": 0.464,
+ "grad_norm": 1.0909446477890015,
+ "learning_rate": 9.40130409010077e-07,
+ "logits/chosen": 1.6112158298492432,
+ "logits/rejected": 1.6040490865707397,
+ "logps/chosen": -155.4887237548828,
+ "logps/rejected": -149.31924438476562,
+ "loss": 0.6928027629852295,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.07096447050571442,
+ "rewards/margins": 0.0013116076588630676,
+ "rewards/rejected": 0.06965287029743195,
+ "step": 290
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 0.8769014477729797,
+ "learning_rate": 9.34202726733847e-07,
+ "logits/chosen": 1.4986810684204102,
+ "logits/rejected": 1.524269700050354,
+ "logps/chosen": -138.91099548339844,
+ "logps/rejected": -141.48495483398438,
+ "loss": 0.6924718379974365,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.060377366840839386,
+ "rewards/margins": 0.0016540264477953315,
+ "rewards/rejected": 0.058723341673612595,
+ "step": 300
+ },
+ {
+ "epoch": 0.48,
+ "eval_logits/chosen": 1.5390068292617798,
+ "eval_logits/rejected": 1.573019027709961,
+ "eval_logps/chosen": -152.91844177246094,
+ "eval_logps/rejected": -147.44703674316406,
+ "eval_loss": 0.6906248331069946,
+ "eval_rewards/accuracies": 0.5680000185966492,
+ "eval_rewards/chosen": 0.06304025650024414,
+ "eval_rewards/margins": 0.005528101697564125,
+ "eval_rewards/rejected": 0.057512152940034866,
+ "eval_runtime": 91.3495,
+ "eval_samples_per_second": 5.473,
+ "eval_steps_per_second": 2.737,
+ "step": 300
+ },
+ {
+ "epoch": 0.496,
+ "grad_norm": 1.050471305847168,
+ "learning_rate": 9.28275044457617e-07,
+ "logits/chosen": 1.7194430828094482,
+ "logits/rejected": 1.807745337486267,
+ "logps/chosen": -160.05563354492188,
+ "logps/rejected": -138.70767211914062,
+ "loss": 0.6887433052062988,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0673876479268074,
+ "rewards/margins": 0.009202037006616592,
+ "rewards/rejected": 0.058185599744319916,
+ "step": 310
+ },
+ {
+ "epoch": 0.512,
+ "grad_norm": 1.1666924953460693,
+ "learning_rate": 9.223473621813871e-07,
+ "logits/chosen": 1.737310767173767,
+ "logits/rejected": 1.6316074132919312,
+ "logps/chosen": -151.58229064941406,
+ "logps/rejected": -142.8289794921875,
+ "loss": 0.6919591426849365,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.06055358797311783,
+ "rewards/margins": 0.0026850434951484203,
+ "rewards/rejected": 0.057868532836437225,
+ "step": 320
+ },
+ {
+ "epoch": 0.528,
+ "grad_norm": 1.2360730171203613,
+ "learning_rate": 9.16419679905157e-07,
+ "logits/chosen": 1.4524610042572021,
+ "logits/rejected": 1.490668535232544,
+ "logps/chosen": -136.93104553222656,
+ "logps/rejected": -155.99844360351562,
+ "loss": 0.6895017623901367,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.06680475920438766,
+ "rewards/margins": 0.007821018807590008,
+ "rewards/rejected": 0.05898374319076538,
+ "step": 330
+ },
+ {
+ "epoch": 0.544,
+ "grad_norm": 1.0067647695541382,
+ "learning_rate": 9.10491997628927e-07,
+ "logits/chosen": 1.5482432842254639,
+ "logits/rejected": 1.596333622932434,
+ "logps/chosen": -142.59603881835938,
+ "logps/rejected": -120.35069274902344,
+ "loss": 0.6895721912384033,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.0686279758810997,
+ "rewards/margins": 0.007503635250031948,
+ "rewards/rejected": 0.06112435460090637,
+ "step": 340
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 1.2567517757415771,
+ "learning_rate": 9.04564315352697e-07,
+ "logits/chosen": 1.5976614952087402,
+ "logits/rejected": 1.3774999380111694,
+ "logps/chosen": -137.83567810058594,
+ "logps/rejected": -134.2790985107422,
+ "loss": 0.6924222946166992,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06484080851078033,
+ "rewards/margins": 0.0020756719168275595,
+ "rewards/rejected": 0.06276513636112213,
+ "step": 350
+ },
+ {
+ "epoch": 0.56,
+ "eval_logits/chosen": 1.5381687879562378,
+ "eval_logits/rejected": 1.5718625783920288,
+ "eval_logps/chosen": -152.91278076171875,
+ "eval_logps/rejected": -147.4539794921875,
+ "eval_loss": 0.6900433301925659,
+ "eval_rewards/accuracies": 0.5559999942779541,
+ "eval_rewards/chosen": 0.06360553950071335,
+ "eval_rewards/margins": 0.006787541322410107,
+ "eval_rewards/rejected": 0.05681798979640007,
+ "eval_runtime": 91.4768,
+ "eval_samples_per_second": 5.466,
+ "eval_steps_per_second": 2.733,
+ "step": 350
+ },
+ {
+ "epoch": 0.576,
+ "grad_norm": 1.3803397417068481,
+ "learning_rate": 8.986366330764671e-07,
+ "logits/chosen": 1.6290550231933594,
+ "logits/rejected": 1.7498657703399658,
+ "logps/chosen": -163.21676635742188,
+ "logps/rejected": -170.33084106445312,
+ "loss": 0.6899758338928222,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06653806567192078,
+ "rewards/margins": 0.006778226234018803,
+ "rewards/rejected": 0.05975983664393425,
+ "step": 360
+ },
+ {
+ "epoch": 0.592,
+ "grad_norm": 1.1363084316253662,
+ "learning_rate": 8.927089508002371e-07,
+ "logits/chosen": 1.6613603830337524,
+ "logits/rejected": 1.7096723318099976,
+ "logps/chosen": -175.3104248046875,
+ "logps/rejected": -176.24655151367188,
+ "loss": 0.690467643737793,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.061445217579603195,
+ "rewards/margins": 0.005937105976045132,
+ "rewards/rejected": 0.05550810694694519,
+ "step": 370
+ },
+ {
+ "epoch": 0.608,
+ "grad_norm": 1.139722228050232,
+ "learning_rate": 8.86781268524007e-07,
+ "logits/chosen": 1.6426604986190796,
+ "logits/rejected": 1.7499873638153076,
+ "logps/chosen": -136.9043731689453,
+ "logps/rejected": -159.98196411132812,
+ "loss": 0.6899127960205078,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.05270732566714287,
+ "rewards/margins": 0.007067640777677298,
+ "rewards/rejected": 0.04563968628644943,
+ "step": 380
+ },
+ {
+ "epoch": 0.624,
+ "grad_norm": 1.2813146114349365,
+ "learning_rate": 8.80853586247777e-07,
+ "logits/chosen": 1.3995320796966553,
+ "logits/rejected": 1.6828539371490479,
+ "logps/chosen": -131.1273651123047,
+ "logps/rejected": -167.51840209960938,
+ "loss": 0.6943269729614258,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.05107322335243225,
+ "rewards/margins": -0.0017629960784688592,
+ "rewards/rejected": 0.05283621698617935,
+ "step": 390
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 1.1288373470306396,
+ "learning_rate": 8.749259039715471e-07,
+ "logits/chosen": 1.6386661529541016,
+ "logits/rejected": 1.6272175312042236,
+ "logps/chosen": -150.4084014892578,
+ "logps/rejected": -161.52236938476562,
+ "loss": 0.6923639297485351,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.039307981729507446,
+ "rewards/margins": 0.001983280060812831,
+ "rewards/rejected": 0.037324704229831696,
+ "step": 400
+ },
+ {
+ "epoch": 0.64,
+ "eval_logits/chosen": 1.519494652748108,
+ "eval_logits/rejected": 1.5528141260147095,
+ "eval_logps/chosen": -153.15609741210938,
+ "eval_logps/rejected": -147.69198608398438,
+ "eval_loss": 0.6903249025344849,
+ "eval_rewards/accuracies": 0.5460000038146973,
+ "eval_rewards/chosen": 0.03927500545978546,
+ "eval_rewards/margins": 0.006258465815335512,
+ "eval_rewards/rejected": 0.033016543835401535,
+ "eval_runtime": 91.4257,
+ "eval_samples_per_second": 5.469,
+ "eval_steps_per_second": 2.734,
+ "step": 400
+ },
+ {
+ "epoch": 0.656,
+ "grad_norm": 1.355100154876709,
+ "learning_rate": 8.689982216953171e-07,
+ "logits/chosen": 1.5310468673706055,
+ "logits/rejected": 1.6076139211654663,
+ "logps/chosen": -157.39776611328125,
+ "logps/rejected": -145.91275024414062,
+ "loss": 0.68691987991333,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.04244391992688179,
+ "rewards/margins": 0.01302963774651289,
+ "rewards/rejected": 0.029414281249046326,
+ "step": 410
+ },
+ {
+ "epoch": 0.672,
+ "grad_norm": 1.1569719314575195,
+ "learning_rate": 8.630705394190871e-07,
+ "logits/chosen": 1.5669739246368408,
+ "logits/rejected": 1.5956265926361084,
+ "logps/chosen": -132.22671508789062,
+ "logps/rejected": -132.46981811523438,
+ "loss": 0.6866061210632324,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.04782433062791824,
+ "rewards/margins": 0.013592484407126904,
+ "rewards/rejected": 0.034231849014759064,
+ "step": 420
+ },
+ {
+ "epoch": 0.688,
+ "grad_norm": 1.1847658157348633,
+ "learning_rate": 8.57142857142857e-07,
+ "logits/chosen": 1.6808805465698242,
+ "logits/rejected": 1.7047704458236694,
+ "logps/chosen": -140.24200439453125,
+ "logps/rejected": -140.81124877929688,
+ "loss": 0.6933694362640381,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.047805462032556534,
+ "rewards/margins": 0.00025355032994411886,
+ "rewards/rejected": 0.04755191132426262,
+ "step": 430
+ },
+ {
+ "epoch": 0.704,
+ "grad_norm": 0.9608289003372192,
+ "learning_rate": 8.512151748666271e-07,
+ "logits/chosen": 1.663816213607788,
+ "logits/rejected": 1.6789140701293945,
+ "logps/chosen": -155.818603515625,
+ "logps/rejected": -145.31468200683594,
+ "loss": 0.6929332733154296,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.042977917939424515,
+ "rewards/margins": 0.0008615514962002635,
+ "rewards/rejected": 0.04211636632680893,
+ "step": 440
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 1.3275092840194702,
+ "learning_rate": 8.452874925903971e-07,
+ "logits/chosen": 1.4646484851837158,
+ "logits/rejected": 1.6175050735473633,
+ "logps/chosen": -151.04867553710938,
+ "logps/rejected": -147.8173828125,
+ "loss": 0.6903160572052002,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.03235078603029251,
+ "rewards/margins": 0.00654552411288023,
+ "rewards/rejected": 0.025805264711380005,
+ "step": 450
+ },
+ {
+ "epoch": 0.72,
+ "eval_logits/chosen": 1.5208625793457031,
+ "eval_logits/rejected": 1.5539581775665283,
+ "eval_logps/chosen": -153.08497619628906,
+ "eval_logps/rejected": -147.63291931152344,
+ "eval_loss": 0.6898273229598999,
+ "eval_rewards/accuracies": 0.5680000185966492,
+ "eval_rewards/chosen": 0.046387214213609695,
+ "eval_rewards/margins": 0.0074648563750088215,
+ "eval_rewards/rejected": 0.038922350853681564,
+ "eval_runtime": 91.384,
+ "eval_samples_per_second": 5.471,
+ "eval_steps_per_second": 2.736,
+ "step": 450
+ },
+ {
+ "epoch": 0.736,
+ "grad_norm": 1.3754528760910034,
+ "learning_rate": 8.393598103141671e-07,
+ "logits/chosen": 1.5359246730804443,
+ "logits/rejected": 1.604922890663147,
+ "logps/chosen": -137.5425262451172,
+ "logps/rejected": -161.14151000976562,
+ "loss": 0.6894284725189209,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.03450053930282593,
+ "rewards/margins": 0.007881317287683487,
+ "rewards/rejected": 0.02661922201514244,
+ "step": 460
+ },
+ {
+ "epoch": 0.752,
+ "grad_norm": 1.045366883277893,
+ "learning_rate": 8.334321280379371e-07,
+ "logits/chosen": 1.4338958263397217,
+ "logits/rejected": 1.5732755661010742,
+ "logps/chosen": -142.7367401123047,
+ "logps/rejected": -129.0875244140625,
+ "loss": 0.687935209274292,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03967561572790146,
+ "rewards/margins": 0.01102046575397253,
+ "rewards/rejected": 0.028655150905251503,
+ "step": 470
+ },
+ {
+ "epoch": 0.768,
+ "grad_norm": 1.232444167137146,
+ "learning_rate": 8.275044457617071e-07,
+ "logits/chosen": 1.7112070322036743,
+ "logits/rejected": 1.8045260906219482,
+ "logps/chosen": -170.90280151367188,
+ "logps/rejected": -164.41964721679688,
+ "loss": 0.6896602630615234,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.04624806344509125,
+ "rewards/margins": 0.0076986560598015785,
+ "rewards/rejected": 0.038549408316612244,
+ "step": 480
+ },
+ {
+ "epoch": 0.784,
+ "grad_norm": 1.2316750288009644,
+ "learning_rate": 8.215767634854771e-07,
+ "logits/chosen": 1.660528540611267,
+ "logits/rejected": 1.5905355215072632,
+ "logps/chosen": -166.0521697998047,
+ "logps/rejected": -164.01641845703125,
+ "loss": 0.6967196464538574,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": 0.022944677621126175,
+ "rewards/margins": -0.006097245961427689,
+ "rewards/rejected": 0.029041921719908714,
+ "step": 490
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 1.0982184410095215,
+ "learning_rate": 8.156490812092471e-07,
+ "logits/chosen": 1.5519205331802368,
+ "logits/rejected": 1.6819387674331665,
+ "logps/chosen": -137.09713745117188,
+ "logps/rejected": -139.62698364257812,
+ "loss": 0.6872400283813477,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.027622828260064125,
+ "rewards/margins": 0.012441830709576607,
+ "rewards/rejected": 0.015180995687842369,
+ "step": 500
+ },
+ {
+ "epoch": 0.8,
+ "eval_logits/chosen": 1.513393521308899,
+ "eval_logits/rejected": 1.5464614629745483,
+ "eval_logps/chosen": -153.2435302734375,
+ "eval_logps/rejected": -147.79180908203125,
+ "eval_loss": 0.6898292303085327,
+ "eval_rewards/accuracies": 0.5559999942779541,
+ "eval_rewards/chosen": 0.030531544238328934,
+ "eval_rewards/margins": 0.0074988617561757565,
+ "eval_rewards/rejected": 0.02303268015384674,
+ "eval_runtime": 91.4769,
+ "eval_samples_per_second": 5.466,
+ "eval_steps_per_second": 2.733,
+ "step": 500
+ },
+ {
+ "epoch": 0.816,
+ "grad_norm": 0.9861111640930176,
+ "learning_rate": 8.097213989330171e-07,
+ "logits/chosen": 1.7147877216339111,
+ "logits/rejected": 1.6248664855957031,
+ "logps/chosen": -166.3791961669922,
+ "logps/rejected": -152.08753967285156,
+ "loss": 0.6864038944244385,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.0399431511759758,
+ "rewards/margins": 0.014103055000305176,
+ "rewards/rejected": 0.025840098038315773,
+ "step": 510
+ },
+ {
+ "epoch": 0.832,
+ "grad_norm": 1.118251919746399,
+ "learning_rate": 8.037937166567872e-07,
+ "logits/chosen": 1.6375319957733154,
+ "logits/rejected": 1.6414417028427124,
+ "logps/chosen": -150.5874786376953,
+ "logps/rejected": -179.5028076171875,
+ "loss": 0.6879529476165771,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.03941132128238678,
+ "rewards/margins": 0.011486245319247246,
+ "rewards/rejected": 0.027925079688429832,
+ "step": 520
+ },
+ {
+ "epoch": 0.848,
+ "grad_norm": 1.354157567024231,
+ "learning_rate": 7.978660343805572e-07,
+ "logits/chosen": 1.5268175601959229,
+ "logits/rejected": 1.5682529211044312,
+ "logps/chosen": -144.65402221679688,
+ "logps/rejected": -136.36709594726562,
+ "loss": 0.6815488815307618,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.03592699021100998,
+ "rewards/margins": 0.02444549836218357,
+ "rewards/rejected": 0.011481491848826408,
+ "step": 530
+ },
+ {
+ "epoch": 0.864,
+ "grad_norm": 1.0916801691055298,
+ "learning_rate": 7.919383521043271e-07,
+ "logits/chosen": 1.4136483669281006,
+ "logits/rejected": 1.5612545013427734,
+ "logps/chosen": -128.98541259765625,
+ "logps/rejected": -149.62124633789062,
+ "loss": 0.6931351661682129,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.014613496139645576,
+ "rewards/margins": 0.0007547929999418557,
+ "rewards/rejected": 0.013858701102435589,
+ "step": 540
+ },
+ {
+ "epoch": 0.88,
+ "grad_norm": 1.2422736883163452,
+ "learning_rate": 7.860106698280971e-07,
+ "logits/chosen": 1.6898448467254639,
+ "logits/rejected": 1.7594168186187744,
+ "logps/chosen": -175.46859741210938,
+ "logps/rejected": -185.50271606445312,
+ "loss": 0.6864456176757813,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.030645186081528664,
+ "rewards/margins": 0.01440943218767643,
+ "rewards/rejected": 0.016235752031207085,
+ "step": 550
+ },
+ {
+ "epoch": 0.88,
+ "eval_logits/chosen": 1.5022823810577393,
+ "eval_logits/rejected": 1.5351035594940186,
+ "eval_logps/chosen": -153.427490234375,
+ "eval_logps/rejected": -147.97894287109375,
+ "eval_loss": 0.6898054480552673,
+ "eval_rewards/accuracies": 0.5580000281333923,
+ "eval_rewards/chosen": 0.012135841883718967,
+ "eval_rewards/margins": 0.007815876975655556,
+ "eval_rewards/rejected": 0.004319964908063412,
+ "eval_runtime": 91.5476,
+ "eval_samples_per_second": 5.462,
+ "eval_steps_per_second": 2.731,
+ "step": 550
+ },
+ {
+ "epoch": 0.896,
+ "grad_norm": 1.1276408433914185,
+ "learning_rate": 7.800829875518672e-07,
+ "logits/chosen": 1.466566801071167,
+ "logits/rejected": 1.3821640014648438,
+ "logps/chosen": -145.90614318847656,
+ "logps/rejected": -152.33578491210938,
+ "loss": 0.6930467128753662,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.01578265056014061,
+ "rewards/margins": 0.001082198810763657,
+ "rewards/rejected": 0.014700451865792274,
+ "step": 560
+ },
+ {
+ "epoch": 0.912,
+ "grad_norm": 1.0809053182601929,
+ "learning_rate": 7.741553052756372e-07,
+ "logits/chosen": 1.6201200485229492,
+ "logits/rejected": 1.6171897649765015,
+ "logps/chosen": -150.1614990234375,
+ "logps/rejected": -137.40005493164062,
+ "loss": 0.6915182590484619,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.015585039742290974,
+ "rewards/margins": 0.003875966649502516,
+ "rewards/rejected": 0.01170907262712717,
+ "step": 570
+ },
+ {
+ "epoch": 0.928,
+ "grad_norm": 1.3044863939285278,
+ "learning_rate": 7.682276229994072e-07,
+ "logits/chosen": 1.688551902770996,
+ "logits/rejected": 1.6596629619598389,
+ "logps/chosen": -168.21104431152344,
+ "logps/rejected": -137.5209197998047,
+ "loss": 0.6888096332550049,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.00975596159696579,
+ "rewards/margins": 0.009401545859873295,
+ "rewards/rejected": 0.00035441629006527364,
+ "step": 580
+ },
+ {
+ "epoch": 0.944,
+ "grad_norm": 1.3138145208358765,
+ "learning_rate": 7.622999407231771e-07,
+ "logits/chosen": 1.5617314577102661,
+ "logits/rejected": 1.4952644109725952,
+ "logps/chosen": -177.6552734375,
+ "logps/rejected": -156.41143798828125,
+ "loss": 0.6811863422393799,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.023103689774870872,
+ "rewards/margins": 0.02511446550488472,
+ "rewards/rejected": -0.0020107750315219164,
+ "step": 590
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 1.284184217453003,
+ "learning_rate": 7.563722584469472e-07,
+ "logits/chosen": 1.5531961917877197,
+ "logits/rejected": 1.5904725790023804,
+ "logps/chosen": -163.69615173339844,
+ "logps/rejected": -155.56857299804688,
+ "loss": 0.6912278652191162,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.021354522556066513,
+ "rewards/margins": 0.005431803874671459,
+ "rewards/rejected": 0.01592271961271763,
+ "step": 600
+ },
+ {
+ "epoch": 0.96,
+ "eval_logits/chosen": 1.5025317668914795,
+ "eval_logits/rejected": 1.5352839231491089,
+ "eval_logps/chosen": -153.3654022216797,
+ "eval_logps/rejected": -147.92263793945312,
+ "eval_loss": 0.6896048784255981,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": 0.018345065414905548,
+ "eval_rewards/margins": 0.008394664153456688,
+ "eval_rewards/rejected": 0.009950404055416584,
+ "eval_runtime": 91.5004,
+ "eval_samples_per_second": 5.464,
+ "eval_steps_per_second": 2.732,
+ "step": 600
+ },
+ {
+ "epoch": 0.976,
+ "grad_norm": 1.136492371559143,
+ "learning_rate": 7.504445761707172e-07,
+ "logits/chosen": 1.5187523365020752,
+ "logits/rejected": 1.5115095376968384,
+ "logps/chosen": -123.2374267578125,
+ "logps/rejected": -121.64241790771484,
+ "loss": 0.6866262912750244,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.04122164472937584,
+ "rewards/margins": 0.013987530954182148,
+ "rewards/rejected": 0.027234116569161415,
+ "step": 610
+ },
+ {
+ "epoch": 0.992,
+ "grad_norm": 1.3648184537887573,
+ "learning_rate": 7.445168938944872e-07,
+ "logits/chosen": 1.2267796993255615,
+ "logits/rejected": 1.318672776222229,
+ "logps/chosen": -125.6270751953125,
+ "logps/rejected": -139.3863067626953,
+ "loss": 0.6841369152069092,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.02149296924471855,
+ "rewards/margins": 0.02048521675169468,
+ "rewards/rejected": 0.001007752143777907,
+ "step": 620
+ },
+ {
+ "epoch": 1.008,
+ "grad_norm": 0.8776348233222961,
+ "learning_rate": 7.385892116182572e-07,
+ "logits/chosen": 1.4997355937957764,
+ "logits/rejected": 1.376514196395874,
+ "logps/chosen": -152.75949096679688,
+ "logps/rejected": -142.19186401367188,
+ "loss": 0.6843707084655761,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.03699764981865883,
+ "rewards/margins": 0.01918930374085903,
+ "rewards/rejected": 0.01780834048986435,
+ "step": 630
+ },
+ {
+ "epoch": 1.024,
+ "grad_norm": 1.2698174715042114,
+ "learning_rate": 7.326615293420272e-07,
+ "logits/chosen": 1.4477511644363403,
+ "logits/rejected": 1.5352327823638916,
+ "logps/chosen": -146.83685302734375,
+ "logps/rejected": -151.5980224609375,
+ "loss": 0.6780704021453857,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.04432147741317749,
+ "rewards/margins": 0.031893618404865265,
+ "rewards/rejected": 0.012427863664925098,
+ "step": 640
+ },
+ {
+ "epoch": 1.04,
+ "grad_norm": 1.309323787689209,
+ "learning_rate": 7.267338470657972e-07,
+ "logits/chosen": 1.6456422805786133,
+ "logits/rejected": 1.7331817150115967,
+ "logps/chosen": -148.64895629882812,
+ "logps/rejected": -160.71920776367188,
+ "loss": 0.6779833793640136,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.049473900347948074,
+ "rewards/margins": 0.0323341004550457,
+ "rewards/rejected": 0.017139799892902374,
+ "step": 650
+ },
+ {
+ "epoch": 1.04,
+ "eval_logits/chosen": 1.4995282888412476,
+ "eval_logits/rejected": 1.5318833589553833,
+ "eval_logps/chosen": -153.2689208984375,
+ "eval_logps/rejected": -147.83871459960938,
+ "eval_loss": 0.689191997051239,
+ "eval_rewards/accuracies": 0.5460000038146973,
+ "eval_rewards/chosen": 0.02799258381128311,
+ "eval_rewards/margins": 0.009650195948779583,
+ "eval_rewards/rejected": 0.018342390656471252,
+ "eval_runtime": 91.5151,
+ "eval_samples_per_second": 5.464,
+ "eval_steps_per_second": 2.732,
+ "step": 650
+ },
+ {
+ "epoch": 1.056,
+ "grad_norm": 1.1198511123657227,
+ "learning_rate": 7.208061647895672e-07,
+ "logits/chosen": 1.393791675567627,
+ "logits/rejected": 1.537705659866333,
+ "logps/chosen": -141.07870483398438,
+ "logps/rejected": -151.03375244140625,
+ "loss": 0.6852189064025879,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.028253447264432907,
+ "rewards/margins": 0.01726841926574707,
+ "rewards/rejected": 0.010985029861330986,
+ "step": 660
+ },
+ {
+ "epoch": 1.072,
+ "grad_norm": 1.2956689596176147,
+ "learning_rate": 7.148784825133372e-07,
+ "logits/chosen": 1.5089902877807617,
+ "logits/rejected": 1.5116627216339111,
+ "logps/chosen": -153.54898071289062,
+ "logps/rejected": -154.49032592773438,
+ "loss": 0.6772716045379639,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.04329819604754448,
+ "rewards/margins": 0.03352857381105423,
+ "rewards/rejected": 0.0097696203738451,
+ "step": 670
+ },
+ {
+ "epoch": 1.088,
+ "grad_norm": 1.0869190692901611,
+ "learning_rate": 7.089508002371073e-07,
+ "logits/chosen": 1.5537410974502563,
+ "logits/rejected": 1.4676992893218994,
+ "logps/chosen": -128.73751831054688,
+ "logps/rejected": -131.25894165039062,
+ "loss": 0.6872283458709717,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.02748076617717743,
+ "rewards/margins": 0.013209682889282703,
+ "rewards/rejected": 0.01427108608186245,
+ "step": 680
+ },
+ {
+ "epoch": 1.104,
+ "grad_norm": 1.2243199348449707,
+ "learning_rate": 7.030231179608772e-07,
+ "logits/chosen": 1.6543573141098022,
+ "logits/rejected": 1.7242790460586548,
+ "logps/chosen": -176.51280212402344,
+ "logps/rejected": -174.8518524169922,
+ "loss": 0.675547981262207,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.05593103915452957,
+ "rewards/margins": 0.0371597483754158,
+ "rewards/rejected": 0.01877129077911377,
+ "step": 690
+ },
+ {
+ "epoch": 1.12,
+ "grad_norm": 1.3488147258758545,
+ "learning_rate": 6.970954356846472e-07,
+ "logits/chosen": 1.5460538864135742,
+ "logits/rejected": 1.5656627416610718,
+ "logps/chosen": -146.79302978515625,
+ "logps/rejected": -140.67349243164062,
+ "loss": 0.669365406036377,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.04432321712374687,
+ "rewards/margins": 0.05032513290643692,
+ "rewards/rejected": -0.006001918110996485,
+ "step": 700
+ },
+ {
+ "epoch": 1.12,
+ "eval_logits/chosen": 1.5015324354171753,
+ "eval_logits/rejected": 1.5338712930679321,
+ "eval_logps/chosen": -153.185791015625,
+ "eval_logps/rejected": -147.7559814453125,
+ "eval_loss": 0.6892233490943909,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.03630613535642624,
+ "eval_rewards/margins": 0.009690148755908012,
+ "eval_rewards/rejected": 0.026615988463163376,
+ "eval_runtime": 91.5231,
+ "eval_samples_per_second": 5.463,
+ "eval_steps_per_second": 2.732,
+ "step": 700
+ },
+ {
+ "epoch": 1.1360000000000001,
+ "grad_norm": 0.9973899722099304,
+ "learning_rate": 6.911677534084172e-07,
+ "logits/chosen": 1.6786168813705444,
+ "logits/rejected": 1.6147639751434326,
+ "logps/chosen": -175.9976806640625,
+ "logps/rejected": -145.07785034179688,
+ "loss": 0.6673734664916993,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.0721244290471077,
+ "rewards/margins": 0.054663002490997314,
+ "rewards/rejected": 0.017461419105529785,
+ "step": 710
+ },
+ {
+ "epoch": 1.152,
+ "grad_norm": 1.1944832801818848,
+ "learning_rate": 6.852400711321873e-07,
+ "logits/chosen": 1.452962040901184,
+ "logits/rejected": 1.4640306234359741,
+ "logps/chosen": -139.7812042236328,
+ "logps/rejected": -159.2677001953125,
+ "loss": 0.6787771224975586,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.07180429995059967,
+ "rewards/margins": 0.03057795763015747,
+ "rewards/rejected": 0.0412263348698616,
+ "step": 720
+ },
+ {
+ "epoch": 1.168,
+ "grad_norm": 1.4376368522644043,
+ "learning_rate": 6.793123888559573e-07,
+ "logits/chosen": 1.579025387763977,
+ "logits/rejected": 1.5297110080718994,
+ "logps/chosen": -161.11328125,
+ "logps/rejected": -164.56582641601562,
+ "loss": 0.680543327331543,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.07446161657571793,
+ "rewards/margins": 0.027710452675819397,
+ "rewards/rejected": 0.04675116389989853,
+ "step": 730
+ },
+ {
+ "epoch": 1.184,
+ "grad_norm": 1.1249836683273315,
+ "learning_rate": 6.733847065797273e-07,
+ "logits/chosen": 1.7372865676879883,
+ "logits/rejected": 1.7558908462524414,
+ "logps/chosen": -147.6786346435547,
+ "logps/rejected": -155.9397430419922,
+ "loss": 0.667045783996582,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.08958978950977325,
+ "rewards/margins": 0.05507674068212509,
+ "rewards/rejected": 0.03451304882764816,
+ "step": 740
+ },
+ {
+ "epoch": 1.2,
+ "grad_norm": 1.0353419780731201,
+ "learning_rate": 6.674570243034972e-07,
+ "logits/chosen": 1.6369037628173828,
+ "logits/rejected": 1.6020278930664062,
+ "logps/chosen": -166.5580291748047,
+ "logps/rejected": -144.23370361328125,
+ "loss": 0.6636831760406494,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.06881646066904068,
+ "rewards/margins": 0.06279782950878143,
+ "rewards/rejected": 0.0060186260379850864,
+ "step": 750
+ },
+ {
+ "epoch": 1.2,
+ "eval_logits/chosen": 1.4953525066375732,
+ "eval_logits/rejected": 1.5272784233093262,
+ "eval_logps/chosen": -153.105712890625,
+ "eval_logps/rejected": -147.6992950439453,
+ "eval_loss": 0.6883665919303894,
+ "eval_rewards/accuracies": 0.5460000038146973,
+ "eval_rewards/chosen": 0.044312380254268646,
+ "eval_rewards/margins": 0.012026800774037838,
+ "eval_rewards/rejected": 0.032285578548908234,
+ "eval_runtime": 91.4577,
+ "eval_samples_per_second": 5.467,
+ "eval_steps_per_second": 2.734,
+ "step": 750
+ },
+ {
+ "epoch": 1.216,
+ "grad_norm": 1.1240723133087158,
+ "learning_rate": 6.615293420272673e-07,
+ "logits/chosen": 1.4654918909072876,
+ "logits/rejected": 1.4545985460281372,
+ "logps/chosen": -156.47726440429688,
+ "logps/rejected": -138.5869598388672,
+ "loss": 0.6732029438018798,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.05434625223278999,
+ "rewards/margins": 0.04249870777130127,
+ "rewards/rejected": 0.011847544461488724,
+ "step": 760
+ },
+ {
+ "epoch": 1.232,
+ "grad_norm": 1.1530578136444092,
+ "learning_rate": 6.556016597510373e-07,
+ "logits/chosen": 1.5636401176452637,
+ "logits/rejected": 1.603150725364685,
+ "logps/chosen": -147.6873779296875,
+ "logps/rejected": -127.45448303222656,
+ "loss": 0.6720876216888427,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.07123544067144394,
+ "rewards/margins": 0.04455225542187691,
+ "rewards/rejected": 0.026683185249567032,
+ "step": 770
+ },
+ {
+ "epoch": 1.248,
+ "grad_norm": 1.0530054569244385,
+ "learning_rate": 6.496739774748073e-07,
+ "logits/chosen": 1.672349214553833,
+ "logits/rejected": 1.773258924484253,
+ "logps/chosen": -171.55245971679688,
+ "logps/rejected": -185.20631408691406,
+ "loss": 0.6770391464233398,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.06762111186981201,
+ "rewards/margins": 0.03522135317325592,
+ "rewards/rejected": 0.032399751245975494,
+ "step": 780
+ },
+ {
+ "epoch": 1.264,
+ "grad_norm": 0.9587673544883728,
+ "learning_rate": 6.437462951985773e-07,
+ "logits/chosen": 1.6674445867538452,
+ "logits/rejected": 1.5399541854858398,
+ "logps/chosen": -148.35202026367188,
+ "logps/rejected": -141.05218505859375,
+ "loss": 0.688765001296997,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.059925489127635956,
+ "rewards/margins": 0.011203404515981674,
+ "rewards/rejected": 0.04872208088636398,
+ "step": 790
+ },
+ {
+ "epoch": 1.28,
+ "grad_norm": 1.1872327327728271,
+ "learning_rate": 6.378186129223473e-07,
+ "logits/chosen": 1.4424043893814087,
+ "logits/rejected": 1.5345561504364014,
+ "logps/chosen": -142.57899475097656,
+ "logps/rejected": -138.53045654296875,
+ "loss": 0.6739780902862549,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.07486443221569061,
+ "rewards/margins": 0.04266515001654625,
+ "rewards/rejected": 0.032199278473854065,
+ "step": 800
+ },
+ {
+ "epoch": 1.28,
+ "eval_logits/chosen": 1.502331256866455,
+ "eval_logits/rejected": 1.5342037677764893,
+ "eval_logps/chosen": -152.94044494628906,
+ "eval_logps/rejected": -147.55020141601562,
+ "eval_loss": 0.6877647042274475,
+ "eval_rewards/accuracies": 0.5540000200271606,
+ "eval_rewards/chosen": 0.060839928686618805,
+ "eval_rewards/margins": 0.013645462691783905,
+ "eval_rewards/rejected": 0.0471944697201252,
+ "eval_runtime": 91.4749,
+ "eval_samples_per_second": 5.466,
+ "eval_steps_per_second": 2.733,
+ "step": 800
+ },
+ {
+ "epoch": 1.296,
+ "grad_norm": 1.1386040449142456,
+ "learning_rate": 6.318909306461173e-07,
+ "logits/chosen": 1.4093509912490845,
+ "logits/rejected": 1.4580585956573486,
+ "logps/chosen": -164.720947265625,
+ "logps/rejected": -142.99754333496094,
+ "loss": 0.6797928810119629,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.07539167255163193,
+ "rewards/margins": 0.0292592104524374,
+ "rewards/rejected": 0.046132463961839676,
+ "step": 810
+ },
+ {
+ "epoch": 1.312,
+ "grad_norm": 1.3657523393630981,
+ "learning_rate": 6.259632483698873e-07,
+ "logits/chosen": 1.3146798610687256,
+ "logits/rejected": 1.4988536834716797,
+ "logps/chosen": -141.5091552734375,
+ "logps/rejected": -138.624755859375,
+ "loss": 0.6713742733001709,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.06485269963741302,
+ "rewards/margins": 0.04707767441868782,
+ "rewards/rejected": 0.017775025218725204,
+ "step": 820
+ },
+ {
+ "epoch": 1.328,
+ "grad_norm": 1.207746148109436,
+ "learning_rate": 6.200355660936573e-07,
+ "logits/chosen": 1.5304832458496094,
+ "logits/rejected": 1.567846655845642,
+ "logps/chosen": -145.65390014648438,
+ "logps/rejected": -134.02752685546875,
+ "loss": 0.6716270923614502,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08788949996232986,
+ "rewards/margins": 0.04611582309007645,
+ "rewards/rejected": 0.041773684322834015,
+ "step": 830
+ },
+ {
+ "epoch": 1.3439999999999999,
+ "grad_norm": 1.3036965131759644,
+ "learning_rate": 6.141078838174274e-07,
+ "logits/chosen": 1.5840891599655151,
+ "logits/rejected": 1.500603199005127,
+ "logps/chosen": -138.60763549804688,
+ "logps/rejected": -125.70695495605469,
+ "loss": 0.6759689331054688,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10840122401714325,
+ "rewards/margins": 0.037392571568489075,
+ "rewards/rejected": 0.07100865244865417,
+ "step": 840
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "grad_norm": 1.1700375080108643,
+ "learning_rate": 6.081802015411973e-07,
+ "logits/chosen": 1.5052621364593506,
+ "logits/rejected": 1.5524170398712158,
+ "logps/chosen": -133.3436279296875,
+ "logps/rejected": -137.5284423828125,
+ "loss": 0.6731984615325928,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.09424515068531036,
+ "rewards/margins": 0.04275429993867874,
+ "rewards/rejected": 0.051490843296051025,
+ "step": 850
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "eval_logits/chosen": 1.514055609703064,
+ "eval_logits/rejected": 1.545797348022461,
+ "eval_logps/chosen": -152.71902465820312,
+ "eval_logps/rejected": -147.34742736816406,
+ "eval_loss": 0.6870437264442444,
+ "eval_rewards/accuracies": 0.5540000200271606,
+ "eval_rewards/chosen": 0.08298191428184509,
+ "eval_rewards/margins": 0.015508526936173439,
+ "eval_rewards/rejected": 0.0674733817577362,
+ "eval_runtime": 91.4891,
+ "eval_samples_per_second": 5.465,
+ "eval_steps_per_second": 2.733,
+ "step": 850
+ },
+ {
+ "epoch": 1.376,
+ "grad_norm": 1.080214023590088,
+ "learning_rate": 6.022525192649673e-07,
+ "logits/chosen": 1.4343178272247314,
+ "logits/rejected": 1.5825798511505127,
+ "logps/chosen": -148.39657592773438,
+ "logps/rejected": -129.2698516845703,
+ "loss": 0.6728739261627197,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.10437977313995361,
+ "rewards/margins": 0.04413595423102379,
+ "rewards/rejected": 0.06024380773305893,
+ "step": 860
+ },
+ {
+ "epoch": 1.392,
+ "grad_norm": 1.3011701107025146,
+ "learning_rate": 5.963248369887373e-07,
+ "logits/chosen": 1.6486666202545166,
+ "logits/rejected": 1.636071801185608,
+ "logps/chosen": -155.72866821289062,
+ "logps/rejected": -156.0375213623047,
+ "loss": 0.6663424491882324,
+ "rewards/accuracies": 0.7875000238418579,
+ "rewards/chosen": 0.0980696976184845,
+ "rewards/margins": 0.05624913424253464,
+ "rewards/rejected": 0.04182056710124016,
+ "step": 870
+ },
+ {
+ "epoch": 1.408,
+ "grad_norm": 1.1527386903762817,
+ "learning_rate": 5.903971547125074e-07,
+ "logits/chosen": 1.6375783681869507,
+ "logits/rejected": 1.5879249572753906,
+ "logps/chosen": -138.84786987304688,
+ "logps/rejected": -149.22756958007812,
+ "loss": 0.6661319255828857,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.1091323122382164,
+ "rewards/margins": 0.05697597935795784,
+ "rewards/rejected": 0.05215633660554886,
+ "step": 880
+ },
+ {
+ "epoch": 1.424,
+ "grad_norm": 1.2128634452819824,
+ "learning_rate": 5.844694724362774e-07,
+ "logits/chosen": 1.643206238746643,
+ "logits/rejected": 1.529200792312622,
+ "logps/chosen": -156.31326293945312,
+ "logps/rejected": -144.78689575195312,
+ "loss": 0.6645416736602783,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.11794288456439972,
+ "rewards/margins": 0.06102023646235466,
+ "rewards/rejected": 0.05692264437675476,
+ "step": 890
+ },
+ {
+ "epoch": 1.44,
+ "grad_norm": 0.9876027703285217,
+ "learning_rate": 5.785417901600474e-07,
+ "logits/chosen": 1.3582518100738525,
+ "logits/rejected": 1.5023400783538818,
+ "logps/chosen": -137.19717407226562,
+ "logps/rejected": -167.57350158691406,
+ "loss": 0.6799286842346192,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.09652381390333176,
+ "rewards/margins": 0.02946191094815731,
+ "rewards/rejected": 0.0670619010925293,
+ "step": 900
+ },
+ {
+ "epoch": 1.44,
+ "eval_logits/chosen": 1.5112248659133911,
+ "eval_logits/rejected": 1.542572021484375,
+ "eval_logps/chosen": -152.71963500976562,
+ "eval_logps/rejected": -147.3575897216797,
+ "eval_loss": 0.6868481636047363,
+ "eval_rewards/accuracies": 0.5720000267028809,
+ "eval_rewards/chosen": 0.08292035758495331,
+ "eval_rewards/margins": 0.016464442014694214,
+ "eval_rewards/rejected": 0.0664559081196785,
+ "eval_runtime": 91.463,
+ "eval_samples_per_second": 5.467,
+ "eval_steps_per_second": 2.733,
+ "step": 900
+ },
+ {
+ "epoch": 1.456,
+ "grad_norm": 1.0684723854064941,
+ "learning_rate": 5.726141078838173e-07,
+ "logits/chosen": 1.5465466976165771,
+ "logits/rejected": 1.500496506690979,
+ "logps/chosen": -132.5164337158203,
+ "logps/rejected": -129.40220642089844,
+ "loss": 0.6799171924591064,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08448641002178192,
+ "rewards/margins": 0.028616121038794518,
+ "rewards/rejected": 0.05587029457092285,
+ "step": 910
+ },
+ {
+ "epoch": 1.472,
+ "grad_norm": 1.0342527627944946,
+ "learning_rate": 5.666864256075874e-07,
+ "logits/chosen": 1.6495583057403564,
+ "logits/rejected": 1.7119420766830444,
+ "logps/chosen": -139.18484497070312,
+ "logps/rejected": -154.33407592773438,
+ "loss": 0.6807973861694336,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.09809015691280365,
+ "rewards/margins": 0.027513395994901657,
+ "rewards/rejected": 0.0705767497420311,
+ "step": 920
+ },
+ {
+ "epoch": 1.488,
+ "grad_norm": 1.0507510900497437,
+ "learning_rate": 5.607587433313574e-07,
+ "logits/chosen": 1.5217338800430298,
+ "logits/rejected": 1.5648618936538696,
+ "logps/chosen": -131.5649871826172,
+ "logps/rejected": -122.32954406738281,
+ "loss": 0.681494951248169,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.08206742256879807,
+ "rewards/margins": 0.026795843616127968,
+ "rewards/rejected": 0.055271584540605545,
+ "step": 930
+ },
+ {
+ "epoch": 1.504,
+ "grad_norm": 1.3102085590362549,
+ "learning_rate": 5.548310610551274e-07,
+ "logits/chosen": 1.6052824258804321,
+ "logits/rejected": 1.6236565113067627,
+ "logps/chosen": -149.2860565185547,
+ "logps/rejected": -153.01657104492188,
+ "loss": 0.6783152103424073,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.10417164862155914,
+ "rewards/margins": 0.03314245864748955,
+ "rewards/rejected": 0.0710291862487793,
+ "step": 940
+ },
+ {
+ "epoch": 1.52,
+ "grad_norm": 1.3876780271530151,
+ "learning_rate": 5.489033787788974e-07,
+ "logits/chosen": 1.689612627029419,
+ "logits/rejected": 1.6765525341033936,
+ "logps/chosen": -164.98193359375,
+ "logps/rejected": -164.33975219726562,
+ "loss": 0.6782557964324951,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.09384694695472717,
+ "rewards/margins": 0.03179098665714264,
+ "rewards/rejected": 0.062055956572294235,
+ "step": 950
+ },
+ {
+ "epoch": 1.52,
+ "eval_logits/chosen": 1.5106096267700195,
+ "eval_logits/rejected": 1.5420260429382324,
+ "eval_logps/chosen": -152.7333526611328,
+ "eval_logps/rejected": -147.37344360351562,
+ "eval_loss": 0.6867280602455139,
+ "eval_rewards/accuracies": 0.5540000200271606,
+ "eval_rewards/chosen": 0.08154840767383575,
+ "eval_rewards/margins": 0.01667776331305504,
+ "eval_rewards/rejected": 0.06487063318490982,
+ "eval_runtime": 91.5472,
+ "eval_samples_per_second": 5.462,
+ "eval_steps_per_second": 2.731,
+ "step": 950
+ },
+ {
+ "epoch": 1.536,
+ "grad_norm": 1.0544084310531616,
+ "learning_rate": 5.429756965026674e-07,
+ "logits/chosen": 1.696502685546875,
+ "logits/rejected": 1.722957968711853,
+ "logps/chosen": -177.16172790527344,
+ "logps/rejected": -160.9434051513672,
+ "loss": 0.6731346130371094,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.09499023109674454,
+ "rewards/margins": 0.04484598711133003,
+ "rewards/rejected": 0.050144243985414505,
+ "step": 960
+ },
+ {
+ "epoch": 1.552,
+ "grad_norm": 1.2123764753341675,
+ "learning_rate": 5.370480142264374e-07,
+ "logits/chosen": 1.5045363903045654,
+ "logits/rejected": 1.4909541606903076,
+ "logps/chosen": -144.6968994140625,
+ "logps/rejected": -160.55947875976562,
+ "loss": 0.6804170131683349,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.08150655776262283,
+ "rewards/margins": 0.029505986720323563,
+ "rewards/rejected": 0.05200057104229927,
+ "step": 970
+ },
+ {
+ "epoch": 1.568,
+ "grad_norm": 1.0709542036056519,
+ "learning_rate": 5.311203319502074e-07,
+ "logits/chosen": 1.5131323337554932,
+ "logits/rejected": 1.5731028318405151,
+ "logps/chosen": -157.4567108154297,
+ "logps/rejected": -149.07107543945312,
+ "loss": 0.6691867351531983,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.08909507840871811,
+ "rewards/margins": 0.05152255296707153,
+ "rewards/rejected": 0.03757254034280777,
+ "step": 980
+ },
+ {
+ "epoch": 1.584,
+ "grad_norm": 1.329915165901184,
+ "learning_rate": 5.251926496739774e-07,
+ "logits/chosen": 1.574861764907837,
+ "logits/rejected": 1.5829271078109741,
+ "logps/chosen": -143.85403442382812,
+ "logps/rejected": -140.85009765625,
+ "loss": 0.6716786384582519,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08492995798587799,
+ "rewards/margins": 0.04930455982685089,
+ "rewards/rejected": 0.0356253944337368,
+ "step": 990
+ },
+ {
+ "epoch": 1.6,
+ "grad_norm": 1.3974183797836304,
+ "learning_rate": 5.192649673977475e-07,
+ "logits/chosen": 1.5636709928512573,
+ "logits/rejected": 1.3984944820404053,
+ "logps/chosen": -148.8516845703125,
+ "logps/rejected": -143.12136840820312,
+ "loss": 0.676781702041626,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.05405520275235176,
+ "rewards/margins": 0.035432908684015274,
+ "rewards/rejected": 0.018622295930981636,
+ "step": 1000
+ },
+ {
+ "epoch": 1.6,
+ "eval_logits/chosen": 1.5051932334899902,
+ "eval_logits/rejected": 1.5366464853286743,
+ "eval_logps/chosen": -152.83253479003906,
+ "eval_logps/rejected": -147.4718475341797,
+ "eval_loss": 0.6868343353271484,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.07163143903017044,
+ "eval_rewards/margins": 0.016600240021944046,
+ "eval_rewards/rejected": 0.055031195282936096,
+ "eval_runtime": 91.6291,
+ "eval_samples_per_second": 5.457,
+ "eval_steps_per_second": 2.728,
+ "step": 1000
+ },
+ {
+ "epoch": 1.616,
+ "grad_norm": 1.2400332689285278,
+ "learning_rate": 5.133372851215174e-07,
+ "logits/chosen": 1.5299904346466064,
+ "logits/rejected": 1.6691240072250366,
+ "logps/chosen": -150.43557739257812,
+ "logps/rejected": -170.71127319335938,
+ "loss": 0.6721617221832276,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.08803285658359528,
+ "rewards/margins": 0.04526625573635101,
+ "rewards/rejected": 0.04276660084724426,
+ "step": 1010
+ },
+ {
+ "epoch": 1.6320000000000001,
+ "grad_norm": 0.8514426350593567,
+ "learning_rate": 5.074096028452874e-07,
+ "logits/chosen": 1.4948112964630127,
+ "logits/rejected": 1.515966773033142,
+ "logps/chosen": -118.82080078125,
+ "logps/rejected": -139.77511596679688,
+ "loss": 0.6717492580413819,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.05406653881072998,
+ "rewards/margins": 0.04632322117686272,
+ "rewards/rejected": 0.007743313908576965,
+ "step": 1020
+ },
+ {
+ "epoch": 1.6480000000000001,
+ "grad_norm": 1.5450512170791626,
+ "learning_rate": 5.014819205690574e-07,
+ "logits/chosen": 1.5520379543304443,
+ "logits/rejected": 1.5921989679336548,
+ "logps/chosen": -147.900634765625,
+ "logps/rejected": -138.66140747070312,
+ "loss": 0.6880045413970948,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.06136086583137512,
+ "rewards/margins": 0.014043760485947132,
+ "rewards/rejected": 0.04731711000204086,
+ "step": 1030
+ },
+ {
+ "epoch": 1.6640000000000001,
+ "grad_norm": 1.1967573165893555,
+ "learning_rate": 4.955542382928275e-07,
+ "logits/chosen": 1.5113961696624756,
+ "logits/rejected": 1.4426608085632324,
+ "logps/chosen": -139.90713500976562,
+ "logps/rejected": -155.5555877685547,
+ "loss": 0.6754290580749511,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.06368472427129745,
+ "rewards/margins": 0.04256365820765495,
+ "rewards/rejected": 0.021121066063642502,
+ "step": 1040
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "grad_norm": 1.758066177368164,
+ "learning_rate": 4.896265560165974e-07,
+ "logits/chosen": 1.6157357692718506,
+ "logits/rejected": 1.578230857849121,
+ "logps/chosen": -141.13455200195312,
+ "logps/rejected": -166.70489501953125,
+ "loss": 0.6755188941955567,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.05974128097295761,
+ "rewards/margins": 0.03907640278339386,
+ "rewards/rejected": 0.0206648800522089,
+ "step": 1050
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "eval_logits/chosen": 1.4930020570755005,
+ "eval_logits/rejected": 1.5244908332824707,
+ "eval_logps/chosen": -153.0406036376953,
+ "eval_logps/rejected": -147.66622924804688,
+ "eval_loss": 0.6875224709510803,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": 0.05082407966256142,
+ "eval_rewards/margins": 0.01523309201002121,
+ "eval_rewards/rejected": 0.03559098765254021,
+ "eval_runtime": 91.5108,
+ "eval_samples_per_second": 5.464,
+ "eval_steps_per_second": 2.732,
+ "step": 1050
+ },
+ {
+ "epoch": 1.696,
+ "grad_norm": 1.2342020273208618,
+ "learning_rate": 4.836988737403675e-07,
+ "logits/chosen": 1.2796365022659302,
+ "logits/rejected": 1.3894035816192627,
+ "logps/chosen": -145.9099578857422,
+ "logps/rejected": -148.60482788085938,
+ "loss": 0.6661100864410401,
+ "rewards/accuracies": 0.800000011920929,
+ "rewards/chosen": 0.055561065673828125,
+ "rewards/margins": 0.057345401495695114,
+ "rewards/rejected": -0.0017843395471572876,
+ "step": 1060
+ },
+ {
+ "epoch": 1.712,
+ "grad_norm": 1.3287626504898071,
+ "learning_rate": 4.777711914641375e-07,
+ "logits/chosen": 1.792177438735962,
+ "logits/rejected": 1.648444414138794,
+ "logps/chosen": -159.8272247314453,
+ "logps/rejected": -182.2501983642578,
+ "loss": 0.6666016101837158,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.08213181793689728,
+ "rewards/margins": 0.05758446455001831,
+ "rewards/rejected": 0.024547358974814415,
+ "step": 1070
+ },
+ {
+ "epoch": 1.728,
+ "grad_norm": 0.9578614234924316,
+ "learning_rate": 4.718435091879075e-07,
+ "logits/chosen": 1.55452561378479,
+ "logits/rejected": 1.5692381858825684,
+ "logps/chosen": -154.53590393066406,
+ "logps/rejected": -142.9732666015625,
+ "loss": 0.6688636302947998,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08060242235660553,
+ "rewards/margins": 0.053381383419036865,
+ "rewards/rejected": 0.02722104825079441,
+ "step": 1080
+ },
+ {
+ "epoch": 1.744,
+ "grad_norm": 1.1178059577941895,
+ "learning_rate": 4.6591582691167753e-07,
+ "logits/chosen": 1.607924461364746,
+ "logits/rejected": 1.7212623357772827,
+ "logps/chosen": -152.4365234375,
+ "logps/rejected": -157.656494140625,
+ "loss": 0.6775742053985596,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.0737459659576416,
+ "rewards/margins": 0.03416898101568222,
+ "rewards/rejected": 0.03957698121666908,
+ "step": 1090
+ },
+ {
+ "epoch": 1.76,
+ "grad_norm": 1.4560215473175049,
+ "learning_rate": 4.599881446354475e-07,
+ "logits/chosen": 1.6074415445327759,
+ "logits/rejected": 1.5955140590667725,
+ "logps/chosen": -171.25421142578125,
+ "logps/rejected": -124.63997650146484,
+ "loss": 0.6695756435394287,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.040774114429950714,
+ "rewards/margins": 0.05189450830221176,
+ "rewards/rejected": -0.01112039852887392,
+ "step": 1100
+ },
+ {
+ "epoch": 1.76,
+ "eval_logits/chosen": 1.4804071187973022,
+ "eval_logits/rejected": 1.5116674900054932,
+ "eval_logps/chosen": -153.20590209960938,
+ "eval_logps/rejected": -147.82687377929688,
+ "eval_loss": 0.6880003809928894,
+ "eval_rewards/accuracies": 0.5479999780654907,
+ "eval_rewards/chosen": 0.03429403901100159,
+ "eval_rewards/margins": 0.014766983687877655,
+ "eval_rewards/rejected": 0.019527053460478783,
+ "eval_runtime": 91.5827,
+ "eval_samples_per_second": 5.46,
+ "eval_steps_per_second": 2.73,
+ "step": 1100
+ },
+ {
+ "epoch": 1.776,
+ "grad_norm": 1.113887071609497,
+ "learning_rate": 4.5406046235921753e-07,
+ "logits/chosen": 1.6315574645996094,
+ "logits/rejected": 1.5989868640899658,
+ "logps/chosen": -155.7100067138672,
+ "logps/rejected": -157.9479522705078,
+ "loss": 0.6718159675598144,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.05936906486749649,
+ "rewards/margins": 0.04729412868618965,
+ "rewards/rejected": 0.012074939906597137,
+ "step": 1110
+ },
+ {
+ "epoch": 1.792,
+ "grad_norm": 0.9409530162811279,
+ "learning_rate": 4.481327800829875e-07,
+ "logits/chosen": 1.47927987575531,
+ "logits/rejected": 1.6135873794555664,
+ "logps/chosen": -143.21559143066406,
+ "logps/rejected": -155.91262817382812,
+ "loss": 0.6721374988555908,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.06062264367938042,
+ "rewards/margins": 0.04633890092372894,
+ "rewards/rejected": 0.0142837418243289,
+ "step": 1120
+ },
+ {
+ "epoch": 1.808,
+ "grad_norm": 1.1281055212020874,
+ "learning_rate": 4.422050978067575e-07,
+ "logits/chosen": 1.6088281869888306,
+ "logits/rejected": 1.6764856576919556,
+ "logps/chosen": -143.90219116210938,
+ "logps/rejected": -145.31483459472656,
+ "loss": 0.6717781066894531,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.037483494728803635,
+ "rewards/margins": 0.04687192291021347,
+ "rewards/rejected": -0.009388430044054985,
+ "step": 1130
+ },
+ {
+ "epoch": 1.8239999999999998,
+ "grad_norm": 1.2734681367874146,
+ "learning_rate": 4.362774155305275e-07,
+ "logits/chosen": 1.3558837175369263,
+ "logits/rejected": 1.4319185018539429,
+ "logps/chosen": -133.63270568847656,
+ "logps/rejected": -133.51321411132812,
+ "loss": 0.6760762214660645,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03574688360095024,
+ "rewards/margins": 0.03954789787530899,
+ "rewards/rejected": -0.0038010149728506804,
+ "step": 1140
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "grad_norm": 1.5277918577194214,
+ "learning_rate": 4.303497332542976e-07,
+ "logits/chosen": 1.45877206325531,
+ "logits/rejected": 1.5461132526397705,
+ "logps/chosen": -146.4346466064453,
+ "logps/rejected": -157.70391845703125,
+ "loss": 0.676148509979248,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.05790795013308525,
+ "rewards/margins": 0.03729299455881119,
+ "rewards/rejected": 0.020614955574274063,
+ "step": 1150
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "eval_logits/chosen": 1.4798471927642822,
+ "eval_logits/rejected": 1.5109448432922363,
+ "eval_logps/chosen": -153.19979858398438,
+ "eval_logps/rejected": -147.82022094726562,
+ "eval_loss": 0.6881932616233826,
+ "eval_rewards/accuracies": 0.5479999780654907,
+ "eval_rewards/chosen": 0.03490476310253143,
+ "eval_rewards/margins": 0.01471200492233038,
+ "eval_rewards/rejected": 0.020192759111523628,
+ "eval_runtime": 91.6364,
+ "eval_samples_per_second": 5.456,
+ "eval_steps_per_second": 2.728,
+ "step": 1150
+ },
+ {
+ "epoch": 1.8559999999999999,
+ "grad_norm": 1.274863839149475,
+ "learning_rate": 4.244220509780675e-07,
+ "logits/chosen": 1.4154646396636963,
+ "logits/rejected": 1.4470824003219604,
+ "logps/chosen": -126.20066833496094,
+ "logps/rejected": -151.97293090820312,
+ "loss": 0.6712587356567383,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.043645404279232025,
+ "rewards/margins": 0.047874242067337036,
+ "rewards/rejected": -0.004228837788105011,
+ "step": 1160
+ },
+ {
+ "epoch": 1.8719999999999999,
+ "grad_norm": 1.5296283960342407,
+ "learning_rate": 4.184943687018376e-07,
+ "logits/chosen": 1.6431888341903687,
+ "logits/rejected": 1.6058006286621094,
+ "logps/chosen": -148.9349822998047,
+ "logps/rejected": -147.87075805664062,
+ "loss": 0.6697592735290527,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.04735144227743149,
+ "rewards/margins": 0.05177399516105652,
+ "rewards/rejected": -0.0044225468300282955,
+ "step": 1170
+ },
+ {
+ "epoch": 1.888,
+ "grad_norm": 1.1624146699905396,
+ "learning_rate": 4.125666864256075e-07,
+ "logits/chosen": 1.5232517719268799,
+ "logits/rejected": 1.6060616970062256,
+ "logps/chosen": -162.28726196289062,
+ "logps/rejected": -156.43606567382812,
+ "loss": 0.6648381233215332,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.08147656917572021,
+ "rewards/margins": 0.06185563653707504,
+ "rewards/rejected": 0.01962094008922577,
+ "step": 1180
+ },
+ {
+ "epoch": 1.904,
+ "grad_norm": 1.1841973066329956,
+ "learning_rate": 4.066390041493776e-07,
+ "logits/chosen": 1.5964248180389404,
+ "logits/rejected": 1.490767240524292,
+ "logps/chosen": -147.4954071044922,
+ "logps/rejected": -130.23780822753906,
+ "loss": 0.6871873378753662,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.03555892035365105,
+ "rewards/margins": 0.016584066674113274,
+ "rewards/rejected": 0.018974855542182922,
+ "step": 1190
+ },
+ {
+ "epoch": 1.92,
+ "grad_norm": 1.2493818998336792,
+ "learning_rate": 4.007113218731476e-07,
+ "logits/chosen": 1.7136554718017578,
+ "logits/rejected": 1.5512396097183228,
+ "logps/chosen": -144.41561889648438,
+ "logps/rejected": -155.35577392578125,
+ "loss": 0.6688260078430176,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.056659065186977386,
+ "rewards/margins": 0.05496113747358322,
+ "rewards/rejected": 0.0016979314386844635,
+ "step": 1200
+ },
+ {
+ "epoch": 1.92,
+ "eval_logits/chosen": 1.4767016172409058,
+ "eval_logits/rejected": 1.5077458620071411,
+ "eval_logps/chosen": -153.31640625,
+ "eval_logps/rejected": -147.94268798828125,
+ "eval_loss": 0.6879809498786926,
+ "eval_rewards/accuracies": 0.5540000200271606,
+ "eval_rewards/chosen": 0.023242462426424026,
+ "eval_rewards/margins": 0.015296407975256443,
+ "eval_rewards/rejected": 0.007946056313812733,
+ "eval_runtime": 91.5558,
+ "eval_samples_per_second": 5.461,
+ "eval_steps_per_second": 2.731,
+ "step": 1200
+ },
+ {
+ "epoch": 1.936,
+ "grad_norm": 1.3514587879180908,
+ "learning_rate": 3.947836395969176e-07,
+ "logits/chosen": 1.4940625429153442,
+ "logits/rejected": 1.528282880783081,
+ "logps/chosen": -143.37240600585938,
+ "logps/rejected": -145.9002227783203,
+ "loss": 0.6773603439331055,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0432279035449028,
+ "rewards/margins": 0.036386217921972275,
+ "rewards/rejected": 0.006841684691607952,
+ "step": 1210
+ },
+ {
+ "epoch": 1.952,
+ "grad_norm": 1.3247877359390259,
+ "learning_rate": 3.888559573206876e-07,
+ "logits/chosen": 1.3842494487762451,
+ "logits/rejected": 1.551335334777832,
+ "logps/chosen": -137.93670654296875,
+ "logps/rejected": -165.12350463867188,
+ "loss": 0.6811867713928222,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.021465692669153214,
+ "rewards/margins": 0.028191816061735153,
+ "rewards/rejected": -0.006726124789565802,
+ "step": 1220
+ },
+ {
+ "epoch": 1.968,
+ "grad_norm": 1.294998288154602,
+ "learning_rate": 3.8292827504445763e-07,
+ "logits/chosen": 1.6616626977920532,
+ "logits/rejected": 1.5197885036468506,
+ "logps/chosen": -163.02841186523438,
+ "logps/rejected": -146.42318725585938,
+ "loss": 0.6719709396362304,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.04652197286486626,
+ "rewards/margins": 0.04806648939847946,
+ "rewards/rejected": -0.0015445180470123887,
+ "step": 1230
+ },
+ {
+ "epoch": 1.984,
+ "grad_norm": 1.0518370866775513,
+ "learning_rate": 3.770005927682276e-07,
+ "logits/chosen": 1.498633623123169,
+ "logits/rejected": 1.407947301864624,
+ "logps/chosen": -147.41317749023438,
+ "logps/rejected": -142.83969116210938,
+ "loss": 0.668755054473877,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.060599617660045624,
+ "rewards/margins": 0.05342463403940201,
+ "rewards/rejected": 0.007174992468208075,
+ "step": 1240
+ },
+ {
+ "epoch": 2.0,
+ "grad_norm": 1.274714708328247,
+ "learning_rate": 3.710729104919976e-07,
+ "logits/chosen": 1.6258140802383423,
+ "logits/rejected": 1.5810320377349854,
+ "logps/chosen": -179.64002990722656,
+ "logps/rejected": -180.4091796875,
+ "loss": 0.6710570812225342,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.030506063252687454,
+ "rewards/margins": 0.05016648769378662,
+ "rewards/rejected": -0.019660424441099167,
+ "step": 1250
+ },
+ {
+ "epoch": 2.0,
+ "eval_logits/chosen": 1.4780300855636597,
+ "eval_logits/rejected": 1.5088509321212769,
+ "eval_logps/chosen": -153.2786865234375,
+ "eval_logps/rejected": -147.9156036376953,
+ "eval_loss": 0.6875351071357727,
+ "eval_rewards/accuracies": 0.5580000281333923,
+ "eval_rewards/chosen": 0.027016151696443558,
+ "eval_rewards/margins": 0.01636195369064808,
+ "eval_rewards/rejected": 0.010654192417860031,
+ "eval_runtime": 91.4999,
+ "eval_samples_per_second": 5.464,
+ "eval_steps_per_second": 2.732,
+ "step": 1250
+ },
+ {
+ "epoch": 2.016,
+ "grad_norm": 1.2844147682189941,
+ "learning_rate": 3.6514522821576757e-07,
+ "logits/chosen": 1.5406789779663086,
+ "logits/rejected": 1.6422452926635742,
+ "logps/chosen": -156.5453338623047,
+ "logps/rejected": -134.34414672851562,
+ "loss": 0.6678662300109863,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.06532102078199387,
+ "rewards/margins": 0.05562450364232063,
+ "rewards/rejected": 0.009696529246866703,
+ "step": 1260
+ },
+ {
+ "epoch": 2.032,
+ "grad_norm": 1.933401346206665,
+ "learning_rate": 3.592175459395376e-07,
+ "logits/chosen": 1.4194307327270508,
+ "logits/rejected": 1.4650447368621826,
+ "logps/chosen": -160.74501037597656,
+ "logps/rejected": -164.00306701660156,
+ "loss": 0.6694974422454834,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.0568288154900074,
+ "rewards/margins": 0.05134005472064018,
+ "rewards/rejected": 0.005488761700689793,
+ "step": 1270
+ },
+ {
+ "epoch": 2.048,
+ "grad_norm": 1.0357041358947754,
+ "learning_rate": 3.532898636633077e-07,
+ "logits/chosen": 1.3812814950942993,
+ "logits/rejected": 1.624565839767456,
+ "logps/chosen": -137.08206176757812,
+ "logps/rejected": -162.38833618164062,
+ "loss": 0.6587967872619629,
+ "rewards/accuracies": 0.762499988079071,
+ "rewards/chosen": 0.06582682579755783,
+ "rewards/margins": 0.0754324197769165,
+ "rewards/rejected": -0.009605596773326397,
+ "step": 1280
+ },
+ {
+ "epoch": 2.064,
+ "grad_norm": 1.196915864944458,
+ "learning_rate": 3.473621813870776e-07,
+ "logits/chosen": 1.5303500890731812,
+ "logits/rejected": 1.539674997329712,
+ "logps/chosen": -143.24818420410156,
+ "logps/rejected": -139.9320831298828,
+ "loss": 0.6571191310882568,
+ "rewards/accuracies": 0.7749999761581421,
+ "rewards/chosen": 0.06632508337497711,
+ "rewards/margins": 0.07795976102352142,
+ "rewards/rejected": -0.011634674854576588,
+ "step": 1290
+ },
+ {
+ "epoch": 2.08,
+ "grad_norm": 1.1703696250915527,
+ "learning_rate": 3.414344991108477e-07,
+ "logits/chosen": 1.629018783569336,
+ "logits/rejected": 1.6967315673828125,
+ "logps/chosen": -167.43856811523438,
+ "logps/rejected": -176.7183074951172,
+ "loss": 0.657524299621582,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.08362282812595367,
+ "rewards/margins": 0.07772316038608551,
+ "rewards/rejected": 0.0058996789157390594,
+ "step": 1300
+ },
+ {
+ "epoch": 2.08,
+ "eval_logits/chosen": 1.4791160821914673,
+ "eval_logits/rejected": 1.5099083185195923,
+ "eval_logps/chosen": -153.2497100830078,
+ "eval_logps/rejected": -147.89149475097656,
+ "eval_loss": 0.6873863935470581,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": 0.029913639649748802,
+ "eval_rewards/margins": 0.016850139945745468,
+ "eval_rewards/rejected": 0.013063502497971058,
+ "eval_runtime": 91.6423,
+ "eval_samples_per_second": 5.456,
+ "eval_steps_per_second": 2.728,
+ "step": 1300
+ },
+ {
+ "epoch": 2.096,
+ "grad_norm": 1.3917887210845947,
+ "learning_rate": 3.355068168346176e-07,
+ "logits/chosen": 1.4511966705322266,
+ "logits/rejected": 1.5072139501571655,
+ "logps/chosen": -152.747802734375,
+ "logps/rejected": -163.4126434326172,
+ "loss": 0.658362340927124,
+ "rewards/accuracies": 0.800000011920929,
+ "rewards/chosen": 0.05855457857251167,
+ "rewards/margins": 0.07485153526067734,
+ "rewards/rejected": -0.016296952962875366,
+ "step": 1310
+ },
+ {
+ "epoch": 2.112,
+ "grad_norm": 1.1995881795883179,
+ "learning_rate": 3.295791345583877e-07,
+ "logits/chosen": 1.5953047275543213,
+ "logits/rejected": 1.5267432928085327,
+ "logps/chosen": -165.5431671142578,
+ "logps/rejected": -176.0441131591797,
+ "loss": 0.6578423976898193,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.07576775550842285,
+ "rewards/margins": 0.07662783563137054,
+ "rewards/rejected": -0.0008600771543569863,
+ "step": 1320
+ },
+ {
+ "epoch": 2.128,
+ "grad_norm": 1.192712664604187,
+ "learning_rate": 3.236514522821577e-07,
+ "logits/chosen": 1.6488456726074219,
+ "logits/rejected": 1.6644923686981201,
+ "logps/chosen": -141.50588989257812,
+ "logps/rejected": -147.02488708496094,
+ "loss": 0.6503303050994873,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.05412819981575012,
+ "rewards/margins": 0.09486084431409836,
+ "rewards/rejected": -0.040732644498348236,
+ "step": 1330
+ },
+ {
+ "epoch": 2.144,
+ "grad_norm": 1.2887846231460571,
+ "learning_rate": 3.177237700059277e-07,
+ "logits/chosen": 1.4427909851074219,
+ "logits/rejected": 1.5052260160446167,
+ "logps/chosen": -133.82730102539062,
+ "logps/rejected": -146.34561157226562,
+ "loss": 0.6659392833709716,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.07661060243844986,
+ "rewards/margins": 0.061056166887283325,
+ "rewards/rejected": 0.015554438345134258,
+ "step": 1340
+ },
+ {
+ "epoch": 2.16,
+ "grad_norm": 1.2006677389144897,
+ "learning_rate": 3.117960877296977e-07,
+ "logits/chosen": 1.5929205417633057,
+ "logits/rejected": 1.4980108737945557,
+ "logps/chosen": -162.6881561279297,
+ "logps/rejected": -131.2484588623047,
+ "loss": 0.6704725742340087,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.051008958369493484,
+ "rewards/margins": 0.05107830837368965,
+ "rewards/rejected": -6.935522105777636e-05,
+ "step": 1350
+ },
+ {
+ "epoch": 2.16,
+ "eval_logits/chosen": 1.4747014045715332,
+ "eval_logits/rejected": 1.5051954984664917,
+ "eval_logps/chosen": -153.24111938476562,
+ "eval_logps/rejected": -147.88681030273438,
+ "eval_loss": 0.6875070929527283,
+ "eval_rewards/accuracies": 0.5519999861717224,
+ "eval_rewards/chosen": 0.03077210672199726,
+ "eval_rewards/margins": 0.017239386215806007,
+ "eval_rewards/rejected": 0.013532716780900955,
+ "eval_runtime": 91.625,
+ "eval_samples_per_second": 5.457,
+ "eval_steps_per_second": 2.729,
+ "step": 1350
+ },
+ {
+ "epoch": 2.176,
+ "grad_norm": 1.0531398057937622,
+ "learning_rate": 3.0586840545346773e-07,
+ "logits/chosen": 1.5693953037261963,
+ "logits/rejected": 1.4765347242355347,
+ "logps/chosen": -138.3097686767578,
+ "logps/rejected": -131.7692108154297,
+ "loss": 0.6577227592468262,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.05366247892379761,
+ "rewards/margins": 0.07729668915271759,
+ "rewards/rejected": -0.023634206503629684,
+ "step": 1360
+ },
+ {
+ "epoch": 2.192,
+ "grad_norm": 1.231702446937561,
+ "learning_rate": 2.999407231772377e-07,
+ "logits/chosen": 1.2275021076202393,
+ "logits/rejected": 1.3573815822601318,
+ "logps/chosen": -132.48486328125,
+ "logps/rejected": -135.35678100585938,
+ "loss": 0.6756887435913086,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.05586853623390198,
+ "rewards/margins": 0.04077490419149399,
+ "rewards/rejected": 0.015093636699020863,
+ "step": 1370
+ },
+ {
+ "epoch": 2.208,
+ "grad_norm": 1.4397618770599365,
+ "learning_rate": 2.940130409010077e-07,
+ "logits/chosen": 1.7811187505722046,
+ "logits/rejected": 1.6808115243911743,
+ "logps/chosen": -162.24937438964844,
+ "logps/rejected": -161.79856872558594,
+ "loss": 0.6767025947570801,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0448589026927948,
+ "rewards/margins": 0.036365605890750885,
+ "rewards/rejected": 0.008493305183947086,
+ "step": 1380
+ },
+ {
+ "epoch": 2.224,
+ "grad_norm": 1.3940778970718384,
+ "learning_rate": 2.8808535862477767e-07,
+ "logits/chosen": 1.4333906173706055,
+ "logits/rejected": 1.5776273012161255,
+ "logps/chosen": -139.67471313476562,
+ "logps/rejected": -145.5760498046875,
+ "loss": 0.6700955867767334,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.0032391161657869816,
+ "rewards/margins": 0.051826756447553635,
+ "rewards/rejected": -0.04858764261007309,
+ "step": 1390
+ },
+ {
+ "epoch": 2.24,
+ "grad_norm": 1.035470724105835,
+ "learning_rate": 2.821576763485477e-07,
+ "logits/chosen": 1.362156629562378,
+ "logits/rejected": 1.5719056129455566,
+ "logps/chosen": -165.33090209960938,
+ "logps/rejected": -161.99774169921875,
+ "loss": 0.6694781303405761,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.06508176028728485,
+ "rewards/margins": 0.05120532587170601,
+ "rewards/rejected": 0.01387643814086914,
+ "step": 1400
+ },
+ {
+ "epoch": 2.24,
+ "eval_logits/chosen": 1.4757568836212158,
+ "eval_logits/rejected": 1.506177544593811,
+ "eval_logps/chosen": -153.22549438476562,
+ "eval_logps/rejected": -147.8716583251953,
+ "eval_loss": 0.6875126957893372,
+ "eval_rewards/accuracies": 0.5479999780654907,
+ "eval_rewards/chosen": 0.0323345772922039,
+ "eval_rewards/margins": 0.01728663593530655,
+ "eval_rewards/rejected": 0.015047942288219929,
+ "eval_runtime": 91.6393,
+ "eval_samples_per_second": 5.456,
+ "eval_steps_per_second": 2.728,
+ "step": 1400
+ },
+ {
+ "epoch": 2.2560000000000002,
+ "grad_norm": 1.1921639442443848,
+ "learning_rate": 2.762299940723177e-07,
+ "logits/chosen": 1.6182886362075806,
+ "logits/rejected": 1.720829963684082,
+ "logps/chosen": -146.8916778564453,
+ "logps/rejected": -163.6287384033203,
+ "loss": 0.6639984130859375,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.06335891783237457,
+ "rewards/margins": 0.06482325494289398,
+ "rewards/rejected": -0.00146433524787426,
+ "step": 1410
+ },
+ {
+ "epoch": 2.2720000000000002,
+ "grad_norm": 1.0042649507522583,
+ "learning_rate": 2.703023117960877e-07,
+ "logits/chosen": 1.5501604080200195,
+ "logits/rejected": 1.4911870956420898,
+ "logps/chosen": -128.26808166503906,
+ "logps/rejected": -141.57968139648438,
+ "loss": 0.6698122501373291,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.04305802285671234,
+ "rewards/margins": 0.05252720043063164,
+ "rewards/rejected": -0.00946918223053217,
+ "step": 1420
+ },
+ {
+ "epoch": 2.288,
+ "grad_norm": 1.139971137046814,
+ "learning_rate": 2.643746295198577e-07,
+ "logits/chosen": 1.489418864250183,
+ "logits/rejected": 1.484986424446106,
+ "logps/chosen": -148.4396209716797,
+ "logps/rejected": -134.74444580078125,
+ "loss": 0.6659749507904053,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.035206541419029236,
+ "rewards/margins": 0.05955536291003227,
+ "rewards/rejected": -0.024348825216293335,
+ "step": 1430
+ },
+ {
+ "epoch": 2.304,
+ "grad_norm": 1.2615156173706055,
+ "learning_rate": 2.584469472436278e-07,
+ "logits/chosen": 1.4411890506744385,
+ "logits/rejected": 1.517624855041504,
+ "logps/chosen": -150.73739624023438,
+ "logps/rejected": -168.2410888671875,
+ "loss": 0.6469865322113038,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.0349310003221035,
+ "rewards/margins": 0.10427621752023697,
+ "rewards/rejected": -0.06934522092342377,
+ "step": 1440
+ },
+ {
+ "epoch": 2.32,
+ "grad_norm": 1.3218971490859985,
+ "learning_rate": 2.525192649673977e-07,
+ "logits/chosen": 1.4323554039001465,
+ "logits/rejected": 1.5884326696395874,
+ "logps/chosen": -161.39932250976562,
+ "logps/rejected": -170.40005493164062,
+ "loss": 0.6539069652557373,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.047432832419872284,
+ "rewards/margins": 0.08863824605941772,
+ "rewards/rejected": -0.04120541363954544,
+ "step": 1450
+ },
+ {
+ "epoch": 2.32,
+ "eval_logits/chosen": 1.470116138458252,
+ "eval_logits/rejected": 1.500432014465332,
+ "eval_logps/chosen": -153.3093719482422,
+ "eval_logps/rejected": -147.94973754882812,
+ "eval_loss": 0.6879428625106812,
+ "eval_rewards/accuracies": 0.5479999780654907,
+ "eval_rewards/chosen": 0.0239470973610878,
+ "eval_rewards/margins": 0.01670677214860916,
+ "eval_rewards/rejected": 0.007240326143801212,
+ "eval_runtime": 91.6086,
+ "eval_samples_per_second": 5.458,
+ "eval_steps_per_second": 2.729,
+ "step": 1450
+ },
+ {
+ "epoch": 2.336,
+ "grad_norm": 1.5149509906768799,
+ "learning_rate": 2.465915826911678e-07,
+ "logits/chosen": 1.4124549627304077,
+ "logits/rejected": 1.37667977809906,
+ "logps/chosen": -132.296142578125,
+ "logps/rejected": -129.7874755859375,
+ "loss": 0.6767591953277587,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.010999361053109169,
+ "rewards/margins": 0.04050828143954277,
+ "rewards/rejected": -0.0295089241117239,
+ "step": 1460
+ },
+ {
+ "epoch": 2.352,
+ "grad_norm": 1.2253953218460083,
+ "learning_rate": 2.406639004149378e-07,
+ "logits/chosen": 1.6453937292099,
+ "logits/rejected": 1.5322078466415405,
+ "logps/chosen": -144.53091430664062,
+ "logps/rejected": -124.41532897949219,
+ "loss": 0.6565406322479248,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.08202239871025085,
+ "rewards/margins": 0.07841381430625916,
+ "rewards/rejected": 0.0036085874307900667,
+ "step": 1470
+ },
+ {
+ "epoch": 2.368,
+ "grad_norm": 1.341062307357788,
+ "learning_rate": 2.3473621813870775e-07,
+ "logits/chosen": 1.6002895832061768,
+ "logits/rejected": 1.5122597217559814,
+ "logps/chosen": -158.36151123046875,
+ "logps/rejected": -133.71824645996094,
+ "loss": 0.6461643695831298,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.08061931282281876,
+ "rewards/margins": 0.10182766616344452,
+ "rewards/rejected": -0.02120836265385151,
+ "step": 1480
+ },
+ {
+ "epoch": 2.384,
+ "grad_norm": 1.390051007270813,
+ "learning_rate": 2.2880853586247775e-07,
+ "logits/chosen": 1.461780309677124,
+ "logits/rejected": 1.4497120380401611,
+ "logps/chosen": -137.35128784179688,
+ "logps/rejected": -136.74586486816406,
+ "loss": 0.6574192523956299,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.08100035041570663,
+ "rewards/margins": 0.07784504443407059,
+ "rewards/rejected": 0.003155309008434415,
+ "step": 1490
+ },
+ {
+ "epoch": 2.4,
+ "grad_norm": 1.4389512538909912,
+ "learning_rate": 2.2288085358624777e-07,
+ "logits/chosen": 1.4925658702850342,
+ "logits/rejected": 1.5207669734954834,
+ "logps/chosen": -146.5099334716797,
+ "logps/rejected": -134.76077270507812,
+ "loss": 0.6710968971252441,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.04036882147192955,
+ "rewards/margins": 0.05301078408956528,
+ "rewards/rejected": -0.01264196913689375,
+ "step": 1500
+ },
+ {
+ "epoch": 2.4,
+ "eval_logits/chosen": 1.473709225654602,
+ "eval_logits/rejected": 1.5038994550704956,
+ "eval_logps/chosen": -153.1907501220703,
+ "eval_logps/rejected": -147.84222412109375,
+ "eval_loss": 0.6875451803207397,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": 0.035809777677059174,
+ "eval_rewards/margins": 0.017816420644521713,
+ "eval_rewards/rejected": 0.01799335516989231,
+ "eval_runtime": 91.5755,
+ "eval_samples_per_second": 5.46,
+ "eval_steps_per_second": 2.73,
+ "step": 1500
+ }
+ ],
+ "logging_steps": 10,
+ "max_steps": 1875,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1500/training_args.bin b/v3/DPO/DPO_5k/lora/checkpoint-1500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..544215e69bd09b706d2c495c7bb412d0df7f83e7
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afe50d4ac4c5f2af306fafe28b341897775b898324082e319b56c66893dceabb
+size 6097
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1875/README.md b/v3/DPO/DPO_5k/lora/checkpoint-1875/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..250500f798b4682b2cd2c35cd1fc366677c215dd
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1875/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1875/adapter_config.json b/v3/DPO/DPO_5k/lora/checkpoint-1875/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..695e5e16d4a5436a17790fe2e31fc2a9e1631d03
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1875/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "gate_proj",
+ "k_proj",
+ "q_proj",
+ "down_proj",
+ "o_proj",
+ "v_proj",
+ "up_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1875/adapter_model.safetensors b/v3/DPO/DPO_5k/lora/checkpoint-1875/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6635f4cb505655e646a63ebae1b9d26e21c5f9d2
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1875/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:504da931dd735d5a7d203b6528f0ef0550f7e365d2cd61c78dfefdcd3360562b
+size 180385008
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1875/chat_template.jinja b/v3/DPO/DPO_5k/lora/checkpoint-1875/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1875/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1875/optimizer.pt b/v3/DPO/DPO_5k/lora/checkpoint-1875/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3181541b5ed93d6946b9097ad510989e7cdb928c
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1875/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:861348584cdea990bdff032b6cf045e1dc90a66a099787dccf681a1a36c24ee9
+size 360902475
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1875/rng_state.pth b/v3/DPO/DPO_5k/lora/checkpoint-1875/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..7859360e6e5ad44f3ed47e7c17f4cafb2aebd41a
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1875/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50dd579ef1d2d932c3d7726ef0f6ec72d0e25b023ae2fd8b67112ac75e1041b9
+size 14645
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1875/scaler.pt b/v3/DPO/DPO_5k/lora/checkpoint-1875/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..62674612b56e28ca98da704825f72028fdee802e
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1875/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0dcc47514d330a99944bf3bc8eb7381f257363691e003bc57a61172f4d6a6cfe
+size 1383
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1875/scheduler.pt b/v3/DPO/DPO_5k/lora/checkpoint-1875/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4ed990519d0a220767f04589f05e887bb5615374
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1875/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:491983a7bbf746f355582f6257be2432c110e4e5bc26e821f3f84720363fd2b9
+size 1465
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1875/tokenizer.json b/v3/DPO/DPO_5k/lora/checkpoint-1875/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1875/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1875/tokenizer_config.json b/v3/DPO/DPO_5k/lora/checkpoint-1875/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1875/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1875/trainer_state.json b/v3/DPO/DPO_5k/lora/checkpoint-1875/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..932e5021fa7d9482cbe9845031a703b7fe5bc07a
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1875/trainer_state.json
@@ -0,0 +1,3431 @@
+{
+ "best_global_step": 950,
+ "best_metric": 0.6867280602455139,
+ "best_model_checkpoint": null,
+ "epoch": 3.0,
+ "eval_steps": 50,
+ "global_step": 1875,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.016,
+ "grad_norm": 1.1517287492752075,
+ "learning_rate": 4.787234042553191e-08,
+ "logits/chosen": 1.6246497631072998,
+ "logits/rejected": 1.5396833419799805,
+ "logps/chosen": -145.91531372070312,
+ "logps/rejected": -151.4051971435547,
+ "loss": 0.693363094329834,
+ "rewards/accuracies": 0.2874999940395355,
+ "rewards/chosen": -3.5572534216044005e-06,
+ "rewards/margins": -0.0004222107236273587,
+ "rewards/rejected": 0.00041865353705361485,
+ "step": 10
+ },
+ {
+ "epoch": 0.032,
+ "grad_norm": 1.3633077144622803,
+ "learning_rate": 1.0106382978723404e-07,
+ "logits/chosen": 1.313926339149475,
+ "logits/rejected": 1.4038684368133545,
+ "logps/chosen": -120.60591888427734,
+ "logps/rejected": -134.05252075195312,
+ "loss": 0.6930661201477051,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 9.576084994478151e-05,
+ "rewards/margins": 0.0001732158416416496,
+ "rewards/rejected": -7.745502807665616e-05,
+ "step": 20
+ },
+ {
+ "epoch": 0.048,
+ "grad_norm": 1.209151029586792,
+ "learning_rate": 1.5425531914893615e-07,
+ "logits/chosen": 1.5228456258773804,
+ "logits/rejected": 1.6869430541992188,
+ "logps/chosen": -162.40280151367188,
+ "logps/rejected": -179.38192749023438,
+ "loss": 0.692751693725586,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0007011269335635006,
+ "rewards/margins": 0.0008053636993281543,
+ "rewards/rejected": -0.00010423659114167094,
+ "step": 30
+ },
+ {
+ "epoch": 0.064,
+ "grad_norm": 1.3589205741882324,
+ "learning_rate": 2.074468085106383e-07,
+ "logits/chosen": 1.544001817703247,
+ "logits/rejected": 1.4715760946273804,
+ "logps/chosen": -152.7266845703125,
+ "logps/rejected": -150.9158477783203,
+ "loss": 0.6925192832946777,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.00026422025985084474,
+ "rewards/margins": 0.0012672842713072896,
+ "rewards/rejected": -0.0010030639823526144,
+ "step": 40
+ },
+ {
+ "epoch": 0.08,
+ "grad_norm": 1.202810525894165,
+ "learning_rate": 2.6063829787234044e-07,
+ "logits/chosen": 1.6794925928115845,
+ "logits/rejected": 1.7414724826812744,
+ "logps/chosen": -169.71751403808594,
+ "logps/rejected": -156.7571258544922,
+ "loss": 0.6930821895599365,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 9.479046275373548e-05,
+ "rewards/margins": 0.00014605997421313077,
+ "rewards/rejected": -5.126944597577676e-05,
+ "step": 50
+ },
+ {
+ "epoch": 0.08,
+ "eval_logits/chosen": 1.5141410827636719,
+ "eval_logits/rejected": 1.5485105514526367,
+ "eval_logps/chosen": -153.53335571289062,
+ "eval_logps/rejected": -148.00894165039062,
+ "eval_loss": 0.6930396556854248,
+ "eval_rewards/accuracies": 0.5120000243186951,
+ "eval_rewards/chosen": 0.0015483711613342166,
+ "eval_rewards/margins": 0.00022836039715912193,
+ "eval_rewards/rejected": 0.0013200106332078576,
+ "eval_runtime": 91.1536,
+ "eval_samples_per_second": 5.485,
+ "eval_steps_per_second": 2.743,
+ "step": 50
+ },
+ {
+ "epoch": 0.096,
+ "grad_norm": 1.2080731391906738,
+ "learning_rate": 3.1382978723404253e-07,
+ "logits/chosen": 1.796142339706421,
+ "logits/rejected": 1.7786738872528076,
+ "logps/chosen": -158.88607788085938,
+ "logps/rejected": -149.43331909179688,
+ "loss": 0.6934396743774414,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.001977279083803296,
+ "rewards/margins": -0.0005727410316467285,
+ "rewards/rejected": 0.0025500201154500246,
+ "step": 60
+ },
+ {
+ "epoch": 0.112,
+ "grad_norm": 0.9988052248954773,
+ "learning_rate": 3.6702127659574467e-07,
+ "logits/chosen": 1.7978185415267944,
+ "logits/rejected": 1.7301257848739624,
+ "logps/chosen": -158.67416381835938,
+ "logps/rejected": -165.23361206054688,
+ "loss": 0.6937376976013183,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.0031797506380826235,
+ "rewards/margins": -0.0011613152455538511,
+ "rewards/rejected": 0.004341065417975187,
+ "step": 70
+ },
+ {
+ "epoch": 0.128,
+ "grad_norm": 1.2025635242462158,
+ "learning_rate": 4.202127659574468e-07,
+ "logits/chosen": 1.4933072328567505,
+ "logits/rejected": 1.4721126556396484,
+ "logps/chosen": -146.546875,
+ "logps/rejected": -138.2855224609375,
+ "loss": 0.6930079936981202,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.005923466756939888,
+ "rewards/margins": 0.000295188365271315,
+ "rewards/rejected": 0.005628278478980064,
+ "step": 80
+ },
+ {
+ "epoch": 0.144,
+ "grad_norm": 1.1192086935043335,
+ "learning_rate": 4.734042553191489e-07,
+ "logits/chosen": 1.4889073371887207,
+ "logits/rejected": 1.4786438941955566,
+ "logps/chosen": -163.24220275878906,
+ "logps/rejected": -160.1715850830078,
+ "loss": 0.6927919387817383,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.00839515682309866,
+ "rewards/margins": 0.0007321214652620256,
+ "rewards/rejected": 0.007663035299628973,
+ "step": 90
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 1.2289488315582275,
+ "learning_rate": 5.26595744680851e-07,
+ "logits/chosen": 1.5521520376205444,
+ "logits/rejected": 1.620697259902954,
+ "logps/chosen": -143.54342651367188,
+ "logps/rejected": -166.68838500976562,
+ "loss": 0.6924111843109131,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.010075005702674389,
+ "rewards/margins": 0.00150866259355098,
+ "rewards/rejected": 0.008566344156861305,
+ "step": 100
+ },
+ {
+ "epoch": 0.16,
+ "eval_logits/chosen": 1.513370156288147,
+ "eval_logits/rejected": 1.5475622415542603,
+ "eval_logps/chosen": -153.44686889648438,
+ "eval_logps/rejected": -147.92510986328125,
+ "eval_loss": 0.6929221153259277,
+ "eval_rewards/accuracies": 0.4699999988079071,
+ "eval_rewards/chosen": 0.010197031311690807,
+ "eval_rewards/margins": 0.0004922974039800465,
+ "eval_rewards/rejected": 0.00970473513007164,
+ "eval_runtime": 91.4962,
+ "eval_samples_per_second": 5.465,
+ "eval_steps_per_second": 2.732,
+ "step": 100
+ },
+ {
+ "epoch": 0.176,
+ "grad_norm": 1.0589202642440796,
+ "learning_rate": 5.797872340425531e-07,
+ "logits/chosen": 1.5165660381317139,
+ "logits/rejected": 1.5812733173370361,
+ "logps/chosen": -144.24609375,
+ "logps/rejected": -138.3242645263672,
+ "loss": 0.6928606033325195,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.011964901350438595,
+ "rewards/margins": 0.0006008840864524245,
+ "rewards/rejected": 0.011364017613232136,
+ "step": 110
+ },
+ {
+ "epoch": 0.192,
+ "grad_norm": 1.1458054780960083,
+ "learning_rate": 6.329787234042553e-07,
+ "logits/chosen": 1.5244230031967163,
+ "logits/rejected": 1.594994306564331,
+ "logps/chosen": -143.27723693847656,
+ "logps/rejected": -143.82754516601562,
+ "loss": 0.6933906078338623,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": 0.009245705790817738,
+ "rewards/margins": -0.0004399990721140057,
+ "rewards/rejected": 0.009685704484581947,
+ "step": 120
+ },
+ {
+ "epoch": 0.208,
+ "grad_norm": 1.3152341842651367,
+ "learning_rate": 6.861702127659574e-07,
+ "logits/chosen": 1.6518194675445557,
+ "logits/rejected": 1.7864271402359009,
+ "logps/chosen": -156.05361938476562,
+ "logps/rejected": -151.7067108154297,
+ "loss": 0.6933954238891602,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.004633297678083181,
+ "rewards/margins": -0.00046000973088666797,
+ "rewards/rejected": 0.005093307700008154,
+ "step": 130
+ },
+ {
+ "epoch": 0.224,
+ "grad_norm": 1.1007953882217407,
+ "learning_rate": 7.393617021276596e-07,
+ "logits/chosen": 1.4447275400161743,
+ "logits/rejected": 1.522552251815796,
+ "logps/chosen": -134.9702911376953,
+ "logps/rejected": -133.505615234375,
+ "loss": 0.6926079750061035,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0014777708565816283,
+ "rewards/margins": 0.0011204290203750134,
+ "rewards/rejected": 0.00035734166158363223,
+ "step": 140
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 0.92185378074646,
+ "learning_rate": 7.925531914893616e-07,
+ "logits/chosen": 1.7024447917938232,
+ "logits/rejected": 1.6758934259414673,
+ "logps/chosen": -142.47024536132812,
+ "logps/rejected": -141.05670166015625,
+ "loss": 0.6922515869140625,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.007132537662982941,
+ "rewards/margins": 0.0018380668479949236,
+ "rewards/rejected": 0.005294471047818661,
+ "step": 150
+ },
+ {
+ "epoch": 0.24,
+ "eval_logits/chosen": 1.5091065168380737,
+ "eval_logits/rejected": 1.5432627201080322,
+ "eval_logps/chosen": -153.50772094726562,
+ "eval_logps/rejected": -147.98837280273438,
+ "eval_loss": 0.6928078532218933,
+ "eval_rewards/accuracies": 0.5139999985694885,
+ "eval_rewards/chosen": 0.004112891852855682,
+ "eval_rewards/margins": 0.0007364260382018983,
+ "eval_rewards/rejected": 0.0033764655236154795,
+ "eval_runtime": 91.3903,
+ "eval_samples_per_second": 5.471,
+ "eval_steps_per_second": 2.736,
+ "step": 150
+ },
+ {
+ "epoch": 0.256,
+ "grad_norm": 1.2662791013717651,
+ "learning_rate": 8.457446808510637e-07,
+ "logits/chosen": 1.6849098205566406,
+ "logits/rejected": 1.620693564414978,
+ "logps/chosen": -164.6056365966797,
+ "logps/rejected": -148.01400756835938,
+ "loss": 0.6930419445037842,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": 0.009472684934735298,
+ "rewards/margins": 0.0002518748806323856,
+ "rewards/rejected": 0.009220810607075691,
+ "step": 160
+ },
+ {
+ "epoch": 0.272,
+ "grad_norm": 1.0822441577911377,
+ "learning_rate": 8.989361702127659e-07,
+ "logits/chosen": 1.4992691278457642,
+ "logits/rejected": 1.5570002794265747,
+ "logps/chosen": -141.10240173339844,
+ "logps/rejected": -125.11993408203125,
+ "loss": 0.692972469329834,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.009402485564351082,
+ "rewards/margins": 0.0004053188022226095,
+ "rewards/rejected": 0.008997166529297829,
+ "step": 170
+ },
+ {
+ "epoch": 0.288,
+ "grad_norm": 0.9809871315956116,
+ "learning_rate": 9.52127659574468e-07,
+ "logits/chosen": 1.5153322219848633,
+ "logits/rejected": 1.5223143100738525,
+ "logps/chosen": -154.1062469482422,
+ "logps/rejected": -155.04188537597656,
+ "loss": 0.6921162605285645,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.016383877024054527,
+ "rewards/margins": 0.002122154226526618,
+ "rewards/rejected": 0.014261722564697266,
+ "step": 180
+ },
+ {
+ "epoch": 0.304,
+ "grad_norm": 1.1610571146011353,
+ "learning_rate": 9.99407231772377e-07,
+ "logits/chosen": 1.6443488597869873,
+ "logits/rejected": 1.5873346328735352,
+ "logps/chosen": -157.59298706054688,
+ "logps/rejected": -155.9975128173828,
+ "loss": 0.6925239086151123,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.025818049907684326,
+ "rewards/margins": 0.0013327361084520817,
+ "rewards/rejected": 0.024485310539603233,
+ "step": 190
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 1.2108772993087769,
+ "learning_rate": 9.93479549496147e-07,
+ "logits/chosen": 1.3679401874542236,
+ "logits/rejected": 1.5478118658065796,
+ "logps/chosen": -127.5545654296875,
+ "logps/rejected": -134.0937042236328,
+ "loss": 0.6889955520629882,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.033060748130083084,
+ "rewards/margins": 0.008453840389847755,
+ "rewards/rejected": 0.02460690215229988,
+ "step": 200
+ },
+ {
+ "epoch": 0.32,
+ "eval_logits/chosen": 1.5197134017944336,
+ "eval_logits/rejected": 1.5536755323410034,
+ "eval_logps/chosen": -153.1664581298828,
+ "eval_logps/rejected": -147.66543579101562,
+ "eval_loss": 0.6919757127761841,
+ "eval_rewards/accuracies": 0.5239999890327454,
+ "eval_rewards/chosen": 0.03824040666222572,
+ "eval_rewards/margins": 0.0025693033821880817,
+ "eval_rewards/rejected": 0.03567110374569893,
+ "eval_runtime": 91.3674,
+ "eval_samples_per_second": 5.472,
+ "eval_steps_per_second": 2.736,
+ "step": 200
+ },
+ {
+ "epoch": 0.336,
+ "grad_norm": 1.4075576066970825,
+ "learning_rate": 9.87551867219917e-07,
+ "logits/chosen": 1.6747233867645264,
+ "logits/rejected": 1.557538628578186,
+ "logps/chosen": -151.46202087402344,
+ "logps/rejected": -126.70817565917969,
+ "loss": 0.6903837203979493,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.0453563891351223,
+ "rewards/margins": 0.005785389803349972,
+ "rewards/rejected": 0.03957099840044975,
+ "step": 210
+ },
+ {
+ "epoch": 0.352,
+ "grad_norm": 1.05863356590271,
+ "learning_rate": 9.81624184943687e-07,
+ "logits/chosen": 1.7199232578277588,
+ "logits/rejected": 1.6891214847564697,
+ "logps/chosen": -164.67184448242188,
+ "logps/rejected": -143.46157836914062,
+ "loss": 0.6919777870178223,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.0621187686920166,
+ "rewards/margins": 0.002725570695474744,
+ "rewards/rejected": 0.05939319729804993,
+ "step": 220
+ },
+ {
+ "epoch": 0.368,
+ "grad_norm": 0.9537659287452698,
+ "learning_rate": 9.75696502667457e-07,
+ "logits/chosen": 1.5906708240509033,
+ "logits/rejected": 1.5525639057159424,
+ "logps/chosen": -149.23095703125,
+ "logps/rejected": -154.40731811523438,
+ "loss": 0.6942371845245361,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.061252254992723465,
+ "rewards/margins": -0.0017460950184613466,
+ "rewards/rejected": 0.06299833953380585,
+ "step": 230
+ },
+ {
+ "epoch": 0.384,
+ "grad_norm": 0.9180606603622437,
+ "learning_rate": 9.69768820391227e-07,
+ "logits/chosen": 1.5654420852661133,
+ "logits/rejected": 1.6795461177825928,
+ "logps/chosen": -158.65875244140625,
+ "logps/rejected": -165.13951110839844,
+ "loss": 0.692426061630249,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.06773029267787933,
+ "rewards/margins": 0.0019084472442045808,
+ "rewards/rejected": 0.06582184135913849,
+ "step": 240
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.7486060857772827,
+ "learning_rate": 9.63841138114997e-07,
+ "logits/chosen": 1.5377140045166016,
+ "logits/rejected": 1.6011888980865479,
+ "logps/chosen": -138.14199829101562,
+ "logps/rejected": -151.1943817138672,
+ "loss": 0.6908403396606445,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.06411711871623993,
+ "rewards/margins": 0.004912695847451687,
+ "rewards/rejected": 0.05920442193746567,
+ "step": 250
+ },
+ {
+ "epoch": 0.4,
+ "eval_logits/chosen": 1.534494161605835,
+ "eval_logits/rejected": 1.5683013200759888,
+ "eval_logps/chosen": -152.8787078857422,
+ "eval_logps/rejected": -147.39991760253906,
+ "eval_loss": 0.6910172700881958,
+ "eval_rewards/accuracies": 0.5460000038146973,
+ "eval_rewards/chosen": 0.06701447814702988,
+ "eval_rewards/margins": 0.004791789222508669,
+ "eval_rewards/rejected": 0.06222269684076309,
+ "eval_runtime": 91.3518,
+ "eval_samples_per_second": 5.473,
+ "eval_steps_per_second": 2.737,
+ "step": 250
+ },
+ {
+ "epoch": 0.416,
+ "grad_norm": 1.0861977338790894,
+ "learning_rate": 9.579134558387671e-07,
+ "logits/chosen": 1.5887064933776855,
+ "logits/rejected": 1.6124156713485718,
+ "logps/chosen": -145.68692016601562,
+ "logps/rejected": -150.8065948486328,
+ "loss": 0.6914952278137207,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.07477976381778717,
+ "rewards/margins": 0.00367984501644969,
+ "rewards/rejected": 0.0710999146103859,
+ "step": 260
+ },
+ {
+ "epoch": 0.432,
+ "grad_norm": 1.2531821727752686,
+ "learning_rate": 9.519857735625369e-07,
+ "logits/chosen": 1.552918553352356,
+ "logits/rejected": 1.6837631464004517,
+ "logps/chosen": -158.31802368164062,
+ "logps/rejected": -151.29379272460938,
+ "loss": 0.691353178024292,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.07334339618682861,
+ "rewards/margins": 0.004158434458076954,
+ "rewards/rejected": 0.06918496638536453,
+ "step": 270
+ },
+ {
+ "epoch": 0.448,
+ "grad_norm": 0.8373661637306213,
+ "learning_rate": 9.460580912863069e-07,
+ "logits/chosen": 1.6604083776474,
+ "logits/rejected": 1.7040210962295532,
+ "logps/chosen": -132.56422424316406,
+ "logps/rejected": -151.414794921875,
+ "loss": 0.6947796821594239,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.07179545611143112,
+ "rewards/margins": -0.002721407450735569,
+ "rewards/rejected": 0.07451687008142471,
+ "step": 280
+ },
+ {
+ "epoch": 0.464,
+ "grad_norm": 1.0909446477890015,
+ "learning_rate": 9.40130409010077e-07,
+ "logits/chosen": 1.6112158298492432,
+ "logits/rejected": 1.6040490865707397,
+ "logps/chosen": -155.4887237548828,
+ "logps/rejected": -149.31924438476562,
+ "loss": 0.6928027629852295,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.07096447050571442,
+ "rewards/margins": 0.0013116076588630676,
+ "rewards/rejected": 0.06965287029743195,
+ "step": 290
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 0.8769014477729797,
+ "learning_rate": 9.34202726733847e-07,
+ "logits/chosen": 1.4986810684204102,
+ "logits/rejected": 1.524269700050354,
+ "logps/chosen": -138.91099548339844,
+ "logps/rejected": -141.48495483398438,
+ "loss": 0.6924718379974365,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.060377366840839386,
+ "rewards/margins": 0.0016540264477953315,
+ "rewards/rejected": 0.058723341673612595,
+ "step": 300
+ },
+ {
+ "epoch": 0.48,
+ "eval_logits/chosen": 1.5390068292617798,
+ "eval_logits/rejected": 1.573019027709961,
+ "eval_logps/chosen": -152.91844177246094,
+ "eval_logps/rejected": -147.44703674316406,
+ "eval_loss": 0.6906248331069946,
+ "eval_rewards/accuracies": 0.5680000185966492,
+ "eval_rewards/chosen": 0.06304025650024414,
+ "eval_rewards/margins": 0.005528101697564125,
+ "eval_rewards/rejected": 0.057512152940034866,
+ "eval_runtime": 91.3495,
+ "eval_samples_per_second": 5.473,
+ "eval_steps_per_second": 2.737,
+ "step": 300
+ },
+ {
+ "epoch": 0.496,
+ "grad_norm": 1.050471305847168,
+ "learning_rate": 9.28275044457617e-07,
+ "logits/chosen": 1.7194430828094482,
+ "logits/rejected": 1.807745337486267,
+ "logps/chosen": -160.05563354492188,
+ "logps/rejected": -138.70767211914062,
+ "loss": 0.6887433052062988,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0673876479268074,
+ "rewards/margins": 0.009202037006616592,
+ "rewards/rejected": 0.058185599744319916,
+ "step": 310
+ },
+ {
+ "epoch": 0.512,
+ "grad_norm": 1.1666924953460693,
+ "learning_rate": 9.223473621813871e-07,
+ "logits/chosen": 1.737310767173767,
+ "logits/rejected": 1.6316074132919312,
+ "logps/chosen": -151.58229064941406,
+ "logps/rejected": -142.8289794921875,
+ "loss": 0.6919591426849365,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.06055358797311783,
+ "rewards/margins": 0.0026850434951484203,
+ "rewards/rejected": 0.057868532836437225,
+ "step": 320
+ },
+ {
+ "epoch": 0.528,
+ "grad_norm": 1.2360730171203613,
+ "learning_rate": 9.16419679905157e-07,
+ "logits/chosen": 1.4524610042572021,
+ "logits/rejected": 1.490668535232544,
+ "logps/chosen": -136.93104553222656,
+ "logps/rejected": -155.99844360351562,
+ "loss": 0.6895017623901367,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.06680475920438766,
+ "rewards/margins": 0.007821018807590008,
+ "rewards/rejected": 0.05898374319076538,
+ "step": 330
+ },
+ {
+ "epoch": 0.544,
+ "grad_norm": 1.0067647695541382,
+ "learning_rate": 9.10491997628927e-07,
+ "logits/chosen": 1.5482432842254639,
+ "logits/rejected": 1.596333622932434,
+ "logps/chosen": -142.59603881835938,
+ "logps/rejected": -120.35069274902344,
+ "loss": 0.6895721912384033,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.0686279758810997,
+ "rewards/margins": 0.007503635250031948,
+ "rewards/rejected": 0.06112435460090637,
+ "step": 340
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 1.2567517757415771,
+ "learning_rate": 9.04564315352697e-07,
+ "logits/chosen": 1.5976614952087402,
+ "logits/rejected": 1.3774999380111694,
+ "logps/chosen": -137.83567810058594,
+ "logps/rejected": -134.2790985107422,
+ "loss": 0.6924222946166992,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06484080851078033,
+ "rewards/margins": 0.0020756719168275595,
+ "rewards/rejected": 0.06276513636112213,
+ "step": 350
+ },
+ {
+ "epoch": 0.56,
+ "eval_logits/chosen": 1.5381687879562378,
+ "eval_logits/rejected": 1.5718625783920288,
+ "eval_logps/chosen": -152.91278076171875,
+ "eval_logps/rejected": -147.4539794921875,
+ "eval_loss": 0.6900433301925659,
+ "eval_rewards/accuracies": 0.5559999942779541,
+ "eval_rewards/chosen": 0.06360553950071335,
+ "eval_rewards/margins": 0.006787541322410107,
+ "eval_rewards/rejected": 0.05681798979640007,
+ "eval_runtime": 91.4768,
+ "eval_samples_per_second": 5.466,
+ "eval_steps_per_second": 2.733,
+ "step": 350
+ },
+ {
+ "epoch": 0.576,
+ "grad_norm": 1.3803397417068481,
+ "learning_rate": 8.986366330764671e-07,
+ "logits/chosen": 1.6290550231933594,
+ "logits/rejected": 1.7498657703399658,
+ "logps/chosen": -163.21676635742188,
+ "logps/rejected": -170.33084106445312,
+ "loss": 0.6899758338928222,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06653806567192078,
+ "rewards/margins": 0.006778226234018803,
+ "rewards/rejected": 0.05975983664393425,
+ "step": 360
+ },
+ {
+ "epoch": 0.592,
+ "grad_norm": 1.1363084316253662,
+ "learning_rate": 8.927089508002371e-07,
+ "logits/chosen": 1.6613603830337524,
+ "logits/rejected": 1.7096723318099976,
+ "logps/chosen": -175.3104248046875,
+ "logps/rejected": -176.24655151367188,
+ "loss": 0.690467643737793,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.061445217579603195,
+ "rewards/margins": 0.005937105976045132,
+ "rewards/rejected": 0.05550810694694519,
+ "step": 370
+ },
+ {
+ "epoch": 0.608,
+ "grad_norm": 1.139722228050232,
+ "learning_rate": 8.86781268524007e-07,
+ "logits/chosen": 1.6426604986190796,
+ "logits/rejected": 1.7499873638153076,
+ "logps/chosen": -136.9043731689453,
+ "logps/rejected": -159.98196411132812,
+ "loss": 0.6899127960205078,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.05270732566714287,
+ "rewards/margins": 0.007067640777677298,
+ "rewards/rejected": 0.04563968628644943,
+ "step": 380
+ },
+ {
+ "epoch": 0.624,
+ "grad_norm": 1.2813146114349365,
+ "learning_rate": 8.80853586247777e-07,
+ "logits/chosen": 1.3995320796966553,
+ "logits/rejected": 1.6828539371490479,
+ "logps/chosen": -131.1273651123047,
+ "logps/rejected": -167.51840209960938,
+ "loss": 0.6943269729614258,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.05107322335243225,
+ "rewards/margins": -0.0017629960784688592,
+ "rewards/rejected": 0.05283621698617935,
+ "step": 390
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 1.1288373470306396,
+ "learning_rate": 8.749259039715471e-07,
+ "logits/chosen": 1.6386661529541016,
+ "logits/rejected": 1.6272175312042236,
+ "logps/chosen": -150.4084014892578,
+ "logps/rejected": -161.52236938476562,
+ "loss": 0.6923639297485351,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.039307981729507446,
+ "rewards/margins": 0.001983280060812831,
+ "rewards/rejected": 0.037324704229831696,
+ "step": 400
+ },
+ {
+ "epoch": 0.64,
+ "eval_logits/chosen": 1.519494652748108,
+ "eval_logits/rejected": 1.5528141260147095,
+ "eval_logps/chosen": -153.15609741210938,
+ "eval_logps/rejected": -147.69198608398438,
+ "eval_loss": 0.6903249025344849,
+ "eval_rewards/accuracies": 0.5460000038146973,
+ "eval_rewards/chosen": 0.03927500545978546,
+ "eval_rewards/margins": 0.006258465815335512,
+ "eval_rewards/rejected": 0.033016543835401535,
+ "eval_runtime": 91.4257,
+ "eval_samples_per_second": 5.469,
+ "eval_steps_per_second": 2.734,
+ "step": 400
+ },
+ {
+ "epoch": 0.656,
+ "grad_norm": 1.355100154876709,
+ "learning_rate": 8.689982216953171e-07,
+ "logits/chosen": 1.5310468673706055,
+ "logits/rejected": 1.6076139211654663,
+ "logps/chosen": -157.39776611328125,
+ "logps/rejected": -145.91275024414062,
+ "loss": 0.68691987991333,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.04244391992688179,
+ "rewards/margins": 0.01302963774651289,
+ "rewards/rejected": 0.029414281249046326,
+ "step": 410
+ },
+ {
+ "epoch": 0.672,
+ "grad_norm": 1.1569719314575195,
+ "learning_rate": 8.630705394190871e-07,
+ "logits/chosen": 1.5669739246368408,
+ "logits/rejected": 1.5956265926361084,
+ "logps/chosen": -132.22671508789062,
+ "logps/rejected": -132.46981811523438,
+ "loss": 0.6866061210632324,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.04782433062791824,
+ "rewards/margins": 0.013592484407126904,
+ "rewards/rejected": 0.034231849014759064,
+ "step": 420
+ },
+ {
+ "epoch": 0.688,
+ "grad_norm": 1.1847658157348633,
+ "learning_rate": 8.57142857142857e-07,
+ "logits/chosen": 1.6808805465698242,
+ "logits/rejected": 1.7047704458236694,
+ "logps/chosen": -140.24200439453125,
+ "logps/rejected": -140.81124877929688,
+ "loss": 0.6933694362640381,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.047805462032556534,
+ "rewards/margins": 0.00025355032994411886,
+ "rewards/rejected": 0.04755191132426262,
+ "step": 430
+ },
+ {
+ "epoch": 0.704,
+ "grad_norm": 0.9608289003372192,
+ "learning_rate": 8.512151748666271e-07,
+ "logits/chosen": 1.663816213607788,
+ "logits/rejected": 1.6789140701293945,
+ "logps/chosen": -155.818603515625,
+ "logps/rejected": -145.31468200683594,
+ "loss": 0.6929332733154296,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.042977917939424515,
+ "rewards/margins": 0.0008615514962002635,
+ "rewards/rejected": 0.04211636632680893,
+ "step": 440
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 1.3275092840194702,
+ "learning_rate": 8.452874925903971e-07,
+ "logits/chosen": 1.4646484851837158,
+ "logits/rejected": 1.6175050735473633,
+ "logps/chosen": -151.04867553710938,
+ "logps/rejected": -147.8173828125,
+ "loss": 0.6903160572052002,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.03235078603029251,
+ "rewards/margins": 0.00654552411288023,
+ "rewards/rejected": 0.025805264711380005,
+ "step": 450
+ },
+ {
+ "epoch": 0.72,
+ "eval_logits/chosen": 1.5208625793457031,
+ "eval_logits/rejected": 1.5539581775665283,
+ "eval_logps/chosen": -153.08497619628906,
+ "eval_logps/rejected": -147.63291931152344,
+ "eval_loss": 0.6898273229598999,
+ "eval_rewards/accuracies": 0.5680000185966492,
+ "eval_rewards/chosen": 0.046387214213609695,
+ "eval_rewards/margins": 0.0074648563750088215,
+ "eval_rewards/rejected": 0.038922350853681564,
+ "eval_runtime": 91.384,
+ "eval_samples_per_second": 5.471,
+ "eval_steps_per_second": 2.736,
+ "step": 450
+ },
+ {
+ "epoch": 0.736,
+ "grad_norm": 1.3754528760910034,
+ "learning_rate": 8.393598103141671e-07,
+ "logits/chosen": 1.5359246730804443,
+ "logits/rejected": 1.604922890663147,
+ "logps/chosen": -137.5425262451172,
+ "logps/rejected": -161.14151000976562,
+ "loss": 0.6894284725189209,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.03450053930282593,
+ "rewards/margins": 0.007881317287683487,
+ "rewards/rejected": 0.02661922201514244,
+ "step": 460
+ },
+ {
+ "epoch": 0.752,
+ "grad_norm": 1.045366883277893,
+ "learning_rate": 8.334321280379371e-07,
+ "logits/chosen": 1.4338958263397217,
+ "logits/rejected": 1.5732755661010742,
+ "logps/chosen": -142.7367401123047,
+ "logps/rejected": -129.0875244140625,
+ "loss": 0.687935209274292,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03967561572790146,
+ "rewards/margins": 0.01102046575397253,
+ "rewards/rejected": 0.028655150905251503,
+ "step": 470
+ },
+ {
+ "epoch": 0.768,
+ "grad_norm": 1.232444167137146,
+ "learning_rate": 8.275044457617071e-07,
+ "logits/chosen": 1.7112070322036743,
+ "logits/rejected": 1.8045260906219482,
+ "logps/chosen": -170.90280151367188,
+ "logps/rejected": -164.41964721679688,
+ "loss": 0.6896602630615234,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.04624806344509125,
+ "rewards/margins": 0.0076986560598015785,
+ "rewards/rejected": 0.038549408316612244,
+ "step": 480
+ },
+ {
+ "epoch": 0.784,
+ "grad_norm": 1.2316750288009644,
+ "learning_rate": 8.215767634854771e-07,
+ "logits/chosen": 1.660528540611267,
+ "logits/rejected": 1.5905355215072632,
+ "logps/chosen": -166.0521697998047,
+ "logps/rejected": -164.01641845703125,
+ "loss": 0.6967196464538574,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": 0.022944677621126175,
+ "rewards/margins": -0.006097245961427689,
+ "rewards/rejected": 0.029041921719908714,
+ "step": 490
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 1.0982184410095215,
+ "learning_rate": 8.156490812092471e-07,
+ "logits/chosen": 1.5519205331802368,
+ "logits/rejected": 1.6819387674331665,
+ "logps/chosen": -137.09713745117188,
+ "logps/rejected": -139.62698364257812,
+ "loss": 0.6872400283813477,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.027622828260064125,
+ "rewards/margins": 0.012441830709576607,
+ "rewards/rejected": 0.015180995687842369,
+ "step": 500
+ },
+ {
+ "epoch": 0.8,
+ "eval_logits/chosen": 1.513393521308899,
+ "eval_logits/rejected": 1.5464614629745483,
+ "eval_logps/chosen": -153.2435302734375,
+ "eval_logps/rejected": -147.79180908203125,
+ "eval_loss": 0.6898292303085327,
+ "eval_rewards/accuracies": 0.5559999942779541,
+ "eval_rewards/chosen": 0.030531544238328934,
+ "eval_rewards/margins": 0.0074988617561757565,
+ "eval_rewards/rejected": 0.02303268015384674,
+ "eval_runtime": 91.4769,
+ "eval_samples_per_second": 5.466,
+ "eval_steps_per_second": 2.733,
+ "step": 500
+ },
+ {
+ "epoch": 0.816,
+ "grad_norm": 0.9861111640930176,
+ "learning_rate": 8.097213989330171e-07,
+ "logits/chosen": 1.7147877216339111,
+ "logits/rejected": 1.6248664855957031,
+ "logps/chosen": -166.3791961669922,
+ "logps/rejected": -152.08753967285156,
+ "loss": 0.6864038944244385,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.0399431511759758,
+ "rewards/margins": 0.014103055000305176,
+ "rewards/rejected": 0.025840098038315773,
+ "step": 510
+ },
+ {
+ "epoch": 0.832,
+ "grad_norm": 1.118251919746399,
+ "learning_rate": 8.037937166567872e-07,
+ "logits/chosen": 1.6375319957733154,
+ "logits/rejected": 1.6414417028427124,
+ "logps/chosen": -150.5874786376953,
+ "logps/rejected": -179.5028076171875,
+ "loss": 0.6879529476165771,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.03941132128238678,
+ "rewards/margins": 0.011486245319247246,
+ "rewards/rejected": 0.027925079688429832,
+ "step": 520
+ },
+ {
+ "epoch": 0.848,
+ "grad_norm": 1.354157567024231,
+ "learning_rate": 7.978660343805572e-07,
+ "logits/chosen": 1.5268175601959229,
+ "logits/rejected": 1.5682529211044312,
+ "logps/chosen": -144.65402221679688,
+ "logps/rejected": -136.36709594726562,
+ "loss": 0.6815488815307618,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.03592699021100998,
+ "rewards/margins": 0.02444549836218357,
+ "rewards/rejected": 0.011481491848826408,
+ "step": 530
+ },
+ {
+ "epoch": 0.864,
+ "grad_norm": 1.0916801691055298,
+ "learning_rate": 7.919383521043271e-07,
+ "logits/chosen": 1.4136483669281006,
+ "logits/rejected": 1.5612545013427734,
+ "logps/chosen": -128.98541259765625,
+ "logps/rejected": -149.62124633789062,
+ "loss": 0.6931351661682129,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.014613496139645576,
+ "rewards/margins": 0.0007547929999418557,
+ "rewards/rejected": 0.013858701102435589,
+ "step": 540
+ },
+ {
+ "epoch": 0.88,
+ "grad_norm": 1.2422736883163452,
+ "learning_rate": 7.860106698280971e-07,
+ "logits/chosen": 1.6898448467254639,
+ "logits/rejected": 1.7594168186187744,
+ "logps/chosen": -175.46859741210938,
+ "logps/rejected": -185.50271606445312,
+ "loss": 0.6864456176757813,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.030645186081528664,
+ "rewards/margins": 0.01440943218767643,
+ "rewards/rejected": 0.016235752031207085,
+ "step": 550
+ },
+ {
+ "epoch": 0.88,
+ "eval_logits/chosen": 1.5022823810577393,
+ "eval_logits/rejected": 1.5351035594940186,
+ "eval_logps/chosen": -153.427490234375,
+ "eval_logps/rejected": -147.97894287109375,
+ "eval_loss": 0.6898054480552673,
+ "eval_rewards/accuracies": 0.5580000281333923,
+ "eval_rewards/chosen": 0.012135841883718967,
+ "eval_rewards/margins": 0.007815876975655556,
+ "eval_rewards/rejected": 0.004319964908063412,
+ "eval_runtime": 91.5476,
+ "eval_samples_per_second": 5.462,
+ "eval_steps_per_second": 2.731,
+ "step": 550
+ },
+ {
+ "epoch": 0.896,
+ "grad_norm": 1.1276408433914185,
+ "learning_rate": 7.800829875518672e-07,
+ "logits/chosen": 1.466566801071167,
+ "logits/rejected": 1.3821640014648438,
+ "logps/chosen": -145.90614318847656,
+ "logps/rejected": -152.33578491210938,
+ "loss": 0.6930467128753662,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.01578265056014061,
+ "rewards/margins": 0.001082198810763657,
+ "rewards/rejected": 0.014700451865792274,
+ "step": 560
+ },
+ {
+ "epoch": 0.912,
+ "grad_norm": 1.0809053182601929,
+ "learning_rate": 7.741553052756372e-07,
+ "logits/chosen": 1.6201200485229492,
+ "logits/rejected": 1.6171897649765015,
+ "logps/chosen": -150.1614990234375,
+ "logps/rejected": -137.40005493164062,
+ "loss": 0.6915182590484619,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.015585039742290974,
+ "rewards/margins": 0.003875966649502516,
+ "rewards/rejected": 0.01170907262712717,
+ "step": 570
+ },
+ {
+ "epoch": 0.928,
+ "grad_norm": 1.3044863939285278,
+ "learning_rate": 7.682276229994072e-07,
+ "logits/chosen": 1.688551902770996,
+ "logits/rejected": 1.6596629619598389,
+ "logps/chosen": -168.21104431152344,
+ "logps/rejected": -137.5209197998047,
+ "loss": 0.6888096332550049,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.00975596159696579,
+ "rewards/margins": 0.009401545859873295,
+ "rewards/rejected": 0.00035441629006527364,
+ "step": 580
+ },
+ {
+ "epoch": 0.944,
+ "grad_norm": 1.3138145208358765,
+ "learning_rate": 7.622999407231771e-07,
+ "logits/chosen": 1.5617314577102661,
+ "logits/rejected": 1.4952644109725952,
+ "logps/chosen": -177.6552734375,
+ "logps/rejected": -156.41143798828125,
+ "loss": 0.6811863422393799,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.023103689774870872,
+ "rewards/margins": 0.02511446550488472,
+ "rewards/rejected": -0.0020107750315219164,
+ "step": 590
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 1.284184217453003,
+ "learning_rate": 7.563722584469472e-07,
+ "logits/chosen": 1.5531961917877197,
+ "logits/rejected": 1.5904725790023804,
+ "logps/chosen": -163.69615173339844,
+ "logps/rejected": -155.56857299804688,
+ "loss": 0.6912278652191162,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.021354522556066513,
+ "rewards/margins": 0.005431803874671459,
+ "rewards/rejected": 0.01592271961271763,
+ "step": 600
+ },
+ {
+ "epoch": 0.96,
+ "eval_logits/chosen": 1.5025317668914795,
+ "eval_logits/rejected": 1.5352839231491089,
+ "eval_logps/chosen": -153.3654022216797,
+ "eval_logps/rejected": -147.92263793945312,
+ "eval_loss": 0.6896048784255981,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": 0.018345065414905548,
+ "eval_rewards/margins": 0.008394664153456688,
+ "eval_rewards/rejected": 0.009950404055416584,
+ "eval_runtime": 91.5004,
+ "eval_samples_per_second": 5.464,
+ "eval_steps_per_second": 2.732,
+ "step": 600
+ },
+ {
+ "epoch": 0.976,
+ "grad_norm": 1.136492371559143,
+ "learning_rate": 7.504445761707172e-07,
+ "logits/chosen": 1.5187523365020752,
+ "logits/rejected": 1.5115095376968384,
+ "logps/chosen": -123.2374267578125,
+ "logps/rejected": -121.64241790771484,
+ "loss": 0.6866262912750244,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.04122164472937584,
+ "rewards/margins": 0.013987530954182148,
+ "rewards/rejected": 0.027234116569161415,
+ "step": 610
+ },
+ {
+ "epoch": 0.992,
+ "grad_norm": 1.3648184537887573,
+ "learning_rate": 7.445168938944872e-07,
+ "logits/chosen": 1.2267796993255615,
+ "logits/rejected": 1.318672776222229,
+ "logps/chosen": -125.6270751953125,
+ "logps/rejected": -139.3863067626953,
+ "loss": 0.6841369152069092,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.02149296924471855,
+ "rewards/margins": 0.02048521675169468,
+ "rewards/rejected": 0.001007752143777907,
+ "step": 620
+ },
+ {
+ "epoch": 1.008,
+ "grad_norm": 0.8776348233222961,
+ "learning_rate": 7.385892116182572e-07,
+ "logits/chosen": 1.4997355937957764,
+ "logits/rejected": 1.376514196395874,
+ "logps/chosen": -152.75949096679688,
+ "logps/rejected": -142.19186401367188,
+ "loss": 0.6843707084655761,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.03699764981865883,
+ "rewards/margins": 0.01918930374085903,
+ "rewards/rejected": 0.01780834048986435,
+ "step": 630
+ },
+ {
+ "epoch": 1.024,
+ "grad_norm": 1.2698174715042114,
+ "learning_rate": 7.326615293420272e-07,
+ "logits/chosen": 1.4477511644363403,
+ "logits/rejected": 1.5352327823638916,
+ "logps/chosen": -146.83685302734375,
+ "logps/rejected": -151.5980224609375,
+ "loss": 0.6780704021453857,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.04432147741317749,
+ "rewards/margins": 0.031893618404865265,
+ "rewards/rejected": 0.012427863664925098,
+ "step": 640
+ },
+ {
+ "epoch": 1.04,
+ "grad_norm": 1.309323787689209,
+ "learning_rate": 7.267338470657972e-07,
+ "logits/chosen": 1.6456422805786133,
+ "logits/rejected": 1.7331817150115967,
+ "logps/chosen": -148.64895629882812,
+ "logps/rejected": -160.71920776367188,
+ "loss": 0.6779833793640136,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.049473900347948074,
+ "rewards/margins": 0.0323341004550457,
+ "rewards/rejected": 0.017139799892902374,
+ "step": 650
+ },
+ {
+ "epoch": 1.04,
+ "eval_logits/chosen": 1.4995282888412476,
+ "eval_logits/rejected": 1.5318833589553833,
+ "eval_logps/chosen": -153.2689208984375,
+ "eval_logps/rejected": -147.83871459960938,
+ "eval_loss": 0.689191997051239,
+ "eval_rewards/accuracies": 0.5460000038146973,
+ "eval_rewards/chosen": 0.02799258381128311,
+ "eval_rewards/margins": 0.009650195948779583,
+ "eval_rewards/rejected": 0.018342390656471252,
+ "eval_runtime": 91.5151,
+ "eval_samples_per_second": 5.464,
+ "eval_steps_per_second": 2.732,
+ "step": 650
+ },
+ {
+ "epoch": 1.056,
+ "grad_norm": 1.1198511123657227,
+ "learning_rate": 7.208061647895672e-07,
+ "logits/chosen": 1.393791675567627,
+ "logits/rejected": 1.537705659866333,
+ "logps/chosen": -141.07870483398438,
+ "logps/rejected": -151.03375244140625,
+ "loss": 0.6852189064025879,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.028253447264432907,
+ "rewards/margins": 0.01726841926574707,
+ "rewards/rejected": 0.010985029861330986,
+ "step": 660
+ },
+ {
+ "epoch": 1.072,
+ "grad_norm": 1.2956689596176147,
+ "learning_rate": 7.148784825133372e-07,
+ "logits/chosen": 1.5089902877807617,
+ "logits/rejected": 1.5116627216339111,
+ "logps/chosen": -153.54898071289062,
+ "logps/rejected": -154.49032592773438,
+ "loss": 0.6772716045379639,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.04329819604754448,
+ "rewards/margins": 0.03352857381105423,
+ "rewards/rejected": 0.0097696203738451,
+ "step": 670
+ },
+ {
+ "epoch": 1.088,
+ "grad_norm": 1.0869190692901611,
+ "learning_rate": 7.089508002371073e-07,
+ "logits/chosen": 1.5537410974502563,
+ "logits/rejected": 1.4676992893218994,
+ "logps/chosen": -128.73751831054688,
+ "logps/rejected": -131.25894165039062,
+ "loss": 0.6872283458709717,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.02748076617717743,
+ "rewards/margins": 0.013209682889282703,
+ "rewards/rejected": 0.01427108608186245,
+ "step": 680
+ },
+ {
+ "epoch": 1.104,
+ "grad_norm": 1.2243199348449707,
+ "learning_rate": 7.030231179608772e-07,
+ "logits/chosen": 1.6543573141098022,
+ "logits/rejected": 1.7242790460586548,
+ "logps/chosen": -176.51280212402344,
+ "logps/rejected": -174.8518524169922,
+ "loss": 0.675547981262207,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.05593103915452957,
+ "rewards/margins": 0.0371597483754158,
+ "rewards/rejected": 0.01877129077911377,
+ "step": 690
+ },
+ {
+ "epoch": 1.12,
+ "grad_norm": 1.3488147258758545,
+ "learning_rate": 6.970954356846472e-07,
+ "logits/chosen": 1.5460538864135742,
+ "logits/rejected": 1.5656627416610718,
+ "logps/chosen": -146.79302978515625,
+ "logps/rejected": -140.67349243164062,
+ "loss": 0.669365406036377,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.04432321712374687,
+ "rewards/margins": 0.05032513290643692,
+ "rewards/rejected": -0.006001918110996485,
+ "step": 700
+ },
+ {
+ "epoch": 1.12,
+ "eval_logits/chosen": 1.5015324354171753,
+ "eval_logits/rejected": 1.5338712930679321,
+ "eval_logps/chosen": -153.185791015625,
+ "eval_logps/rejected": -147.7559814453125,
+ "eval_loss": 0.6892233490943909,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": 0.03630613535642624,
+ "eval_rewards/margins": 0.009690148755908012,
+ "eval_rewards/rejected": 0.026615988463163376,
+ "eval_runtime": 91.5231,
+ "eval_samples_per_second": 5.463,
+ "eval_steps_per_second": 2.732,
+ "step": 700
+ },
+ {
+ "epoch": 1.1360000000000001,
+ "grad_norm": 0.9973899722099304,
+ "learning_rate": 6.911677534084172e-07,
+ "logits/chosen": 1.6786168813705444,
+ "logits/rejected": 1.6147639751434326,
+ "logps/chosen": -175.9976806640625,
+ "logps/rejected": -145.07785034179688,
+ "loss": 0.6673734664916993,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.0721244290471077,
+ "rewards/margins": 0.054663002490997314,
+ "rewards/rejected": 0.017461419105529785,
+ "step": 710
+ },
+ {
+ "epoch": 1.152,
+ "grad_norm": 1.1944832801818848,
+ "learning_rate": 6.852400711321873e-07,
+ "logits/chosen": 1.452962040901184,
+ "logits/rejected": 1.4640306234359741,
+ "logps/chosen": -139.7812042236328,
+ "logps/rejected": -159.2677001953125,
+ "loss": 0.6787771224975586,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.07180429995059967,
+ "rewards/margins": 0.03057795763015747,
+ "rewards/rejected": 0.0412263348698616,
+ "step": 720
+ },
+ {
+ "epoch": 1.168,
+ "grad_norm": 1.4376368522644043,
+ "learning_rate": 6.793123888559573e-07,
+ "logits/chosen": 1.579025387763977,
+ "logits/rejected": 1.5297110080718994,
+ "logps/chosen": -161.11328125,
+ "logps/rejected": -164.56582641601562,
+ "loss": 0.680543327331543,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.07446161657571793,
+ "rewards/margins": 0.027710452675819397,
+ "rewards/rejected": 0.04675116389989853,
+ "step": 730
+ },
+ {
+ "epoch": 1.184,
+ "grad_norm": 1.1249836683273315,
+ "learning_rate": 6.733847065797273e-07,
+ "logits/chosen": 1.7372865676879883,
+ "logits/rejected": 1.7558908462524414,
+ "logps/chosen": -147.6786346435547,
+ "logps/rejected": -155.9397430419922,
+ "loss": 0.667045783996582,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.08958978950977325,
+ "rewards/margins": 0.05507674068212509,
+ "rewards/rejected": 0.03451304882764816,
+ "step": 740
+ },
+ {
+ "epoch": 1.2,
+ "grad_norm": 1.0353419780731201,
+ "learning_rate": 6.674570243034972e-07,
+ "logits/chosen": 1.6369037628173828,
+ "logits/rejected": 1.6020278930664062,
+ "logps/chosen": -166.5580291748047,
+ "logps/rejected": -144.23370361328125,
+ "loss": 0.6636831760406494,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.06881646066904068,
+ "rewards/margins": 0.06279782950878143,
+ "rewards/rejected": 0.0060186260379850864,
+ "step": 750
+ },
+ {
+ "epoch": 1.2,
+ "eval_logits/chosen": 1.4953525066375732,
+ "eval_logits/rejected": 1.5272784233093262,
+ "eval_logps/chosen": -153.105712890625,
+ "eval_logps/rejected": -147.6992950439453,
+ "eval_loss": 0.6883665919303894,
+ "eval_rewards/accuracies": 0.5460000038146973,
+ "eval_rewards/chosen": 0.044312380254268646,
+ "eval_rewards/margins": 0.012026800774037838,
+ "eval_rewards/rejected": 0.032285578548908234,
+ "eval_runtime": 91.4577,
+ "eval_samples_per_second": 5.467,
+ "eval_steps_per_second": 2.734,
+ "step": 750
+ },
+ {
+ "epoch": 1.216,
+ "grad_norm": 1.1240723133087158,
+ "learning_rate": 6.615293420272673e-07,
+ "logits/chosen": 1.4654918909072876,
+ "logits/rejected": 1.4545985460281372,
+ "logps/chosen": -156.47726440429688,
+ "logps/rejected": -138.5869598388672,
+ "loss": 0.6732029438018798,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.05434625223278999,
+ "rewards/margins": 0.04249870777130127,
+ "rewards/rejected": 0.011847544461488724,
+ "step": 760
+ },
+ {
+ "epoch": 1.232,
+ "grad_norm": 1.1530578136444092,
+ "learning_rate": 6.556016597510373e-07,
+ "logits/chosen": 1.5636401176452637,
+ "logits/rejected": 1.603150725364685,
+ "logps/chosen": -147.6873779296875,
+ "logps/rejected": -127.45448303222656,
+ "loss": 0.6720876216888427,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.07123544067144394,
+ "rewards/margins": 0.04455225542187691,
+ "rewards/rejected": 0.026683185249567032,
+ "step": 770
+ },
+ {
+ "epoch": 1.248,
+ "grad_norm": 1.0530054569244385,
+ "learning_rate": 6.496739774748073e-07,
+ "logits/chosen": 1.672349214553833,
+ "logits/rejected": 1.773258924484253,
+ "logps/chosen": -171.55245971679688,
+ "logps/rejected": -185.20631408691406,
+ "loss": 0.6770391464233398,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.06762111186981201,
+ "rewards/margins": 0.03522135317325592,
+ "rewards/rejected": 0.032399751245975494,
+ "step": 780
+ },
+ {
+ "epoch": 1.264,
+ "grad_norm": 0.9587673544883728,
+ "learning_rate": 6.437462951985773e-07,
+ "logits/chosen": 1.6674445867538452,
+ "logits/rejected": 1.5399541854858398,
+ "logps/chosen": -148.35202026367188,
+ "logps/rejected": -141.05218505859375,
+ "loss": 0.688765001296997,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.059925489127635956,
+ "rewards/margins": 0.011203404515981674,
+ "rewards/rejected": 0.04872208088636398,
+ "step": 790
+ },
+ {
+ "epoch": 1.28,
+ "grad_norm": 1.1872327327728271,
+ "learning_rate": 6.378186129223473e-07,
+ "logits/chosen": 1.4424043893814087,
+ "logits/rejected": 1.5345561504364014,
+ "logps/chosen": -142.57899475097656,
+ "logps/rejected": -138.53045654296875,
+ "loss": 0.6739780902862549,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.07486443221569061,
+ "rewards/margins": 0.04266515001654625,
+ "rewards/rejected": 0.032199278473854065,
+ "step": 800
+ },
+ {
+ "epoch": 1.28,
+ "eval_logits/chosen": 1.502331256866455,
+ "eval_logits/rejected": 1.5342037677764893,
+ "eval_logps/chosen": -152.94044494628906,
+ "eval_logps/rejected": -147.55020141601562,
+ "eval_loss": 0.6877647042274475,
+ "eval_rewards/accuracies": 0.5540000200271606,
+ "eval_rewards/chosen": 0.060839928686618805,
+ "eval_rewards/margins": 0.013645462691783905,
+ "eval_rewards/rejected": 0.0471944697201252,
+ "eval_runtime": 91.4749,
+ "eval_samples_per_second": 5.466,
+ "eval_steps_per_second": 2.733,
+ "step": 800
+ },
+ {
+ "epoch": 1.296,
+ "grad_norm": 1.1386040449142456,
+ "learning_rate": 6.318909306461173e-07,
+ "logits/chosen": 1.4093509912490845,
+ "logits/rejected": 1.4580585956573486,
+ "logps/chosen": -164.720947265625,
+ "logps/rejected": -142.99754333496094,
+ "loss": 0.6797928810119629,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.07539167255163193,
+ "rewards/margins": 0.0292592104524374,
+ "rewards/rejected": 0.046132463961839676,
+ "step": 810
+ },
+ {
+ "epoch": 1.312,
+ "grad_norm": 1.3657523393630981,
+ "learning_rate": 6.259632483698873e-07,
+ "logits/chosen": 1.3146798610687256,
+ "logits/rejected": 1.4988536834716797,
+ "logps/chosen": -141.5091552734375,
+ "logps/rejected": -138.624755859375,
+ "loss": 0.6713742733001709,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.06485269963741302,
+ "rewards/margins": 0.04707767441868782,
+ "rewards/rejected": 0.017775025218725204,
+ "step": 820
+ },
+ {
+ "epoch": 1.328,
+ "grad_norm": 1.207746148109436,
+ "learning_rate": 6.200355660936573e-07,
+ "logits/chosen": 1.5304832458496094,
+ "logits/rejected": 1.567846655845642,
+ "logps/chosen": -145.65390014648438,
+ "logps/rejected": -134.02752685546875,
+ "loss": 0.6716270923614502,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08788949996232986,
+ "rewards/margins": 0.04611582309007645,
+ "rewards/rejected": 0.041773684322834015,
+ "step": 830
+ },
+ {
+ "epoch": 1.3439999999999999,
+ "grad_norm": 1.3036965131759644,
+ "learning_rate": 6.141078838174274e-07,
+ "logits/chosen": 1.5840891599655151,
+ "logits/rejected": 1.500603199005127,
+ "logps/chosen": -138.60763549804688,
+ "logps/rejected": -125.70695495605469,
+ "loss": 0.6759689331054688,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.10840122401714325,
+ "rewards/margins": 0.037392571568489075,
+ "rewards/rejected": 0.07100865244865417,
+ "step": 840
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "grad_norm": 1.1700375080108643,
+ "learning_rate": 6.081802015411973e-07,
+ "logits/chosen": 1.5052621364593506,
+ "logits/rejected": 1.5524170398712158,
+ "logps/chosen": -133.3436279296875,
+ "logps/rejected": -137.5284423828125,
+ "loss": 0.6731984615325928,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.09424515068531036,
+ "rewards/margins": 0.04275429993867874,
+ "rewards/rejected": 0.051490843296051025,
+ "step": 850
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "eval_logits/chosen": 1.514055609703064,
+ "eval_logits/rejected": 1.545797348022461,
+ "eval_logps/chosen": -152.71902465820312,
+ "eval_logps/rejected": -147.34742736816406,
+ "eval_loss": 0.6870437264442444,
+ "eval_rewards/accuracies": 0.5540000200271606,
+ "eval_rewards/chosen": 0.08298191428184509,
+ "eval_rewards/margins": 0.015508526936173439,
+ "eval_rewards/rejected": 0.0674733817577362,
+ "eval_runtime": 91.4891,
+ "eval_samples_per_second": 5.465,
+ "eval_steps_per_second": 2.733,
+ "step": 850
+ },
+ {
+ "epoch": 1.376,
+ "grad_norm": 1.080214023590088,
+ "learning_rate": 6.022525192649673e-07,
+ "logits/chosen": 1.4343178272247314,
+ "logits/rejected": 1.5825798511505127,
+ "logps/chosen": -148.39657592773438,
+ "logps/rejected": -129.2698516845703,
+ "loss": 0.6728739261627197,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.10437977313995361,
+ "rewards/margins": 0.04413595423102379,
+ "rewards/rejected": 0.06024380773305893,
+ "step": 860
+ },
+ {
+ "epoch": 1.392,
+ "grad_norm": 1.3011701107025146,
+ "learning_rate": 5.963248369887373e-07,
+ "logits/chosen": 1.6486666202545166,
+ "logits/rejected": 1.636071801185608,
+ "logps/chosen": -155.72866821289062,
+ "logps/rejected": -156.0375213623047,
+ "loss": 0.6663424491882324,
+ "rewards/accuracies": 0.7875000238418579,
+ "rewards/chosen": 0.0980696976184845,
+ "rewards/margins": 0.05624913424253464,
+ "rewards/rejected": 0.04182056710124016,
+ "step": 870
+ },
+ {
+ "epoch": 1.408,
+ "grad_norm": 1.1527386903762817,
+ "learning_rate": 5.903971547125074e-07,
+ "logits/chosen": 1.6375783681869507,
+ "logits/rejected": 1.5879249572753906,
+ "logps/chosen": -138.84786987304688,
+ "logps/rejected": -149.22756958007812,
+ "loss": 0.6661319255828857,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.1091323122382164,
+ "rewards/margins": 0.05697597935795784,
+ "rewards/rejected": 0.05215633660554886,
+ "step": 880
+ },
+ {
+ "epoch": 1.424,
+ "grad_norm": 1.2128634452819824,
+ "learning_rate": 5.844694724362774e-07,
+ "logits/chosen": 1.643206238746643,
+ "logits/rejected": 1.529200792312622,
+ "logps/chosen": -156.31326293945312,
+ "logps/rejected": -144.78689575195312,
+ "loss": 0.6645416736602783,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.11794288456439972,
+ "rewards/margins": 0.06102023646235466,
+ "rewards/rejected": 0.05692264437675476,
+ "step": 890
+ },
+ {
+ "epoch": 1.44,
+ "grad_norm": 0.9876027703285217,
+ "learning_rate": 5.785417901600474e-07,
+ "logits/chosen": 1.3582518100738525,
+ "logits/rejected": 1.5023400783538818,
+ "logps/chosen": -137.19717407226562,
+ "logps/rejected": -167.57350158691406,
+ "loss": 0.6799286842346192,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.09652381390333176,
+ "rewards/margins": 0.02946191094815731,
+ "rewards/rejected": 0.0670619010925293,
+ "step": 900
+ },
+ {
+ "epoch": 1.44,
+ "eval_logits/chosen": 1.5112248659133911,
+ "eval_logits/rejected": 1.542572021484375,
+ "eval_logps/chosen": -152.71963500976562,
+ "eval_logps/rejected": -147.3575897216797,
+ "eval_loss": 0.6868481636047363,
+ "eval_rewards/accuracies": 0.5720000267028809,
+ "eval_rewards/chosen": 0.08292035758495331,
+ "eval_rewards/margins": 0.016464442014694214,
+ "eval_rewards/rejected": 0.0664559081196785,
+ "eval_runtime": 91.463,
+ "eval_samples_per_second": 5.467,
+ "eval_steps_per_second": 2.733,
+ "step": 900
+ },
+ {
+ "epoch": 1.456,
+ "grad_norm": 1.0684723854064941,
+ "learning_rate": 5.726141078838173e-07,
+ "logits/chosen": 1.5465466976165771,
+ "logits/rejected": 1.500496506690979,
+ "logps/chosen": -132.5164337158203,
+ "logps/rejected": -129.40220642089844,
+ "loss": 0.6799171924591064,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08448641002178192,
+ "rewards/margins": 0.028616121038794518,
+ "rewards/rejected": 0.05587029457092285,
+ "step": 910
+ },
+ {
+ "epoch": 1.472,
+ "grad_norm": 1.0342527627944946,
+ "learning_rate": 5.666864256075874e-07,
+ "logits/chosen": 1.6495583057403564,
+ "logits/rejected": 1.7119420766830444,
+ "logps/chosen": -139.18484497070312,
+ "logps/rejected": -154.33407592773438,
+ "loss": 0.6807973861694336,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.09809015691280365,
+ "rewards/margins": 0.027513395994901657,
+ "rewards/rejected": 0.0705767497420311,
+ "step": 920
+ },
+ {
+ "epoch": 1.488,
+ "grad_norm": 1.0507510900497437,
+ "learning_rate": 5.607587433313574e-07,
+ "logits/chosen": 1.5217338800430298,
+ "logits/rejected": 1.5648618936538696,
+ "logps/chosen": -131.5649871826172,
+ "logps/rejected": -122.32954406738281,
+ "loss": 0.681494951248169,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.08206742256879807,
+ "rewards/margins": 0.026795843616127968,
+ "rewards/rejected": 0.055271584540605545,
+ "step": 930
+ },
+ {
+ "epoch": 1.504,
+ "grad_norm": 1.3102085590362549,
+ "learning_rate": 5.548310610551274e-07,
+ "logits/chosen": 1.6052824258804321,
+ "logits/rejected": 1.6236565113067627,
+ "logps/chosen": -149.2860565185547,
+ "logps/rejected": -153.01657104492188,
+ "loss": 0.6783152103424073,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.10417164862155914,
+ "rewards/margins": 0.03314245864748955,
+ "rewards/rejected": 0.0710291862487793,
+ "step": 940
+ },
+ {
+ "epoch": 1.52,
+ "grad_norm": 1.3876780271530151,
+ "learning_rate": 5.489033787788974e-07,
+ "logits/chosen": 1.689612627029419,
+ "logits/rejected": 1.6765525341033936,
+ "logps/chosen": -164.98193359375,
+ "logps/rejected": -164.33975219726562,
+ "loss": 0.6782557964324951,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.09384694695472717,
+ "rewards/margins": 0.03179098665714264,
+ "rewards/rejected": 0.062055956572294235,
+ "step": 950
+ },
+ {
+ "epoch": 1.52,
+ "eval_logits/chosen": 1.5106096267700195,
+ "eval_logits/rejected": 1.5420260429382324,
+ "eval_logps/chosen": -152.7333526611328,
+ "eval_logps/rejected": -147.37344360351562,
+ "eval_loss": 0.6867280602455139,
+ "eval_rewards/accuracies": 0.5540000200271606,
+ "eval_rewards/chosen": 0.08154840767383575,
+ "eval_rewards/margins": 0.01667776331305504,
+ "eval_rewards/rejected": 0.06487063318490982,
+ "eval_runtime": 91.5472,
+ "eval_samples_per_second": 5.462,
+ "eval_steps_per_second": 2.731,
+ "step": 950
+ },
+ {
+ "epoch": 1.536,
+ "grad_norm": 1.0544084310531616,
+ "learning_rate": 5.429756965026674e-07,
+ "logits/chosen": 1.696502685546875,
+ "logits/rejected": 1.722957968711853,
+ "logps/chosen": -177.16172790527344,
+ "logps/rejected": -160.9434051513672,
+ "loss": 0.6731346130371094,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.09499023109674454,
+ "rewards/margins": 0.04484598711133003,
+ "rewards/rejected": 0.050144243985414505,
+ "step": 960
+ },
+ {
+ "epoch": 1.552,
+ "grad_norm": 1.2123764753341675,
+ "learning_rate": 5.370480142264374e-07,
+ "logits/chosen": 1.5045363903045654,
+ "logits/rejected": 1.4909541606903076,
+ "logps/chosen": -144.6968994140625,
+ "logps/rejected": -160.55947875976562,
+ "loss": 0.6804170131683349,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.08150655776262283,
+ "rewards/margins": 0.029505986720323563,
+ "rewards/rejected": 0.05200057104229927,
+ "step": 970
+ },
+ {
+ "epoch": 1.568,
+ "grad_norm": 1.0709542036056519,
+ "learning_rate": 5.311203319502074e-07,
+ "logits/chosen": 1.5131323337554932,
+ "logits/rejected": 1.5731028318405151,
+ "logps/chosen": -157.4567108154297,
+ "logps/rejected": -149.07107543945312,
+ "loss": 0.6691867351531983,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.08909507840871811,
+ "rewards/margins": 0.05152255296707153,
+ "rewards/rejected": 0.03757254034280777,
+ "step": 980
+ },
+ {
+ "epoch": 1.584,
+ "grad_norm": 1.329915165901184,
+ "learning_rate": 5.251926496739774e-07,
+ "logits/chosen": 1.574861764907837,
+ "logits/rejected": 1.5829271078109741,
+ "logps/chosen": -143.85403442382812,
+ "logps/rejected": -140.85009765625,
+ "loss": 0.6716786384582519,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08492995798587799,
+ "rewards/margins": 0.04930455982685089,
+ "rewards/rejected": 0.0356253944337368,
+ "step": 990
+ },
+ {
+ "epoch": 1.6,
+ "grad_norm": 1.3974183797836304,
+ "learning_rate": 5.192649673977475e-07,
+ "logits/chosen": 1.5636709928512573,
+ "logits/rejected": 1.3984944820404053,
+ "logps/chosen": -148.8516845703125,
+ "logps/rejected": -143.12136840820312,
+ "loss": 0.676781702041626,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.05405520275235176,
+ "rewards/margins": 0.035432908684015274,
+ "rewards/rejected": 0.018622295930981636,
+ "step": 1000
+ },
+ {
+ "epoch": 1.6,
+ "eval_logits/chosen": 1.5051932334899902,
+ "eval_logits/rejected": 1.5366464853286743,
+ "eval_logps/chosen": -152.83253479003906,
+ "eval_logps/rejected": -147.4718475341797,
+ "eval_loss": 0.6868343353271484,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.07163143903017044,
+ "eval_rewards/margins": 0.016600240021944046,
+ "eval_rewards/rejected": 0.055031195282936096,
+ "eval_runtime": 91.6291,
+ "eval_samples_per_second": 5.457,
+ "eval_steps_per_second": 2.728,
+ "step": 1000
+ },
+ {
+ "epoch": 1.616,
+ "grad_norm": 1.2400332689285278,
+ "learning_rate": 5.133372851215174e-07,
+ "logits/chosen": 1.5299904346466064,
+ "logits/rejected": 1.6691240072250366,
+ "logps/chosen": -150.43557739257812,
+ "logps/rejected": -170.71127319335938,
+ "loss": 0.6721617221832276,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.08803285658359528,
+ "rewards/margins": 0.04526625573635101,
+ "rewards/rejected": 0.04276660084724426,
+ "step": 1010
+ },
+ {
+ "epoch": 1.6320000000000001,
+ "grad_norm": 0.8514426350593567,
+ "learning_rate": 5.074096028452874e-07,
+ "logits/chosen": 1.4948112964630127,
+ "logits/rejected": 1.515966773033142,
+ "logps/chosen": -118.82080078125,
+ "logps/rejected": -139.77511596679688,
+ "loss": 0.6717492580413819,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.05406653881072998,
+ "rewards/margins": 0.04632322117686272,
+ "rewards/rejected": 0.007743313908576965,
+ "step": 1020
+ },
+ {
+ "epoch": 1.6480000000000001,
+ "grad_norm": 1.5450512170791626,
+ "learning_rate": 5.014819205690574e-07,
+ "logits/chosen": 1.5520379543304443,
+ "logits/rejected": 1.5921989679336548,
+ "logps/chosen": -147.900634765625,
+ "logps/rejected": -138.66140747070312,
+ "loss": 0.6880045413970948,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.06136086583137512,
+ "rewards/margins": 0.014043760485947132,
+ "rewards/rejected": 0.04731711000204086,
+ "step": 1030
+ },
+ {
+ "epoch": 1.6640000000000001,
+ "grad_norm": 1.1967573165893555,
+ "learning_rate": 4.955542382928275e-07,
+ "logits/chosen": 1.5113961696624756,
+ "logits/rejected": 1.4426608085632324,
+ "logps/chosen": -139.90713500976562,
+ "logps/rejected": -155.5555877685547,
+ "loss": 0.6754290580749511,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.06368472427129745,
+ "rewards/margins": 0.04256365820765495,
+ "rewards/rejected": 0.021121066063642502,
+ "step": 1040
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "grad_norm": 1.758066177368164,
+ "learning_rate": 4.896265560165974e-07,
+ "logits/chosen": 1.6157357692718506,
+ "logits/rejected": 1.578230857849121,
+ "logps/chosen": -141.13455200195312,
+ "logps/rejected": -166.70489501953125,
+ "loss": 0.6755188941955567,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.05974128097295761,
+ "rewards/margins": 0.03907640278339386,
+ "rewards/rejected": 0.0206648800522089,
+ "step": 1050
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "eval_logits/chosen": 1.4930020570755005,
+ "eval_logits/rejected": 1.5244908332824707,
+ "eval_logps/chosen": -153.0406036376953,
+ "eval_logps/rejected": -147.66622924804688,
+ "eval_loss": 0.6875224709510803,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": 0.05082407966256142,
+ "eval_rewards/margins": 0.01523309201002121,
+ "eval_rewards/rejected": 0.03559098765254021,
+ "eval_runtime": 91.5108,
+ "eval_samples_per_second": 5.464,
+ "eval_steps_per_second": 2.732,
+ "step": 1050
+ },
+ {
+ "epoch": 1.696,
+ "grad_norm": 1.2342020273208618,
+ "learning_rate": 4.836988737403675e-07,
+ "logits/chosen": 1.2796365022659302,
+ "logits/rejected": 1.3894035816192627,
+ "logps/chosen": -145.9099578857422,
+ "logps/rejected": -148.60482788085938,
+ "loss": 0.6661100864410401,
+ "rewards/accuracies": 0.800000011920929,
+ "rewards/chosen": 0.055561065673828125,
+ "rewards/margins": 0.057345401495695114,
+ "rewards/rejected": -0.0017843395471572876,
+ "step": 1060
+ },
+ {
+ "epoch": 1.712,
+ "grad_norm": 1.3287626504898071,
+ "learning_rate": 4.777711914641375e-07,
+ "logits/chosen": 1.792177438735962,
+ "logits/rejected": 1.648444414138794,
+ "logps/chosen": -159.8272247314453,
+ "logps/rejected": -182.2501983642578,
+ "loss": 0.6666016101837158,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.08213181793689728,
+ "rewards/margins": 0.05758446455001831,
+ "rewards/rejected": 0.024547358974814415,
+ "step": 1070
+ },
+ {
+ "epoch": 1.728,
+ "grad_norm": 0.9578614234924316,
+ "learning_rate": 4.718435091879075e-07,
+ "logits/chosen": 1.55452561378479,
+ "logits/rejected": 1.5692381858825684,
+ "logps/chosen": -154.53590393066406,
+ "logps/rejected": -142.9732666015625,
+ "loss": 0.6688636302947998,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.08060242235660553,
+ "rewards/margins": 0.053381383419036865,
+ "rewards/rejected": 0.02722104825079441,
+ "step": 1080
+ },
+ {
+ "epoch": 1.744,
+ "grad_norm": 1.1178059577941895,
+ "learning_rate": 4.6591582691167753e-07,
+ "logits/chosen": 1.607924461364746,
+ "logits/rejected": 1.7212623357772827,
+ "logps/chosen": -152.4365234375,
+ "logps/rejected": -157.656494140625,
+ "loss": 0.6775742053985596,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.0737459659576416,
+ "rewards/margins": 0.03416898101568222,
+ "rewards/rejected": 0.03957698121666908,
+ "step": 1090
+ },
+ {
+ "epoch": 1.76,
+ "grad_norm": 1.4560215473175049,
+ "learning_rate": 4.599881446354475e-07,
+ "logits/chosen": 1.6074415445327759,
+ "logits/rejected": 1.5955140590667725,
+ "logps/chosen": -171.25421142578125,
+ "logps/rejected": -124.63997650146484,
+ "loss": 0.6695756435394287,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.040774114429950714,
+ "rewards/margins": 0.05189450830221176,
+ "rewards/rejected": -0.01112039852887392,
+ "step": 1100
+ },
+ {
+ "epoch": 1.76,
+ "eval_logits/chosen": 1.4804071187973022,
+ "eval_logits/rejected": 1.5116674900054932,
+ "eval_logps/chosen": -153.20590209960938,
+ "eval_logps/rejected": -147.82687377929688,
+ "eval_loss": 0.6880003809928894,
+ "eval_rewards/accuracies": 0.5479999780654907,
+ "eval_rewards/chosen": 0.03429403901100159,
+ "eval_rewards/margins": 0.014766983687877655,
+ "eval_rewards/rejected": 0.019527053460478783,
+ "eval_runtime": 91.5827,
+ "eval_samples_per_second": 5.46,
+ "eval_steps_per_second": 2.73,
+ "step": 1100
+ },
+ {
+ "epoch": 1.776,
+ "grad_norm": 1.113887071609497,
+ "learning_rate": 4.5406046235921753e-07,
+ "logits/chosen": 1.6315574645996094,
+ "logits/rejected": 1.5989868640899658,
+ "logps/chosen": -155.7100067138672,
+ "logps/rejected": -157.9479522705078,
+ "loss": 0.6718159675598144,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.05936906486749649,
+ "rewards/margins": 0.04729412868618965,
+ "rewards/rejected": 0.012074939906597137,
+ "step": 1110
+ },
+ {
+ "epoch": 1.792,
+ "grad_norm": 0.9409530162811279,
+ "learning_rate": 4.481327800829875e-07,
+ "logits/chosen": 1.47927987575531,
+ "logits/rejected": 1.6135873794555664,
+ "logps/chosen": -143.21559143066406,
+ "logps/rejected": -155.91262817382812,
+ "loss": 0.6721374988555908,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.06062264367938042,
+ "rewards/margins": 0.04633890092372894,
+ "rewards/rejected": 0.0142837418243289,
+ "step": 1120
+ },
+ {
+ "epoch": 1.808,
+ "grad_norm": 1.1281055212020874,
+ "learning_rate": 4.422050978067575e-07,
+ "logits/chosen": 1.6088281869888306,
+ "logits/rejected": 1.6764856576919556,
+ "logps/chosen": -143.90219116210938,
+ "logps/rejected": -145.31483459472656,
+ "loss": 0.6717781066894531,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.037483494728803635,
+ "rewards/margins": 0.04687192291021347,
+ "rewards/rejected": -0.009388430044054985,
+ "step": 1130
+ },
+ {
+ "epoch": 1.8239999999999998,
+ "grad_norm": 1.2734681367874146,
+ "learning_rate": 4.362774155305275e-07,
+ "logits/chosen": 1.3558837175369263,
+ "logits/rejected": 1.4319185018539429,
+ "logps/chosen": -133.63270568847656,
+ "logps/rejected": -133.51321411132812,
+ "loss": 0.6760762214660645,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03574688360095024,
+ "rewards/margins": 0.03954789787530899,
+ "rewards/rejected": -0.0038010149728506804,
+ "step": 1140
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "grad_norm": 1.5277918577194214,
+ "learning_rate": 4.303497332542976e-07,
+ "logits/chosen": 1.45877206325531,
+ "logits/rejected": 1.5461132526397705,
+ "logps/chosen": -146.4346466064453,
+ "logps/rejected": -157.70391845703125,
+ "loss": 0.676148509979248,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.05790795013308525,
+ "rewards/margins": 0.03729299455881119,
+ "rewards/rejected": 0.020614955574274063,
+ "step": 1150
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "eval_logits/chosen": 1.4798471927642822,
+ "eval_logits/rejected": 1.5109448432922363,
+ "eval_logps/chosen": -153.19979858398438,
+ "eval_logps/rejected": -147.82022094726562,
+ "eval_loss": 0.6881932616233826,
+ "eval_rewards/accuracies": 0.5479999780654907,
+ "eval_rewards/chosen": 0.03490476310253143,
+ "eval_rewards/margins": 0.01471200492233038,
+ "eval_rewards/rejected": 0.020192759111523628,
+ "eval_runtime": 91.6364,
+ "eval_samples_per_second": 5.456,
+ "eval_steps_per_second": 2.728,
+ "step": 1150
+ },
+ {
+ "epoch": 1.8559999999999999,
+ "grad_norm": 1.274863839149475,
+ "learning_rate": 4.244220509780675e-07,
+ "logits/chosen": 1.4154646396636963,
+ "logits/rejected": 1.4470824003219604,
+ "logps/chosen": -126.20066833496094,
+ "logps/rejected": -151.97293090820312,
+ "loss": 0.6712587356567383,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.043645404279232025,
+ "rewards/margins": 0.047874242067337036,
+ "rewards/rejected": -0.004228837788105011,
+ "step": 1160
+ },
+ {
+ "epoch": 1.8719999999999999,
+ "grad_norm": 1.5296283960342407,
+ "learning_rate": 4.184943687018376e-07,
+ "logits/chosen": 1.6431888341903687,
+ "logits/rejected": 1.6058006286621094,
+ "logps/chosen": -148.9349822998047,
+ "logps/rejected": -147.87075805664062,
+ "loss": 0.6697592735290527,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.04735144227743149,
+ "rewards/margins": 0.05177399516105652,
+ "rewards/rejected": -0.0044225468300282955,
+ "step": 1170
+ },
+ {
+ "epoch": 1.888,
+ "grad_norm": 1.1624146699905396,
+ "learning_rate": 4.125666864256075e-07,
+ "logits/chosen": 1.5232517719268799,
+ "logits/rejected": 1.6060616970062256,
+ "logps/chosen": -162.28726196289062,
+ "logps/rejected": -156.43606567382812,
+ "loss": 0.6648381233215332,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.08147656917572021,
+ "rewards/margins": 0.06185563653707504,
+ "rewards/rejected": 0.01962094008922577,
+ "step": 1180
+ },
+ {
+ "epoch": 1.904,
+ "grad_norm": 1.1841973066329956,
+ "learning_rate": 4.066390041493776e-07,
+ "logits/chosen": 1.5964248180389404,
+ "logits/rejected": 1.490767240524292,
+ "logps/chosen": -147.4954071044922,
+ "logps/rejected": -130.23780822753906,
+ "loss": 0.6871873378753662,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.03555892035365105,
+ "rewards/margins": 0.016584066674113274,
+ "rewards/rejected": 0.018974855542182922,
+ "step": 1190
+ },
+ {
+ "epoch": 1.92,
+ "grad_norm": 1.2493818998336792,
+ "learning_rate": 4.007113218731476e-07,
+ "logits/chosen": 1.7136554718017578,
+ "logits/rejected": 1.5512396097183228,
+ "logps/chosen": -144.41561889648438,
+ "logps/rejected": -155.35577392578125,
+ "loss": 0.6688260078430176,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.056659065186977386,
+ "rewards/margins": 0.05496113747358322,
+ "rewards/rejected": 0.0016979314386844635,
+ "step": 1200
+ },
+ {
+ "epoch": 1.92,
+ "eval_logits/chosen": 1.4767016172409058,
+ "eval_logits/rejected": 1.5077458620071411,
+ "eval_logps/chosen": -153.31640625,
+ "eval_logps/rejected": -147.94268798828125,
+ "eval_loss": 0.6879809498786926,
+ "eval_rewards/accuracies": 0.5540000200271606,
+ "eval_rewards/chosen": 0.023242462426424026,
+ "eval_rewards/margins": 0.015296407975256443,
+ "eval_rewards/rejected": 0.007946056313812733,
+ "eval_runtime": 91.5558,
+ "eval_samples_per_second": 5.461,
+ "eval_steps_per_second": 2.731,
+ "step": 1200
+ },
+ {
+ "epoch": 1.936,
+ "grad_norm": 1.3514587879180908,
+ "learning_rate": 3.947836395969176e-07,
+ "logits/chosen": 1.4940625429153442,
+ "logits/rejected": 1.528282880783081,
+ "logps/chosen": -143.37240600585938,
+ "logps/rejected": -145.9002227783203,
+ "loss": 0.6773603439331055,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0432279035449028,
+ "rewards/margins": 0.036386217921972275,
+ "rewards/rejected": 0.006841684691607952,
+ "step": 1210
+ },
+ {
+ "epoch": 1.952,
+ "grad_norm": 1.3247877359390259,
+ "learning_rate": 3.888559573206876e-07,
+ "logits/chosen": 1.3842494487762451,
+ "logits/rejected": 1.551335334777832,
+ "logps/chosen": -137.93670654296875,
+ "logps/rejected": -165.12350463867188,
+ "loss": 0.6811867713928222,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.021465692669153214,
+ "rewards/margins": 0.028191816061735153,
+ "rewards/rejected": -0.006726124789565802,
+ "step": 1220
+ },
+ {
+ "epoch": 1.968,
+ "grad_norm": 1.294998288154602,
+ "learning_rate": 3.8292827504445763e-07,
+ "logits/chosen": 1.6616626977920532,
+ "logits/rejected": 1.5197885036468506,
+ "logps/chosen": -163.02841186523438,
+ "logps/rejected": -146.42318725585938,
+ "loss": 0.6719709396362304,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.04652197286486626,
+ "rewards/margins": 0.04806648939847946,
+ "rewards/rejected": -0.0015445180470123887,
+ "step": 1230
+ },
+ {
+ "epoch": 1.984,
+ "grad_norm": 1.0518370866775513,
+ "learning_rate": 3.770005927682276e-07,
+ "logits/chosen": 1.498633623123169,
+ "logits/rejected": 1.407947301864624,
+ "logps/chosen": -147.41317749023438,
+ "logps/rejected": -142.83969116210938,
+ "loss": 0.668755054473877,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.060599617660045624,
+ "rewards/margins": 0.05342463403940201,
+ "rewards/rejected": 0.007174992468208075,
+ "step": 1240
+ },
+ {
+ "epoch": 2.0,
+ "grad_norm": 1.274714708328247,
+ "learning_rate": 3.710729104919976e-07,
+ "logits/chosen": 1.6258140802383423,
+ "logits/rejected": 1.5810320377349854,
+ "logps/chosen": -179.64002990722656,
+ "logps/rejected": -180.4091796875,
+ "loss": 0.6710570812225342,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.030506063252687454,
+ "rewards/margins": 0.05016648769378662,
+ "rewards/rejected": -0.019660424441099167,
+ "step": 1250
+ },
+ {
+ "epoch": 2.0,
+ "eval_logits/chosen": 1.4780300855636597,
+ "eval_logits/rejected": 1.5088509321212769,
+ "eval_logps/chosen": -153.2786865234375,
+ "eval_logps/rejected": -147.9156036376953,
+ "eval_loss": 0.6875351071357727,
+ "eval_rewards/accuracies": 0.5580000281333923,
+ "eval_rewards/chosen": 0.027016151696443558,
+ "eval_rewards/margins": 0.01636195369064808,
+ "eval_rewards/rejected": 0.010654192417860031,
+ "eval_runtime": 91.4999,
+ "eval_samples_per_second": 5.464,
+ "eval_steps_per_second": 2.732,
+ "step": 1250
+ },
+ {
+ "epoch": 2.016,
+ "grad_norm": 1.2844147682189941,
+ "learning_rate": 3.6514522821576757e-07,
+ "logits/chosen": 1.5406789779663086,
+ "logits/rejected": 1.6422452926635742,
+ "logps/chosen": -156.5453338623047,
+ "logps/rejected": -134.34414672851562,
+ "loss": 0.6678662300109863,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": 0.06532102078199387,
+ "rewards/margins": 0.05562450364232063,
+ "rewards/rejected": 0.009696529246866703,
+ "step": 1260
+ },
+ {
+ "epoch": 2.032,
+ "grad_norm": 1.933401346206665,
+ "learning_rate": 3.592175459395376e-07,
+ "logits/chosen": 1.4194307327270508,
+ "logits/rejected": 1.4650447368621826,
+ "logps/chosen": -160.74501037597656,
+ "logps/rejected": -164.00306701660156,
+ "loss": 0.6694974422454834,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.0568288154900074,
+ "rewards/margins": 0.05134005472064018,
+ "rewards/rejected": 0.005488761700689793,
+ "step": 1270
+ },
+ {
+ "epoch": 2.048,
+ "grad_norm": 1.0357041358947754,
+ "learning_rate": 3.532898636633077e-07,
+ "logits/chosen": 1.3812814950942993,
+ "logits/rejected": 1.624565839767456,
+ "logps/chosen": -137.08206176757812,
+ "logps/rejected": -162.38833618164062,
+ "loss": 0.6587967872619629,
+ "rewards/accuracies": 0.762499988079071,
+ "rewards/chosen": 0.06582682579755783,
+ "rewards/margins": 0.0754324197769165,
+ "rewards/rejected": -0.009605596773326397,
+ "step": 1280
+ },
+ {
+ "epoch": 2.064,
+ "grad_norm": 1.196915864944458,
+ "learning_rate": 3.473621813870776e-07,
+ "logits/chosen": 1.5303500890731812,
+ "logits/rejected": 1.539674997329712,
+ "logps/chosen": -143.24818420410156,
+ "logps/rejected": -139.9320831298828,
+ "loss": 0.6571191310882568,
+ "rewards/accuracies": 0.7749999761581421,
+ "rewards/chosen": 0.06632508337497711,
+ "rewards/margins": 0.07795976102352142,
+ "rewards/rejected": -0.011634674854576588,
+ "step": 1290
+ },
+ {
+ "epoch": 2.08,
+ "grad_norm": 1.1703696250915527,
+ "learning_rate": 3.414344991108477e-07,
+ "logits/chosen": 1.629018783569336,
+ "logits/rejected": 1.6967315673828125,
+ "logps/chosen": -167.43856811523438,
+ "logps/rejected": -176.7183074951172,
+ "loss": 0.657524299621582,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.08362282812595367,
+ "rewards/margins": 0.07772316038608551,
+ "rewards/rejected": 0.0058996789157390594,
+ "step": 1300
+ },
+ {
+ "epoch": 2.08,
+ "eval_logits/chosen": 1.4791160821914673,
+ "eval_logits/rejected": 1.5099083185195923,
+ "eval_logps/chosen": -153.2497100830078,
+ "eval_logps/rejected": -147.89149475097656,
+ "eval_loss": 0.6873863935470581,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": 0.029913639649748802,
+ "eval_rewards/margins": 0.016850139945745468,
+ "eval_rewards/rejected": 0.013063502497971058,
+ "eval_runtime": 91.6423,
+ "eval_samples_per_second": 5.456,
+ "eval_steps_per_second": 2.728,
+ "step": 1300
+ },
+ {
+ "epoch": 2.096,
+ "grad_norm": 1.3917887210845947,
+ "learning_rate": 3.355068168346176e-07,
+ "logits/chosen": 1.4511966705322266,
+ "logits/rejected": 1.5072139501571655,
+ "logps/chosen": -152.747802734375,
+ "logps/rejected": -163.4126434326172,
+ "loss": 0.658362340927124,
+ "rewards/accuracies": 0.800000011920929,
+ "rewards/chosen": 0.05855457857251167,
+ "rewards/margins": 0.07485153526067734,
+ "rewards/rejected": -0.016296952962875366,
+ "step": 1310
+ },
+ {
+ "epoch": 2.112,
+ "grad_norm": 1.1995881795883179,
+ "learning_rate": 3.295791345583877e-07,
+ "logits/chosen": 1.5953047275543213,
+ "logits/rejected": 1.5267432928085327,
+ "logps/chosen": -165.5431671142578,
+ "logps/rejected": -176.0441131591797,
+ "loss": 0.6578423976898193,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.07576775550842285,
+ "rewards/margins": 0.07662783563137054,
+ "rewards/rejected": -0.0008600771543569863,
+ "step": 1320
+ },
+ {
+ "epoch": 2.128,
+ "grad_norm": 1.192712664604187,
+ "learning_rate": 3.236514522821577e-07,
+ "logits/chosen": 1.6488456726074219,
+ "logits/rejected": 1.6644923686981201,
+ "logps/chosen": -141.50588989257812,
+ "logps/rejected": -147.02488708496094,
+ "loss": 0.6503303050994873,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.05412819981575012,
+ "rewards/margins": 0.09486084431409836,
+ "rewards/rejected": -0.040732644498348236,
+ "step": 1330
+ },
+ {
+ "epoch": 2.144,
+ "grad_norm": 1.2887846231460571,
+ "learning_rate": 3.177237700059277e-07,
+ "logits/chosen": 1.4427909851074219,
+ "logits/rejected": 1.5052260160446167,
+ "logps/chosen": -133.82730102539062,
+ "logps/rejected": -146.34561157226562,
+ "loss": 0.6659392833709716,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.07661060243844986,
+ "rewards/margins": 0.061056166887283325,
+ "rewards/rejected": 0.015554438345134258,
+ "step": 1340
+ },
+ {
+ "epoch": 2.16,
+ "grad_norm": 1.2006677389144897,
+ "learning_rate": 3.117960877296977e-07,
+ "logits/chosen": 1.5929205417633057,
+ "logits/rejected": 1.4980108737945557,
+ "logps/chosen": -162.6881561279297,
+ "logps/rejected": -131.2484588623047,
+ "loss": 0.6704725742340087,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.051008958369493484,
+ "rewards/margins": 0.05107830837368965,
+ "rewards/rejected": -6.935522105777636e-05,
+ "step": 1350
+ },
+ {
+ "epoch": 2.16,
+ "eval_logits/chosen": 1.4747014045715332,
+ "eval_logits/rejected": 1.5051954984664917,
+ "eval_logps/chosen": -153.24111938476562,
+ "eval_logps/rejected": -147.88681030273438,
+ "eval_loss": 0.6875070929527283,
+ "eval_rewards/accuracies": 0.5519999861717224,
+ "eval_rewards/chosen": 0.03077210672199726,
+ "eval_rewards/margins": 0.017239386215806007,
+ "eval_rewards/rejected": 0.013532716780900955,
+ "eval_runtime": 91.625,
+ "eval_samples_per_second": 5.457,
+ "eval_steps_per_second": 2.729,
+ "step": 1350
+ },
+ {
+ "epoch": 2.176,
+ "grad_norm": 1.0531398057937622,
+ "learning_rate": 3.0586840545346773e-07,
+ "logits/chosen": 1.5693953037261963,
+ "logits/rejected": 1.4765347242355347,
+ "logps/chosen": -138.3097686767578,
+ "logps/rejected": -131.7692108154297,
+ "loss": 0.6577227592468262,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.05366247892379761,
+ "rewards/margins": 0.07729668915271759,
+ "rewards/rejected": -0.023634206503629684,
+ "step": 1360
+ },
+ {
+ "epoch": 2.192,
+ "grad_norm": 1.231702446937561,
+ "learning_rate": 2.999407231772377e-07,
+ "logits/chosen": 1.2275021076202393,
+ "logits/rejected": 1.3573815822601318,
+ "logps/chosen": -132.48486328125,
+ "logps/rejected": -135.35678100585938,
+ "loss": 0.6756887435913086,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.05586853623390198,
+ "rewards/margins": 0.04077490419149399,
+ "rewards/rejected": 0.015093636699020863,
+ "step": 1370
+ },
+ {
+ "epoch": 2.208,
+ "grad_norm": 1.4397618770599365,
+ "learning_rate": 2.940130409010077e-07,
+ "logits/chosen": 1.7811187505722046,
+ "logits/rejected": 1.6808115243911743,
+ "logps/chosen": -162.24937438964844,
+ "logps/rejected": -161.79856872558594,
+ "loss": 0.6767025947570801,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0448589026927948,
+ "rewards/margins": 0.036365605890750885,
+ "rewards/rejected": 0.008493305183947086,
+ "step": 1380
+ },
+ {
+ "epoch": 2.224,
+ "grad_norm": 1.3940778970718384,
+ "learning_rate": 2.8808535862477767e-07,
+ "logits/chosen": 1.4333906173706055,
+ "logits/rejected": 1.5776273012161255,
+ "logps/chosen": -139.67471313476562,
+ "logps/rejected": -145.5760498046875,
+ "loss": 0.6700955867767334,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.0032391161657869816,
+ "rewards/margins": 0.051826756447553635,
+ "rewards/rejected": -0.04858764261007309,
+ "step": 1390
+ },
+ {
+ "epoch": 2.24,
+ "grad_norm": 1.035470724105835,
+ "learning_rate": 2.821576763485477e-07,
+ "logits/chosen": 1.362156629562378,
+ "logits/rejected": 1.5719056129455566,
+ "logps/chosen": -165.33090209960938,
+ "logps/rejected": -161.99774169921875,
+ "loss": 0.6694781303405761,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.06508176028728485,
+ "rewards/margins": 0.05120532587170601,
+ "rewards/rejected": 0.01387643814086914,
+ "step": 1400
+ },
+ {
+ "epoch": 2.24,
+ "eval_logits/chosen": 1.4757568836212158,
+ "eval_logits/rejected": 1.506177544593811,
+ "eval_logps/chosen": -153.22549438476562,
+ "eval_logps/rejected": -147.8716583251953,
+ "eval_loss": 0.6875126957893372,
+ "eval_rewards/accuracies": 0.5479999780654907,
+ "eval_rewards/chosen": 0.0323345772922039,
+ "eval_rewards/margins": 0.01728663593530655,
+ "eval_rewards/rejected": 0.015047942288219929,
+ "eval_runtime": 91.6393,
+ "eval_samples_per_second": 5.456,
+ "eval_steps_per_second": 2.728,
+ "step": 1400
+ },
+ {
+ "epoch": 2.2560000000000002,
+ "grad_norm": 1.1921639442443848,
+ "learning_rate": 2.762299940723177e-07,
+ "logits/chosen": 1.6182886362075806,
+ "logits/rejected": 1.720829963684082,
+ "logps/chosen": -146.8916778564453,
+ "logps/rejected": -163.6287384033203,
+ "loss": 0.6639984130859375,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.06335891783237457,
+ "rewards/margins": 0.06482325494289398,
+ "rewards/rejected": -0.00146433524787426,
+ "step": 1410
+ },
+ {
+ "epoch": 2.2720000000000002,
+ "grad_norm": 1.0042649507522583,
+ "learning_rate": 2.703023117960877e-07,
+ "logits/chosen": 1.5501604080200195,
+ "logits/rejected": 1.4911870956420898,
+ "logps/chosen": -128.26808166503906,
+ "logps/rejected": -141.57968139648438,
+ "loss": 0.6698122501373291,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.04305802285671234,
+ "rewards/margins": 0.05252720043063164,
+ "rewards/rejected": -0.00946918223053217,
+ "step": 1420
+ },
+ {
+ "epoch": 2.288,
+ "grad_norm": 1.139971137046814,
+ "learning_rate": 2.643746295198577e-07,
+ "logits/chosen": 1.489418864250183,
+ "logits/rejected": 1.484986424446106,
+ "logps/chosen": -148.4396209716797,
+ "logps/rejected": -134.74444580078125,
+ "loss": 0.6659749507904053,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.035206541419029236,
+ "rewards/margins": 0.05955536291003227,
+ "rewards/rejected": -0.024348825216293335,
+ "step": 1430
+ },
+ {
+ "epoch": 2.304,
+ "grad_norm": 1.2615156173706055,
+ "learning_rate": 2.584469472436278e-07,
+ "logits/chosen": 1.4411890506744385,
+ "logits/rejected": 1.517624855041504,
+ "logps/chosen": -150.73739624023438,
+ "logps/rejected": -168.2410888671875,
+ "loss": 0.6469865322113038,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.0349310003221035,
+ "rewards/margins": 0.10427621752023697,
+ "rewards/rejected": -0.06934522092342377,
+ "step": 1440
+ },
+ {
+ "epoch": 2.32,
+ "grad_norm": 1.3218971490859985,
+ "learning_rate": 2.525192649673977e-07,
+ "logits/chosen": 1.4323554039001465,
+ "logits/rejected": 1.5884326696395874,
+ "logps/chosen": -161.39932250976562,
+ "logps/rejected": -170.40005493164062,
+ "loss": 0.6539069652557373,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.047432832419872284,
+ "rewards/margins": 0.08863824605941772,
+ "rewards/rejected": -0.04120541363954544,
+ "step": 1450
+ },
+ {
+ "epoch": 2.32,
+ "eval_logits/chosen": 1.470116138458252,
+ "eval_logits/rejected": 1.500432014465332,
+ "eval_logps/chosen": -153.3093719482422,
+ "eval_logps/rejected": -147.94973754882812,
+ "eval_loss": 0.6879428625106812,
+ "eval_rewards/accuracies": 0.5479999780654907,
+ "eval_rewards/chosen": 0.0239470973610878,
+ "eval_rewards/margins": 0.01670677214860916,
+ "eval_rewards/rejected": 0.007240326143801212,
+ "eval_runtime": 91.6086,
+ "eval_samples_per_second": 5.458,
+ "eval_steps_per_second": 2.729,
+ "step": 1450
+ },
+ {
+ "epoch": 2.336,
+ "grad_norm": 1.5149509906768799,
+ "learning_rate": 2.465915826911678e-07,
+ "logits/chosen": 1.4124549627304077,
+ "logits/rejected": 1.37667977809906,
+ "logps/chosen": -132.296142578125,
+ "logps/rejected": -129.7874755859375,
+ "loss": 0.6767591953277587,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.010999361053109169,
+ "rewards/margins": 0.04050828143954277,
+ "rewards/rejected": -0.0295089241117239,
+ "step": 1460
+ },
+ {
+ "epoch": 2.352,
+ "grad_norm": 1.2253953218460083,
+ "learning_rate": 2.406639004149378e-07,
+ "logits/chosen": 1.6453937292099,
+ "logits/rejected": 1.5322078466415405,
+ "logps/chosen": -144.53091430664062,
+ "logps/rejected": -124.41532897949219,
+ "loss": 0.6565406322479248,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.08202239871025085,
+ "rewards/margins": 0.07841381430625916,
+ "rewards/rejected": 0.0036085874307900667,
+ "step": 1470
+ },
+ {
+ "epoch": 2.368,
+ "grad_norm": 1.341062307357788,
+ "learning_rate": 2.3473621813870775e-07,
+ "logits/chosen": 1.6002895832061768,
+ "logits/rejected": 1.5122597217559814,
+ "logps/chosen": -158.36151123046875,
+ "logps/rejected": -133.71824645996094,
+ "loss": 0.6461643695831298,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.08061931282281876,
+ "rewards/margins": 0.10182766616344452,
+ "rewards/rejected": -0.02120836265385151,
+ "step": 1480
+ },
+ {
+ "epoch": 2.384,
+ "grad_norm": 1.390051007270813,
+ "learning_rate": 2.2880853586247775e-07,
+ "logits/chosen": 1.461780309677124,
+ "logits/rejected": 1.4497120380401611,
+ "logps/chosen": -137.35128784179688,
+ "logps/rejected": -136.74586486816406,
+ "loss": 0.6574192523956299,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.08100035041570663,
+ "rewards/margins": 0.07784504443407059,
+ "rewards/rejected": 0.003155309008434415,
+ "step": 1490
+ },
+ {
+ "epoch": 2.4,
+ "grad_norm": 1.4389512538909912,
+ "learning_rate": 2.2288085358624777e-07,
+ "logits/chosen": 1.4925658702850342,
+ "logits/rejected": 1.5207669734954834,
+ "logps/chosen": -146.5099334716797,
+ "logps/rejected": -134.76077270507812,
+ "loss": 0.6710968971252441,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.04036882147192955,
+ "rewards/margins": 0.05301078408956528,
+ "rewards/rejected": -0.01264196913689375,
+ "step": 1500
+ },
+ {
+ "epoch": 2.4,
+ "eval_logits/chosen": 1.473709225654602,
+ "eval_logits/rejected": 1.5038994550704956,
+ "eval_logps/chosen": -153.1907501220703,
+ "eval_logps/rejected": -147.84222412109375,
+ "eval_loss": 0.6875451803207397,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": 0.035809777677059174,
+ "eval_rewards/margins": 0.017816420644521713,
+ "eval_rewards/rejected": 0.01799335516989231,
+ "eval_runtime": 91.5755,
+ "eval_samples_per_second": 5.46,
+ "eval_steps_per_second": 2.73,
+ "step": 1500
+ },
+ {
+ "epoch": 2.416,
+ "grad_norm": 1.2243754863739014,
+ "learning_rate": 2.1695317131001777e-07,
+ "logits/chosen": 1.4834078550338745,
+ "logits/rejected": 1.4932808876037598,
+ "logps/chosen": -133.7617950439453,
+ "logps/rejected": -132.2259063720703,
+ "loss": 0.6513554573059082,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.06009601801633835,
+ "rewards/margins": 0.09164053946733475,
+ "rewards/rejected": -0.0315445214509964,
+ "step": 1510
+ },
+ {
+ "epoch": 2.432,
+ "grad_norm": 1.090537190437317,
+ "learning_rate": 2.1102548903378777e-07,
+ "logits/chosen": 1.532271146774292,
+ "logits/rejected": 1.5871059894561768,
+ "logps/chosen": -137.46914672851562,
+ "logps/rejected": -153.37509155273438,
+ "loss": 0.6659578323364258,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.03360649198293686,
+ "rewards/margins": 0.06050106883049011,
+ "rewards/rejected": -0.026894578710198402,
+ "step": 1520
+ },
+ {
+ "epoch": 2.448,
+ "grad_norm": 1.2809340953826904,
+ "learning_rate": 2.0509780675755777e-07,
+ "logits/chosen": 1.5342439413070679,
+ "logits/rejected": 1.557093858718872,
+ "logps/chosen": -145.97274780273438,
+ "logps/rejected": -150.5465850830078,
+ "loss": 0.6720418930053711,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0609341561794281,
+ "rewards/margins": 0.047071993350982666,
+ "rewards/rejected": 0.013862164691090584,
+ "step": 1530
+ },
+ {
+ "epoch": 2.464,
+ "grad_norm": 1.1325640678405762,
+ "learning_rate": 1.9917012448132777e-07,
+ "logits/chosen": 1.5809669494628906,
+ "logits/rejected": 1.550691843032837,
+ "logps/chosen": -153.353759765625,
+ "logps/rejected": -163.1613311767578,
+ "loss": 0.6606723308563233,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": 0.06856328994035721,
+ "rewards/margins": 0.07200589776039124,
+ "rewards/rejected": -0.003442607820034027,
+ "step": 1540
+ },
+ {
+ "epoch": 2.48,
+ "grad_norm": 1.2451732158660889,
+ "learning_rate": 1.932424422050978e-07,
+ "logits/chosen": 1.5336394309997559,
+ "logits/rejected": 1.4750980138778687,
+ "logps/chosen": -141.4608154296875,
+ "logps/rejected": -168.307861328125,
+ "loss": 0.6642096996307373,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.05440855771303177,
+ "rewards/margins": 0.06600542366504669,
+ "rewards/rejected": -0.011596867814660072,
+ "step": 1550
+ },
+ {
+ "epoch": 2.48,
+ "eval_logits/chosen": 1.4721124172210693,
+ "eval_logits/rejected": 1.502312421798706,
+ "eval_logps/chosen": -153.19175720214844,
+ "eval_logps/rejected": -147.8402862548828,
+ "eval_loss": 0.68780517578125,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": 0.03571058809757233,
+ "eval_rewards/margins": 0.017523877322673798,
+ "eval_rewards/rejected": 0.01818670891225338,
+ "eval_runtime": 91.719,
+ "eval_samples_per_second": 5.451,
+ "eval_steps_per_second": 2.726,
+ "step": 1550
+ },
+ {
+ "epoch": 2.496,
+ "grad_norm": 1.186293601989746,
+ "learning_rate": 1.873147599288678e-07,
+ "logits/chosen": 1.6083301305770874,
+ "logits/rejected": 1.5244642496109009,
+ "logps/chosen": -133.59799194335938,
+ "logps/rejected": -130.28594970703125,
+ "loss": 0.6528758049011231,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.0587749108672142,
+ "rewards/margins": 0.08881707489490509,
+ "rewards/rejected": -0.030042162165045738,
+ "step": 1560
+ },
+ {
+ "epoch": 2.512,
+ "grad_norm": 1.276349663734436,
+ "learning_rate": 1.813870776526378e-07,
+ "logits/chosen": 1.6797683238983154,
+ "logits/rejected": 1.5076231956481934,
+ "logps/chosen": -153.5227813720703,
+ "logps/rejected": -155.51109313964844,
+ "loss": 0.6569789409637451,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.07746303081512451,
+ "rewards/margins": 0.0799773782491684,
+ "rewards/rejected": -0.0025143474340438843,
+ "step": 1570
+ },
+ {
+ "epoch": 2.528,
+ "grad_norm": 1.0541719198226929,
+ "learning_rate": 1.754593953764078e-07,
+ "logits/chosen": 1.8211376667022705,
+ "logits/rejected": 1.6606826782226562,
+ "logps/chosen": -159.01956176757812,
+ "logps/rejected": -158.61135864257812,
+ "loss": 0.6697492122650146,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.051909469068050385,
+ "rewards/margins": 0.05238593369722366,
+ "rewards/rejected": -0.00047646090388298035,
+ "step": 1580
+ },
+ {
+ "epoch": 2.544,
+ "grad_norm": 1.538649320602417,
+ "learning_rate": 1.6953171310017782e-07,
+ "logits/chosen": 1.5991268157958984,
+ "logits/rejected": 1.5752675533294678,
+ "logps/chosen": -150.49803161621094,
+ "logps/rejected": -144.097412109375,
+ "loss": 0.6589127540588379,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.06415747851133347,
+ "rewards/margins": 0.07586739212274551,
+ "rewards/rejected": -0.011709906160831451,
+ "step": 1590
+ },
+ {
+ "epoch": 2.56,
+ "grad_norm": 1.2445380687713623,
+ "learning_rate": 1.6360403082394782e-07,
+ "logits/chosen": 1.536322832107544,
+ "logits/rejected": 1.5229716300964355,
+ "logps/chosen": -160.0904541015625,
+ "logps/rejected": -147.23117065429688,
+ "loss": 0.6632528305053711,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.060222309082746506,
+ "rewards/margins": 0.06837073713541031,
+ "rewards/rejected": -0.008148419670760632,
+ "step": 1600
+ },
+ {
+ "epoch": 2.56,
+ "eval_logits/chosen": 1.4679710865020752,
+ "eval_logits/rejected": 1.4978972673416138,
+ "eval_logps/chosen": -153.241943359375,
+ "eval_logps/rejected": -147.89138793945312,
+ "eval_loss": 0.6878563165664673,
+ "eval_rewards/accuracies": 0.5440000295639038,
+ "eval_rewards/chosen": 0.030691616237163544,
+ "eval_rewards/margins": 0.01761641353368759,
+ "eval_rewards/rejected": 0.013075205497443676,
+ "eval_runtime": 91.5482,
+ "eval_samples_per_second": 5.462,
+ "eval_steps_per_second": 2.731,
+ "step": 1600
+ },
+ {
+ "epoch": 2.576,
+ "grad_norm": 1.299237847328186,
+ "learning_rate": 1.5767634854771782e-07,
+ "logits/chosen": 1.4606423377990723,
+ "logits/rejected": 1.5331600904464722,
+ "logps/chosen": -156.78688049316406,
+ "logps/rejected": -174.20486450195312,
+ "loss": 0.6687525272369385,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": 0.03773252293467522,
+ "rewards/margins": 0.05658789351582527,
+ "rewards/rejected": -0.018855368718504906,
+ "step": 1610
+ },
+ {
+ "epoch": 2.592,
+ "grad_norm": 1.252417802810669,
+ "learning_rate": 1.5174866627148782e-07,
+ "logits/chosen": 1.531015157699585,
+ "logits/rejected": 1.4835556745529175,
+ "logps/chosen": -153.46090698242188,
+ "logps/rejected": -138.4996337890625,
+ "loss": 0.6781048774719238,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.022028688341379166,
+ "rewards/margins": 0.03805949538946152,
+ "rewards/rejected": -0.0160308089107275,
+ "step": 1620
+ },
+ {
+ "epoch": 2.608,
+ "grad_norm": 1.4750654697418213,
+ "learning_rate": 1.4582098399525785e-07,
+ "logits/chosen": 1.4760229587554932,
+ "logits/rejected": 1.5888662338256836,
+ "logps/chosen": -152.83285522460938,
+ "logps/rejected": -160.96487426757812,
+ "loss": 0.6529106616973877,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.0814286321401596,
+ "rewards/margins": 0.08959604799747467,
+ "rewards/rejected": -0.008167424239218235,
+ "step": 1630
+ },
+ {
+ "epoch": 2.624,
+ "grad_norm": 1.3131834268569946,
+ "learning_rate": 1.3989330171902785e-07,
+ "logits/chosen": 1.7102229595184326,
+ "logits/rejected": 1.5242104530334473,
+ "logps/chosen": -149.85971069335938,
+ "logps/rejected": -136.90907287597656,
+ "loss": 0.6715713500976562,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.06474443525075912,
+ "rewards/margins": 0.0555412583053112,
+ "rewards/rejected": 0.009203184396028519,
+ "step": 1640
+ },
+ {
+ "epoch": 2.64,
+ "grad_norm": 1.1318562030792236,
+ "learning_rate": 1.3396561944279785e-07,
+ "logits/chosen": 1.5815187692642212,
+ "logits/rejected": 1.5685369968414307,
+ "logps/chosen": -155.14144897460938,
+ "logps/rejected": -148.47853088378906,
+ "loss": 0.6686881065368653,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.04677755385637283,
+ "rewards/margins": 0.05729418992996216,
+ "rewards/rejected": -0.010516637936234474,
+ "step": 1650
+ },
+ {
+ "epoch": 2.64,
+ "eval_logits/chosen": 1.4654622077941895,
+ "eval_logits/rejected": 1.4955151081085205,
+ "eval_logps/chosen": -153.30543518066406,
+ "eval_logps/rejected": -147.95297241210938,
+ "eval_loss": 0.687913715839386,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": 0.024341551586985588,
+ "eval_rewards/margins": 0.01742393709719181,
+ "eval_rewards/rejected": 0.006917611230164766,
+ "eval_runtime": 91.5909,
+ "eval_samples_per_second": 5.459,
+ "eval_steps_per_second": 2.73,
+ "step": 1650
+ },
+ {
+ "epoch": 2.656,
+ "grad_norm": 1.366890788078308,
+ "learning_rate": 1.2803793716656785e-07,
+ "logits/chosen": 1.4456878900527954,
+ "logits/rejected": 1.499376654624939,
+ "logps/chosen": -152.6118621826172,
+ "logps/rejected": -130.8150634765625,
+ "loss": 0.680852222442627,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.018559083342552185,
+ "rewards/margins": 0.037391066551208496,
+ "rewards/rejected": -0.01883198320865631,
+ "step": 1660
+ },
+ {
+ "epoch": 2.672,
+ "grad_norm": 1.420353889465332,
+ "learning_rate": 1.2211025489033787e-07,
+ "logits/chosen": 1.4291784763336182,
+ "logits/rejected": 1.38071608543396,
+ "logps/chosen": -138.0047149658203,
+ "logps/rejected": -140.8662872314453,
+ "loss": 0.6645754337310791,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.039429180324077606,
+ "rewards/margins": 0.06526370346546173,
+ "rewards/rejected": -0.025834526866674423,
+ "step": 1670
+ },
+ {
+ "epoch": 2.6879999999999997,
+ "grad_norm": 1.2673463821411133,
+ "learning_rate": 1.1618257261410788e-07,
+ "logits/chosen": 1.6407334804534912,
+ "logits/rejected": 1.525295615196228,
+ "logps/chosen": -154.69979858398438,
+ "logps/rejected": -141.7242889404297,
+ "loss": 0.666029405593872,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.0484330952167511,
+ "rewards/margins": 0.06104108691215515,
+ "rewards/rejected": -0.012607984244823456,
+ "step": 1680
+ },
+ {
+ "epoch": 2.7039999999999997,
+ "grad_norm": 1.077925682067871,
+ "learning_rate": 1.1025489033787788e-07,
+ "logits/chosen": 1.565926432609558,
+ "logits/rejected": 1.531680703163147,
+ "logps/chosen": -137.2372589111328,
+ "logps/rejected": -163.35003662109375,
+ "loss": 0.663050651550293,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.056818485260009766,
+ "rewards/margins": 0.0716979056596756,
+ "rewards/rejected": -0.014879422262310982,
+ "step": 1690
+ },
+ {
+ "epoch": 2.7199999999999998,
+ "grad_norm": 1.5206483602523804,
+ "learning_rate": 1.043272080616479e-07,
+ "logits/chosen": 1.4674041271209717,
+ "logits/rejected": 1.3627955913543701,
+ "logps/chosen": -160.63465881347656,
+ "logps/rejected": -152.9118194580078,
+ "loss": 0.6480361938476562,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.04499155282974243,
+ "rewards/margins": 0.10087253898382187,
+ "rewards/rejected": -0.05588098242878914,
+ "step": 1700
+ },
+ {
+ "epoch": 2.7199999999999998,
+ "eval_logits/chosen": 1.4640889167785645,
+ "eval_logits/rejected": 1.4942444562911987,
+ "eval_logps/chosen": -153.35006713867188,
+ "eval_logps/rejected": -147.99827575683594,
+ "eval_loss": 0.6878501772880554,
+ "eval_rewards/accuracies": 0.5479999780654907,
+ "eval_rewards/chosen": 0.019878296181559563,
+ "eval_rewards/margins": 0.017491795122623444,
+ "eval_rewards/rejected": 0.002386500360444188,
+ "eval_runtime": 91.6069,
+ "eval_samples_per_second": 5.458,
+ "eval_steps_per_second": 2.729,
+ "step": 1700
+ },
+ {
+ "epoch": 2.7359999999999998,
+ "grad_norm": 1.2565103769302368,
+ "learning_rate": 9.83995257854179e-08,
+ "logits/chosen": 1.5768482685089111,
+ "logits/rejected": 1.6890283823013306,
+ "logps/chosen": -143.33065795898438,
+ "logps/rejected": -147.1623077392578,
+ "loss": 0.6540540218353271,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.06702157109975815,
+ "rewards/margins": 0.08658405393362045,
+ "rewards/rejected": -0.019562482833862305,
+ "step": 1710
+ },
+ {
+ "epoch": 2.752,
+ "grad_norm": 1.4862720966339111,
+ "learning_rate": 9.247184350918791e-08,
+ "logits/chosen": 1.457726240158081,
+ "logits/rejected": 1.4858250617980957,
+ "logps/chosen": -144.1268310546875,
+ "logps/rejected": -151.44485473632812,
+ "loss": 0.6899446010589599,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.009164285846054554,
+ "rewards/margins": 0.010964317247271538,
+ "rewards/rejected": -0.0018000311683863401,
+ "step": 1720
+ },
+ {
+ "epoch": 2.768,
+ "grad_norm": 1.2616908550262451,
+ "learning_rate": 8.654416123295791e-08,
+ "logits/chosen": 1.406851053237915,
+ "logits/rejected": 1.3306976556777954,
+ "logps/chosen": -126.03456115722656,
+ "logps/rejected": -141.63986206054688,
+ "loss": 0.6686761856079102,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": 0.03257106989622116,
+ "rewards/margins": 0.05362869054079056,
+ "rewards/rejected": -0.02105761505663395,
+ "step": 1730
+ },
+ {
+ "epoch": 2.784,
+ "grad_norm": 1.096355676651001,
+ "learning_rate": 8.061647895672792e-08,
+ "logits/chosen": 1.4490602016448975,
+ "logits/rejected": 1.6556310653686523,
+ "logps/chosen": -155.9315185546875,
+ "logps/rejected": -135.6969757080078,
+ "loss": 0.6610583782196044,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.07227732986211777,
+ "rewards/margins": 0.07086370885372162,
+ "rewards/rejected": 0.0014136269455775619,
+ "step": 1740
+ },
+ {
+ "epoch": 2.8,
+ "grad_norm": 1.291092872619629,
+ "learning_rate": 7.468879668049792e-08,
+ "logits/chosen": 1.3888136148452759,
+ "logits/rejected": 1.5601799488067627,
+ "logps/chosen": -126.0166244506836,
+ "logps/rejected": -155.1581268310547,
+ "loss": 0.670356273651123,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": 0.014735782518982887,
+ "rewards/margins": 0.052715569734573364,
+ "rewards/rejected": -0.037979789078235626,
+ "step": 1750
+ },
+ {
+ "epoch": 2.8,
+ "eval_logits/chosen": 1.4647109508514404,
+ "eval_logits/rejected": 1.4947710037231445,
+ "eval_logps/chosen": -153.3578643798828,
+ "eval_logps/rejected": -148.00909423828125,
+ "eval_loss": 0.6876614093780518,
+ "eval_rewards/accuracies": 0.5559999942779541,
+ "eval_rewards/chosen": 0.01909835822880268,
+ "eval_rewards/margins": 0.01779349334537983,
+ "eval_rewards/rejected": 0.0013048641849309206,
+ "eval_runtime": 91.6184,
+ "eval_samples_per_second": 5.457,
+ "eval_steps_per_second": 2.729,
+ "step": 1750
+ },
+ {
+ "epoch": 2.816,
+ "grad_norm": 1.280806303024292,
+ "learning_rate": 6.876111440426793e-08,
+ "logits/chosen": 1.706970453262329,
+ "logits/rejected": 1.6935030221939087,
+ "logps/chosen": -155.4099578857422,
+ "logps/rejected": -157.07286071777344,
+ "loss": 0.660371971130371,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.08173780888319016,
+ "rewards/margins": 0.07147711515426636,
+ "rewards/rejected": 0.01026069838553667,
+ "step": 1760
+ },
+ {
+ "epoch": 2.832,
+ "grad_norm": 1.2481036186218262,
+ "learning_rate": 6.283343212803793e-08,
+ "logits/chosen": 1.6746574640274048,
+ "logits/rejected": 1.6040579080581665,
+ "logps/chosen": -163.3013916015625,
+ "logps/rejected": -175.66671752929688,
+ "loss": 0.6617218971252441,
+ "rewards/accuracies": 0.7250000238418579,
+ "rewards/chosen": 0.06778182834386826,
+ "rewards/margins": 0.07007112354040146,
+ "rewards/rejected": -0.002289295196533203,
+ "step": 1770
+ },
+ {
+ "epoch": 2.848,
+ "grad_norm": 0.9844895005226135,
+ "learning_rate": 5.690574985180794e-08,
+ "logits/chosen": 1.5059736967086792,
+ "logits/rejected": 1.3353385925292969,
+ "logps/chosen": -141.01809692382812,
+ "logps/rejected": -129.45187377929688,
+ "loss": 0.6617399692535401,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": 0.049883510917425156,
+ "rewards/margins": 0.07276918739080429,
+ "rewards/rejected": -0.022885670885443687,
+ "step": 1780
+ },
+ {
+ "epoch": 2.864,
+ "grad_norm": 1.0773348808288574,
+ "learning_rate": 5.0978067575577946e-08,
+ "logits/chosen": 1.557916283607483,
+ "logits/rejected": 1.5435125827789307,
+ "logps/chosen": -168.74581909179688,
+ "logps/rejected": -161.37094116210938,
+ "loss": 0.655879020690918,
+ "rewards/accuracies": 0.762499988079071,
+ "rewards/chosen": 0.07581307739019394,
+ "rewards/margins": 0.08062167465686798,
+ "rewards/rejected": -0.004808598663657904,
+ "step": 1790
+ },
+ {
+ "epoch": 2.88,
+ "grad_norm": 1.5534027814865112,
+ "learning_rate": 4.505038529934795e-08,
+ "logits/chosen": 1.4658396244049072,
+ "logits/rejected": 1.461570143699646,
+ "logps/chosen": -173.49241638183594,
+ "logps/rejected": -177.7185821533203,
+ "loss": 0.6556243419647216,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.08077023923397064,
+ "rewards/margins": 0.08332471549510956,
+ "rewards/rejected": -0.0025544664822518826,
+ "step": 1800
+ },
+ {
+ "epoch": 2.88,
+ "eval_logits/chosen": 1.463594913482666,
+ "eval_logits/rejected": 1.4937729835510254,
+ "eval_logps/chosen": -153.3700408935547,
+ "eval_logps/rejected": -148.02059936523438,
+ "eval_loss": 0.6877142190933228,
+ "eval_rewards/accuracies": 0.5600000023841858,
+ "eval_rewards/chosen": 0.017879322171211243,
+ "eval_rewards/margins": 0.017723429948091507,
+ "eval_rewards/rejected": 0.00015589475515298545,
+ "eval_runtime": 91.592,
+ "eval_samples_per_second": 5.459,
+ "eval_steps_per_second": 2.729,
+ "step": 1800
+ },
+ {
+ "epoch": 2.896,
+ "grad_norm": 1.4052973985671997,
+ "learning_rate": 3.912270302311796e-08,
+ "logits/chosen": 1.665469765663147,
+ "logits/rejected": 1.591843843460083,
+ "logps/chosen": -156.02999877929688,
+ "logps/rejected": -143.94110107421875,
+ "loss": 0.6743856906890869,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.07271949201822281,
+ "rewards/margins": 0.0441838875412941,
+ "rewards/rejected": 0.02853560447692871,
+ "step": 1810
+ },
+ {
+ "epoch": 2.912,
+ "grad_norm": 1.5630269050598145,
+ "learning_rate": 3.3195020746887964e-08,
+ "logits/chosen": 1.4642374515533447,
+ "logits/rejected": 1.5267945528030396,
+ "logps/chosen": -144.21298217773438,
+ "logps/rejected": -146.3809051513672,
+ "loss": 0.6608489513397217,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": 0.029606735333800316,
+ "rewards/margins": 0.07201993465423584,
+ "rewards/rejected": -0.042413197457790375,
+ "step": 1820
+ },
+ {
+ "epoch": 2.928,
+ "grad_norm": 1.2600101232528687,
+ "learning_rate": 2.726733847065797e-08,
+ "logits/chosen": 1.547340989112854,
+ "logits/rejected": 1.608282446861267,
+ "logps/chosen": -157.01394653320312,
+ "logps/rejected": -151.60873413085938,
+ "loss": 0.6710472583770752,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03543226420879364,
+ "rewards/margins": 0.049915216863155365,
+ "rewards/rejected": -0.014482955448329449,
+ "step": 1830
+ },
+ {
+ "epoch": 2.944,
+ "grad_norm": 1.0947951078414917,
+ "learning_rate": 2.133965619442798e-08,
+ "logits/chosen": 1.303476333618164,
+ "logits/rejected": 1.4889559745788574,
+ "logps/chosen": -145.82022094726562,
+ "logps/rejected": -145.65536499023438,
+ "loss": 0.6459365367889405,
+ "rewards/accuracies": 0.762499988079071,
+ "rewards/chosen": 0.042109210044145584,
+ "rewards/margins": 0.10741522163152695,
+ "rewards/rejected": -0.06530602276325226,
+ "step": 1840
+ },
+ {
+ "epoch": 2.96,
+ "grad_norm": 1.0814207792282104,
+ "learning_rate": 1.5411973918197986e-08,
+ "logits/chosen": 1.508532166481018,
+ "logits/rejected": 1.6741304397583008,
+ "logps/chosen": -147.40884399414062,
+ "logps/rejected": -180.92013549804688,
+ "loss": 0.6481856346130371,
+ "rewards/accuracies": 0.800000011920929,
+ "rewards/chosen": 0.06129009276628494,
+ "rewards/margins": 0.09948200732469559,
+ "rewards/rejected": -0.038191910833120346,
+ "step": 1850
+ },
+ {
+ "epoch": 2.96,
+ "eval_logits/chosen": 1.4641175270080566,
+ "eval_logits/rejected": 1.494136095046997,
+ "eval_logps/chosen": -153.36550903320312,
+ "eval_logps/rejected": -148.01507568359375,
+ "eval_loss": 0.6877753734588623,
+ "eval_rewards/accuracies": 0.5540000200271606,
+ "eval_rewards/chosen": 0.01833285205066204,
+ "eval_rewards/margins": 0.017625929787755013,
+ "eval_rewards/rejected": 0.0007069256389513612,
+ "eval_runtime": 91.6569,
+ "eval_samples_per_second": 5.455,
+ "eval_steps_per_second": 2.728,
+ "step": 1850
+ },
+ {
+ "epoch": 2.976,
+ "grad_norm": 1.2660458087921143,
+ "learning_rate": 9.484291641967989e-09,
+ "logits/chosen": 1.5423072576522827,
+ "logits/rejected": 1.413693904876709,
+ "logps/chosen": -166.9947052001953,
+ "logps/rejected": -131.19735717773438,
+ "loss": 0.6499662876129151,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": 0.05684496834874153,
+ "rewards/margins": 0.09581321477890015,
+ "rewards/rejected": -0.03896826133131981,
+ "step": 1860
+ },
+ {
+ "epoch": 2.992,
+ "grad_norm": 1.0133707523345947,
+ "learning_rate": 3.5566093657379962e-09,
+ "logits/chosen": 1.5958634614944458,
+ "logits/rejected": 1.6392433643341064,
+ "logps/chosen": -139.81076049804688,
+ "logps/rejected": -147.24954223632812,
+ "loss": 0.6680123805999756,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.0883885845541954,
+ "rewards/margins": 0.05749764293432236,
+ "rewards/rejected": 0.030890945345163345,
+ "step": 1870
+ }
+ ],
+ "logging_steps": 10,
+ "max_steps": 1875,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": true
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-1875/training_args.bin b/v3/DPO/DPO_5k/lora/checkpoint-1875/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..544215e69bd09b706d2c495c7bb412d0df7f83e7
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-1875/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afe50d4ac4c5f2af306fafe28b341897775b898324082e319b56c66893dceabb
+size 6097
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-500/README.md b/v3/DPO/DPO_5k/lora/checkpoint-500/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..250500f798b4682b2cd2c35cd1fc366677c215dd
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-500/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-500/adapter_config.json b/v3/DPO/DPO_5k/lora/checkpoint-500/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..695e5e16d4a5436a17790fe2e31fc2a9e1631d03
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-500/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "gate_proj",
+ "k_proj",
+ "q_proj",
+ "down_proj",
+ "o_proj",
+ "v_proj",
+ "up_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-500/adapter_model.safetensors b/v3/DPO/DPO_5k/lora/checkpoint-500/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b1423d7a963f4a10e33ea85a2ef53edb5deea390
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-500/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8445b21a52bbbddf79065f3113e019da6709ec64376031c294763fec12830d4f
+size 180385008
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-500/chat_template.jinja b/v3/DPO/DPO_5k/lora/checkpoint-500/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-500/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-500/optimizer.pt b/v3/DPO/DPO_5k/lora/checkpoint-500/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ecc7a9910bae0ce599d2da1e4518e67fdb530912
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-500/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36000149339d71802283b869530df54ff984d5b5407507a415be6f09d1933dd7
+size 360902475
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-500/rng_state.pth b/v3/DPO/DPO_5k/lora/checkpoint-500/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..d5e20c9b1725549f13f103bd76fa989eb82478a1
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-500/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea11996454b5587fcf33ae0ab5cf14b2031bf5f53f8c2ed5a48e87de31e29c84
+size 14645
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-500/scaler.pt b/v3/DPO/DPO_5k/lora/checkpoint-500/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..86c857e8514e5db52a765434b135695dac4c9c36
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-500/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f77569c2e850b04af982cc8c1389f1430851448915c593b69e5da36ce05b71d7
+size 1383
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-500/scheduler.pt b/v3/DPO/DPO_5k/lora/checkpoint-500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..99c690ab1047d3564bf90f12492ef23829077712
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c8f6ac42a57ed5178a0d9846f86a7baa877efa3da3c0e514365dab9b7f4f2a3
+size 1465
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-500/tokenizer.json b/v3/DPO/DPO_5k/lora/checkpoint-500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-500/tokenizer_config.json b/v3/DPO/DPO_5k/lora/checkpoint-500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-500/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-500/trainer_state.json b/v3/DPO/DPO_5k/lora/checkpoint-500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..17fd259d65f422f131bc9fb6cc09aaad0c3db34d
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-500/trainer_state.json
@@ -0,0 +1,944 @@
+{
+ "best_global_step": 450,
+ "best_metric": 0.6898273229598999,
+ "best_model_checkpoint": null,
+ "epoch": 0.8,
+ "eval_steps": 50,
+ "global_step": 500,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.016,
+ "grad_norm": 1.1517287492752075,
+ "learning_rate": 4.787234042553191e-08,
+ "logits/chosen": 1.6246497631072998,
+ "logits/rejected": 1.5396833419799805,
+ "logps/chosen": -145.91531372070312,
+ "logps/rejected": -151.4051971435547,
+ "loss": 0.693363094329834,
+ "rewards/accuracies": 0.2874999940395355,
+ "rewards/chosen": -3.5572534216044005e-06,
+ "rewards/margins": -0.0004222107236273587,
+ "rewards/rejected": 0.00041865353705361485,
+ "step": 10
+ },
+ {
+ "epoch": 0.032,
+ "grad_norm": 1.3633077144622803,
+ "learning_rate": 1.0106382978723404e-07,
+ "logits/chosen": 1.313926339149475,
+ "logits/rejected": 1.4038684368133545,
+ "logps/chosen": -120.60591888427734,
+ "logps/rejected": -134.05252075195312,
+ "loss": 0.6930661201477051,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 9.576084994478151e-05,
+ "rewards/margins": 0.0001732158416416496,
+ "rewards/rejected": -7.745502807665616e-05,
+ "step": 20
+ },
+ {
+ "epoch": 0.048,
+ "grad_norm": 1.209151029586792,
+ "learning_rate": 1.5425531914893615e-07,
+ "logits/chosen": 1.5228456258773804,
+ "logits/rejected": 1.6869430541992188,
+ "logps/chosen": -162.40280151367188,
+ "logps/rejected": -179.38192749023438,
+ "loss": 0.692751693725586,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.0007011269335635006,
+ "rewards/margins": 0.0008053636993281543,
+ "rewards/rejected": -0.00010423659114167094,
+ "step": 30
+ },
+ {
+ "epoch": 0.064,
+ "grad_norm": 1.3589205741882324,
+ "learning_rate": 2.074468085106383e-07,
+ "logits/chosen": 1.544001817703247,
+ "logits/rejected": 1.4715760946273804,
+ "logps/chosen": -152.7266845703125,
+ "logps/rejected": -150.9158477783203,
+ "loss": 0.6925192832946777,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.00026422025985084474,
+ "rewards/margins": 0.0012672842713072896,
+ "rewards/rejected": -0.0010030639823526144,
+ "step": 40
+ },
+ {
+ "epoch": 0.08,
+ "grad_norm": 1.202810525894165,
+ "learning_rate": 2.6063829787234044e-07,
+ "logits/chosen": 1.6794925928115845,
+ "logits/rejected": 1.7414724826812744,
+ "logps/chosen": -169.71751403808594,
+ "logps/rejected": -156.7571258544922,
+ "loss": 0.6930821895599365,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 9.479046275373548e-05,
+ "rewards/margins": 0.00014605997421313077,
+ "rewards/rejected": -5.126944597577676e-05,
+ "step": 50
+ },
+ {
+ "epoch": 0.08,
+ "eval_logits/chosen": 1.5141410827636719,
+ "eval_logits/rejected": 1.5485105514526367,
+ "eval_logps/chosen": -153.53335571289062,
+ "eval_logps/rejected": -148.00894165039062,
+ "eval_loss": 0.6930396556854248,
+ "eval_rewards/accuracies": 0.5120000243186951,
+ "eval_rewards/chosen": 0.0015483711613342166,
+ "eval_rewards/margins": 0.00022836039715912193,
+ "eval_rewards/rejected": 0.0013200106332078576,
+ "eval_runtime": 91.1536,
+ "eval_samples_per_second": 5.485,
+ "eval_steps_per_second": 2.743,
+ "step": 50
+ },
+ {
+ "epoch": 0.096,
+ "grad_norm": 1.2080731391906738,
+ "learning_rate": 3.1382978723404253e-07,
+ "logits/chosen": 1.796142339706421,
+ "logits/rejected": 1.7786738872528076,
+ "logps/chosen": -158.88607788085938,
+ "logps/rejected": -149.43331909179688,
+ "loss": 0.6934396743774414,
+ "rewards/accuracies": 0.4124999940395355,
+ "rewards/chosen": 0.001977279083803296,
+ "rewards/margins": -0.0005727410316467285,
+ "rewards/rejected": 0.0025500201154500246,
+ "step": 60
+ },
+ {
+ "epoch": 0.112,
+ "grad_norm": 0.9988052248954773,
+ "learning_rate": 3.6702127659574467e-07,
+ "logits/chosen": 1.7978185415267944,
+ "logits/rejected": 1.7301257848739624,
+ "logps/chosen": -158.67416381835938,
+ "logps/rejected": -165.23361206054688,
+ "loss": 0.6937376976013183,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.0031797506380826235,
+ "rewards/margins": -0.0011613152455538511,
+ "rewards/rejected": 0.004341065417975187,
+ "step": 70
+ },
+ {
+ "epoch": 0.128,
+ "grad_norm": 1.2025635242462158,
+ "learning_rate": 4.202127659574468e-07,
+ "logits/chosen": 1.4933072328567505,
+ "logits/rejected": 1.4721126556396484,
+ "logps/chosen": -146.546875,
+ "logps/rejected": -138.2855224609375,
+ "loss": 0.6930079936981202,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.005923466756939888,
+ "rewards/margins": 0.000295188365271315,
+ "rewards/rejected": 0.005628278478980064,
+ "step": 80
+ },
+ {
+ "epoch": 0.144,
+ "grad_norm": 1.1192086935043335,
+ "learning_rate": 4.734042553191489e-07,
+ "logits/chosen": 1.4889073371887207,
+ "logits/rejected": 1.4786438941955566,
+ "logps/chosen": -163.24220275878906,
+ "logps/rejected": -160.1715850830078,
+ "loss": 0.6927919387817383,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.00839515682309866,
+ "rewards/margins": 0.0007321214652620256,
+ "rewards/rejected": 0.007663035299628973,
+ "step": 90
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 1.2289488315582275,
+ "learning_rate": 5.26595744680851e-07,
+ "logits/chosen": 1.5521520376205444,
+ "logits/rejected": 1.620697259902954,
+ "logps/chosen": -143.54342651367188,
+ "logps/rejected": -166.68838500976562,
+ "loss": 0.6924111843109131,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.010075005702674389,
+ "rewards/margins": 0.00150866259355098,
+ "rewards/rejected": 0.008566344156861305,
+ "step": 100
+ },
+ {
+ "epoch": 0.16,
+ "eval_logits/chosen": 1.513370156288147,
+ "eval_logits/rejected": 1.5475622415542603,
+ "eval_logps/chosen": -153.44686889648438,
+ "eval_logps/rejected": -147.92510986328125,
+ "eval_loss": 0.6929221153259277,
+ "eval_rewards/accuracies": 0.4699999988079071,
+ "eval_rewards/chosen": 0.010197031311690807,
+ "eval_rewards/margins": 0.0004922974039800465,
+ "eval_rewards/rejected": 0.00970473513007164,
+ "eval_runtime": 91.4962,
+ "eval_samples_per_second": 5.465,
+ "eval_steps_per_second": 2.732,
+ "step": 100
+ },
+ {
+ "epoch": 0.176,
+ "grad_norm": 1.0589202642440796,
+ "learning_rate": 5.797872340425531e-07,
+ "logits/chosen": 1.5165660381317139,
+ "logits/rejected": 1.5812733173370361,
+ "logps/chosen": -144.24609375,
+ "logps/rejected": -138.3242645263672,
+ "loss": 0.6928606033325195,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.011964901350438595,
+ "rewards/margins": 0.0006008840864524245,
+ "rewards/rejected": 0.011364017613232136,
+ "step": 110
+ },
+ {
+ "epoch": 0.192,
+ "grad_norm": 1.1458054780960083,
+ "learning_rate": 6.329787234042553e-07,
+ "logits/chosen": 1.5244230031967163,
+ "logits/rejected": 1.594994306564331,
+ "logps/chosen": -143.27723693847656,
+ "logps/rejected": -143.82754516601562,
+ "loss": 0.6933906078338623,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": 0.009245705790817738,
+ "rewards/margins": -0.0004399990721140057,
+ "rewards/rejected": 0.009685704484581947,
+ "step": 120
+ },
+ {
+ "epoch": 0.208,
+ "grad_norm": 1.3152341842651367,
+ "learning_rate": 6.861702127659574e-07,
+ "logits/chosen": 1.6518194675445557,
+ "logits/rejected": 1.7864271402359009,
+ "logps/chosen": -156.05361938476562,
+ "logps/rejected": -151.7067108154297,
+ "loss": 0.6933954238891602,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.004633297678083181,
+ "rewards/margins": -0.00046000973088666797,
+ "rewards/rejected": 0.005093307700008154,
+ "step": 130
+ },
+ {
+ "epoch": 0.224,
+ "grad_norm": 1.1007953882217407,
+ "learning_rate": 7.393617021276596e-07,
+ "logits/chosen": 1.4447275400161743,
+ "logits/rejected": 1.522552251815796,
+ "logps/chosen": -134.9702911376953,
+ "logps/rejected": -133.505615234375,
+ "loss": 0.6926079750061035,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.0014777708565816283,
+ "rewards/margins": 0.0011204290203750134,
+ "rewards/rejected": 0.00035734166158363223,
+ "step": 140
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 0.92185378074646,
+ "learning_rate": 7.925531914893616e-07,
+ "logits/chosen": 1.7024447917938232,
+ "logits/rejected": 1.6758934259414673,
+ "logps/chosen": -142.47024536132812,
+ "logps/rejected": -141.05670166015625,
+ "loss": 0.6922515869140625,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.007132537662982941,
+ "rewards/margins": 0.0018380668479949236,
+ "rewards/rejected": 0.005294471047818661,
+ "step": 150
+ },
+ {
+ "epoch": 0.24,
+ "eval_logits/chosen": 1.5091065168380737,
+ "eval_logits/rejected": 1.5432627201080322,
+ "eval_logps/chosen": -153.50772094726562,
+ "eval_logps/rejected": -147.98837280273438,
+ "eval_loss": 0.6928078532218933,
+ "eval_rewards/accuracies": 0.5139999985694885,
+ "eval_rewards/chosen": 0.004112891852855682,
+ "eval_rewards/margins": 0.0007364260382018983,
+ "eval_rewards/rejected": 0.0033764655236154795,
+ "eval_runtime": 91.3903,
+ "eval_samples_per_second": 5.471,
+ "eval_steps_per_second": 2.736,
+ "step": 150
+ },
+ {
+ "epoch": 0.256,
+ "grad_norm": 1.2662791013717651,
+ "learning_rate": 8.457446808510637e-07,
+ "logits/chosen": 1.6849098205566406,
+ "logits/rejected": 1.620693564414978,
+ "logps/chosen": -164.6056365966797,
+ "logps/rejected": -148.01400756835938,
+ "loss": 0.6930419445037842,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": 0.009472684934735298,
+ "rewards/margins": 0.0002518748806323856,
+ "rewards/rejected": 0.009220810607075691,
+ "step": 160
+ },
+ {
+ "epoch": 0.272,
+ "grad_norm": 1.0822441577911377,
+ "learning_rate": 8.989361702127659e-07,
+ "logits/chosen": 1.4992691278457642,
+ "logits/rejected": 1.5570002794265747,
+ "logps/chosen": -141.10240173339844,
+ "logps/rejected": -125.11993408203125,
+ "loss": 0.692972469329834,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.009402485564351082,
+ "rewards/margins": 0.0004053188022226095,
+ "rewards/rejected": 0.008997166529297829,
+ "step": 170
+ },
+ {
+ "epoch": 0.288,
+ "grad_norm": 0.9809871315956116,
+ "learning_rate": 9.52127659574468e-07,
+ "logits/chosen": 1.5153322219848633,
+ "logits/rejected": 1.5223143100738525,
+ "logps/chosen": -154.1062469482422,
+ "logps/rejected": -155.04188537597656,
+ "loss": 0.6921162605285645,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.016383877024054527,
+ "rewards/margins": 0.002122154226526618,
+ "rewards/rejected": 0.014261722564697266,
+ "step": 180
+ },
+ {
+ "epoch": 0.304,
+ "grad_norm": 1.1610571146011353,
+ "learning_rate": 9.99407231772377e-07,
+ "logits/chosen": 1.6443488597869873,
+ "logits/rejected": 1.5873346328735352,
+ "logps/chosen": -157.59298706054688,
+ "logps/rejected": -155.9975128173828,
+ "loss": 0.6925239086151123,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.025818049907684326,
+ "rewards/margins": 0.0013327361084520817,
+ "rewards/rejected": 0.024485310539603233,
+ "step": 190
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 1.2108772993087769,
+ "learning_rate": 9.93479549496147e-07,
+ "logits/chosen": 1.3679401874542236,
+ "logits/rejected": 1.5478118658065796,
+ "logps/chosen": -127.5545654296875,
+ "logps/rejected": -134.0937042236328,
+ "loss": 0.6889955520629882,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": 0.033060748130083084,
+ "rewards/margins": 0.008453840389847755,
+ "rewards/rejected": 0.02460690215229988,
+ "step": 200
+ },
+ {
+ "epoch": 0.32,
+ "eval_logits/chosen": 1.5197134017944336,
+ "eval_logits/rejected": 1.5536755323410034,
+ "eval_logps/chosen": -153.1664581298828,
+ "eval_logps/rejected": -147.66543579101562,
+ "eval_loss": 0.6919757127761841,
+ "eval_rewards/accuracies": 0.5239999890327454,
+ "eval_rewards/chosen": 0.03824040666222572,
+ "eval_rewards/margins": 0.0025693033821880817,
+ "eval_rewards/rejected": 0.03567110374569893,
+ "eval_runtime": 91.3674,
+ "eval_samples_per_second": 5.472,
+ "eval_steps_per_second": 2.736,
+ "step": 200
+ },
+ {
+ "epoch": 0.336,
+ "grad_norm": 1.4075576066970825,
+ "learning_rate": 9.87551867219917e-07,
+ "logits/chosen": 1.6747233867645264,
+ "logits/rejected": 1.557538628578186,
+ "logps/chosen": -151.46202087402344,
+ "logps/rejected": -126.70817565917969,
+ "loss": 0.6903837203979493,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.0453563891351223,
+ "rewards/margins": 0.005785389803349972,
+ "rewards/rejected": 0.03957099840044975,
+ "step": 210
+ },
+ {
+ "epoch": 0.352,
+ "grad_norm": 1.05863356590271,
+ "learning_rate": 9.81624184943687e-07,
+ "logits/chosen": 1.7199232578277588,
+ "logits/rejected": 1.6891214847564697,
+ "logps/chosen": -164.67184448242188,
+ "logps/rejected": -143.46157836914062,
+ "loss": 0.6919777870178223,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.0621187686920166,
+ "rewards/margins": 0.002725570695474744,
+ "rewards/rejected": 0.05939319729804993,
+ "step": 220
+ },
+ {
+ "epoch": 0.368,
+ "grad_norm": 0.9537659287452698,
+ "learning_rate": 9.75696502667457e-07,
+ "logits/chosen": 1.5906708240509033,
+ "logits/rejected": 1.5525639057159424,
+ "logps/chosen": -149.23095703125,
+ "logps/rejected": -154.40731811523438,
+ "loss": 0.6942371845245361,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.061252254992723465,
+ "rewards/margins": -0.0017460950184613466,
+ "rewards/rejected": 0.06299833953380585,
+ "step": 230
+ },
+ {
+ "epoch": 0.384,
+ "grad_norm": 0.9180606603622437,
+ "learning_rate": 9.69768820391227e-07,
+ "logits/chosen": 1.5654420852661133,
+ "logits/rejected": 1.6795461177825928,
+ "logps/chosen": -158.65875244140625,
+ "logps/rejected": -165.13951110839844,
+ "loss": 0.692426061630249,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.06773029267787933,
+ "rewards/margins": 0.0019084472442045808,
+ "rewards/rejected": 0.06582184135913849,
+ "step": 240
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.7486060857772827,
+ "learning_rate": 9.63841138114997e-07,
+ "logits/chosen": 1.5377140045166016,
+ "logits/rejected": 1.6011888980865479,
+ "logps/chosen": -138.14199829101562,
+ "logps/rejected": -151.1943817138672,
+ "loss": 0.6908403396606445,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": 0.06411711871623993,
+ "rewards/margins": 0.004912695847451687,
+ "rewards/rejected": 0.05920442193746567,
+ "step": 250
+ },
+ {
+ "epoch": 0.4,
+ "eval_logits/chosen": 1.534494161605835,
+ "eval_logits/rejected": 1.5683013200759888,
+ "eval_logps/chosen": -152.8787078857422,
+ "eval_logps/rejected": -147.39991760253906,
+ "eval_loss": 0.6910172700881958,
+ "eval_rewards/accuracies": 0.5460000038146973,
+ "eval_rewards/chosen": 0.06701447814702988,
+ "eval_rewards/margins": 0.004791789222508669,
+ "eval_rewards/rejected": 0.06222269684076309,
+ "eval_runtime": 91.3518,
+ "eval_samples_per_second": 5.473,
+ "eval_steps_per_second": 2.737,
+ "step": 250
+ },
+ {
+ "epoch": 0.416,
+ "grad_norm": 1.0861977338790894,
+ "learning_rate": 9.579134558387671e-07,
+ "logits/chosen": 1.5887064933776855,
+ "logits/rejected": 1.6124156713485718,
+ "logps/chosen": -145.68692016601562,
+ "logps/rejected": -150.8065948486328,
+ "loss": 0.6914952278137207,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.07477976381778717,
+ "rewards/margins": 0.00367984501644969,
+ "rewards/rejected": 0.0710999146103859,
+ "step": 260
+ },
+ {
+ "epoch": 0.432,
+ "grad_norm": 1.2531821727752686,
+ "learning_rate": 9.519857735625369e-07,
+ "logits/chosen": 1.552918553352356,
+ "logits/rejected": 1.6837631464004517,
+ "logps/chosen": -158.31802368164062,
+ "logps/rejected": -151.29379272460938,
+ "loss": 0.691353178024292,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.07334339618682861,
+ "rewards/margins": 0.004158434458076954,
+ "rewards/rejected": 0.06918496638536453,
+ "step": 270
+ },
+ {
+ "epoch": 0.448,
+ "grad_norm": 0.8373661637306213,
+ "learning_rate": 9.460580912863069e-07,
+ "logits/chosen": 1.6604083776474,
+ "logits/rejected": 1.7040210962295532,
+ "logps/chosen": -132.56422424316406,
+ "logps/rejected": -151.414794921875,
+ "loss": 0.6947796821594239,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": 0.07179545611143112,
+ "rewards/margins": -0.002721407450735569,
+ "rewards/rejected": 0.07451687008142471,
+ "step": 280
+ },
+ {
+ "epoch": 0.464,
+ "grad_norm": 1.0909446477890015,
+ "learning_rate": 9.40130409010077e-07,
+ "logits/chosen": 1.6112158298492432,
+ "logits/rejected": 1.6040490865707397,
+ "logps/chosen": -155.4887237548828,
+ "logps/rejected": -149.31924438476562,
+ "loss": 0.6928027629852295,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.07096447050571442,
+ "rewards/margins": 0.0013116076588630676,
+ "rewards/rejected": 0.06965287029743195,
+ "step": 290
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 0.8769014477729797,
+ "learning_rate": 9.34202726733847e-07,
+ "logits/chosen": 1.4986810684204102,
+ "logits/rejected": 1.524269700050354,
+ "logps/chosen": -138.91099548339844,
+ "logps/rejected": -141.48495483398438,
+ "loss": 0.6924718379974365,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.060377366840839386,
+ "rewards/margins": 0.0016540264477953315,
+ "rewards/rejected": 0.058723341673612595,
+ "step": 300
+ },
+ {
+ "epoch": 0.48,
+ "eval_logits/chosen": 1.5390068292617798,
+ "eval_logits/rejected": 1.573019027709961,
+ "eval_logps/chosen": -152.91844177246094,
+ "eval_logps/rejected": -147.44703674316406,
+ "eval_loss": 0.6906248331069946,
+ "eval_rewards/accuracies": 0.5680000185966492,
+ "eval_rewards/chosen": 0.06304025650024414,
+ "eval_rewards/margins": 0.005528101697564125,
+ "eval_rewards/rejected": 0.057512152940034866,
+ "eval_runtime": 91.3495,
+ "eval_samples_per_second": 5.473,
+ "eval_steps_per_second": 2.737,
+ "step": 300
+ },
+ {
+ "epoch": 0.496,
+ "grad_norm": 1.050471305847168,
+ "learning_rate": 9.28275044457617e-07,
+ "logits/chosen": 1.7194430828094482,
+ "logits/rejected": 1.807745337486267,
+ "logps/chosen": -160.05563354492188,
+ "logps/rejected": -138.70767211914062,
+ "loss": 0.6887433052062988,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.0673876479268074,
+ "rewards/margins": 0.009202037006616592,
+ "rewards/rejected": 0.058185599744319916,
+ "step": 310
+ },
+ {
+ "epoch": 0.512,
+ "grad_norm": 1.1666924953460693,
+ "learning_rate": 9.223473621813871e-07,
+ "logits/chosen": 1.737310767173767,
+ "logits/rejected": 1.6316074132919312,
+ "logps/chosen": -151.58229064941406,
+ "logps/rejected": -142.8289794921875,
+ "loss": 0.6919591426849365,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.06055358797311783,
+ "rewards/margins": 0.0026850434951484203,
+ "rewards/rejected": 0.057868532836437225,
+ "step": 320
+ },
+ {
+ "epoch": 0.528,
+ "grad_norm": 1.2360730171203613,
+ "learning_rate": 9.16419679905157e-07,
+ "logits/chosen": 1.4524610042572021,
+ "logits/rejected": 1.490668535232544,
+ "logps/chosen": -136.93104553222656,
+ "logps/rejected": -155.99844360351562,
+ "loss": 0.6895017623901367,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.06680475920438766,
+ "rewards/margins": 0.007821018807590008,
+ "rewards/rejected": 0.05898374319076538,
+ "step": 330
+ },
+ {
+ "epoch": 0.544,
+ "grad_norm": 1.0067647695541382,
+ "learning_rate": 9.10491997628927e-07,
+ "logits/chosen": 1.5482432842254639,
+ "logits/rejected": 1.596333622932434,
+ "logps/chosen": -142.59603881835938,
+ "logps/rejected": -120.35069274902344,
+ "loss": 0.6895721912384033,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": 0.0686279758810997,
+ "rewards/margins": 0.007503635250031948,
+ "rewards/rejected": 0.06112435460090637,
+ "step": 340
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 1.2567517757415771,
+ "learning_rate": 9.04564315352697e-07,
+ "logits/chosen": 1.5976614952087402,
+ "logits/rejected": 1.3774999380111694,
+ "logps/chosen": -137.83567810058594,
+ "logps/rejected": -134.2790985107422,
+ "loss": 0.6924222946166992,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06484080851078033,
+ "rewards/margins": 0.0020756719168275595,
+ "rewards/rejected": 0.06276513636112213,
+ "step": 350
+ },
+ {
+ "epoch": 0.56,
+ "eval_logits/chosen": 1.5381687879562378,
+ "eval_logits/rejected": 1.5718625783920288,
+ "eval_logps/chosen": -152.91278076171875,
+ "eval_logps/rejected": -147.4539794921875,
+ "eval_loss": 0.6900433301925659,
+ "eval_rewards/accuracies": 0.5559999942779541,
+ "eval_rewards/chosen": 0.06360553950071335,
+ "eval_rewards/margins": 0.006787541322410107,
+ "eval_rewards/rejected": 0.05681798979640007,
+ "eval_runtime": 91.4768,
+ "eval_samples_per_second": 5.466,
+ "eval_steps_per_second": 2.733,
+ "step": 350
+ },
+ {
+ "epoch": 0.576,
+ "grad_norm": 1.3803397417068481,
+ "learning_rate": 8.986366330764671e-07,
+ "logits/chosen": 1.6290550231933594,
+ "logits/rejected": 1.7498657703399658,
+ "logps/chosen": -163.21676635742188,
+ "logps/rejected": -170.33084106445312,
+ "loss": 0.6899758338928222,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.06653806567192078,
+ "rewards/margins": 0.006778226234018803,
+ "rewards/rejected": 0.05975983664393425,
+ "step": 360
+ },
+ {
+ "epoch": 0.592,
+ "grad_norm": 1.1363084316253662,
+ "learning_rate": 8.927089508002371e-07,
+ "logits/chosen": 1.6613603830337524,
+ "logits/rejected": 1.7096723318099976,
+ "logps/chosen": -175.3104248046875,
+ "logps/rejected": -176.24655151367188,
+ "loss": 0.690467643737793,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.061445217579603195,
+ "rewards/margins": 0.005937105976045132,
+ "rewards/rejected": 0.05550810694694519,
+ "step": 370
+ },
+ {
+ "epoch": 0.608,
+ "grad_norm": 1.139722228050232,
+ "learning_rate": 8.86781268524007e-07,
+ "logits/chosen": 1.6426604986190796,
+ "logits/rejected": 1.7499873638153076,
+ "logps/chosen": -136.9043731689453,
+ "logps/rejected": -159.98196411132812,
+ "loss": 0.6899127960205078,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.05270732566714287,
+ "rewards/margins": 0.007067640777677298,
+ "rewards/rejected": 0.04563968628644943,
+ "step": 380
+ },
+ {
+ "epoch": 0.624,
+ "grad_norm": 1.2813146114349365,
+ "learning_rate": 8.80853586247777e-07,
+ "logits/chosen": 1.3995320796966553,
+ "logits/rejected": 1.6828539371490479,
+ "logps/chosen": -131.1273651123047,
+ "logps/rejected": -167.51840209960938,
+ "loss": 0.6943269729614258,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": 0.05107322335243225,
+ "rewards/margins": -0.0017629960784688592,
+ "rewards/rejected": 0.05283621698617935,
+ "step": 390
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 1.1288373470306396,
+ "learning_rate": 8.749259039715471e-07,
+ "logits/chosen": 1.6386661529541016,
+ "logits/rejected": 1.6272175312042236,
+ "logps/chosen": -150.4084014892578,
+ "logps/rejected": -161.52236938476562,
+ "loss": 0.6923639297485351,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": 0.039307981729507446,
+ "rewards/margins": 0.001983280060812831,
+ "rewards/rejected": 0.037324704229831696,
+ "step": 400
+ },
+ {
+ "epoch": 0.64,
+ "eval_logits/chosen": 1.519494652748108,
+ "eval_logits/rejected": 1.5528141260147095,
+ "eval_logps/chosen": -153.15609741210938,
+ "eval_logps/rejected": -147.69198608398438,
+ "eval_loss": 0.6903249025344849,
+ "eval_rewards/accuracies": 0.5460000038146973,
+ "eval_rewards/chosen": 0.03927500545978546,
+ "eval_rewards/margins": 0.006258465815335512,
+ "eval_rewards/rejected": 0.033016543835401535,
+ "eval_runtime": 91.4257,
+ "eval_samples_per_second": 5.469,
+ "eval_steps_per_second": 2.734,
+ "step": 400
+ },
+ {
+ "epoch": 0.656,
+ "grad_norm": 1.355100154876709,
+ "learning_rate": 8.689982216953171e-07,
+ "logits/chosen": 1.5310468673706055,
+ "logits/rejected": 1.6076139211654663,
+ "logps/chosen": -157.39776611328125,
+ "logps/rejected": -145.91275024414062,
+ "loss": 0.68691987991333,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.04244391992688179,
+ "rewards/margins": 0.01302963774651289,
+ "rewards/rejected": 0.029414281249046326,
+ "step": 410
+ },
+ {
+ "epoch": 0.672,
+ "grad_norm": 1.1569719314575195,
+ "learning_rate": 8.630705394190871e-07,
+ "logits/chosen": 1.5669739246368408,
+ "logits/rejected": 1.5956265926361084,
+ "logps/chosen": -132.22671508789062,
+ "logps/rejected": -132.46981811523438,
+ "loss": 0.6866061210632324,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.04782433062791824,
+ "rewards/margins": 0.013592484407126904,
+ "rewards/rejected": 0.034231849014759064,
+ "step": 420
+ },
+ {
+ "epoch": 0.688,
+ "grad_norm": 1.1847658157348633,
+ "learning_rate": 8.57142857142857e-07,
+ "logits/chosen": 1.6808805465698242,
+ "logits/rejected": 1.7047704458236694,
+ "logps/chosen": -140.24200439453125,
+ "logps/rejected": -140.81124877929688,
+ "loss": 0.6933694362640381,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": 0.047805462032556534,
+ "rewards/margins": 0.00025355032994411886,
+ "rewards/rejected": 0.04755191132426262,
+ "step": 430
+ },
+ {
+ "epoch": 0.704,
+ "grad_norm": 0.9608289003372192,
+ "learning_rate": 8.512151748666271e-07,
+ "logits/chosen": 1.663816213607788,
+ "logits/rejected": 1.6789140701293945,
+ "logps/chosen": -155.818603515625,
+ "logps/rejected": -145.31468200683594,
+ "loss": 0.6929332733154296,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": 0.042977917939424515,
+ "rewards/margins": 0.0008615514962002635,
+ "rewards/rejected": 0.04211636632680893,
+ "step": 440
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 1.3275092840194702,
+ "learning_rate": 8.452874925903971e-07,
+ "logits/chosen": 1.4646484851837158,
+ "logits/rejected": 1.6175050735473633,
+ "logps/chosen": -151.04867553710938,
+ "logps/rejected": -147.8173828125,
+ "loss": 0.6903160572052002,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": 0.03235078603029251,
+ "rewards/margins": 0.00654552411288023,
+ "rewards/rejected": 0.025805264711380005,
+ "step": 450
+ },
+ {
+ "epoch": 0.72,
+ "eval_logits/chosen": 1.5208625793457031,
+ "eval_logits/rejected": 1.5539581775665283,
+ "eval_logps/chosen": -153.08497619628906,
+ "eval_logps/rejected": -147.63291931152344,
+ "eval_loss": 0.6898273229598999,
+ "eval_rewards/accuracies": 0.5680000185966492,
+ "eval_rewards/chosen": 0.046387214213609695,
+ "eval_rewards/margins": 0.0074648563750088215,
+ "eval_rewards/rejected": 0.038922350853681564,
+ "eval_runtime": 91.384,
+ "eval_samples_per_second": 5.471,
+ "eval_steps_per_second": 2.736,
+ "step": 450
+ },
+ {
+ "epoch": 0.736,
+ "grad_norm": 1.3754528760910034,
+ "learning_rate": 8.393598103141671e-07,
+ "logits/chosen": 1.5359246730804443,
+ "logits/rejected": 1.604922890663147,
+ "logps/chosen": -137.5425262451172,
+ "logps/rejected": -161.14151000976562,
+ "loss": 0.6894284725189209,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.03450053930282593,
+ "rewards/margins": 0.007881317287683487,
+ "rewards/rejected": 0.02661922201514244,
+ "step": 460
+ },
+ {
+ "epoch": 0.752,
+ "grad_norm": 1.045366883277893,
+ "learning_rate": 8.334321280379371e-07,
+ "logits/chosen": 1.4338958263397217,
+ "logits/rejected": 1.5732755661010742,
+ "logps/chosen": -142.7367401123047,
+ "logps/rejected": -129.0875244140625,
+ "loss": 0.687935209274292,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": 0.03967561572790146,
+ "rewards/margins": 0.01102046575397253,
+ "rewards/rejected": 0.028655150905251503,
+ "step": 470
+ },
+ {
+ "epoch": 0.768,
+ "grad_norm": 1.232444167137146,
+ "learning_rate": 8.275044457617071e-07,
+ "logits/chosen": 1.7112070322036743,
+ "logits/rejected": 1.8045260906219482,
+ "logps/chosen": -170.90280151367188,
+ "logps/rejected": -164.41964721679688,
+ "loss": 0.6896602630615234,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": 0.04624806344509125,
+ "rewards/margins": 0.0076986560598015785,
+ "rewards/rejected": 0.038549408316612244,
+ "step": 480
+ },
+ {
+ "epoch": 0.784,
+ "grad_norm": 1.2316750288009644,
+ "learning_rate": 8.215767634854771e-07,
+ "logits/chosen": 1.660528540611267,
+ "logits/rejected": 1.5905355215072632,
+ "logps/chosen": -166.0521697998047,
+ "logps/rejected": -164.01641845703125,
+ "loss": 0.6967196464538574,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": 0.022944677621126175,
+ "rewards/margins": -0.006097245961427689,
+ "rewards/rejected": 0.029041921719908714,
+ "step": 490
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 1.0982184410095215,
+ "learning_rate": 8.156490812092471e-07,
+ "logits/chosen": 1.5519205331802368,
+ "logits/rejected": 1.6819387674331665,
+ "logps/chosen": -137.09713745117188,
+ "logps/rejected": -139.62698364257812,
+ "loss": 0.6872400283813477,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": 0.027622828260064125,
+ "rewards/margins": 0.012441830709576607,
+ "rewards/rejected": 0.015180995687842369,
+ "step": 500
+ },
+ {
+ "epoch": 0.8,
+ "eval_logits/chosen": 1.513393521308899,
+ "eval_logits/rejected": 1.5464614629745483,
+ "eval_logps/chosen": -153.2435302734375,
+ "eval_logps/rejected": -147.79180908203125,
+ "eval_loss": 0.6898292303085327,
+ "eval_rewards/accuracies": 0.5559999942779541,
+ "eval_rewards/chosen": 0.030531544238328934,
+ "eval_rewards/margins": 0.0074988617561757565,
+ "eval_rewards/rejected": 0.02303268015384674,
+ "eval_runtime": 91.4769,
+ "eval_samples_per_second": 5.466,
+ "eval_steps_per_second": 2.733,
+ "step": 500
+ }
+ ],
+ "logging_steps": 10,
+ "max_steps": 1875,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/v3/DPO/DPO_5k/lora/checkpoint-500/training_args.bin b/v3/DPO/DPO_5k/lora/checkpoint-500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..544215e69bd09b706d2c495c7bb412d0df7f83e7
--- /dev/null
+++ b/v3/DPO/DPO_5k/lora/checkpoint-500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afe50d4ac4c5f2af306fafe28b341897775b898324082e319b56c66893dceabb
+size 6097
diff --git a/v3/DPO/gen-output/DPO_10k/data-00000-of-00001.arrow b/v3/DPO/gen-output/DPO_10k/data-00000-of-00001.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..7e9a3d1ca2f21288a66a2d16d27b40a9bdaebc18
--- /dev/null
+++ b/v3/DPO/gen-output/DPO_10k/data-00000-of-00001.arrow
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d58ea41e489535b8cb4eb1d6dbb6de2aabe7c2c9763ec8dcd9d4c834bf8459dc
+size 981616
diff --git a/v3/DPO/gen-output/DPO_10k/dataset_info.json b/v3/DPO/gen-output/DPO_10k/dataset_info.json
new file mode 100644
index 0000000000000000000000000000000000000000..523b881cd388b7ceb2401ad4759de06c0653cbef
--- /dev/null
+++ b/v3/DPO/gen-output/DPO_10k/dataset_info.json
@@ -0,0 +1,34 @@
+{
+ "citation": "",
+ "description": "",
+ "features": {
+ "prompt": {
+ "feature": {
+ "content": {
+ "dtype": "string",
+ "_type": "Value"
+ },
+ "role": {
+ "dtype": "string",
+ "_type": "Value"
+ }
+ },
+ "_type": "List"
+ },
+ "generated_text": {
+ "feature": {
+ "content": {
+ "dtype": "string",
+ "_type": "Value"
+ },
+ "role": {
+ "dtype": "string",
+ "_type": "Value"
+ }
+ },
+ "_type": "List"
+ }
+ },
+ "homepage": "",
+ "license": ""
+}
\ No newline at end of file
diff --git a/v3/DPO/gen-output/DPO_10k/state.json b/v3/DPO/gen-output/DPO_10k/state.json
new file mode 100644
index 0000000000000000000000000000000000000000..59397c566457b2402e568027eb89bc92469cb669
--- /dev/null
+++ b/v3/DPO/gen-output/DPO_10k/state.json
@@ -0,0 +1,13 @@
+{
+ "_data_files": [
+ {
+ "filename": "data-00000-of-00001.arrow"
+ }
+ ],
+ "_fingerprint": "2388df1c911320b2",
+ "_format_columns": null,
+ "_format_kwargs": {},
+ "_format_type": null,
+ "_output_all_columns": false,
+ "_split": null
+}
\ No newline at end of file
diff --git a/v3/DPO/gen-output/DPO_1k/data-00000-of-00001.arrow b/v3/DPO/gen-output/DPO_1k/data-00000-of-00001.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..264afc4e2a585bd6118c29e08a892147bd6d731c
--- /dev/null
+++ b/v3/DPO/gen-output/DPO_1k/data-00000-of-00001.arrow
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd09e18e3de4969e92d6c115cde3a38c13b286f4341fb6dce000726d851e3c6d
+size 994832
diff --git a/v3/DPO/gen-output/DPO_1k/dataset_info.json b/v3/DPO/gen-output/DPO_1k/dataset_info.json
new file mode 100644
index 0000000000000000000000000000000000000000..523b881cd388b7ceb2401ad4759de06c0653cbef
--- /dev/null
+++ b/v3/DPO/gen-output/DPO_1k/dataset_info.json
@@ -0,0 +1,34 @@
+{
+ "citation": "",
+ "description": "",
+ "features": {
+ "prompt": {
+ "feature": {
+ "content": {
+ "dtype": "string",
+ "_type": "Value"
+ },
+ "role": {
+ "dtype": "string",
+ "_type": "Value"
+ }
+ },
+ "_type": "List"
+ },
+ "generated_text": {
+ "feature": {
+ "content": {
+ "dtype": "string",
+ "_type": "Value"
+ },
+ "role": {
+ "dtype": "string",
+ "_type": "Value"
+ }
+ },
+ "_type": "List"
+ }
+ },
+ "homepage": "",
+ "license": ""
+}
\ No newline at end of file
diff --git a/v3/DPO/gen-output/DPO_1k/state.json b/v3/DPO/gen-output/DPO_1k/state.json
new file mode 100644
index 0000000000000000000000000000000000000000..4f8326da5d20abb429ebc78ee92a95034f31336b
--- /dev/null
+++ b/v3/DPO/gen-output/DPO_1k/state.json
@@ -0,0 +1,13 @@
+{
+ "_data_files": [
+ {
+ "filename": "data-00000-of-00001.arrow"
+ }
+ ],
+ "_fingerprint": "de3cd67b7756d1ee",
+ "_format_columns": null,
+ "_format_kwargs": {},
+ "_format_type": null,
+ "_output_all_columns": false,
+ "_split": null
+}
\ No newline at end of file
diff --git a/v3/DPO/gen-output/DPO_5k/data-00000-of-00001.arrow b/v3/DPO/gen-output/DPO_5k/data-00000-of-00001.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..a207197cf85d9c4d92ef2b292b5e54bebbf0abab
--- /dev/null
+++ b/v3/DPO/gen-output/DPO_5k/data-00000-of-00001.arrow
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16d7599dba1c147f4787f3a9572e82f67acd50be7e3b039e3389467cc93af1a9
+size 984520
diff --git a/v3/DPO/gen-output/DPO_5k/dataset_info.json b/v3/DPO/gen-output/DPO_5k/dataset_info.json
new file mode 100644
index 0000000000000000000000000000000000000000..523b881cd388b7ceb2401ad4759de06c0653cbef
--- /dev/null
+++ b/v3/DPO/gen-output/DPO_5k/dataset_info.json
@@ -0,0 +1,34 @@
+{
+ "citation": "",
+ "description": "",
+ "features": {
+ "prompt": {
+ "feature": {
+ "content": {
+ "dtype": "string",
+ "_type": "Value"
+ },
+ "role": {
+ "dtype": "string",
+ "_type": "Value"
+ }
+ },
+ "_type": "List"
+ },
+ "generated_text": {
+ "feature": {
+ "content": {
+ "dtype": "string",
+ "_type": "Value"
+ },
+ "role": {
+ "dtype": "string",
+ "_type": "Value"
+ }
+ },
+ "_type": "List"
+ }
+ },
+ "homepage": "",
+ "license": ""
+}
\ No newline at end of file
diff --git a/v3/DPO/gen-output/DPO_5k/state.json b/v3/DPO/gen-output/DPO_5k/state.json
new file mode 100644
index 0000000000000000000000000000000000000000..cb4071bd5be66b2ccb5b59b6516a99d278f88d54
--- /dev/null
+++ b/v3/DPO/gen-output/DPO_5k/state.json
@@ -0,0 +1,13 @@
+{
+ "_data_files": [
+ {
+ "filename": "data-00000-of-00001.arrow"
+ }
+ ],
+ "_fingerprint": "d6d257bd4f18b87c",
+ "_format_columns": null,
+ "_format_kwargs": {},
+ "_format_type": null,
+ "_output_all_columns": false,
+ "_split": null
+}
\ No newline at end of file
diff --git a/v3/ORPO/ORPO_1k/MORPO_1k/chat_template.jinja b/v3/ORPO/ORPO_1k/MORPO_1k/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/MORPO_1k/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/ORPO/ORPO_1k/MORPO_1k/config.json b/v3/ORPO/ORPO_1k/MORPO_1k/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e9ec18bc1ae18922052a57480e28401f3b9c6b84
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/MORPO_1k/config.json
@@ -0,0 +1,40 @@
+{
+ "architectures": [
+ "LlamaForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 128000,
+ "dtype": "float16",
+ "eos_token_id": [
+ 128001,
+ 128008,
+ 128009
+ ],
+ "head_dim": 64,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 8192,
+ "max_position_embeddings": 131072,
+ "mlp_bias": false,
+ "model_type": "llama",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 16,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "pretraining_tp": 1,
+ "rms_norm_eps": 1e-05,
+ "rope_parameters": {
+ "factor": 32.0,
+ "high_freq_factor": 4.0,
+ "low_freq_factor": 1.0,
+ "original_max_position_embeddings": 8192,
+ "rope_theta": 500000.0,
+ "rope_type": "llama3"
+ },
+ "tie_word_embeddings": true,
+ "transformers_version": "5.2.0",
+ "use_cache": true,
+ "vocab_size": 128256
+}
diff --git a/v3/ORPO/ORPO_1k/MORPO_1k/generation_config.json b/v3/ORPO/ORPO_1k/MORPO_1k/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..898a2e748a41a4b7a931b754f8abbdac02039fae
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/MORPO_1k/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "bos_token_id": 128000,
+ "do_sample": true,
+ "eos_token_id": [
+ 128001,
+ 128008,
+ 128009
+ ],
+ "temperature": 0.6,
+ "top_p": 0.9,
+ "transformers_version": "5.2.0"
+}
diff --git a/v3/ORPO/ORPO_1k/MORPO_1k/model.safetensors b/v3/ORPO/ORPO_1k/MORPO_1k/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5e10568afa1157ed34cba77886aaef8a459b57da
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/MORPO_1k/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef878d1b9201fbf4adc8fd6c34c4fc26e95f0274006586e6b1dcd8e764b45384
+size 2471645464
diff --git a/v3/ORPO/ORPO_1k/MORPO_1k/tokenizer.json b/v3/ORPO/ORPO_1k/MORPO_1k/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/MORPO_1k/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/ORPO/ORPO_1k/MORPO_1k/tokenizer_config.json b/v3/ORPO/ORPO_1k/MORPO_1k/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/MORPO_1k/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/ORPO/ORPO_1k/ORPO_1k/README.md b/v3/ORPO/ORPO_1k/ORPO_1k/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..df73bb4ed6d6a8e69c79ba9761e4cdd4272cc1ab
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/ORPO_1k/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- lora
+- orpo
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/ORPO/ORPO_1k/ORPO_1k/adapter_config.json b/v3/ORPO/ORPO_1k/ORPO_1k/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..616cb58efc0471854578815c3abb05efee7a2ddd
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/ORPO_1k/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "o_proj",
+ "q_proj",
+ "k_proj",
+ "down_proj",
+ "v_proj",
+ "up_proj",
+ "gate_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/ORPO/ORPO_1k/ORPO_1k/adapter_model.safetensors b/v3/ORPO/ORPO_1k/ORPO_1k/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..32bff08dbfe26c4304957c7da8740fad0746d445
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/ORPO_1k/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13e4669fa4f8c3f338a87ecedc9a888c4dcec9718e21584b4283b1b6272161cc
+size 180385008
diff --git a/v3/ORPO/ORPO_1k/lora/README.md b/v3/ORPO/ORPO_1k/lora/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..c096746c0210754cd3a5d5b433750bcee1d0e7a1
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/lora/README.md
@@ -0,0 +1,66 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: transformers
+model_name: lora
+tags:
+- generated_from_trainer
+- orpo
+- trl
+licence: license
+---
+
+# Model Card for lora
+
+This model is a fine-tuned version of [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[](https://wandb.ai/sea-rod/huggingface/runs/e0sprdw9)
+
+
+This model was trained with ORPO, a method introduced in [ORPO: Monolithic Preference Optimization without Reference Model](https://huggingface.co/papers/2403.07691).
+
+### Framework versions
+
+- TRL: 0.28.0
+- Transformers: 5.2.0
+- Pytorch: 2.10.0
+- Datasets: 4.5.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite ORPO as:
+
+```bibtex
+@article{hong2024orpo,
+ title = {{ORPO: Monolithic Preference Optimization without Reference Model}},
+ author = {Jiwoo Hong and Noah Lee and James Thorne},
+ year = 2024,
+ eprint = {arXiv:2403.07691}
+}
+```
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+ title = {{TRL: Transformers Reinforcement Learning}},
+ author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+ license = {Apache-2.0},
+ url = {https://github.com/huggingface/trl},
+ year = {2020}
+}
+```
\ No newline at end of file
diff --git a/v3/ORPO/ORPO_1k/lora/checkpoint-375/README.md b/v3/ORPO/ORPO_1k/lora/checkpoint-375/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..df73bb4ed6d6a8e69c79ba9761e4cdd4272cc1ab
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/lora/checkpoint-375/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- lora
+- orpo
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/ORPO/ORPO_1k/lora/checkpoint-375/adapter_config.json b/v3/ORPO/ORPO_1k/lora/checkpoint-375/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..616cb58efc0471854578815c3abb05efee7a2ddd
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/lora/checkpoint-375/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "o_proj",
+ "q_proj",
+ "k_proj",
+ "down_proj",
+ "v_proj",
+ "up_proj",
+ "gate_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/ORPO/ORPO_1k/lora/checkpoint-375/adapter_model.safetensors b/v3/ORPO/ORPO_1k/lora/checkpoint-375/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..32bff08dbfe26c4304957c7da8740fad0746d445
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/lora/checkpoint-375/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13e4669fa4f8c3f338a87ecedc9a888c4dcec9718e21584b4283b1b6272161cc
+size 180385008
diff --git a/v3/ORPO/ORPO_1k/lora/checkpoint-375/chat_template.jinja b/v3/ORPO/ORPO_1k/lora/checkpoint-375/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/lora/checkpoint-375/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/ORPO/ORPO_1k/lora/checkpoint-375/optimizer.pt b/v3/ORPO/ORPO_1k/lora/checkpoint-375/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9ec1fa5f1cac37bcac925b73cfeee0f8848216c1
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/lora/checkpoint-375/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01a5cd21c0dc2da3c5220db33e3df3e085f357245063582ca02e4420a2048ee5
+size 360902475
diff --git a/v3/ORPO/ORPO_1k/lora/checkpoint-375/rng_state.pth b/v3/ORPO/ORPO_1k/lora/checkpoint-375/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..7859360e6e5ad44f3ed47e7c17f4cafb2aebd41a
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/lora/checkpoint-375/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50dd579ef1d2d932c3d7726ef0f6ec72d0e25b023ae2fd8b67112ac75e1041b9
+size 14645
diff --git a/v3/ORPO/ORPO_1k/lora/checkpoint-375/scaler.pt b/v3/ORPO/ORPO_1k/lora/checkpoint-375/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..096ec41fd7ece1a6761c13f49e4aeb268a4a2e4a
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/lora/checkpoint-375/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac58ef7454dc327cdd1497d560e3c1cadc5881183c106e69fda6701af10248e8
+size 1383
diff --git a/v3/ORPO/ORPO_1k/lora/checkpoint-375/scheduler.pt b/v3/ORPO/ORPO_1k/lora/checkpoint-375/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c05f8e2eac217c8b08d1c093d8ff7b389c86c3f1
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/lora/checkpoint-375/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:222a99bba6717d83d9b3500c6d4e096af1c89e61499d4ad9f714430b83f58d7b
+size 1465
diff --git a/v3/ORPO/ORPO_1k/lora/checkpoint-375/tokenizer.json b/v3/ORPO/ORPO_1k/lora/checkpoint-375/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/lora/checkpoint-375/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/ORPO/ORPO_1k/lora/checkpoint-375/tokenizer_config.json b/v3/ORPO/ORPO_1k/lora/checkpoint-375/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/lora/checkpoint-375/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/ORPO/ORPO_1k/lora/checkpoint-375/trainer_state.json b/v3/ORPO/ORPO_1k/lora/checkpoint-375/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..81f3f39f0302f6dc4bd677c19e89da4f64d04ffd
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/lora/checkpoint-375/trainer_state.json
@@ -0,0 +1,1403 @@
+{
+ "best_global_step": 370,
+ "best_metric": 1.8383456468582153,
+ "best_model_checkpoint": null,
+ "epoch": 3.0,
+ "eval_steps": 10,
+ "global_step": 375,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.08,
+ "grad_norm": 0.9772794246673584,
+ "learning_rate": 1.894736842105263e-06,
+ "log_odds_chosen": 0.19650818407535553,
+ "log_odds_ratio": -0.8589460253715515,
+ "logits/chosen": 1.1004374027252197,
+ "logits/rejected": 1.0792971849441528,
+ "logps/chosen": -3.056217670440674,
+ "logps/rejected": -3.2498669624328613,
+ "loss": 3.5441307067871093,
+ "nll_loss": 3.4582359790802,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.30562177300453186,
+ "rewards/margins": 0.019364940002560616,
+ "rewards/rejected": -0.32498669624328613,
+ "step": 10
+ },
+ {
+ "epoch": 0.08,
+ "eval_log_odds_chosen": 0.1487705260515213,
+ "eval_log_odds_ratio": -0.8249135613441467,
+ "eval_logits/chosen": 1.0748957395553589,
+ "eval_logits/rejected": 1.0697605609893799,
+ "eval_logps/chosen": -3.040273427963257,
+ "eval_logps/rejected": -3.189253091812134,
+ "eval_loss": 3.5089430809020996,
+ "eval_nll_loss": 3.4264516830444336,
+ "eval_rewards/accuracies": 0.5440000295639038,
+ "eval_rewards/chosen": -0.30402734875679016,
+ "eval_rewards/margins": 0.014897952787578106,
+ "eval_rewards/rejected": -0.31892529129981995,
+ "eval_runtime": 52.952,
+ "eval_samples_per_second": 9.443,
+ "eval_steps_per_second": 4.721,
+ "step": 10
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 0.595796525478363,
+ "learning_rate": 4e-06,
+ "log_odds_chosen": 0.05564836412668228,
+ "log_odds_ratio": -0.8168161511421204,
+ "logits/chosen": 1.0918205976486206,
+ "logits/rejected": 1.1515363454818726,
+ "logps/chosen": -3.0827183723449707,
+ "logps/rejected": -3.138075351715088,
+ "loss": 3.3813629150390625,
+ "nll_loss": 3.2996826171875,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.3082718849182129,
+ "rewards/margins": 0.005535666830837727,
+ "rewards/rejected": -0.31380754709243774,
+ "step": 20
+ },
+ {
+ "epoch": 0.16,
+ "eval_log_odds_chosen": 0.1419471800327301,
+ "eval_log_odds_ratio": -0.816088855266571,
+ "eval_logits/chosen": 1.0871241092681885,
+ "eval_logits/rejected": 1.07681143283844,
+ "eval_logps/chosen": -2.9660096168518066,
+ "eval_logps/rejected": -3.108426332473755,
+ "eval_loss": 3.275442361831665,
+ "eval_nll_loss": 3.193833589553833,
+ "eval_rewards/accuracies": 0.5400000214576721,
+ "eval_rewards/chosen": -0.29660093784332275,
+ "eval_rewards/margins": 0.014241669327020645,
+ "eval_rewards/rejected": -0.3108426630496979,
+ "eval_runtime": 53.5605,
+ "eval_samples_per_second": 9.335,
+ "eval_steps_per_second": 4.668,
+ "step": 20
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 0.48330172896385193,
+ "learning_rate": 6.105263157894737e-06,
+ "log_odds_chosen": 0.06886305660009384,
+ "log_odds_ratio": -0.7635106444358826,
+ "logits/chosen": 1.188002109527588,
+ "logits/rejected": 1.155263900756836,
+ "logps/chosen": -2.844672441482544,
+ "logps/rejected": -2.9038586616516113,
+ "loss": 3.0850343704223633,
+ "nll_loss": 3.0086829662323,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.2844671905040741,
+ "rewards/margins": 0.005918659269809723,
+ "rewards/rejected": -0.290385901927948,
+ "step": 30
+ },
+ {
+ "epoch": 0.24,
+ "eval_log_odds_chosen": 0.1258297562599182,
+ "eval_log_odds_ratio": -0.8003272414207458,
+ "eval_logits/chosen": 1.111700415611267,
+ "eval_logits/rejected": 1.0921913385391235,
+ "eval_logps/chosen": -2.82503080368042,
+ "eval_logps/rejected": -2.952216386795044,
+ "eval_loss": 3.034886121749878,
+ "eval_nll_loss": 2.9548532962799072,
+ "eval_rewards/accuracies": 0.5479999780654907,
+ "eval_rewards/chosen": -0.2825030982494354,
+ "eval_rewards/margins": 0.012718543410301208,
+ "eval_rewards/rejected": -0.29522162675857544,
+ "eval_runtime": 53.5129,
+ "eval_samples_per_second": 9.344,
+ "eval_steps_per_second": 4.672,
+ "step": 30
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 0.37701094150543213,
+ "learning_rate": 7.976261127596439e-06,
+ "log_odds_chosen": 0.16706740856170654,
+ "log_odds_ratio": -0.7840421795845032,
+ "logits/chosen": 1.1109424829483032,
+ "logits/rejected": 1.111083745956421,
+ "logps/chosen": -2.6344399452209473,
+ "logps/rejected": -2.769670009613037,
+ "loss": 2.8504947662353515,
+ "nll_loss": 2.772090435028076,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.2634440064430237,
+ "rewards/margins": 0.01352303009480238,
+ "rewards/rejected": -0.27696701884269714,
+ "step": 40
+ },
+ {
+ "epoch": 0.32,
+ "eval_log_odds_chosen": 0.09920188039541245,
+ "eval_log_odds_ratio": -0.780275821685791,
+ "eval_logits/chosen": 1.1098300218582153,
+ "eval_logits/rejected": 1.078803300857544,
+ "eval_logps/chosen": -2.622143268585205,
+ "eval_logps/rejected": -2.724855661392212,
+ "eval_loss": 2.7800722122192383,
+ "eval_nll_loss": 2.70204496383667,
+ "eval_rewards/accuracies": 0.5479999780654907,
+ "eval_rewards/chosen": -0.2622143626213074,
+ "eval_rewards/margins": 0.010271222330629826,
+ "eval_rewards/rejected": -0.27248555421829224,
+ "eval_runtime": 53.4631,
+ "eval_samples_per_second": 9.352,
+ "eval_steps_per_second": 4.676,
+ "step": 40
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.3793655335903168,
+ "learning_rate": 7.738872403560831e-06,
+ "log_odds_chosen": 0.07008141279220581,
+ "log_odds_ratio": -0.7406946420669556,
+ "logits/chosen": 1.1617451906204224,
+ "logits/rejected": 1.0783092975616455,
+ "logps/chosen": -2.499058246612549,
+ "logps/rejected": -2.5704269409179688,
+ "loss": 2.5753040313720703,
+ "nll_loss": 2.501234531402588,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.2499057948589325,
+ "rewards/margins": 0.007136875297874212,
+ "rewards/rejected": -0.25704267621040344,
+ "step": 50
+ },
+ {
+ "epoch": 0.4,
+ "eval_log_odds_chosen": 0.06802652031183243,
+ "eval_log_odds_ratio": -0.7690593004226685,
+ "eval_logits/chosen": 1.05830717086792,
+ "eval_logits/rejected": 1.0166288614273071,
+ "eval_logps/chosen": -2.449127197265625,
+ "eval_logps/rejected": -2.522343158721924,
+ "eval_loss": 2.536269187927246,
+ "eval_nll_loss": 2.4593632221221924,
+ "eval_rewards/accuracies": 0.5400000214576721,
+ "eval_rewards/chosen": -0.24491269886493683,
+ "eval_rewards/margins": 0.007321594282984734,
+ "eval_rewards/rejected": -0.2522343099117279,
+ "eval_runtime": 53.6172,
+ "eval_samples_per_second": 9.325,
+ "eval_steps_per_second": 4.663,
+ "step": 50
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 0.22407379746437073,
+ "learning_rate": 7.501483679525222e-06,
+ "log_odds_chosen": 0.06782802194356918,
+ "log_odds_ratio": -0.7393356561660767,
+ "logits/chosen": 0.9188723564147949,
+ "logits/rejected": 1.0408287048339844,
+ "logps/chosen": -2.295084238052368,
+ "logps/rejected": -2.338684558868408,
+ "loss": 2.4012365341186523,
+ "nll_loss": 2.327303171157837,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.2295083999633789,
+ "rewards/margins": 0.0043600560165941715,
+ "rewards/rejected": -0.23386844992637634,
+ "step": 60
+ },
+ {
+ "epoch": 0.48,
+ "eval_log_odds_chosen": 0.04691058769822121,
+ "eval_log_odds_ratio": -0.7670963406562805,
+ "eval_logits/chosen": 0.9979300498962402,
+ "eval_logits/rejected": 0.9495062232017517,
+ "eval_logps/chosen": -2.354620933532715,
+ "eval_logps/rejected": -2.4075090885162354,
+ "eval_loss": 2.3141748905181885,
+ "eval_nll_loss": 2.2374651432037354,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": -0.23546208441257477,
+ "eval_rewards/margins": 0.005288826301693916,
+ "eval_rewards/rejected": -0.24075090885162354,
+ "eval_runtime": 53.4669,
+ "eval_samples_per_second": 9.352,
+ "eval_steps_per_second": 4.676,
+ "step": 60
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 0.20351995527744293,
+ "learning_rate": 7.264094955489614e-06,
+ "log_odds_chosen": 0.010356808081269264,
+ "log_odds_ratio": -0.7408021092414856,
+ "logits/chosen": 1.050209403038025,
+ "logits/rejected": 1.077514886856079,
+ "logps/chosen": -2.2567782402038574,
+ "logps/rejected": -2.2628250122070312,
+ "loss": 2.2166725158691407,
+ "nll_loss": 2.142592430114746,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.22567781805992126,
+ "rewards/margins": 0.0006046619964763522,
+ "rewards/rejected": -0.22628247737884521,
+ "step": 70
+ },
+ {
+ "epoch": 0.56,
+ "eval_log_odds_chosen": 0.0382089838385582,
+ "eval_log_odds_ratio": -0.766432523727417,
+ "eval_logits/chosen": 0.976753294467926,
+ "eval_logits/rejected": 0.9264048337936401,
+ "eval_logps/chosen": -2.3048932552337646,
+ "eval_logps/rejected": -2.3492259979248047,
+ "eval_loss": 2.19045352935791,
+ "eval_nll_loss": 2.1138103008270264,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.2304893285036087,
+ "eval_rewards/margins": 0.004433275666087866,
+ "eval_rewards/rejected": -0.2349226027727127,
+ "eval_runtime": 53.4447,
+ "eval_samples_per_second": 9.355,
+ "eval_steps_per_second": 4.678,
+ "step": 70
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 0.2011009007692337,
+ "learning_rate": 7.026706231454006e-06,
+ "log_odds_chosen": 0.1629197597503662,
+ "log_odds_ratio": -0.7066301107406616,
+ "logits/chosen": 0.966200053691864,
+ "logits/rejected": 0.9356706738471985,
+ "logps/chosen": -2.2345035076141357,
+ "logps/rejected": -2.393071174621582,
+ "loss": 2.156692886352539,
+ "nll_loss": 2.0860297679901123,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.22345037758350372,
+ "rewards/margins": 0.015856750309467316,
+ "rewards/rejected": -0.23930712044239044,
+ "step": 80
+ },
+ {
+ "epoch": 0.64,
+ "eval_log_odds_chosen": 0.03499959409236908,
+ "eval_log_odds_ratio": -0.7663364410400391,
+ "eval_logits/chosen": 1.0133678913116455,
+ "eval_logits/rejected": 0.9637854695320129,
+ "eval_logps/chosen": -2.268923044204712,
+ "eval_logps/rejected": -2.309880256652832,
+ "eval_loss": 2.1032519340515137,
+ "eval_nll_loss": 2.026618480682373,
+ "eval_rewards/accuracies": 0.5260000228881836,
+ "eval_rewards/chosen": -0.22689230740070343,
+ "eval_rewards/margins": 0.004095741547644138,
+ "eval_rewards/rejected": -0.2309880554676056,
+ "eval_runtime": 53.4168,
+ "eval_samples_per_second": 9.36,
+ "eval_steps_per_second": 4.68,
+ "step": 80
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 0.1581200510263443,
+ "learning_rate": 6.7893175074183975e-06,
+ "log_odds_chosen": 0.050972986966371536,
+ "log_odds_ratio": -0.7442581653594971,
+ "logits/chosen": 1.043745994567871,
+ "logits/rejected": 1.0103332996368408,
+ "logps/chosen": -2.2243306636810303,
+ "logps/rejected": -2.281054735183716,
+ "loss": 2.0534223556518554,
+ "nll_loss": 1.9789966344833374,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.22243309020996094,
+ "rewards/margins": 0.00567237613722682,
+ "rewards/rejected": -0.22810545563697815,
+ "step": 90
+ },
+ {
+ "epoch": 0.72,
+ "eval_log_odds_chosen": 0.03331448510289192,
+ "eval_log_odds_ratio": -0.765789806842804,
+ "eval_logits/chosen": 1.0755304098129272,
+ "eval_logits/rejected": 1.026427984237671,
+ "eval_logps/chosen": -2.2377917766571045,
+ "eval_logps/rejected": -2.2765679359436035,
+ "eval_loss": 2.0383899211883545,
+ "eval_nll_loss": 1.961810827255249,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.22377915680408478,
+ "eval_rewards/margins": 0.00387762114405632,
+ "eval_rewards/rejected": -0.2276567965745926,
+ "eval_runtime": 53.532,
+ "eval_samples_per_second": 9.34,
+ "eval_steps_per_second": 4.67,
+ "step": 90
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 0.2415573000907898,
+ "learning_rate": 6.551928783382788e-06,
+ "log_odds_chosen": 0.041442085057497025,
+ "log_odds_ratio": -0.7387251257896423,
+ "logits/chosen": 0.9986271858215332,
+ "logits/rejected": 1.049234390258789,
+ "logps/chosen": -2.2130329608917236,
+ "logps/rejected": -2.238391637802124,
+ "loss": 2.0246566772460937,
+ "nll_loss": 1.9507839679718018,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.22130326926708221,
+ "rewards/margins": 0.0025358875282108784,
+ "rewards/rejected": -0.2238391637802124,
+ "step": 100
+ },
+ {
+ "epoch": 0.8,
+ "eval_log_odds_chosen": 0.031448494642972946,
+ "eval_log_odds_ratio": -0.7643815279006958,
+ "eval_logits/chosen": 1.0942497253417969,
+ "eval_logits/rejected": 1.0446091890335083,
+ "eval_logps/chosen": -2.2041985988616943,
+ "eval_logps/rejected": -2.240527629852295,
+ "eval_loss": 1.9893497228622437,
+ "eval_nll_loss": 1.9129114151000977,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.22041983902454376,
+ "eval_rewards/margins": 0.0036329119466245174,
+ "eval_rewards/rejected": -0.2240527719259262,
+ "eval_runtime": 53.6172,
+ "eval_samples_per_second": 9.325,
+ "eval_steps_per_second": 4.663,
+ "step": 100
+ },
+ {
+ "epoch": 0.88,
+ "grad_norm": 0.1447879523038864,
+ "learning_rate": 6.314540059347181e-06,
+ "log_odds_chosen": -0.10262440145015717,
+ "log_odds_ratio": -0.8409156799316406,
+ "logits/chosen": 1.1213892698287964,
+ "logits/rejected": 1.089329481124878,
+ "logps/chosen": -2.1380906105041504,
+ "logps/rejected": -2.0553345680236816,
+ "loss": 1.9718734741210937,
+ "nll_loss": 1.8877818584442139,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.2138090580701828,
+ "rewards/margins": -0.008275633677840233,
+ "rewards/rejected": -0.20553342998027802,
+ "step": 110
+ },
+ {
+ "epoch": 0.88,
+ "eval_log_odds_chosen": 0.029969975352287292,
+ "eval_log_odds_ratio": -0.7633962631225586,
+ "eval_logits/chosen": 1.1052088737487793,
+ "eval_logits/rejected": 1.0555152893066406,
+ "eval_logps/chosen": -2.1711573600769043,
+ "eval_logps/rejected": -2.2057836055755615,
+ "eval_loss": 1.9516394138336182,
+ "eval_nll_loss": 1.8752998113632202,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.21711574494838715,
+ "eval_rewards/margins": 0.00346262170933187,
+ "eval_rewards/rejected": -0.2205783724784851,
+ "eval_runtime": 53.4193,
+ "eval_samples_per_second": 9.36,
+ "eval_steps_per_second": 4.68,
+ "step": 110
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 0.17102450132369995,
+ "learning_rate": 6.077151335311573e-06,
+ "log_odds_chosen": 0.07301244884729385,
+ "log_odds_ratio": -0.7499282956123352,
+ "logits/chosen": 1.1306703090667725,
+ "logits/rejected": 1.1610009670257568,
+ "logps/chosen": -2.093169689178467,
+ "logps/rejected": -2.158841609954834,
+ "loss": 1.935629653930664,
+ "nll_loss": 1.8606367111206055,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.20931696891784668,
+ "rewards/margins": 0.006567192263901234,
+ "rewards/rejected": -0.21588416397571564,
+ "step": 120
+ },
+ {
+ "epoch": 0.96,
+ "eval_log_odds_chosen": 0.029722029343247414,
+ "eval_log_odds_ratio": -0.7631016373634338,
+ "eval_logits/chosen": 1.0973773002624512,
+ "eval_logits/rejected": 1.0479023456573486,
+ "eval_logps/chosen": -2.14707350730896,
+ "eval_logps/rejected": -2.1812427043914795,
+ "eval_loss": 1.926309585571289,
+ "eval_nll_loss": 1.8499994277954102,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": -0.2147073596715927,
+ "eval_rewards/margins": 0.003416918683797121,
+ "eval_rewards/rejected": -0.21812427043914795,
+ "eval_runtime": 53.3805,
+ "eval_samples_per_second": 9.367,
+ "eval_steps_per_second": 4.683,
+ "step": 120
+ },
+ {
+ "epoch": 1.04,
+ "grad_norm": 0.17677651345729828,
+ "learning_rate": 5.839762611275964e-06,
+ "log_odds_chosen": -0.0031179129146039486,
+ "log_odds_ratio": -0.7423644065856934,
+ "logits/chosen": 1.2274668216705322,
+ "logits/rejected": 1.0976972579956055,
+ "logps/chosen": -1.9559234380722046,
+ "logps/rejected": -1.9523088932037354,
+ "loss": 1.915380096435547,
+ "nll_loss": 1.8411436080932617,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.19559237360954285,
+ "rewards/margins": -0.00036150217056274414,
+ "rewards/rejected": -0.1952308714389801,
+ "step": 130
+ },
+ {
+ "epoch": 1.04,
+ "eval_log_odds_chosen": 0.029143329709768295,
+ "eval_log_odds_ratio": -0.7634750008583069,
+ "eval_logits/chosen": 1.1202707290649414,
+ "eval_logits/rejected": 1.0722858905792236,
+ "eval_logps/chosen": -2.133115291595459,
+ "eval_logps/rejected": -2.166740655899048,
+ "eval_loss": 1.907346248626709,
+ "eval_nll_loss": 1.8309987783432007,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": -0.21331150829792023,
+ "eval_rewards/margins": 0.0033625485375523567,
+ "eval_rewards/rejected": -0.2166740596294403,
+ "eval_runtime": 53.3936,
+ "eval_samples_per_second": 9.364,
+ "eval_steps_per_second": 4.682,
+ "step": 130
+ },
+ {
+ "epoch": 1.12,
+ "grad_norm": 0.16196487843990326,
+ "learning_rate": 5.6023738872403555e-06,
+ "log_odds_chosen": -0.11776870489120483,
+ "log_odds_ratio": -0.8455734252929688,
+ "logits/chosen": 1.1195478439331055,
+ "logits/rejected": 1.1206276416778564,
+ "logps/chosen": -2.1271960735321045,
+ "logps/rejected": -2.0324878692626953,
+ "loss": 1.9241960525512696,
+ "nll_loss": 1.8396384716033936,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": -0.2127196043729782,
+ "rewards/margins": -0.009470805525779724,
+ "rewards/rejected": -0.20324881374835968,
+ "step": 140
+ },
+ {
+ "epoch": 1.12,
+ "eval_log_odds_chosen": 0.02882545068860054,
+ "eval_log_odds_ratio": -0.7633554339408875,
+ "eval_logits/chosen": 1.1695773601531982,
+ "eval_logits/rejected": 1.1225638389587402,
+ "eval_logps/chosen": -2.1247518062591553,
+ "eval_logps/rejected": -2.1582350730895996,
+ "eval_loss": 1.8932896852493286,
+ "eval_nll_loss": 1.8169541358947754,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": -0.21247518062591553,
+ "eval_rewards/margins": 0.003348324680700898,
+ "eval_rewards/rejected": -0.21582350134849548,
+ "eval_runtime": 53.4082,
+ "eval_samples_per_second": 9.362,
+ "eval_steps_per_second": 4.681,
+ "step": 140
+ },
+ {
+ "epoch": 1.2,
+ "grad_norm": 0.12368938326835632,
+ "learning_rate": 5.364985163204747e-06,
+ "log_odds_chosen": 0.11099217087030411,
+ "log_odds_ratio": -0.6863120794296265,
+ "logits/chosen": 1.225134253501892,
+ "logits/rejected": 1.2558250427246094,
+ "logps/chosen": -2.0426182746887207,
+ "logps/rejected": -2.1387341022491455,
+ "loss": 1.9164009094238281,
+ "nll_loss": 1.8477699756622314,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.20426180958747864,
+ "rewards/margins": 0.00961159449070692,
+ "rewards/rejected": -0.21387341618537903,
+ "step": 150
+ },
+ {
+ "epoch": 1.2,
+ "eval_log_odds_chosen": 0.029773231595754623,
+ "eval_log_odds_ratio": -0.7633053660392761,
+ "eval_logits/chosen": 1.2088301181793213,
+ "eval_logits/rejected": 1.1618542671203613,
+ "eval_logps/chosen": -2.1208908557891846,
+ "eval_logps/rejected": -2.155456781387329,
+ "eval_loss": 1.884640097618103,
+ "eval_nll_loss": 1.80830979347229,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.21208904683589935,
+ "eval_rewards/margins": 0.0034566062968224287,
+ "eval_rewards/rejected": -0.215545654296875,
+ "eval_runtime": 53.4248,
+ "eval_samples_per_second": 9.359,
+ "eval_steps_per_second": 4.679,
+ "step": 150
+ },
+ {
+ "epoch": 1.28,
+ "grad_norm": 0.1698087453842163,
+ "learning_rate": 5.12759643916914e-06,
+ "log_odds_chosen": 0.14880229532718658,
+ "log_odds_ratio": -0.6961595416069031,
+ "logits/chosen": 1.1544950008392334,
+ "logits/rejected": 1.241871953010559,
+ "logps/chosen": -2.0620102882385254,
+ "logps/rejected": -2.1889777183532715,
+ "loss": 1.8679191589355468,
+ "nll_loss": 1.7983033657073975,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.20620103180408478,
+ "rewards/margins": 0.01269673090428114,
+ "rewards/rejected": -0.2188977748155594,
+ "step": 160
+ },
+ {
+ "epoch": 1.28,
+ "eval_log_odds_chosen": 0.029680760577321053,
+ "eval_log_odds_ratio": -0.763420820236206,
+ "eval_logits/chosen": 1.228447437286377,
+ "eval_logits/rejected": 1.1817827224731445,
+ "eval_logps/chosen": -2.114239454269409,
+ "eval_logps/rejected": -2.1486246585845947,
+ "eval_loss": 1.877873182296753,
+ "eval_nll_loss": 1.8015310764312744,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": -0.21142394840717316,
+ "eval_rewards/margins": 0.003438530955463648,
+ "eval_rewards/rejected": -0.21486248075962067,
+ "eval_runtime": 53.382,
+ "eval_samples_per_second": 9.366,
+ "eval_steps_per_second": 4.683,
+ "step": 160
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "grad_norm": 0.16546542942523956,
+ "learning_rate": 4.890207715133531e-06,
+ "log_odds_chosen": 0.14024165272712708,
+ "log_odds_ratio": -0.6900363564491272,
+ "logits/chosen": 1.1234607696533203,
+ "logits/rejected": 1.2519900798797607,
+ "logps/chosen": -2.0526554584503174,
+ "logps/rejected": -2.160978317260742,
+ "loss": 1.8593597412109375,
+ "nll_loss": 1.7903560400009155,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": -0.2052655667066574,
+ "rewards/margins": 0.010832270607352257,
+ "rewards/rejected": -0.21609783172607422,
+ "step": 170
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "eval_log_odds_chosen": 0.02900824137032032,
+ "eval_log_odds_ratio": -0.7633247971534729,
+ "eval_logits/chosen": 1.2155014276504517,
+ "eval_logits/rejected": 1.16861891746521,
+ "eval_logps/chosen": -2.1065866947174072,
+ "eval_logps/rejected": -2.1401009559631348,
+ "eval_loss": 1.8723371028900146,
+ "eval_nll_loss": 1.7960048913955688,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.21065863966941833,
+ "eval_rewards/margins": 0.003351445309817791,
+ "eval_rewards/rejected": -0.2140101194381714,
+ "eval_runtime": 53.3237,
+ "eval_samples_per_second": 9.377,
+ "eval_steps_per_second": 4.688,
+ "step": 170
+ },
+ {
+ "epoch": 1.44,
+ "grad_norm": 0.14689764380455017,
+ "learning_rate": 4.652818991097923e-06,
+ "log_odds_chosen": -0.014471495524048805,
+ "log_odds_ratio": -0.8246002197265625,
+ "logits/chosen": 1.259467363357544,
+ "logits/rejected": 1.216888427734375,
+ "logps/chosen": -2.0859415531158447,
+ "logps/rejected": -2.084341526031494,
+ "loss": 1.884648895263672,
+ "nll_loss": 1.8021886348724365,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.2085941731929779,
+ "rewards/margins": -0.00016001760377548635,
+ "rewards/rejected": -0.20843414962291718,
+ "step": 180
+ },
+ {
+ "epoch": 1.44,
+ "eval_log_odds_chosen": 0.030826514586806297,
+ "eval_log_odds_ratio": -0.7627649903297424,
+ "eval_logits/chosen": 1.2350919246673584,
+ "eval_logits/rejected": 1.188149333000183,
+ "eval_logps/chosen": -2.1029579639434814,
+ "eval_logps/rejected": -2.1379969120025635,
+ "eval_loss": 1.8677619695663452,
+ "eval_nll_loss": 1.79148530960083,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": -0.21029581129550934,
+ "eval_rewards/margins": 0.003503883723169565,
+ "eval_rewards/rejected": -0.21379968523979187,
+ "eval_runtime": 53.3344,
+ "eval_samples_per_second": 9.375,
+ "eval_steps_per_second": 4.687,
+ "step": 180
+ },
+ {
+ "epoch": 1.52,
+ "grad_norm": 0.1614168882369995,
+ "learning_rate": 4.415430267062314e-06,
+ "log_odds_chosen": 0.08284492045640945,
+ "log_odds_ratio": -0.6944996118545532,
+ "logits/chosen": 1.2591311931610107,
+ "logits/rejected": 1.2080775499343872,
+ "logps/chosen": -2.018791675567627,
+ "logps/rejected": -2.0822417736053467,
+ "loss": 1.898875617980957,
+ "nll_loss": 1.829425573348999,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.20187918841838837,
+ "rewards/margins": 0.006344993598759174,
+ "rewards/rejected": -0.20822420716285706,
+ "step": 190
+ },
+ {
+ "epoch": 1.52,
+ "eval_log_odds_chosen": 0.03216712921857834,
+ "eval_log_odds_ratio": -0.7628231048583984,
+ "eval_logits/chosen": 1.2684508562088013,
+ "eval_logits/rejected": 1.2221770286560059,
+ "eval_logps/chosen": -2.0986759662628174,
+ "eval_logps/rejected": -2.1348414421081543,
+ "eval_loss": 1.8638546466827393,
+ "eval_nll_loss": 1.7875722646713257,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": -0.20986759662628174,
+ "eval_rewards/margins": 0.003616561181843281,
+ "eval_rewards/rejected": -0.21348415315151215,
+ "eval_runtime": 53.3751,
+ "eval_samples_per_second": 9.368,
+ "eval_steps_per_second": 4.684,
+ "step": 190
+ },
+ {
+ "epoch": 1.6,
+ "grad_norm": 0.15446655452251434,
+ "learning_rate": 4.178041543026706e-06,
+ "log_odds_chosen": -0.10362448543310165,
+ "log_odds_ratio": -0.8201931715011597,
+ "logits/chosen": 1.2476270198822021,
+ "logits/rejected": 1.1510899066925049,
+ "logps/chosen": -2.097903251647949,
+ "logps/rejected": -2.0198020935058594,
+ "loss": 1.933295440673828,
+ "nll_loss": 1.851276159286499,
+ "rewards/accuracies": 0.38749998807907104,
+ "rewards/chosen": -0.20979031920433044,
+ "rewards/margins": -0.007810118608176708,
+ "rewards/rejected": -0.20198020339012146,
+ "step": 200
+ },
+ {
+ "epoch": 1.6,
+ "eval_log_odds_chosen": 0.032845109701156616,
+ "eval_log_odds_ratio": -0.7628770470619202,
+ "eval_logits/chosen": 1.280562400817871,
+ "eval_logits/rejected": 1.2350828647613525,
+ "eval_logps/chosen": -2.0954396724700928,
+ "eval_logps/rejected": -2.1321818828582764,
+ "eval_loss": 1.8606969118118286,
+ "eval_nll_loss": 1.7844091653823853,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": -0.20954395830631256,
+ "eval_rewards/margins": 0.003674217266961932,
+ "eval_rewards/rejected": -0.21321816742420197,
+ "eval_runtime": 53.4169,
+ "eval_samples_per_second": 9.36,
+ "eval_steps_per_second": 4.68,
+ "step": 200
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "grad_norm": 0.16359363496303558,
+ "learning_rate": 3.940652818991098e-06,
+ "log_odds_chosen": 0.14669272303581238,
+ "log_odds_ratio": -0.6812139749526978,
+ "logits/chosen": 1.2737199068069458,
+ "logits/rejected": 1.218265414237976,
+ "logps/chosen": -2.0203709602355957,
+ "logps/rejected": -2.1482558250427246,
+ "loss": 1.8169612884521484,
+ "nll_loss": 1.7488397359848022,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": -0.20203709602355957,
+ "rewards/margins": 0.012788480147719383,
+ "rewards/rejected": -0.2148255854845047,
+ "step": 210
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "eval_log_odds_chosen": 0.031184684485197067,
+ "eval_log_odds_ratio": -0.7630588412284851,
+ "eval_logits/chosen": 1.295477271080017,
+ "eval_logits/rejected": 1.2507117986679077,
+ "eval_logps/chosen": -2.0950186252593994,
+ "eval_logps/rejected": -2.130385160446167,
+ "eval_loss": 1.8578598499298096,
+ "eval_nll_loss": 1.781554102897644,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": -0.20950186252593994,
+ "eval_rewards/margins": 0.0035366546362638474,
+ "eval_rewards/rejected": -0.21303851902484894,
+ "eval_runtime": 53.3933,
+ "eval_samples_per_second": 9.364,
+ "eval_steps_per_second": 4.682,
+ "step": 210
+ },
+ {
+ "epoch": 1.76,
+ "grad_norm": 0.13849231600761414,
+ "learning_rate": 3.7032640949554893e-06,
+ "log_odds_chosen": -0.0053191520273685455,
+ "log_odds_ratio": -0.7406737804412842,
+ "logits/chosen": 1.4272799491882324,
+ "logits/rejected": 1.4022886753082275,
+ "logps/chosen": -2.137610673904419,
+ "logps/rejected": -2.137540340423584,
+ "loss": 1.9181819915771485,
+ "nll_loss": 1.8441146612167358,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.21376104652881622,
+ "rewards/margins": -7.031113000266487e-06,
+ "rewards/rejected": -0.21375402808189392,
+ "step": 220
+ },
+ {
+ "epoch": 1.76,
+ "eval_log_odds_chosen": 0.03328011557459831,
+ "eval_log_odds_ratio": -0.7628946304321289,
+ "eval_logits/chosen": 1.315618634223938,
+ "eval_logits/rejected": 1.2708230018615723,
+ "eval_logps/chosen": -2.091679573059082,
+ "eval_logps/rejected": -2.1288042068481445,
+ "eval_loss": 1.8550817966461182,
+ "eval_nll_loss": 1.7787925004959106,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": -0.2091679722070694,
+ "eval_rewards/margins": 0.0037124231457710266,
+ "eval_rewards/rejected": -0.21288041770458221,
+ "eval_runtime": 53.3472,
+ "eval_samples_per_second": 9.373,
+ "eval_steps_per_second": 4.686,
+ "step": 220
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "grad_norm": 0.15999111533164978,
+ "learning_rate": 3.4658753709198815e-06,
+ "log_odds_chosen": 0.06285012513399124,
+ "log_odds_ratio": -0.782901406288147,
+ "logits/chosen": 1.2496129274368286,
+ "logits/rejected": 1.2683494091033936,
+ "logps/chosen": -2.0192151069641113,
+ "logps/rejected": -2.062490224838257,
+ "loss": 1.8350942611694336,
+ "nll_loss": 1.7568042278289795,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.20192153751850128,
+ "rewards/margins": 0.004327498376369476,
+ "rewards/rejected": -0.20624902844429016,
+ "step": 230
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "eval_log_odds_chosen": 0.034007344394922256,
+ "eval_log_odds_ratio": -0.762892484664917,
+ "eval_logits/chosen": 1.3318549394607544,
+ "eval_logits/rejected": 1.2868961095809937,
+ "eval_logps/chosen": -2.0880751609802246,
+ "eval_logps/rejected": -2.125743865966797,
+ "eval_loss": 1.8529021739959717,
+ "eval_nll_loss": 1.7766127586364746,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": -0.2088075429201126,
+ "eval_rewards/margins": 0.003766861045733094,
+ "eval_rewards/rejected": -0.21257439255714417,
+ "eval_runtime": 53.5663,
+ "eval_samples_per_second": 9.334,
+ "eval_steps_per_second": 4.667,
+ "step": 230
+ },
+ {
+ "epoch": 1.92,
+ "grad_norm": 0.24993999302387238,
+ "learning_rate": 3.228486646884273e-06,
+ "log_odds_chosen": 0.14747142791748047,
+ "log_odds_ratio": -0.7056423425674438,
+ "logits/chosen": 1.3445780277252197,
+ "logits/rejected": 1.3658064603805542,
+ "logps/chosen": -1.9867181777954102,
+ "logps/rejected": -2.1228175163269043,
+ "loss": 1.8942413330078125,
+ "nll_loss": 1.8236770629882812,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.1986718326807022,
+ "rewards/margins": 0.013609920628368855,
+ "rewards/rejected": -0.2122817486524582,
+ "step": 240
+ },
+ {
+ "epoch": 1.92,
+ "eval_log_odds_chosen": 0.032722458243370056,
+ "eval_log_odds_ratio": -0.7631469964981079,
+ "eval_logits/chosen": 1.3472635746002197,
+ "eval_logits/rejected": 1.3025418519973755,
+ "eval_logps/chosen": -2.086138963699341,
+ "eval_logps/rejected": -2.1226885318756104,
+ "eval_loss": 1.8508049249649048,
+ "eval_nll_loss": 1.774490237236023,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": -0.20861388742923737,
+ "eval_rewards/margins": 0.003654928645119071,
+ "eval_rewards/rejected": -0.21226882934570312,
+ "eval_runtime": 53.531,
+ "eval_samples_per_second": 9.34,
+ "eval_steps_per_second": 4.67,
+ "step": 240
+ },
+ {
+ "epoch": 2.0,
+ "grad_norm": 0.13049109280109406,
+ "learning_rate": 2.9910979228486642e-06,
+ "log_odds_chosen": 0.11018727719783783,
+ "log_odds_ratio": -0.6979345083236694,
+ "logits/chosen": 1.461464285850525,
+ "logits/rejected": 1.559509038925171,
+ "logps/chosen": -2.01706600189209,
+ "logps/rejected": -2.1191089153289795,
+ "loss": 1.8566574096679687,
+ "nll_loss": 1.786863923072815,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.20170661807060242,
+ "rewards/margins": 0.010204277001321316,
+ "rewards/rejected": -0.2119108885526657,
+ "step": 250
+ },
+ {
+ "epoch": 2.0,
+ "eval_log_odds_chosen": 0.031319759786129,
+ "eval_log_odds_ratio": -0.7631150484085083,
+ "eval_logits/chosen": 1.3540582656860352,
+ "eval_logits/rejected": 1.3091437816619873,
+ "eval_logps/chosen": -2.0833497047424316,
+ "eval_logps/rejected": -2.118629217147827,
+ "eval_loss": 1.848903775215149,
+ "eval_nll_loss": 1.772592306137085,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": -0.20833496749401093,
+ "eval_rewards/margins": 0.0035279702860862017,
+ "eval_rewards/rejected": -0.21186292171478271,
+ "eval_runtime": 53.4977,
+ "eval_samples_per_second": 9.346,
+ "eval_steps_per_second": 4.673,
+ "step": 250
+ },
+ {
+ "epoch": 2.08,
+ "grad_norm": 0.18017899990081787,
+ "learning_rate": 2.7537091988130565e-06,
+ "log_odds_chosen": -0.06458185613155365,
+ "log_odds_ratio": -0.8291934132575989,
+ "logits/chosen": 1.259045124053955,
+ "logits/rejected": 1.27291738986969,
+ "logps/chosen": -2.1065449714660645,
+ "logps/rejected": -2.0579888820648193,
+ "loss": 1.8788591384887696,
+ "nll_loss": 1.7959396839141846,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.21065452694892883,
+ "rewards/margins": -0.004855643957853317,
+ "rewards/rejected": -0.20579886436462402,
+ "step": 260
+ },
+ {
+ "epoch": 2.08,
+ "eval_log_odds_chosen": 0.029519477859139442,
+ "eval_log_odds_ratio": -0.7636783123016357,
+ "eval_logits/chosen": 1.3371659517288208,
+ "eval_logits/rejected": 1.2922630310058594,
+ "eval_logps/chosen": -2.0797743797302246,
+ "eval_logps/rejected": -2.113273859024048,
+ "eval_loss": 1.8470534086227417,
+ "eval_nll_loss": 1.7706855535507202,
+ "eval_rewards/accuracies": 0.5239999890327454,
+ "eval_rewards/chosen": -0.20797744393348694,
+ "eval_rewards/margins": 0.00334998220205307,
+ "eval_rewards/rejected": -0.2113274186849594,
+ "eval_runtime": 53.5083,
+ "eval_samples_per_second": 9.344,
+ "eval_steps_per_second": 4.672,
+ "step": 260
+ },
+ {
+ "epoch": 2.16,
+ "grad_norm": 0.21924053132534027,
+ "learning_rate": 2.516320474777448e-06,
+ "log_odds_chosen": 0.19592975080013275,
+ "log_odds_ratio": -0.6901609301567078,
+ "logits/chosen": 1.3916186094284058,
+ "logits/rejected": 1.3600447177886963,
+ "logps/chosen": -2.0902369022369385,
+ "logps/rejected": -2.2713229656219482,
+ "loss": 1.8826128005981446,
+ "nll_loss": 1.8135969638824463,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.20902368426322937,
+ "rewards/margins": 0.01810864731669426,
+ "rewards/rejected": -0.22713232040405273,
+ "step": 270
+ },
+ {
+ "epoch": 2.16,
+ "eval_log_odds_chosen": 0.030172234401106834,
+ "eval_log_odds_ratio": -0.7637178897857666,
+ "eval_logits/chosen": 1.3338592052459717,
+ "eval_logits/rejected": 1.2886252403259277,
+ "eval_logps/chosen": -2.0776026248931885,
+ "eval_logps/rejected": -2.1116316318511963,
+ "eval_loss": 1.8454630374908447,
+ "eval_nll_loss": 1.769091010093689,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.2077602595090866,
+ "eval_rewards/margins": 0.0034029120579361916,
+ "eval_rewards/rejected": -0.21116317808628082,
+ "eval_runtime": 53.4938,
+ "eval_samples_per_second": 9.347,
+ "eval_steps_per_second": 4.673,
+ "step": 270
+ },
+ {
+ "epoch": 2.24,
+ "grad_norm": 0.12223289161920547,
+ "learning_rate": 2.2789317507418396e-06,
+ "log_odds_chosen": 0.26546868681907654,
+ "log_odds_ratio": -0.6293281316757202,
+ "logits/chosen": 1.4367039203643799,
+ "logits/rejected": 1.3482847213745117,
+ "logps/chosen": -1.8553558588027954,
+ "logps/rejected": -2.0716209411621094,
+ "loss": 1.812876319885254,
+ "nll_loss": 1.749943494796753,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": -0.18553557991981506,
+ "rewards/margins": 0.02162652276456356,
+ "rewards/rejected": -0.20716209709644318,
+ "step": 280
+ },
+ {
+ "epoch": 2.24,
+ "eval_log_odds_chosen": 0.02894444204866886,
+ "eval_log_odds_ratio": -0.7643565535545349,
+ "eval_logits/chosen": 1.341394305229187,
+ "eval_logits/rejected": 1.2972102165222168,
+ "eval_logps/chosen": -2.0761563777923584,
+ "eval_logps/rejected": -2.1090288162231445,
+ "eval_loss": 1.8440659046173096,
+ "eval_nll_loss": 1.7676302194595337,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": -0.2076156586408615,
+ "eval_rewards/margins": 0.003287235740572214,
+ "eval_rewards/rejected": -0.21090292930603027,
+ "eval_runtime": 53.4191,
+ "eval_samples_per_second": 9.36,
+ "eval_steps_per_second": 4.68,
+ "step": 280
+ },
+ {
+ "epoch": 2.32,
+ "grad_norm": 0.18634724617004395,
+ "learning_rate": 2.0415430267062314e-06,
+ "log_odds_chosen": -0.009359505027532578,
+ "log_odds_ratio": -0.7599523663520813,
+ "logits/chosen": 1.350929856300354,
+ "logits/rejected": 1.3596473932266235,
+ "logps/chosen": -2.0085995197296143,
+ "logps/rejected": -2.003897190093994,
+ "loss": 1.8252199172973633,
+ "nll_loss": 1.7492249011993408,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.200859934091568,
+ "rewards/margins": -0.00047021376667544246,
+ "rewards/rejected": -0.20038974285125732,
+ "step": 290
+ },
+ {
+ "epoch": 2.32,
+ "eval_log_odds_chosen": 0.029105709865689278,
+ "eval_log_odds_ratio": -0.7642828226089478,
+ "eval_logits/chosen": 1.3378888368606567,
+ "eval_logits/rejected": 1.2935822010040283,
+ "eval_logps/chosen": -2.0764894485473633,
+ "eval_logps/rejected": -2.1094634532928467,
+ "eval_loss": 1.843017816543579,
+ "eval_nll_loss": 1.766589641571045,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": -0.20764894783496857,
+ "eval_rewards/margins": 0.0032973980996757746,
+ "eval_rewards/rejected": -0.21094635128974915,
+ "eval_runtime": 53.4847,
+ "eval_samples_per_second": 9.348,
+ "eval_steps_per_second": 4.674,
+ "step": 290
+ },
+ {
+ "epoch": 2.4,
+ "grad_norm": 0.14173384010791779,
+ "learning_rate": 1.804154302670623e-06,
+ "log_odds_chosen": -0.07970528304576874,
+ "log_odds_ratio": -0.7945966124534607,
+ "logits/chosen": 1.3024346828460693,
+ "logits/rejected": 1.2510418891906738,
+ "logps/chosen": -1.9631941318511963,
+ "logps/rejected": -1.9063323736190796,
+ "loss": 1.8475601196289062,
+ "nll_loss": 1.7681005001068115,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": -0.19631941616535187,
+ "rewards/margins": -0.005686186254024506,
+ "rewards/rejected": -0.19063322246074677,
+ "step": 300
+ },
+ {
+ "epoch": 2.4,
+ "eval_log_odds_chosen": 0.028889847919344902,
+ "eval_log_odds_ratio": -0.7642103433609009,
+ "eval_logits/chosen": 1.3468135595321655,
+ "eval_logits/rejected": 1.3025881052017212,
+ "eval_logps/chosen": -2.076691150665283,
+ "eval_logps/rejected": -2.109506368637085,
+ "eval_loss": 1.8420958518981934,
+ "eval_nll_loss": 1.7656748294830322,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.20766912400722504,
+ "eval_rewards/margins": 0.003281528363004327,
+ "eval_rewards/rejected": -0.21095065772533417,
+ "eval_runtime": 53.4576,
+ "eval_samples_per_second": 9.353,
+ "eval_steps_per_second": 4.677,
+ "step": 300
+ },
+ {
+ "epoch": 2.48,
+ "grad_norm": 0.1275371015071869,
+ "learning_rate": 1.5667655786350147e-06,
+ "log_odds_chosen": 0.06028425693511963,
+ "log_odds_ratio": -0.7511486411094666,
+ "logits/chosen": 1.385467529296875,
+ "logits/rejected": 1.3117777109146118,
+ "logps/chosen": -2.0457942485809326,
+ "logps/rejected": -2.1070237159729004,
+ "loss": 1.815831184387207,
+ "nll_loss": 1.7407163381576538,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.2045794278383255,
+ "rewards/margins": 0.006122945342212915,
+ "rewards/rejected": -0.21070237457752228,
+ "step": 310
+ },
+ {
+ "epoch": 2.48,
+ "eval_log_odds_chosen": 0.02872202731668949,
+ "eval_log_odds_ratio": -0.7640355229377747,
+ "eval_logits/chosen": 1.3565622568130493,
+ "eval_logits/rejected": 1.3121683597564697,
+ "eval_logps/chosen": -2.075730800628662,
+ "eval_logps/rejected": -2.108355760574341,
+ "eval_loss": 1.8412187099456787,
+ "eval_nll_loss": 1.764815092086792,
+ "eval_rewards/accuracies": 0.5260000228881836,
+ "eval_rewards/chosen": -0.2075730860233307,
+ "eval_rewards/margins": 0.0032624828163534403,
+ "eval_rewards/rejected": -0.21083557605743408,
+ "eval_runtime": 53.398,
+ "eval_samples_per_second": 9.364,
+ "eval_steps_per_second": 4.682,
+ "step": 310
+ },
+ {
+ "epoch": 2.56,
+ "grad_norm": 0.15991471707820892,
+ "learning_rate": 1.3293768545994065e-06,
+ "log_odds_chosen": -0.0012696400517597795,
+ "log_odds_ratio": -0.7692364454269409,
+ "logits/chosen": 1.35402512550354,
+ "logits/rejected": 1.3285179138183594,
+ "logps/chosen": -2.1521174907684326,
+ "logps/rejected": -2.1507022380828857,
+ "loss": 1.9172073364257813,
+ "nll_loss": 1.840283989906311,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.21521171927452087,
+ "rewards/margins": -0.00014152470976114273,
+ "rewards/rejected": -0.2150702178478241,
+ "step": 320
+ },
+ {
+ "epoch": 2.56,
+ "eval_log_odds_chosen": 0.03067120723426342,
+ "eval_log_odds_ratio": -0.7632718682289124,
+ "eval_logits/chosen": 1.354533314704895,
+ "eval_logits/rejected": 1.3092080354690552,
+ "eval_logps/chosen": -2.0744073390960693,
+ "eval_logps/rejected": -2.108569383621216,
+ "eval_loss": 1.840209722518921,
+ "eval_nll_loss": 1.7638823986053467,
+ "eval_rewards/accuracies": 0.5239999890327454,
+ "eval_rewards/chosen": -0.20744071900844574,
+ "eval_rewards/margins": 0.0034162160009145737,
+ "eval_rewards/rejected": -0.21085695922374725,
+ "eval_runtime": 53.4069,
+ "eval_samples_per_second": 9.362,
+ "eval_steps_per_second": 4.681,
+ "step": 320
+ },
+ {
+ "epoch": 2.64,
+ "grad_norm": 0.16354456543922424,
+ "learning_rate": 1.091988130563798e-06,
+ "log_odds_chosen": 0.09046351909637451,
+ "log_odds_ratio": -0.7273394465446472,
+ "logits/chosen": 1.3274592161178589,
+ "logits/rejected": 1.3187423944473267,
+ "logps/chosen": -2.0142929553985596,
+ "logps/rejected": -2.081125259399414,
+ "loss": 1.8718931198120117,
+ "nll_loss": 1.7991588115692139,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.20142929255962372,
+ "rewards/margins": 0.006683236453682184,
+ "rewards/rejected": -0.20811252295970917,
+ "step": 330
+ },
+ {
+ "epoch": 2.64,
+ "eval_log_odds_chosen": 0.03132975473999977,
+ "eval_log_odds_ratio": -0.7632184624671936,
+ "eval_logits/chosen": 1.3558892011642456,
+ "eval_logits/rejected": 1.3105283975601196,
+ "eval_logps/chosen": -2.0740418434143066,
+ "eval_logps/rejected": -2.1086978912353516,
+ "eval_loss": 1.8395516872406006,
+ "eval_nll_loss": 1.7632297277450562,
+ "eval_rewards/accuracies": 0.5239999890327454,
+ "eval_rewards/chosen": -0.20740418136119843,
+ "eval_rewards/margins": 0.003465648042038083,
+ "eval_rewards/rejected": -0.21086980402469635,
+ "eval_runtime": 53.4027,
+ "eval_samples_per_second": 9.363,
+ "eval_steps_per_second": 4.681,
+ "step": 330
+ },
+ {
+ "epoch": 2.7199999999999998,
+ "grad_norm": 0.15135407447814941,
+ "learning_rate": 8.545994065281898e-07,
+ "log_odds_chosen": 0.0617956705391407,
+ "log_odds_ratio": -0.7476121783256531,
+ "logits/chosen": 1.3852603435516357,
+ "logits/rejected": 1.4368553161621094,
+ "logps/chosen": -2.073819637298584,
+ "logps/rejected": -2.1341233253479004,
+ "loss": 1.8446620941162108,
+ "nll_loss": 1.769900918006897,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.2073819637298584,
+ "rewards/margins": 0.00603040074929595,
+ "rewards/rejected": -0.21341237425804138,
+ "step": 340
+ },
+ {
+ "epoch": 2.7199999999999998,
+ "eval_log_odds_chosen": 0.031140869483351707,
+ "eval_log_odds_ratio": -0.7634550333023071,
+ "eval_logits/chosen": 1.362491250038147,
+ "eval_logits/rejected": 1.3176865577697754,
+ "eval_logps/chosen": -2.074117660522461,
+ "eval_logps/rejected": -2.108567237854004,
+ "eval_loss": 1.8390932083129883,
+ "eval_nll_loss": 1.7627475261688232,
+ "eval_rewards/accuracies": 0.5239999890327454,
+ "eval_rewards/chosen": -0.2074117809534073,
+ "eval_rewards/margins": 0.003444940084591508,
+ "eval_rewards/rejected": -0.21085670590400696,
+ "eval_runtime": 53.3911,
+ "eval_samples_per_second": 9.365,
+ "eval_steps_per_second": 4.682,
+ "step": 340
+ },
+ {
+ "epoch": 2.8,
+ "grad_norm": 0.13217419385910034,
+ "learning_rate": 6.172106824925815e-07,
+ "log_odds_chosen": 0.1088455468416214,
+ "log_odds_ratio": -0.7027878165245056,
+ "logits/chosen": 1.3703031539916992,
+ "logits/rejected": 1.4056010246276855,
+ "logps/chosen": -1.9828360080718994,
+ "logps/rejected": -2.0673537254333496,
+ "loss": 1.8847023010253907,
+ "nll_loss": 1.8144232034683228,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.1982835829257965,
+ "rewards/margins": 0.008451784029603004,
+ "rewards/rejected": -0.20673534274101257,
+ "step": 350
+ },
+ {
+ "epoch": 2.8,
+ "eval_log_odds_chosen": 0.030954066663980484,
+ "eval_log_odds_ratio": -0.7633995413780212,
+ "eval_logits/chosen": 1.3685437440872192,
+ "eval_logits/rejected": 1.3237495422363281,
+ "eval_logps/chosen": -2.073606491088867,
+ "eval_logps/rejected": -2.107947826385498,
+ "eval_loss": 1.8386913537979126,
+ "eval_nll_loss": 1.7623512744903564,
+ "eval_rewards/accuracies": 0.5239999890327454,
+ "eval_rewards/chosen": -0.2073606550693512,
+ "eval_rewards/margins": 0.0034341313876211643,
+ "eval_rewards/rejected": -0.21079477667808533,
+ "eval_runtime": 53.4376,
+ "eval_samples_per_second": 9.357,
+ "eval_steps_per_second": 4.678,
+ "step": 350
+ },
+ {
+ "epoch": 2.88,
+ "grad_norm": 0.13802677392959595,
+ "learning_rate": 3.798219584569733e-07,
+ "log_odds_chosen": -0.05070577189326286,
+ "log_odds_ratio": -0.7754552364349365,
+ "logits/chosen": 1.2700990438461304,
+ "logits/rejected": 1.2876553535461426,
+ "logps/chosen": -2.019728660583496,
+ "logps/rejected": -1.9804366827011108,
+ "loss": 1.8514394760131836,
+ "nll_loss": 1.7738940715789795,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": -0.2019728720188141,
+ "rewards/margins": -0.003929179161787033,
+ "rewards/rejected": -0.19804367423057556,
+ "step": 360
+ },
+ {
+ "epoch": 2.88,
+ "eval_log_odds_chosen": 0.03096388839185238,
+ "eval_log_odds_ratio": -0.763260006904602,
+ "eval_logits/chosen": 1.3725476264953613,
+ "eval_logits/rejected": 1.3276945352554321,
+ "eval_logps/chosen": -2.073659896850586,
+ "eval_logps/rejected": -2.108074188232422,
+ "eval_loss": 1.8384569883346558,
+ "eval_nll_loss": 1.7621310949325562,
+ "eval_rewards/accuracies": 0.5239999890327454,
+ "eval_rewards/chosen": -0.2073660045862198,
+ "eval_rewards/margins": 0.0034414229448884726,
+ "eval_rewards/rejected": -0.2108074277639389,
+ "eval_runtime": 53.4383,
+ "eval_samples_per_second": 9.357,
+ "eval_steps_per_second": 4.678,
+ "step": 360
+ },
+ {
+ "epoch": 2.96,
+ "grad_norm": 0.1975868046283722,
+ "learning_rate": 1.4243323442136497e-07,
+ "log_odds_chosen": 0.2797223925590515,
+ "log_odds_ratio": -0.6124094724655151,
+ "logits/chosen": 1.4003474712371826,
+ "logits/rejected": 1.5073779821395874,
+ "logps/chosen": -1.8499177694320679,
+ "logps/rejected": -2.080507278442383,
+ "loss": 1.7250284194946288,
+ "nll_loss": 1.6637874841690063,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.1849917769432068,
+ "rewards/margins": 0.023058965802192688,
+ "rewards/rejected": -0.20805075764656067,
+ "step": 370
+ },
+ {
+ "epoch": 2.96,
+ "eval_log_odds_chosen": 0.030934542417526245,
+ "eval_log_odds_ratio": -0.7632805109024048,
+ "eval_logits/chosen": 1.3732939958572388,
+ "eval_logits/rejected": 1.3284506797790527,
+ "eval_logps/chosen": -2.073667526245117,
+ "eval_logps/rejected": -2.1080586910247803,
+ "eval_loss": 1.8383456468582153,
+ "eval_nll_loss": 1.762017846107483,
+ "eval_rewards/accuracies": 0.5239999890327454,
+ "eval_rewards/chosen": -0.2073667198419571,
+ "eval_rewards/margins": 0.0034391426015645266,
+ "eval_rewards/rejected": -0.21080586314201355,
+ "eval_runtime": 53.3973,
+ "eval_samples_per_second": 9.364,
+ "eval_steps_per_second": 4.682,
+ "step": 370
+ }
+ ],
+ "logging_steps": 10,
+ "max_steps": 375,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": true
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/v3/ORPO/ORPO_1k/lora/checkpoint-375/training_args.bin b/v3/ORPO/ORPO_1k/lora/checkpoint-375/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..df6546b02d7dcf1b6f5abb65d533d070ee908422
--- /dev/null
+++ b/v3/ORPO/ORPO_1k/lora/checkpoint-375/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c351cac9ae5a9da1e5d680660ed985c0be320dd6d912f8e5e6dd09d8ebe0c00e
+size 5457
diff --git a/v3/ORPO/ORPO_5k/MORPO_5k/chat_template.jinja b/v3/ORPO/ORPO_5k/MORPO_5k/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/MORPO_5k/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/ORPO/ORPO_5k/MORPO_5k/config.json b/v3/ORPO/ORPO_5k/MORPO_5k/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e9ec18bc1ae18922052a57480e28401f3b9c6b84
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/MORPO_5k/config.json
@@ -0,0 +1,40 @@
+{
+ "architectures": [
+ "LlamaForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 128000,
+ "dtype": "float16",
+ "eos_token_id": [
+ 128001,
+ 128008,
+ 128009
+ ],
+ "head_dim": 64,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 8192,
+ "max_position_embeddings": 131072,
+ "mlp_bias": false,
+ "model_type": "llama",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 16,
+ "num_key_value_heads": 8,
+ "pad_token_id": null,
+ "pretraining_tp": 1,
+ "rms_norm_eps": 1e-05,
+ "rope_parameters": {
+ "factor": 32.0,
+ "high_freq_factor": 4.0,
+ "low_freq_factor": 1.0,
+ "original_max_position_embeddings": 8192,
+ "rope_theta": 500000.0,
+ "rope_type": "llama3"
+ },
+ "tie_word_embeddings": true,
+ "transformers_version": "5.2.0",
+ "use_cache": true,
+ "vocab_size": 128256
+}
diff --git a/v3/ORPO/ORPO_5k/MORPO_5k/generation_config.json b/v3/ORPO/ORPO_5k/MORPO_5k/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..898a2e748a41a4b7a931b754f8abbdac02039fae
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/MORPO_5k/generation_config.json
@@ -0,0 +1,12 @@
+{
+ "bos_token_id": 128000,
+ "do_sample": true,
+ "eos_token_id": [
+ 128001,
+ 128008,
+ 128009
+ ],
+ "temperature": 0.6,
+ "top_p": 0.9,
+ "transformers_version": "5.2.0"
+}
diff --git a/v3/ORPO/ORPO_5k/MORPO_5k/model.safetensors b/v3/ORPO/ORPO_5k/MORPO_5k/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0924e888e4df9ce772fc7566b78237e468d50cb8
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/MORPO_5k/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6ec3e19ea57d3ce6932b33fc8f4f52edf179532e5d057fea1e4ab67ab44f90e
+size 2471645464
diff --git a/v3/ORPO/ORPO_5k/MORPO_5k/tokenizer.json b/v3/ORPO/ORPO_5k/MORPO_5k/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/MORPO_5k/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/ORPO/ORPO_5k/MORPO_5k/tokenizer_config.json b/v3/ORPO/ORPO_5k/MORPO_5k/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/MORPO_5k/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/ORPO/ORPO_5k/ORPO_5k/README.md b/v3/ORPO/ORPO_5k/ORPO_5k/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..df73bb4ed6d6a8e69c79ba9761e4cdd4272cc1ab
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/ORPO_5k/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- lora
+- orpo
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/ORPO/ORPO_5k/ORPO_5k/adapter_config.json b/v3/ORPO/ORPO_5k/ORPO_5k/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d02ef66b5d06bfc12ca93c16b0b45a216cff2a6a
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/ORPO_5k/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "k_proj",
+ "down_proj",
+ "up_proj",
+ "v_proj",
+ "q_proj",
+ "o_proj",
+ "gate_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/ORPO/ORPO_5k/ORPO_5k/adapter_model.safetensors b/v3/ORPO/ORPO_5k/ORPO_5k/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..be1a44bed1af3f79acf12aead7942344ce4f55c2
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/ORPO_5k/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2b0dd2b0585724c05b5fb52242aef8196debbb41a4cfcfe13e42fc6ec80e035
+size 180385008
diff --git a/v3/ORPO/ORPO_5k/lora/README.md b/v3/ORPO/ORPO_5k/lora/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..4884f26f3494dcfa8b59b904a5f11b7816bf013c
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/README.md
@@ -0,0 +1,66 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: transformers
+model_name: lora
+tags:
+- generated_from_trainer
+- orpo
+- trl
+licence: license
+---
+
+# Model Card for lora
+
+This model is a fine-tuned version of [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[](https://wandb.ai/sea-rod/huggingface/runs/srbox2tp)
+
+
+This model was trained with ORPO, a method introduced in [ORPO: Monolithic Preference Optimization without Reference Model](https://huggingface.co/papers/2403.07691).
+
+### Framework versions
+
+- TRL: 0.28.0
+- Transformers: 5.2.0
+- Pytorch: 2.10.0
+- Datasets: 4.5.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite ORPO as:
+
+```bibtex
+@article{hong2024orpo,
+ title = {{ORPO: Monolithic Preference Optimization without Reference Model}},
+ author = {Jiwoo Hong and Noah Lee and James Thorne},
+ year = 2024,
+ eprint = {arXiv:2403.07691}
+}
+```
+
+Cite TRL as:
+
+```bibtex
+@software{vonwerra2020trl,
+ title = {{TRL: Transformers Reinforcement Learning}},
+ author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+ license = {Apache-2.0},
+ url = {https://github.com/huggingface/trl},
+ year = {2020}
+}
+```
\ No newline at end of file
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1000/README.md b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..df73bb4ed6d6a8e69c79ba9761e4cdd4272cc1ab
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- lora
+- orpo
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1000/adapter_config.json b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d02ef66b5d06bfc12ca93c16b0b45a216cff2a6a
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "k_proj",
+ "down_proj",
+ "up_proj",
+ "v_proj",
+ "q_proj",
+ "o_proj",
+ "gate_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1000/adapter_model.safetensors b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..8bbd350f0d00adcab174f362cfdd322b7bf959a0
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:313a3a9202ab101657a52e15b303e7649439591149942a62186f4cfc2c014c6e
+size 180385008
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1000/chat_template.jinja b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1000/optimizer.pt b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f2d508cf2e927419f9972d3a7c801eaf58fd82b5
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6afcd8c4f37340c60dd57b222a96d98c1540bbb9e8c1fd6043627586d26061fd
+size 360902475
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1000/rng_state.pth b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..45da84cb9f96139d23fe2df13e9f26310fc87471
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f27403649e37262609f871dba600d7da4f0bfc387b6571584c6bc75289b7c853
+size 14645
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1000/scaler.pt b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0405b11307032f43fcf2212da19c52823cc01ac4
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14ae2a2128444abab378aa06c09a61a84665f758fcc19fc46f5789b0bc1b5665
+size 1383
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1000/scheduler.pt b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b83e56f78a035bdcba394809b82b2fbed3d35f84
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7afa542f04a1cff5c3080232d7071dcf376f743e0c3dbb855e42ef35559b91a
+size 1465
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1000/tokenizer.json b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1000/tokenizer_config.json b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1000/trainer_state.json b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..aaa36bfd2526dbfe9cc4970d4f8add13fa2de4cf
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/trainer_state.json
@@ -0,0 +1,2214 @@
+{
+ "best_global_step": 1000,
+ "best_metric": 1.8202314376831055,
+ "best_model_checkpoint": "output/lora/checkpoint-1000",
+ "epoch": 1.6,
+ "eval_steps": 50,
+ "global_step": 1000,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.016,
+ "grad_norm": 1.373120665550232,
+ "learning_rate": 3.829787234042553e-07,
+ "log_odds_chosen": 0.00716871302574873,
+ "log_odds_ratio": -0.8661719560623169,
+ "logits/chosen": 1.1518588066101074,
+ "logits/rejected": 1.1109380722045898,
+ "logps/chosen": -3.045567512512207,
+ "logps/rejected": -3.052224636077881,
+ "loss": 3.4928749084472654,
+ "nll_loss": 3.4062583446502686,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.3045567572116852,
+ "rewards/margins": 0.000665713450871408,
+ "rewards/rejected": -0.30522245168685913,
+ "step": 10
+ },
+ {
+ "epoch": 0.032,
+ "grad_norm": 1.9124146699905396,
+ "learning_rate": 8.085106382978723e-07,
+ "log_odds_chosen": -0.07713592052459717,
+ "log_odds_ratio": -0.9291092157363892,
+ "logits/chosen": 0.9864249229431152,
+ "logits/rejected": 0.9885759353637695,
+ "logps/chosen": -3.2009873390197754,
+ "logps/rejected": -3.1334564685821533,
+ "loss": 3.693857192993164,
+ "nll_loss": 3.600947141647339,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.32009872794151306,
+ "rewards/margins": -0.0067530544474720955,
+ "rewards/rejected": -0.31334567070007324,
+ "step": 20
+ },
+ {
+ "epoch": 0.048,
+ "grad_norm": 0.8211373686790466,
+ "learning_rate": 1.2340425531914892e-06,
+ "log_odds_chosen": -0.13502930104732513,
+ "log_odds_ratio": -0.9718782305717468,
+ "logits/chosen": 1.1002552509307861,
+ "logits/rejected": 1.1391098499298096,
+ "logps/chosen": -3.2072739601135254,
+ "logps/rejected": -3.0774378776550293,
+ "loss": 3.3957988739013674,
+ "nll_loss": 3.2986106872558594,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.3207274079322815,
+ "rewards/margins": -0.012983566150069237,
+ "rewards/rejected": -0.3077438473701477,
+ "step": 30
+ },
+ {
+ "epoch": 0.064,
+ "grad_norm": 0.768266499042511,
+ "learning_rate": 1.6595744680851064e-06,
+ "log_odds_chosen": 0.06783491373062134,
+ "log_odds_ratio": -0.7825279235839844,
+ "logits/chosen": 1.0638954639434814,
+ "logits/rejected": 1.0289121866226196,
+ "logps/chosen": -2.831758975982666,
+ "logps/rejected": -2.8962886333465576,
+ "loss": 3.35846061706543,
+ "nll_loss": 3.2802085876464844,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.28317588567733765,
+ "rewards/margins": 0.006452961824834347,
+ "rewards/rejected": -0.28962892293930054,
+ "step": 40
+ },
+ {
+ "epoch": 0.08,
+ "grad_norm": 0.686996340751648,
+ "learning_rate": 2.0851063829787235e-06,
+ "log_odds_chosen": -0.1561535745859146,
+ "log_odds_ratio": -0.9219404458999634,
+ "logits/chosen": 1.161010980606079,
+ "logits/rejected": 1.1353846788406372,
+ "logps/chosen": -3.047670841217041,
+ "logps/rejected": -2.895892858505249,
+ "loss": 3.245359420776367,
+ "nll_loss": 3.153165817260742,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.30476707220077515,
+ "rewards/margins": -0.0151777733117342,
+ "rewards/rejected": -0.2895893156528473,
+ "step": 50
+ },
+ {
+ "epoch": 0.08,
+ "eval_log_odds_chosen": 0.13973847031593323,
+ "eval_log_odds_ratio": -0.8127303719520569,
+ "eval_logits/chosen": 1.0891079902648926,
+ "eval_logits/rejected": 1.077158808708191,
+ "eval_logps/chosen": -2.9398701190948486,
+ "eval_logps/rejected": -3.080166816711426,
+ "eval_loss": 3.217437505722046,
+ "eval_nll_loss": 3.1361641883850098,
+ "eval_rewards/accuracies": 0.5460000038146973,
+ "eval_rewards/chosen": -0.2939870059490204,
+ "eval_rewards/margins": 0.01402965560555458,
+ "eval_rewards/rejected": -0.30801668763160706,
+ "eval_runtime": 53.3593,
+ "eval_samples_per_second": 9.37,
+ "eval_steps_per_second": 4.685,
+ "step": 50
+ },
+ {
+ "epoch": 0.096,
+ "grad_norm": 0.5995831489562988,
+ "learning_rate": 2.5106382978723402e-06,
+ "log_odds_chosen": 0.17807230353355408,
+ "log_odds_ratio": -0.7261534333229065,
+ "logits/chosen": 1.1882927417755127,
+ "logits/rejected": 1.1475657224655151,
+ "logps/chosen": -2.6847589015960693,
+ "logps/rejected": -2.866147994995117,
+ "loss": 3.019968795776367,
+ "nll_loss": 2.9473533630371094,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.2684759199619293,
+ "rewards/margins": 0.01813889667391777,
+ "rewards/rejected": -0.2866148054599762,
+ "step": 60
+ },
+ {
+ "epoch": 0.112,
+ "grad_norm": 0.42678770422935486,
+ "learning_rate": 2.9361702127659574e-06,
+ "log_odds_chosen": 0.04040491580963135,
+ "log_odds_ratio": -0.7541126012802124,
+ "logits/chosen": 1.2164686918258667,
+ "logits/rejected": 1.1548993587493896,
+ "logps/chosen": -2.732120990753174,
+ "logps/rejected": -2.7696893215179443,
+ "loss": 2.990768241882324,
+ "nll_loss": 2.915356159210205,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.2732120752334595,
+ "rewards/margins": 0.0037568225525319576,
+ "rewards/rejected": -0.27696892619132996,
+ "step": 70
+ },
+ {
+ "epoch": 0.128,
+ "grad_norm": 0.39134183526039124,
+ "learning_rate": 3.3617021276595745e-06,
+ "log_odds_chosen": 0.2615276277065277,
+ "log_odds_ratio": -0.780361533164978,
+ "logits/chosen": 1.0079171657562256,
+ "logits/rejected": 0.9202032089233398,
+ "logps/chosen": -2.813866138458252,
+ "logps/rejected": -3.050983190536499,
+ "loss": 2.885699653625488,
+ "nll_loss": 2.807663679122925,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.28138667345046997,
+ "rewards/margins": 0.0237116776406765,
+ "rewards/rejected": -0.3050983250141144,
+ "step": 80
+ },
+ {
+ "epoch": 0.144,
+ "grad_norm": 0.3603176474571228,
+ "learning_rate": 3.7872340425531912e-06,
+ "log_odds_chosen": 0.0377628318965435,
+ "log_odds_ratio": -0.8245071172714233,
+ "logits/chosen": 1.0985459089279175,
+ "logits/rejected": 1.0714184045791626,
+ "logps/chosen": -2.8029580116271973,
+ "logps/rejected": -2.8335766792297363,
+ "loss": 2.8207944869995116,
+ "nll_loss": 2.7383437156677246,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.28029578924179077,
+ "rewards/margins": 0.0030618987511843443,
+ "rewards/rejected": -0.2833576798439026,
+ "step": 90
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 0.627916157245636,
+ "learning_rate": 4.212765957446808e-06,
+ "log_odds_chosen": 0.181382954120636,
+ "log_odds_ratio": -0.6789280772209167,
+ "logits/chosen": 1.0331000089645386,
+ "logits/rejected": 1.1242140531539917,
+ "logps/chosen": -2.438321828842163,
+ "logps/rejected": -2.604365825653076,
+ "loss": 2.5767364501953125,
+ "nll_loss": 2.5088438987731934,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.24383218586444855,
+ "rewards/margins": 0.016604404896497726,
+ "rewards/rejected": -0.2604365944862366,
+ "step": 100
+ },
+ {
+ "epoch": 0.16,
+ "eval_log_odds_chosen": 0.07509047538042068,
+ "eval_log_odds_ratio": -0.7700561881065369,
+ "eval_logits/chosen": 1.0837428569793701,
+ "eval_logits/rejected": 1.0448086261749268,
+ "eval_logps/chosen": -2.481499433517456,
+ "eval_logps/rejected": -2.561527967453003,
+ "eval_loss": 2.5757269859313965,
+ "eval_nll_loss": 2.4987213611602783,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": -0.24814994633197784,
+ "eval_rewards/margins": 0.008002860471606255,
+ "eval_rewards/rejected": -0.25615280866622925,
+ "eval_runtime": 53.3273,
+ "eval_samples_per_second": 9.376,
+ "eval_steps_per_second": 4.688,
+ "step": 100
+ },
+ {
+ "epoch": 0.176,
+ "grad_norm": 0.2816033363342285,
+ "learning_rate": 4.638297872340425e-06,
+ "log_odds_chosen": 0.0019359707366675138,
+ "log_odds_ratio": -0.7848639488220215,
+ "logits/chosen": 1.014973759651184,
+ "logits/rejected": 1.0038645267486572,
+ "logps/chosen": -2.474358320236206,
+ "logps/rejected": -2.4702632427215576,
+ "loss": 2.4666120529174806,
+ "nll_loss": 2.3881256580352783,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.24743585288524628,
+ "rewards/margins": -0.0004095211625099182,
+ "rewards/rejected": -0.24702632427215576,
+ "step": 110
+ },
+ {
+ "epoch": 0.192,
+ "grad_norm": 0.3091694712638855,
+ "learning_rate": 5.063829787234042e-06,
+ "log_odds_chosen": 0.18548759818077087,
+ "log_odds_ratio": -0.7040785551071167,
+ "logits/chosen": 1.0326206684112549,
+ "logits/rejected": 1.03751802444458,
+ "logps/chosen": -2.2152860164642334,
+ "logps/rejected": -2.3625495433807373,
+ "loss": 2.2838161468505858,
+ "nll_loss": 2.2134079933166504,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.22152860462665558,
+ "rewards/margins": 0.01472637802362442,
+ "rewards/rejected": -0.2362549751996994,
+ "step": 120
+ },
+ {
+ "epoch": 0.208,
+ "grad_norm": 0.2403600662946701,
+ "learning_rate": 5.489361702127659e-06,
+ "log_odds_chosen": -0.029171090573072433,
+ "log_odds_ratio": -0.7839903831481934,
+ "logits/chosen": 1.124861717224121,
+ "logits/rejected": 1.0936174392700195,
+ "logps/chosen": -2.2851500511169434,
+ "logps/rejected": -2.2685129642486572,
+ "loss": 2.286957359313965,
+ "nll_loss": 2.2085585594177246,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.22851502895355225,
+ "rewards/margins": -0.0016637055668979883,
+ "rewards/rejected": -0.22685131430625916,
+ "step": 130
+ },
+ {
+ "epoch": 0.224,
+ "grad_norm": 0.2264973670244217,
+ "learning_rate": 5.9148936170212766e-06,
+ "log_odds_chosen": 0.008796649985015392,
+ "log_odds_ratio": -0.8179906010627747,
+ "logits/chosen": 1.0210378170013428,
+ "logits/rejected": 1.0923821926116943,
+ "logps/chosen": -2.3308796882629395,
+ "logps/rejected": -2.352290153503418,
+ "loss": 2.1581136703491213,
+ "nll_loss": 2.076314687728882,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.23308798670768738,
+ "rewards/margins": 0.0021410156041383743,
+ "rewards/rejected": -0.2352290153503418,
+ "step": 140
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 0.20365770161151886,
+ "learning_rate": 6.340425531914893e-06,
+ "log_odds_chosen": -0.009427487850189209,
+ "log_odds_ratio": -0.805976390838623,
+ "logits/chosen": 1.1640613079071045,
+ "logits/rejected": 1.076269268989563,
+ "logps/chosen": -2.211876153945923,
+ "logps/rejected": -2.1832690238952637,
+ "loss": 2.04748592376709,
+ "nll_loss": 1.9668887853622437,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.22118762135505676,
+ "rewards/margins": -0.0028607314452528954,
+ "rewards/rejected": -0.2183268964290619,
+ "step": 150
+ },
+ {
+ "epoch": 0.24,
+ "eval_log_odds_chosen": 0.034868109971284866,
+ "eval_log_odds_ratio": -0.7663777470588684,
+ "eval_logits/chosen": 1.1222375631332397,
+ "eval_logits/rejected": 1.0752418041229248,
+ "eval_logps/chosen": -2.245656967163086,
+ "eval_logps/rejected": -2.2865161895751953,
+ "eval_loss": 2.085561990737915,
+ "eval_nll_loss": 2.0089242458343506,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.22456568479537964,
+ "eval_rewards/margins": 0.004085900261998177,
+ "eval_rewards/rejected": -0.22865159809589386,
+ "eval_runtime": 53.2365,
+ "eval_samples_per_second": 9.392,
+ "eval_steps_per_second": 4.696,
+ "step": 150
+ },
+ {
+ "epoch": 0.256,
+ "grad_norm": 0.17429247498512268,
+ "learning_rate": 6.76595744680851e-06,
+ "log_odds_chosen": 0.03718019276857376,
+ "log_odds_ratio": -0.7309927940368652,
+ "logits/chosen": 1.243948221206665,
+ "logits/rejected": 1.155428171157837,
+ "logps/chosen": -2.222162961959839,
+ "logps/rejected": -2.2655863761901855,
+ "loss": 2.079151153564453,
+ "nll_loss": 2.006051778793335,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.22221633791923523,
+ "rewards/margins": 0.004342344123870134,
+ "rewards/rejected": -0.22655868530273438,
+ "step": 160
+ },
+ {
+ "epoch": 0.272,
+ "grad_norm": 0.2044232040643692,
+ "learning_rate": 7.191489361702127e-06,
+ "log_odds_chosen": 0.2378263771533966,
+ "log_odds_ratio": -0.6989302635192871,
+ "logits/chosen": 1.1669152975082397,
+ "logits/rejected": 1.1139662265777588,
+ "logps/chosen": -2.1898856163024902,
+ "logps/rejected": -2.405423641204834,
+ "loss": 2.0023525238037108,
+ "nll_loss": 1.9324592351913452,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.2189885824918747,
+ "rewards/margins": 0.021553806960582733,
+ "rewards/rejected": -0.24054238200187683,
+ "step": 170
+ },
+ {
+ "epoch": 0.288,
+ "grad_norm": 0.17495381832122803,
+ "learning_rate": 7.617021276595744e-06,
+ "log_odds_chosen": -0.06665988266468048,
+ "log_odds_ratio": -0.804401695728302,
+ "logits/chosen": 1.1880234479904175,
+ "logits/rejected": 1.193927526473999,
+ "logps/chosen": -2.1306421756744385,
+ "logps/rejected": -2.082124948501587,
+ "loss": 1.9733341217041016,
+ "nll_loss": 1.8928937911987305,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.21306422352790833,
+ "rewards/margins": -0.004851718433201313,
+ "rewards/rejected": -0.2082124948501587,
+ "step": 180
+ },
+ {
+ "epoch": 0.304,
+ "grad_norm": 0.17830601334571838,
+ "learning_rate": 7.995257854179015e-06,
+ "log_odds_chosen": 0.06894228607416153,
+ "log_odds_ratio": -0.7259312868118286,
+ "logits/chosen": 1.171451449394226,
+ "logits/rejected": 1.1703966856002808,
+ "logps/chosen": -2.0300755500793457,
+ "logps/rejected": -2.1007497310638428,
+ "loss": 1.8905454635620118,
+ "nll_loss": 1.8179525136947632,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.2030075490474701,
+ "rewards/margins": 0.007067441940307617,
+ "rewards/rejected": -0.2100749909877777,
+ "step": 190
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 0.20069865882396698,
+ "learning_rate": 7.947836395969175e-06,
+ "log_odds_chosen": 0.2506960332393646,
+ "log_odds_ratio": -0.6373531222343445,
+ "logits/chosen": 1.1848938465118408,
+ "logits/rejected": 1.1178070306777954,
+ "logps/chosen": -1.992468237876892,
+ "logps/rejected": -2.2120540142059326,
+ "loss": 1.8577205657958984,
+ "nll_loss": 1.79398512840271,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": -0.19924680888652802,
+ "rewards/margins": 0.021958591416478157,
+ "rewards/rejected": -0.2212054282426834,
+ "step": 200
+ },
+ {
+ "epoch": 0.32,
+ "eval_log_odds_chosen": 0.028671210631728172,
+ "eval_log_odds_ratio": -0.7659140825271606,
+ "eval_logits/chosen": 1.267584204673767,
+ "eval_logits/rejected": 1.2251330614089966,
+ "eval_logps/chosen": -2.12668776512146,
+ "eval_logps/rejected": -2.1608970165252686,
+ "eval_loss": 1.9326850175857544,
+ "eval_nll_loss": 1.8560935258865356,
+ "eval_rewards/accuracies": 0.5260000228881836,
+ "eval_rewards/chosen": -0.212668776512146,
+ "eval_rewards/margins": 0.0034209203440696,
+ "eval_rewards/rejected": -0.21608968079090118,
+ "eval_runtime": 53.2553,
+ "eval_samples_per_second": 9.389,
+ "eval_steps_per_second": 4.694,
+ "step": 200
+ },
+ {
+ "epoch": 0.336,
+ "grad_norm": 0.20797842741012573,
+ "learning_rate": 7.900414937759335e-06,
+ "log_odds_chosen": 0.04750821739435196,
+ "log_odds_ratio": -0.702393114566803,
+ "logits/chosen": 1.3723499774932861,
+ "logits/rejected": 1.101571798324585,
+ "logps/chosen": -2.08408784866333,
+ "logps/rejected": -2.1285855770111084,
+ "loss": 1.855466651916504,
+ "nll_loss": 1.7852274179458618,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.20840883255004883,
+ "rewards/margins": 0.004449741914868355,
+ "rewards/rejected": -0.21285855770111084,
+ "step": 210
+ },
+ {
+ "epoch": 0.352,
+ "grad_norm": 0.19125986099243164,
+ "learning_rate": 7.852993479549495e-06,
+ "log_odds_chosen": -0.14522936940193176,
+ "log_odds_ratio": -0.8650233149528503,
+ "logits/chosen": 1.3711931705474854,
+ "logits/rejected": 1.2990773916244507,
+ "logps/chosen": -2.171778440475464,
+ "logps/rejected": -2.039144277572632,
+ "loss": 1.8723089218139648,
+ "nll_loss": 1.7858068943023682,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": -0.2171778380870819,
+ "rewards/margins": -0.013263422064483166,
+ "rewards/rejected": -0.20391443371772766,
+ "step": 220
+ },
+ {
+ "epoch": 0.368,
+ "grad_norm": 0.17002636194229126,
+ "learning_rate": 7.805572021339655e-06,
+ "log_odds_chosen": 0.06036282703280449,
+ "log_odds_ratio": -0.7291742563247681,
+ "logits/chosen": 1.3134567737579346,
+ "logits/rejected": 1.3123853206634521,
+ "logps/chosen": -2.0283656120300293,
+ "logps/rejected": -2.0922353267669678,
+ "loss": 1.8563121795654296,
+ "nll_loss": 1.7833948135375977,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.20283658802509308,
+ "rewards/margins": 0.006386973895132542,
+ "rewards/rejected": -0.20922355353832245,
+ "step": 230
+ },
+ {
+ "epoch": 0.384,
+ "grad_norm": 0.16414593160152435,
+ "learning_rate": 7.758150563129815e-06,
+ "log_odds_chosen": 0.22585150599479675,
+ "log_odds_ratio": -0.6319423913955688,
+ "logits/chosen": 1.317280650138855,
+ "logits/rejected": 1.2409361600875854,
+ "logps/chosen": -1.9960790872573853,
+ "logps/rejected": -2.1858954429626465,
+ "loss": 1.8553081512451173,
+ "nll_loss": 1.792114019393921,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.19960792362689972,
+ "rewards/margins": 0.018981631845235825,
+ "rewards/rejected": -0.21858954429626465,
+ "step": 240
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.14812582731246948,
+ "learning_rate": 7.710729104919975e-06,
+ "log_odds_chosen": -0.09517587721347809,
+ "log_odds_ratio": -0.8104914426803589,
+ "logits/chosen": 1.3900704383850098,
+ "logits/rejected": 1.4599993228912354,
+ "logps/chosen": -2.050788402557373,
+ "logps/rejected": -1.9786239862442017,
+ "loss": 1.8190387725830077,
+ "nll_loss": 1.7379897832870483,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.2050788700580597,
+ "rewards/margins": -0.0072164698503911495,
+ "rewards/rejected": -0.19786237180233002,
+ "step": 250
+ },
+ {
+ "epoch": 0.4,
+ "eval_log_odds_chosen": 0.027970660477876663,
+ "eval_log_odds_ratio": -0.7652882933616638,
+ "eval_logits/chosen": 1.365133285522461,
+ "eval_logits/rejected": 1.3242768049240112,
+ "eval_logps/chosen": -2.095747947692871,
+ "eval_logps/rejected": -2.128530263900757,
+ "eval_loss": 1.9009666442871094,
+ "eval_nll_loss": 1.8244377374649048,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": -0.20957481861114502,
+ "eval_rewards/margins": 0.0032782391645014286,
+ "eval_rewards/rejected": -0.21285304427146912,
+ "eval_runtime": 53.1908,
+ "eval_samples_per_second": 9.4,
+ "eval_steps_per_second": 4.7,
+ "step": 250
+ },
+ {
+ "epoch": 0.416,
+ "grad_norm": 0.18252891302108765,
+ "learning_rate": 7.663307646710137e-06,
+ "log_odds_chosen": 0.07316568493843079,
+ "log_odds_ratio": -0.7166140675544739,
+ "logits/chosen": 1.4400272369384766,
+ "logits/rejected": 1.3449039459228516,
+ "logps/chosen": -2.0099174976348877,
+ "logps/rejected": -2.075176954269409,
+ "loss": 1.8593158721923828,
+ "nll_loss": 1.7876542806625366,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.20099174976348877,
+ "rewards/margins": 0.006525977049022913,
+ "rewards/rejected": -0.20751771330833435,
+ "step": 260
+ },
+ {
+ "epoch": 0.432,
+ "grad_norm": 0.15253905951976776,
+ "learning_rate": 7.6158861885002954e-06,
+ "log_odds_chosen": 0.20866680145263672,
+ "log_odds_ratio": -0.685832142829895,
+ "logits/chosen": 1.3512670993804932,
+ "logits/rejected": 1.3183681964874268,
+ "logps/chosen": -1.9916179180145264,
+ "logps/rejected": -2.1834797859191895,
+ "loss": 1.8131860733032226,
+ "nll_loss": 1.7446027994155884,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.1991618126630783,
+ "rewards/margins": 0.019186144694685936,
+ "rewards/rejected": -0.21834798157215118,
+ "step": 270
+ },
+ {
+ "epoch": 0.448,
+ "grad_norm": 0.3055081069469452,
+ "learning_rate": 7.5684647302904554e-06,
+ "log_odds_chosen": 0.19830426573753357,
+ "log_odds_ratio": -0.6579998135566711,
+ "logits/chosen": 1.3933019638061523,
+ "logits/rejected": 1.433093547821045,
+ "logps/chosen": -1.9509963989257812,
+ "logps/rejected": -2.128086566925049,
+ "loss": 1.8611320495605468,
+ "nll_loss": 1.7953321933746338,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.1950996369123459,
+ "rewards/margins": 0.017709016799926758,
+ "rewards/rejected": -0.21280868351459503,
+ "step": 280
+ },
+ {
+ "epoch": 0.464,
+ "grad_norm": 0.1907416731119156,
+ "learning_rate": 7.521043272080616e-06,
+ "log_odds_chosen": 0.2938005328178406,
+ "log_odds_ratio": -0.6218821406364441,
+ "logits/chosen": 1.454288125038147,
+ "logits/rejected": 1.364547848701477,
+ "logps/chosen": -1.907086968421936,
+ "logps/rejected": -2.168692111968994,
+ "loss": 1.7643163681030274,
+ "nll_loss": 1.7021281719207764,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.1907087117433548,
+ "rewards/margins": 0.026160534471273422,
+ "rewards/rejected": -0.21686923503875732,
+ "step": 290
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 0.20577998459339142,
+ "learning_rate": 7.473621813870776e-06,
+ "log_odds_chosen": 0.0781371220946312,
+ "log_odds_ratio": -0.7277525663375854,
+ "logits/chosen": 1.3441762924194336,
+ "logits/rejected": 1.339400291442871,
+ "logps/chosen": -1.986854910850525,
+ "logps/rejected": -2.0572762489318848,
+ "loss": 1.8164241790771485,
+ "nll_loss": 1.7436487674713135,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.19868549704551697,
+ "rewards/margins": 0.007042132318019867,
+ "rewards/rejected": -0.20572762191295624,
+ "step": 300
+ },
+ {
+ "epoch": 0.48,
+ "eval_log_odds_chosen": 0.027198966592550278,
+ "eval_log_odds_ratio": -0.7654678821563721,
+ "eval_logits/chosen": 1.4181112051010132,
+ "eval_logits/rejected": 1.3766058683395386,
+ "eval_logps/chosen": -2.0740718841552734,
+ "eval_logps/rejected": -2.1053483486175537,
+ "eval_loss": 1.8834025859832764,
+ "eval_nll_loss": 1.8068556785583496,
+ "eval_rewards/accuracies": 0.5260000228881836,
+ "eval_rewards/chosen": -0.2074071615934372,
+ "eval_rewards/margins": 0.0031276745721697807,
+ "eval_rewards/rejected": -0.21053485572338104,
+ "eval_runtime": 53.2972,
+ "eval_samples_per_second": 9.381,
+ "eval_steps_per_second": 4.691,
+ "step": 300
+ },
+ {
+ "epoch": 0.496,
+ "grad_norm": 0.15923626720905304,
+ "learning_rate": 7.426200355660936e-06,
+ "log_odds_chosen": -0.007339484989643097,
+ "log_odds_ratio": -0.7585854530334473,
+ "logits/chosen": 1.4372234344482422,
+ "logits/rejected": 1.3944370746612549,
+ "logps/chosen": -1.9776242971420288,
+ "logps/rejected": -1.9795879125595093,
+ "loss": 1.846565055847168,
+ "nll_loss": 1.7707065343856812,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.19776242971420288,
+ "rewards/margins": 0.00019635623903013766,
+ "rewards/rejected": -0.1979587972164154,
+ "step": 310
+ },
+ {
+ "epoch": 0.512,
+ "grad_norm": 0.2272547334432602,
+ "learning_rate": 7.378778897451097e-06,
+ "log_odds_chosen": 0.06932153552770615,
+ "log_odds_ratio": -0.7089754939079285,
+ "logits/chosen": 1.4551548957824707,
+ "logits/rejected": 1.413223147392273,
+ "logps/chosen": -2.0715298652648926,
+ "logps/rejected": -2.1249260902404785,
+ "loss": 1.8696435928344726,
+ "nll_loss": 1.79874587059021,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.20715300738811493,
+ "rewards/margins": 0.005339615046977997,
+ "rewards/rejected": -0.21249261498451233,
+ "step": 320
+ },
+ {
+ "epoch": 0.528,
+ "grad_norm": 0.17856107652187347,
+ "learning_rate": 7.331357439241256e-06,
+ "log_odds_chosen": 0.01983504556119442,
+ "log_odds_ratio": -0.7833854556083679,
+ "logits/chosen": 1.3731528520584106,
+ "logits/rejected": 1.454303503036499,
+ "logps/chosen": -1.9772266149520874,
+ "logps/rejected": -1.9896529912948608,
+ "loss": 1.7815839767456054,
+ "nll_loss": 1.7032454013824463,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.1977226734161377,
+ "rewards/margins": 0.0012426606845110655,
+ "rewards/rejected": -0.19896534085273743,
+ "step": 330
+ },
+ {
+ "epoch": 0.544,
+ "grad_norm": 0.19845250248908997,
+ "learning_rate": 7.283935981031416e-06,
+ "log_odds_chosen": 0.1632343828678131,
+ "log_odds_ratio": -0.6663784384727478,
+ "logits/chosen": 1.4501641988754272,
+ "logits/rejected": 1.3493249416351318,
+ "logps/chosen": -2.048593521118164,
+ "logps/rejected": -2.186927080154419,
+ "loss": 1.8446569442749023,
+ "nll_loss": 1.7780191898345947,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.20485934615135193,
+ "rewards/margins": 0.013833333738148212,
+ "rewards/rejected": -0.21869266033172607,
+ "step": 340
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 0.21156993508338928,
+ "learning_rate": 7.236514522821576e-06,
+ "log_odds_chosen": 0.15967318415641785,
+ "log_odds_ratio": -0.6973395943641663,
+ "logits/chosen": 1.405096173286438,
+ "logits/rejected": 1.3601795434951782,
+ "logps/chosen": -2.0488533973693848,
+ "logps/rejected": -2.1905434131622314,
+ "loss": 1.8135244369506835,
+ "nll_loss": 1.7437903881072998,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.2048853635787964,
+ "rewards/margins": 0.014168987981975079,
+ "rewards/rejected": -0.21905431151390076,
+ "step": 350
+ },
+ {
+ "epoch": 0.56,
+ "eval_log_odds_chosen": 0.02301771193742752,
+ "eval_log_odds_ratio": -0.7660307288169861,
+ "eval_logits/chosen": 1.4248073101043701,
+ "eval_logits/rejected": 1.3838006258010864,
+ "eval_logps/chosen": -2.0602059364318848,
+ "eval_logps/rejected": -2.0874686241149902,
+ "eval_loss": 1.8709203004837036,
+ "eval_nll_loss": 1.7943171262741089,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.20602059364318848,
+ "eval_rewards/margins": 0.0027262638323009014,
+ "eval_rewards/rejected": -0.20874685049057007,
+ "eval_runtime": 53.1993,
+ "eval_samples_per_second": 9.399,
+ "eval_steps_per_second": 4.699,
+ "step": 350
+ },
+ {
+ "epoch": 0.576,
+ "grad_norm": 0.21077947318553925,
+ "learning_rate": 7.189093064611737e-06,
+ "log_odds_chosen": 0.14307400584220886,
+ "log_odds_ratio": -0.7016319036483765,
+ "logits/chosen": 1.5183265209197998,
+ "logits/rejected": 1.5037349462509155,
+ "logps/chosen": -2.0080947875976562,
+ "logps/rejected": -2.136094570159912,
+ "loss": 1.8600387573242188,
+ "nll_loss": 1.7898753881454468,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.20080947875976562,
+ "rewards/margins": 0.012799995020031929,
+ "rewards/rejected": -0.2136094570159912,
+ "step": 360
+ },
+ {
+ "epoch": 0.592,
+ "grad_norm": 0.21159255504608154,
+ "learning_rate": 7.141671606401897e-06,
+ "log_odds_chosen": 0.05201685428619385,
+ "log_odds_ratio": -0.7168989777565002,
+ "logits/chosen": 1.4038429260253906,
+ "logits/rejected": 1.3609113693237305,
+ "logps/chosen": -2.0137746334075928,
+ "logps/rejected": -2.051905870437622,
+ "loss": 1.864394760131836,
+ "nll_loss": 1.7927048206329346,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.20137743651866913,
+ "rewards/margins": 0.0038131114561110735,
+ "rewards/rejected": -0.20519058406352997,
+ "step": 370
+ },
+ {
+ "epoch": 0.608,
+ "grad_norm": 0.17047084867954254,
+ "learning_rate": 7.094250148192056e-06,
+ "log_odds_chosen": 0.14928987622261047,
+ "log_odds_ratio": -0.7049298286437988,
+ "logits/chosen": 1.4637548923492432,
+ "logits/rejected": 1.5022916793823242,
+ "logps/chosen": -2.015235424041748,
+ "logps/rejected": -2.1330373287200928,
+ "loss": 1.7998531341552735,
+ "nll_loss": 1.7293602228164673,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.2015235424041748,
+ "rewards/margins": 0.011780187487602234,
+ "rewards/rejected": -0.21330371499061584,
+ "step": 380
+ },
+ {
+ "epoch": 0.624,
+ "grad_norm": 0.30234500765800476,
+ "learning_rate": 7.046828689982216e-06,
+ "log_odds_chosen": 0.13443878293037415,
+ "log_odds_ratio": -0.6994689702987671,
+ "logits/chosen": 1.3651323318481445,
+ "logits/rejected": 1.4916176795959473,
+ "logps/chosen": -1.9770710468292236,
+ "logps/rejected": -2.0873372554779053,
+ "loss": 1.8301820755004883,
+ "nll_loss": 1.7602351903915405,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.19770710170269012,
+ "rewards/margins": 0.011026628315448761,
+ "rewards/rejected": -0.20873375236988068,
+ "step": 390
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 0.15642815828323364,
+ "learning_rate": 6.999407231772377e-06,
+ "log_odds_chosen": 0.19918577373027802,
+ "log_odds_ratio": -0.6648043394088745,
+ "logits/chosen": 1.658177137374878,
+ "logits/rejected": 1.6399805545806885,
+ "logps/chosen": -1.924300193786621,
+ "logps/rejected": -2.1058459281921387,
+ "loss": 1.8306264877319336,
+ "nll_loss": 1.764146089553833,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.19243000447750092,
+ "rewards/margins": 0.018154580146074295,
+ "rewards/rejected": -0.2105845957994461,
+ "step": 400
+ },
+ {
+ "epoch": 0.64,
+ "eval_log_odds_chosen": 0.029309768229722977,
+ "eval_log_odds_ratio": -0.7632622718811035,
+ "eval_logits/chosen": 1.523261308670044,
+ "eval_logits/rejected": 1.48293936252594,
+ "eval_logps/chosen": -2.0515761375427246,
+ "eval_logps/rejected": -2.0838050842285156,
+ "eval_loss": 1.8621249198913574,
+ "eval_nll_loss": 1.7857985496520996,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.20515760779380798,
+ "eval_rewards/margins": 0.0032229016069322824,
+ "eval_rewards/rejected": -0.20838052034378052,
+ "eval_runtime": 53.1851,
+ "eval_samples_per_second": 9.401,
+ "eval_steps_per_second": 4.701,
+ "step": 400
+ },
+ {
+ "epoch": 0.656,
+ "grad_norm": 0.1967567503452301,
+ "learning_rate": 6.951985773562537e-06,
+ "log_odds_chosen": 0.1653221845626831,
+ "log_odds_ratio": -0.6918090581893921,
+ "logits/chosen": 1.5250298976898193,
+ "logits/rejected": 1.4470211267471313,
+ "logps/chosen": -2.0086913108825684,
+ "logps/rejected": -2.145521879196167,
+ "loss": 1.8166702270507813,
+ "nll_loss": 1.7474892139434814,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.20086915791034698,
+ "rewards/margins": 0.013683043420314789,
+ "rewards/rejected": -0.21455219388008118,
+ "step": 410
+ },
+ {
+ "epoch": 0.672,
+ "grad_norm": 0.2208651602268219,
+ "learning_rate": 6.904564315352697e-06,
+ "log_odds_chosen": 0.14104491472244263,
+ "log_odds_ratio": -0.7459835410118103,
+ "logits/chosen": 1.605399489402771,
+ "logits/rejected": 1.5775949954986572,
+ "logps/chosen": -1.9965150356292725,
+ "logps/rejected": -2.143871545791626,
+ "loss": 1.749098777770996,
+ "nll_loss": 1.6745002269744873,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.19965147972106934,
+ "rewards/margins": 0.01473566610366106,
+ "rewards/rejected": -0.21438714861869812,
+ "step": 420
+ },
+ {
+ "epoch": 0.688,
+ "grad_norm": 0.2232414036989212,
+ "learning_rate": 6.857142857142856e-06,
+ "log_odds_chosen": 0.10664594173431396,
+ "log_odds_ratio": -0.7230110168457031,
+ "logits/chosen": 1.5467585325241089,
+ "logits/rejected": 1.5005398988723755,
+ "logps/chosen": -1.9454959630966187,
+ "logps/rejected": -2.01902437210083,
+ "loss": 1.771216583251953,
+ "nll_loss": 1.698915719985962,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.1945495903491974,
+ "rewards/margins": 0.0073528350330889225,
+ "rewards/rejected": -0.20190243422985077,
+ "step": 430
+ },
+ {
+ "epoch": 0.704,
+ "grad_norm": 0.19019967317581177,
+ "learning_rate": 6.809721398933017e-06,
+ "log_odds_chosen": -0.022097941488027573,
+ "log_odds_ratio": -0.7597793340682983,
+ "logits/chosen": 1.6924508810043335,
+ "logits/rejected": 1.6551586389541626,
+ "logps/chosen": -1.9923264980316162,
+ "logps/rejected": -1.9681341648101807,
+ "loss": 1.8285696029663085,
+ "nll_loss": 1.7525920867919922,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.19923266768455505,
+ "rewards/margins": -0.002419233787804842,
+ "rewards/rejected": -0.1968134194612503,
+ "step": 440
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 0.15414057672023773,
+ "learning_rate": 6.762299940723177e-06,
+ "log_odds_chosen": -0.07213816046714783,
+ "log_odds_ratio": -0.815600574016571,
+ "logits/chosen": 1.5608292818069458,
+ "logits/rejected": 1.5346075296401978,
+ "logps/chosen": -2.106017589569092,
+ "logps/rejected": -2.0266618728637695,
+ "loss": 1.8748476028442382,
+ "nll_loss": 1.7932875156402588,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": -0.21060176193714142,
+ "rewards/margins": -0.007935596629977226,
+ "rewards/rejected": -0.20266616344451904,
+ "step": 450
+ },
+ {
+ "epoch": 0.72,
+ "eval_log_odds_chosen": 0.02593321166932583,
+ "eval_log_odds_ratio": -0.7637203335762024,
+ "eval_logits/chosen": 1.5849090814590454,
+ "eval_logits/rejected": 1.5462759733200073,
+ "eval_logps/chosen": -2.046966791152954,
+ "eval_logps/rejected": -2.076876401901245,
+ "eval_loss": 1.856321930885315,
+ "eval_nll_loss": 1.7799499034881592,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.2046966552734375,
+ "eval_rewards/margins": 0.0029909671284258366,
+ "eval_rewards/rejected": -0.2076876312494278,
+ "eval_runtime": 53.2289,
+ "eval_samples_per_second": 9.393,
+ "eval_steps_per_second": 4.697,
+ "step": 450
+ },
+ {
+ "epoch": 0.736,
+ "grad_norm": 0.2198966145515442,
+ "learning_rate": 6.714878482513337e-06,
+ "log_odds_chosen": 0.1490866094827652,
+ "log_odds_ratio": -0.6993222832679749,
+ "logits/chosen": 1.530315637588501,
+ "logits/rejected": 1.5981829166412354,
+ "logps/chosen": -2.0311427116394043,
+ "logps/rejected": -2.1558847427368164,
+ "loss": 1.852096939086914,
+ "nll_loss": 1.7821648120880127,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.20311430096626282,
+ "rewards/margins": 0.01247421558946371,
+ "rewards/rejected": -0.21558848023414612,
+ "step": 460
+ },
+ {
+ "epoch": 0.752,
+ "grad_norm": 0.17686985433101654,
+ "learning_rate": 6.667457024303497e-06,
+ "log_odds_chosen": 0.17564800381660461,
+ "log_odds_ratio": -0.6861358880996704,
+ "logits/chosen": 1.559126377105713,
+ "logits/rejected": 1.4911739826202393,
+ "logps/chosen": -1.9338982105255127,
+ "logps/rejected": -2.090195417404175,
+ "loss": 1.7766691207885743,
+ "nll_loss": 1.7080554962158203,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.19338981807231903,
+ "rewards/margins": 0.015629729256033897,
+ "rewards/rejected": -0.20901954174041748,
+ "step": 470
+ },
+ {
+ "epoch": 0.768,
+ "grad_norm": 0.23684880137443542,
+ "learning_rate": 6.620035566093657e-06,
+ "log_odds_chosen": 0.07966659963130951,
+ "log_odds_ratio": -0.7355421781539917,
+ "logits/chosen": 1.610801100730896,
+ "logits/rejected": 1.5581674575805664,
+ "logps/chosen": -1.9333826303482056,
+ "logps/rejected": -2.002697229385376,
+ "loss": 1.8244489669799804,
+ "nll_loss": 1.7508949041366577,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.19333826005458832,
+ "rewards/margins": 0.0069314478896558285,
+ "rewards/rejected": -0.20026972889900208,
+ "step": 480
+ },
+ {
+ "epoch": 0.784,
+ "grad_norm": 0.170082688331604,
+ "learning_rate": 6.572614107883817e-06,
+ "log_odds_chosen": -0.016495058313012123,
+ "log_odds_ratio": -0.7567610740661621,
+ "logits/chosen": 1.4795844554901123,
+ "logits/rejected": 1.5042058229446411,
+ "logps/chosen": -2.035449981689453,
+ "logps/rejected": -2.0238888263702393,
+ "loss": 1.8243209838867187,
+ "nll_loss": 1.7486448287963867,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": -0.2035450041294098,
+ "rewards/margins": -0.001156145939603448,
+ "rewards/rejected": -0.20238885283470154,
+ "step": 490
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 0.1873895525932312,
+ "learning_rate": 6.525192649673977e-06,
+ "log_odds_chosen": 0.1201021671295166,
+ "log_odds_ratio": -0.7277526259422302,
+ "logits/chosen": 1.5447202920913696,
+ "logits/rejected": 1.494986891746521,
+ "logps/chosen": -2.0172603130340576,
+ "logps/rejected": -2.110663890838623,
+ "loss": 1.7948209762573242,
+ "nll_loss": 1.7220455408096313,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": -0.2017260491847992,
+ "rewards/margins": 0.009340338408946991,
+ "rewards/rejected": -0.21106640994548798,
+ "step": 500
+ },
+ {
+ "epoch": 0.8,
+ "eval_log_odds_chosen": 0.02295607700943947,
+ "eval_log_odds_ratio": -0.7634766697883606,
+ "eval_logits/chosen": 1.589399814605713,
+ "eval_logits/rejected": 1.5498627424240112,
+ "eval_logps/chosen": -2.0389089584350586,
+ "eval_logps/rejected": -2.0656278133392334,
+ "eval_loss": 1.851176142692566,
+ "eval_nll_loss": 1.774828314781189,
+ "eval_rewards/accuracies": 0.5379999876022339,
+ "eval_rewards/chosen": -0.20389088988304138,
+ "eval_rewards/margins": 0.0026718771550804377,
+ "eval_rewards/rejected": -0.20656277239322662,
+ "eval_runtime": 53.271,
+ "eval_samples_per_second": 9.386,
+ "eval_steps_per_second": 4.693,
+ "step": 500
+ },
+ {
+ "epoch": 0.816,
+ "grad_norm": 0.18255847692489624,
+ "learning_rate": 6.477771191464137e-06,
+ "log_odds_chosen": 0.0806904286146164,
+ "log_odds_ratio": -0.717610239982605,
+ "logits/chosen": 1.5147536993026733,
+ "logits/rejected": 1.444838285446167,
+ "logps/chosen": -2.0275161266326904,
+ "logps/rejected": -2.100182056427002,
+ "loss": 1.8432020187377929,
+ "nll_loss": 1.771440863609314,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.20275163650512695,
+ "rewards/margins": 0.007266544736921787,
+ "rewards/rejected": -0.21001818776130676,
+ "step": 510
+ },
+ {
+ "epoch": 0.832,
+ "grad_norm": 0.17855268716812134,
+ "learning_rate": 6.430349733254298e-06,
+ "log_odds_chosen": 0.2732783854007721,
+ "log_odds_ratio": -0.6957250237464905,
+ "logits/chosen": 1.557079553604126,
+ "logits/rejected": 1.585305094718933,
+ "logps/chosen": -1.9428716897964478,
+ "logps/rejected": -2.2009921073913574,
+ "loss": 1.809073257446289,
+ "nll_loss": 1.7395007610321045,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.19428716599941254,
+ "rewards/margins": 0.0258120596408844,
+ "rewards/rejected": -0.22009921073913574,
+ "step": 520
+ },
+ {
+ "epoch": 0.848,
+ "grad_norm": 0.27463603019714355,
+ "learning_rate": 6.382928275044458e-06,
+ "log_odds_chosen": -0.053782153874635696,
+ "log_odds_ratio": -0.7889940142631531,
+ "logits/chosen": 1.546355128288269,
+ "logits/rejected": 1.5052711963653564,
+ "logps/chosen": -2.0254311561584473,
+ "logps/rejected": -1.9745910167694092,
+ "loss": 1.8272314071655273,
+ "nll_loss": 1.7483322620391846,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.20254313945770264,
+ "rewards/margins": -0.005084014497697353,
+ "rewards/rejected": -0.1974591165781021,
+ "step": 530
+ },
+ {
+ "epoch": 0.864,
+ "grad_norm": 0.19943900406360626,
+ "learning_rate": 6.335506816834617e-06,
+ "log_odds_chosen": 0.007906230166554451,
+ "log_odds_ratio": -0.7769323587417603,
+ "logits/chosen": 1.686943769454956,
+ "logits/rejected": 1.773566484451294,
+ "logps/chosen": -1.9026868343353271,
+ "logps/rejected": -1.8993861675262451,
+ "loss": 1.8080410003662108,
+ "nll_loss": 1.7303478717803955,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.19026869535446167,
+ "rewards/margins": -0.0003300657554063946,
+ "rewards/rejected": -0.18993863463401794,
+ "step": 540
+ },
+ {
+ "epoch": 0.88,
+ "grad_norm": 0.12966203689575195,
+ "learning_rate": 6.288085358624777e-06,
+ "log_odds_chosen": 0.18394824862480164,
+ "log_odds_ratio": -0.656446099281311,
+ "logits/chosen": 1.578307867050171,
+ "logits/rejected": 1.5473096370697021,
+ "logps/chosen": -1.8790676593780518,
+ "logps/rejected": -2.0375452041625977,
+ "loss": 1.80413818359375,
+ "nll_loss": 1.7384932041168213,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": -0.18790677189826965,
+ "rewards/margins": 0.015847764909267426,
+ "rewards/rejected": -0.20375454425811768,
+ "step": 550
+ },
+ {
+ "epoch": 0.88,
+ "eval_log_odds_chosen": 0.026566974818706512,
+ "eval_log_odds_ratio": -0.7653356790542603,
+ "eval_logits/chosen": 1.5679305791854858,
+ "eval_logits/rejected": 1.532114028930664,
+ "eval_logps/chosen": -2.0328524112701416,
+ "eval_logps/rejected": -2.0620551109313965,
+ "eval_loss": 1.846232533454895,
+ "eval_nll_loss": 1.769698977470398,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": -0.20328520238399506,
+ "eval_rewards/margins": 0.002920305822044611,
+ "eval_rewards/rejected": -0.20620553195476532,
+ "eval_runtime": 53.3132,
+ "eval_samples_per_second": 9.379,
+ "eval_steps_per_second": 4.689,
+ "step": 550
+ },
+ {
+ "epoch": 0.896,
+ "grad_norm": 0.24944448471069336,
+ "learning_rate": 6.240663900414938e-06,
+ "log_odds_chosen": 0.2502843141555786,
+ "log_odds_ratio": -0.6478752493858337,
+ "logits/chosen": 1.4606131315231323,
+ "logits/rejected": 1.3790860176086426,
+ "logps/chosen": -1.9443740844726562,
+ "logps/rejected": -2.1599698066711426,
+ "loss": 1.7577186584472657,
+ "nll_loss": 1.6929309368133545,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.1944374144077301,
+ "rewards/margins": 0.021559562534093857,
+ "rewards/rejected": -0.21599698066711426,
+ "step": 560
+ },
+ {
+ "epoch": 0.912,
+ "grad_norm": 0.1742384284734726,
+ "learning_rate": 6.193242442205098e-06,
+ "log_odds_chosen": 0.0590474009513855,
+ "log_odds_ratio": -0.7506214380264282,
+ "logits/chosen": 1.6640907526016235,
+ "logits/rejected": 1.6049493551254272,
+ "logps/chosen": -1.9722331762313843,
+ "logps/rejected": -2.026829481124878,
+ "loss": 1.7977088928222655,
+ "nll_loss": 1.722646951675415,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19722333550453186,
+ "rewards/margins": 0.005459627602249384,
+ "rewards/rejected": -0.2026829719543457,
+ "step": 570
+ },
+ {
+ "epoch": 0.928,
+ "grad_norm": 0.2213008999824524,
+ "learning_rate": 6.145820983995258e-06,
+ "log_odds_chosen": -0.03383875638246536,
+ "log_odds_ratio": -0.7764871716499329,
+ "logits/chosen": 1.6381727457046509,
+ "logits/rejected": 1.5094597339630127,
+ "logps/chosen": -1.9914861917495728,
+ "logps/rejected": -1.964422583580017,
+ "loss": 1.8107261657714844,
+ "nll_loss": 1.7330776453018188,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.19914861023426056,
+ "rewards/margins": -0.0027063353918492794,
+ "rewards/rejected": -0.19644227623939514,
+ "step": 580
+ },
+ {
+ "epoch": 0.944,
+ "grad_norm": 0.1825234442949295,
+ "learning_rate": 6.098399525785417e-06,
+ "log_odds_chosen": 0.10232333838939667,
+ "log_odds_ratio": -0.7037243843078613,
+ "logits/chosen": 1.5093460083007812,
+ "logits/rejected": 1.4225064516067505,
+ "logps/chosen": -1.9882131814956665,
+ "logps/rejected": -2.0832431316375732,
+ "loss": 1.7983221054077148,
+ "nll_loss": 1.727950096130371,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.1988213062286377,
+ "rewards/margins": 0.009502974338829517,
+ "rewards/rejected": -0.20832428336143494,
+ "step": 590
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 0.17290247976779938,
+ "learning_rate": 6.050978067575578e-06,
+ "log_odds_chosen": 0.06585167348384857,
+ "log_odds_ratio": -0.7220332026481628,
+ "logits/chosen": 1.5872576236724854,
+ "logits/rejected": 1.5169059038162231,
+ "logps/chosen": -1.8601738214492798,
+ "logps/rejected": -1.914232850074768,
+ "loss": 1.7327194213867188,
+ "nll_loss": 1.660516381263733,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.18601737916469574,
+ "rewards/margins": 0.005405900534242392,
+ "rewards/rejected": -0.19142326712608337,
+ "step": 600
+ },
+ {
+ "epoch": 0.96,
+ "eval_log_odds_chosen": 0.031927358359098434,
+ "eval_log_odds_ratio": -0.7613769173622131,
+ "eval_logits/chosen": 1.5999646186828613,
+ "eval_logits/rejected": 1.5608441829681396,
+ "eval_logps/chosen": -2.031277656555176,
+ "eval_logps/rejected": -2.0653114318847656,
+ "eval_loss": 1.842340111732483,
+ "eval_nll_loss": 1.7662022113800049,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.20312777161598206,
+ "eval_rewards/margins": 0.003403406124562025,
+ "eval_rewards/rejected": -0.20653115212917328,
+ "eval_runtime": 53.2205,
+ "eval_samples_per_second": 9.395,
+ "eval_steps_per_second": 4.697,
+ "step": 600
+ },
+ {
+ "epoch": 0.976,
+ "grad_norm": 0.17003707587718964,
+ "learning_rate": 6.003556609365738e-06,
+ "log_odds_chosen": 0.2910720705986023,
+ "log_odds_ratio": -0.6602123975753784,
+ "logits/chosen": 1.622191071510315,
+ "logits/rejected": 1.5799508094787598,
+ "logps/chosen": -1.8309406042099,
+ "logps/rejected": -2.0560221672058105,
+ "loss": 1.74339599609375,
+ "nll_loss": 1.6773748397827148,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": -0.1830940693616867,
+ "rewards/margins": 0.022508159279823303,
+ "rewards/rejected": -0.20560221374034882,
+ "step": 610
+ },
+ {
+ "epoch": 0.992,
+ "grad_norm": 0.23827488720417023,
+ "learning_rate": 5.956135151155898e-06,
+ "log_odds_chosen": 0.09837356209754944,
+ "log_odds_ratio": -0.7269458174705505,
+ "logits/chosen": 1.5235488414764404,
+ "logits/rejected": 1.539660096168518,
+ "logps/chosen": -1.8142712116241455,
+ "logps/rejected": -1.8749595880508423,
+ "loss": 1.7412891387939453,
+ "nll_loss": 1.6685945987701416,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.18142713606357574,
+ "rewards/margins": 0.00606885552406311,
+ "rewards/rejected": -0.18749597668647766,
+ "step": 620
+ },
+ {
+ "epoch": 1.008,
+ "grad_norm": 0.20535889267921448,
+ "learning_rate": 5.908713692946058e-06,
+ "log_odds_chosen": 0.22268417477607727,
+ "log_odds_ratio": -0.7025212049484253,
+ "logits/chosen": 1.5684313774108887,
+ "logits/rejected": 1.452976107597351,
+ "logps/chosen": -1.874678373336792,
+ "logps/rejected": -2.072539806365967,
+ "loss": 1.774791145324707,
+ "nll_loss": 1.7045390605926514,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.18746784329414368,
+ "rewards/margins": 0.019786134362220764,
+ "rewards/rejected": -0.20725397765636444,
+ "step": 630
+ },
+ {
+ "epoch": 1.024,
+ "grad_norm": 0.4164029061794281,
+ "learning_rate": 5.861292234736218e-06,
+ "log_odds_chosen": 0.13269853591918945,
+ "log_odds_ratio": -0.6831094026565552,
+ "logits/chosen": 1.5523912906646729,
+ "logits/rejected": 1.554421067237854,
+ "logps/chosen": -1.9016844034194946,
+ "logps/rejected": -2.0167248249053955,
+ "loss": 1.8452745437622071,
+ "nll_loss": 1.7769638299942017,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.19016845524311066,
+ "rewards/margins": 0.011504052206873894,
+ "rewards/rejected": -0.2016725093126297,
+ "step": 640
+ },
+ {
+ "epoch": 1.04,
+ "grad_norm": 0.33490127325057983,
+ "learning_rate": 5.813870776526378e-06,
+ "log_odds_chosen": -0.07161623239517212,
+ "log_odds_ratio": -0.8163315057754517,
+ "logits/chosen": 1.585946798324585,
+ "logits/rejected": 1.5961542129516602,
+ "logps/chosen": -2.0562708377838135,
+ "logps/rejected": -1.9879754781723022,
+ "loss": 1.7808271408081056,
+ "nll_loss": 1.6991941928863525,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.20562703907489777,
+ "rewards/margins": -0.006829507648944855,
+ "rewards/rejected": -0.1987975537776947,
+ "step": 650
+ },
+ {
+ "epoch": 1.04,
+ "eval_log_odds_chosen": 0.026153452694416046,
+ "eval_log_odds_ratio": -0.7626987099647522,
+ "eval_logits/chosen": 1.5905648469924927,
+ "eval_logits/rejected": 1.5565553903579712,
+ "eval_logps/chosen": -2.0193655490875244,
+ "eval_logps/rejected": -2.047696352005005,
+ "eval_loss": 1.8379414081573486,
+ "eval_nll_loss": 1.7616716623306274,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.20193655788898468,
+ "eval_rewards/margins": 0.002833074890077114,
+ "eval_rewards/rejected": -0.20476962625980377,
+ "eval_runtime": 53.3878,
+ "eval_samples_per_second": 9.365,
+ "eval_steps_per_second": 4.683,
+ "step": 650
+ },
+ {
+ "epoch": 1.056,
+ "grad_norm": 0.21944740414619446,
+ "learning_rate": 5.766449318316538e-06,
+ "log_odds_chosen": 0.09746942669153214,
+ "log_odds_ratio": -0.7083339095115662,
+ "logits/chosen": 1.5755507946014404,
+ "logits/rejected": 1.551966667175293,
+ "logps/chosen": -1.9140907526016235,
+ "logps/rejected": -1.9912245273590088,
+ "loss": 1.7622905731201173,
+ "nll_loss": 1.6914570331573486,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19140908122062683,
+ "rewards/margins": 0.00771337142214179,
+ "rewards/rejected": -0.19912245869636536,
+ "step": 660
+ },
+ {
+ "epoch": 1.072,
+ "grad_norm": 0.20419229567050934,
+ "learning_rate": 5.719027860106698e-06,
+ "log_odds_chosen": 0.10894493013620377,
+ "log_odds_ratio": -0.6956179738044739,
+ "logits/chosen": 1.7187572717666626,
+ "logits/rejected": 1.6571474075317383,
+ "logps/chosen": -1.9920858144760132,
+ "logps/rejected": -2.0984714031219482,
+ "loss": 1.802743148803711,
+ "nll_loss": 1.7331812381744385,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.1992085874080658,
+ "rewards/margins": 0.010638559237122536,
+ "rewards/rejected": -0.20984713733196259,
+ "step": 670
+ },
+ {
+ "epoch": 1.088,
+ "grad_norm": 0.1502259224653244,
+ "learning_rate": 5.671606401896859e-06,
+ "log_odds_chosen": 0.06736402213573456,
+ "log_odds_ratio": -0.7346867322921753,
+ "logits/chosen": 1.662166953086853,
+ "logits/rejected": 1.6347684860229492,
+ "logps/chosen": -1.9538319110870361,
+ "logps/rejected": -2.0279860496520996,
+ "loss": 1.7677694320678712,
+ "nll_loss": 1.694300651550293,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.1953831911087036,
+ "rewards/margins": 0.007415431551635265,
+ "rewards/rejected": -0.20279863476753235,
+ "step": 680
+ },
+ {
+ "epoch": 1.104,
+ "grad_norm": 0.1765543818473816,
+ "learning_rate": 5.624184943687018e-06,
+ "log_odds_chosen": 0.10347869247198105,
+ "log_odds_ratio": -0.7048436403274536,
+ "logits/chosen": 1.7106029987335205,
+ "logits/rejected": 1.6761833429336548,
+ "logps/chosen": -1.9210326671600342,
+ "logps/rejected": -2.0231428146362305,
+ "loss": 1.7810544967651367,
+ "nll_loss": 1.7105700969696045,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.1921032965183258,
+ "rewards/margins": 0.010210996493697166,
+ "rewards/rejected": -0.20231428742408752,
+ "step": 690
+ },
+ {
+ "epoch": 1.12,
+ "grad_norm": 0.26942944526672363,
+ "learning_rate": 5.576763485477178e-06,
+ "log_odds_chosen": 0.2199089527130127,
+ "log_odds_ratio": -0.6759704947471619,
+ "logits/chosen": 1.5313873291015625,
+ "logits/rejected": 1.4286370277404785,
+ "logps/chosen": -1.934305191040039,
+ "logps/rejected": -2.104454517364502,
+ "loss": 1.7423564910888671,
+ "nll_loss": 1.6747591495513916,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19343051314353943,
+ "rewards/margins": 0.01701495423913002,
+ "rewards/rejected": -0.21044547855854034,
+ "step": 700
+ },
+ {
+ "epoch": 1.12,
+ "eval_log_odds_chosen": 0.029196415096521378,
+ "eval_log_odds_ratio": -0.7604461312294006,
+ "eval_logits/chosen": 1.6299145221710205,
+ "eval_logits/rejected": 1.59683358669281,
+ "eval_logps/chosen": -2.016523599624634,
+ "eval_logps/rejected": -2.0469908714294434,
+ "eval_loss": 1.8337444067001343,
+ "eval_nll_loss": 1.757699728012085,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.20165236294269562,
+ "eval_rewards/margins": 0.0030467414762824774,
+ "eval_rewards/rejected": -0.2046990990638733,
+ "eval_runtime": 53.2721,
+ "eval_samples_per_second": 9.386,
+ "eval_steps_per_second": 4.693,
+ "step": 700
+ },
+ {
+ "epoch": 1.1360000000000001,
+ "grad_norm": 0.20357680320739746,
+ "learning_rate": 5.529342027267338e-06,
+ "log_odds_chosen": -0.03970063477754593,
+ "log_odds_ratio": -0.7776111364364624,
+ "logits/chosen": 1.688511848449707,
+ "logits/rejected": 1.5558183193206787,
+ "logps/chosen": -2.083556652069092,
+ "logps/rejected": -2.0501885414123535,
+ "loss": 1.849853515625,
+ "nll_loss": 1.7720924615859985,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.20835569500923157,
+ "rewards/margins": -0.0033368065487593412,
+ "rewards/rejected": -0.20501887798309326,
+ "step": 710
+ },
+ {
+ "epoch": 1.152,
+ "grad_norm": 0.20546825230121613,
+ "learning_rate": 5.481920569057499e-06,
+ "log_odds_chosen": 0.038821566849946976,
+ "log_odds_ratio": -0.7310380935668945,
+ "logits/chosen": 1.694077730178833,
+ "logits/rejected": 1.6808054447174072,
+ "logps/chosen": -1.8798902034759521,
+ "logps/rejected": -1.9120309352874756,
+ "loss": 1.8013761520385743,
+ "nll_loss": 1.728272795677185,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.1879890114068985,
+ "rewards/margins": 0.0032140829134732485,
+ "rewards/rejected": -0.19120310246944427,
+ "step": 720
+ },
+ {
+ "epoch": 1.168,
+ "grad_norm": 0.215702086687088,
+ "learning_rate": 5.434499110847659e-06,
+ "log_odds_chosen": -0.00810262281447649,
+ "log_odds_ratio": -0.7776116132736206,
+ "logits/chosen": 1.7699267864227295,
+ "logits/rejected": 1.8135936260223389,
+ "logps/chosen": -1.9550107717514038,
+ "logps/rejected": -1.9629520177841187,
+ "loss": 1.8207136154174806,
+ "nll_loss": 1.7429521083831787,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.19550108909606934,
+ "rewards/margins": 0.0007941495859995484,
+ "rewards/rejected": -0.19629523158073425,
+ "step": 730
+ },
+ {
+ "epoch": 1.184,
+ "grad_norm": 0.18298740684986115,
+ "learning_rate": 5.387077652637819e-06,
+ "log_odds_chosen": 0.18474040925502777,
+ "log_odds_ratio": -0.672413170337677,
+ "logits/chosen": 1.7683032751083374,
+ "logits/rejected": 1.8081409931182861,
+ "logps/chosen": -1.906714677810669,
+ "logps/rejected": -2.0783538818359375,
+ "loss": 1.7294036865234375,
+ "nll_loss": 1.662162184715271,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.19067144393920898,
+ "rewards/margins": 0.01716393604874611,
+ "rewards/rejected": -0.2078353613615036,
+ "step": 740
+ },
+ {
+ "epoch": 1.2,
+ "grad_norm": 0.13119018077850342,
+ "learning_rate": 5.339656194427978e-06,
+ "log_odds_chosen": 0.3178519606590271,
+ "log_odds_ratio": -0.6739884614944458,
+ "logits/chosen": 1.7732694149017334,
+ "logits/rejected": 1.6666276454925537,
+ "logps/chosen": -1.9240577220916748,
+ "logps/rejected": -2.2177700996398926,
+ "loss": 1.767026710510254,
+ "nll_loss": 1.6996276378631592,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": -0.19240576028823853,
+ "rewards/margins": 0.029371242970228195,
+ "rewards/rejected": -0.2217770367860794,
+ "step": 750
+ },
+ {
+ "epoch": 1.2,
+ "eval_log_odds_chosen": 0.024058617651462555,
+ "eval_log_odds_ratio": -0.763344407081604,
+ "eval_logits/chosen": 1.695021390914917,
+ "eval_logits/rejected": 1.6631685495376587,
+ "eval_logps/chosen": -2.0103158950805664,
+ "eval_logps/rejected": -2.0368571281433105,
+ "eval_loss": 1.8313435316085815,
+ "eval_nll_loss": 1.7550091743469238,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.20103158056735992,
+ "eval_rewards/margins": 0.0026541370898485184,
+ "eval_rewards/rejected": -0.2036857306957245,
+ "eval_runtime": 53.3008,
+ "eval_samples_per_second": 9.381,
+ "eval_steps_per_second": 4.69,
+ "step": 750
+ },
+ {
+ "epoch": 1.216,
+ "grad_norm": 0.1924748718738556,
+ "learning_rate": 5.292234736218139e-06,
+ "log_odds_chosen": 0.07262767851352692,
+ "log_odds_ratio": -0.7266906499862671,
+ "logits/chosen": 1.7665992975234985,
+ "logits/rejected": 1.6683565378189087,
+ "logps/chosen": -1.960656762123108,
+ "logps/rejected": -2.0233612060546875,
+ "loss": 1.7435152053833007,
+ "nll_loss": 1.6708459854125977,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.19606567919254303,
+ "rewards/margins": 0.006270436104387045,
+ "rewards/rejected": -0.20233610272407532,
+ "step": 760
+ },
+ {
+ "epoch": 1.232,
+ "grad_norm": 0.16488949954509735,
+ "learning_rate": 5.244813278008299e-06,
+ "log_odds_chosen": 0.2733311653137207,
+ "log_odds_ratio": -0.6829133629798889,
+ "logits/chosen": 1.707964539527893,
+ "logits/rejected": 1.6122114658355713,
+ "logps/chosen": -1.90310537815094,
+ "logps/rejected": -2.1515767574310303,
+ "loss": 1.7614355087280273,
+ "nll_loss": 1.6931442022323608,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.190310537815094,
+ "rewards/margins": 0.024847140535712242,
+ "rewards/rejected": -0.2151576727628708,
+ "step": 770
+ },
+ {
+ "epoch": 1.248,
+ "grad_norm": 0.1734672635793686,
+ "learning_rate": 5.197391819798459e-06,
+ "log_odds_chosen": 0.13369520008563995,
+ "log_odds_ratio": -0.675721287727356,
+ "logits/chosen": 1.79034423828125,
+ "logits/rejected": 1.811118721961975,
+ "logps/chosen": -1.8553498983383179,
+ "logps/rejected": -1.9695507287979126,
+ "loss": 1.7561630249023437,
+ "nll_loss": 1.6885910034179688,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.1855349838733673,
+ "rewards/margins": 0.011420061811804771,
+ "rewards/rejected": -0.19695505499839783,
+ "step": 780
+ },
+ {
+ "epoch": 1.264,
+ "grad_norm": 0.2151460200548172,
+ "learning_rate": 5.149970361588619e-06,
+ "log_odds_chosen": -0.15310701727867126,
+ "log_odds_ratio": -0.8164137005805969,
+ "logits/chosen": 1.8224090337753296,
+ "logits/rejected": 1.8069593906402588,
+ "logps/chosen": -2.0459067821502686,
+ "logps/rejected": -1.9169931411743164,
+ "loss": 1.8225961685180665,
+ "nll_loss": 1.7409546375274658,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": -0.20459070801734924,
+ "rewards/margins": -0.012891381978988647,
+ "rewards/rejected": -0.1916993111371994,
+ "step": 790
+ },
+ {
+ "epoch": 1.28,
+ "grad_norm": 0.23022110760211945,
+ "learning_rate": 5.102548903378779e-06,
+ "log_odds_chosen": 0.17419373989105225,
+ "log_odds_ratio": -0.665838360786438,
+ "logits/chosen": 1.745001196861267,
+ "logits/rejected": 1.7248725891113281,
+ "logps/chosen": -1.9748042821884155,
+ "logps/rejected": -2.124072313308716,
+ "loss": 1.7632785797119142,
+ "nll_loss": 1.6966947317123413,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.1974804550409317,
+ "rewards/margins": 0.014926811680197716,
+ "rewards/rejected": -0.21240726113319397,
+ "step": 800
+ },
+ {
+ "epoch": 1.28,
+ "eval_log_odds_chosen": 0.0298521239310503,
+ "eval_log_odds_ratio": -0.7603721022605896,
+ "eval_logits/chosen": 1.7159233093261719,
+ "eval_logits/rejected": 1.6853290796279907,
+ "eval_logps/chosen": -2.008664608001709,
+ "eval_logps/rejected": -2.0401124954223633,
+ "eval_loss": 1.8289790153503418,
+ "eval_nll_loss": 1.7529417276382446,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.2008664608001709,
+ "eval_rewards/margins": 0.0031447894871234894,
+ "eval_rewards/rejected": -0.20401126146316528,
+ "eval_runtime": 53.2154,
+ "eval_samples_per_second": 9.396,
+ "eval_steps_per_second": 4.698,
+ "step": 800
+ },
+ {
+ "epoch": 1.296,
+ "grad_norm": 0.1864694058895111,
+ "learning_rate": 5.055127445168939e-06,
+ "log_odds_chosen": -0.02357078157365322,
+ "log_odds_ratio": -0.7400822043418884,
+ "logits/chosen": 1.7981126308441162,
+ "logits/rejected": 1.7508691549301147,
+ "logps/chosen": -2.0388572216033936,
+ "logps/rejected": -2.024317979812622,
+ "loss": 1.8243423461914063,
+ "nll_loss": 1.7503341436386108,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.20388571918010712,
+ "rewards/margins": -0.001453907461836934,
+ "rewards/rejected": -0.20243176817893982,
+ "step": 810
+ },
+ {
+ "epoch": 1.312,
+ "grad_norm": 0.18279023468494415,
+ "learning_rate": 5.007705986959099e-06,
+ "log_odds_chosen": -0.011245409026741982,
+ "log_odds_ratio": -0.8205389976501465,
+ "logits/chosen": 1.726945161819458,
+ "logits/rejected": 1.6804269552230835,
+ "logps/chosen": -2.0612854957580566,
+ "logps/rejected": -2.0304996967315674,
+ "loss": 1.8141677856445313,
+ "nll_loss": 1.7321138381958008,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.2061285525560379,
+ "rewards/margins": -0.00307856360450387,
+ "rewards/rejected": -0.20304997265338898,
+ "step": 820
+ },
+ {
+ "epoch": 1.328,
+ "grad_norm": 0.194142147898674,
+ "learning_rate": 4.960284528749259e-06,
+ "log_odds_chosen": -0.10107870399951935,
+ "log_odds_ratio": -0.8795238733291626,
+ "logits/chosen": 1.6292463541030884,
+ "logits/rejected": 1.6127665042877197,
+ "logps/chosen": -1.9763052463531494,
+ "logps/rejected": -1.872083067893982,
+ "loss": 1.7666927337646485,
+ "nll_loss": 1.67874014377594,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.19763052463531494,
+ "rewards/margins": -0.010422185063362122,
+ "rewards/rejected": -0.18720832467079163,
+ "step": 830
+ },
+ {
+ "epoch": 1.3439999999999999,
+ "grad_norm": 0.18887867033481598,
+ "learning_rate": 4.9128630705394195e-06,
+ "log_odds_chosen": -0.0421462245285511,
+ "log_odds_ratio": -0.7816078662872314,
+ "logits/chosen": 1.6638116836547852,
+ "logits/rejected": 1.5873361825942993,
+ "logps/chosen": -1.980798363685608,
+ "logps/rejected": -1.949537992477417,
+ "loss": 1.793659782409668,
+ "nll_loss": 1.715498924255371,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": -0.19807983934879303,
+ "rewards/margins": -0.003126043127849698,
+ "rewards/rejected": -0.1949537992477417,
+ "step": 840
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "grad_norm": 0.2071056365966797,
+ "learning_rate": 4.865441612329579e-06,
+ "log_odds_chosen": 0.20311307907104492,
+ "log_odds_ratio": -0.6712511777877808,
+ "logits/chosen": 1.5604090690612793,
+ "logits/rejected": 1.6046473979949951,
+ "logps/chosen": -1.8914225101470947,
+ "logps/rejected": -2.0582470893859863,
+ "loss": 1.7601259231567383,
+ "nll_loss": 1.6930010318756104,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.18914227187633514,
+ "rewards/margins": 0.016682462766766548,
+ "rewards/rejected": -0.20582470297813416,
+ "step": 850
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "eval_log_odds_chosen": 0.027712374925613403,
+ "eval_log_odds_ratio": -0.7594860196113586,
+ "eval_logits/chosen": 1.666095495223999,
+ "eval_logits/rejected": 1.633544683456421,
+ "eval_logps/chosen": -2.002187967300415,
+ "eval_logps/rejected": -2.0309674739837646,
+ "eval_loss": 1.824683666229248,
+ "eval_nll_loss": 1.7487353086471558,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": -0.2002188116312027,
+ "eval_rewards/margins": 0.0028779483400285244,
+ "eval_rewards/rejected": -0.20309676229953766,
+ "eval_runtime": 53.1899,
+ "eval_samples_per_second": 9.4,
+ "eval_steps_per_second": 4.7,
+ "step": 850
+ },
+ {
+ "epoch": 1.376,
+ "grad_norm": 0.1590212732553482,
+ "learning_rate": 4.818020154119739e-06,
+ "log_odds_chosen": 0.15208786725997925,
+ "log_odds_ratio": -0.6756098866462708,
+ "logits/chosen": 1.6706883907318115,
+ "logits/rejected": 1.6443389654159546,
+ "logps/chosen": -1.9115455150604248,
+ "logps/rejected": -2.0435924530029297,
+ "loss": 1.7545036315917968,
+ "nll_loss": 1.686942458152771,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.19115455448627472,
+ "rewards/margins": 0.013204696588218212,
+ "rewards/rejected": -0.2043592482805252,
+ "step": 860
+ },
+ {
+ "epoch": 1.392,
+ "grad_norm": 0.15062956511974335,
+ "learning_rate": 4.770598695909899e-06,
+ "log_odds_chosen": 0.2899920344352722,
+ "log_odds_ratio": -0.6323431730270386,
+ "logits/chosen": 1.7012927532196045,
+ "logits/rejected": 1.6665140390396118,
+ "logps/chosen": -1.8794418573379517,
+ "logps/rejected": -2.131627321243286,
+ "loss": 1.817972183227539,
+ "nll_loss": 1.7547378540039062,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": -0.1879442036151886,
+ "rewards/margins": 0.025218535214662552,
+ "rewards/rejected": -0.21316273510456085,
+ "step": 870
+ },
+ {
+ "epoch": 1.408,
+ "grad_norm": 0.23191672563552856,
+ "learning_rate": 4.7231772377000595e-06,
+ "log_odds_chosen": 0.08231823146343231,
+ "log_odds_ratio": -0.7166029810905457,
+ "logits/chosen": 1.739164113998413,
+ "logits/rejected": 1.7106529474258423,
+ "logps/chosen": -1.9510221481323242,
+ "logps/rejected": -2.0197978019714355,
+ "loss": 1.7590656280517578,
+ "nll_loss": 1.6874055862426758,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.1951022446155548,
+ "rewards/margins": 0.006877546664327383,
+ "rewards/rejected": -0.20197978615760803,
+ "step": 880
+ },
+ {
+ "epoch": 1.424,
+ "grad_norm": 0.2074817270040512,
+ "learning_rate": 4.6757557794902194e-06,
+ "log_odds_chosen": 0.09656596183776855,
+ "log_odds_ratio": -0.708441436290741,
+ "logits/chosen": 1.7313311100006104,
+ "logits/rejected": 1.6554332971572876,
+ "logps/chosen": -1.9648983478546143,
+ "logps/rejected": -2.0429916381835938,
+ "loss": 1.77806396484375,
+ "nll_loss": 1.7072200775146484,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.19648981094360352,
+ "rewards/margins": 0.007809334900230169,
+ "rewards/rejected": -0.20429913699626923,
+ "step": 890
+ },
+ {
+ "epoch": 1.44,
+ "grad_norm": 0.17369434237480164,
+ "learning_rate": 4.6283343212803794e-06,
+ "log_odds_chosen": 0.02706352435052395,
+ "log_odds_ratio": -0.7602623701095581,
+ "logits/chosen": 1.6460374593734741,
+ "logits/rejected": 1.6657674312591553,
+ "logps/chosen": -2.0089709758758545,
+ "logps/rejected": -2.0413382053375244,
+ "loss": 1.788330078125,
+ "nll_loss": 1.7123043537139893,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": -0.20089709758758545,
+ "rewards/margins": 0.003236697521060705,
+ "rewards/rejected": -0.2041337937116623,
+ "step": 900
+ },
+ {
+ "epoch": 1.44,
+ "eval_log_odds_chosen": 0.029239021241664886,
+ "eval_log_odds_ratio": -0.7607759237289429,
+ "eval_logits/chosen": 1.7303014993667603,
+ "eval_logits/rejected": 1.7004859447479248,
+ "eval_logps/chosen": -2.001692056655884,
+ "eval_logps/rejected": -2.031625747680664,
+ "eval_loss": 1.8236775398254395,
+ "eval_nll_loss": 1.7475998401641846,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.20016922056674957,
+ "eval_rewards/margins": 0.002993369475007057,
+ "eval_rewards/rejected": -0.20316259562969208,
+ "eval_runtime": 53.2446,
+ "eval_samples_per_second": 9.391,
+ "eval_steps_per_second": 4.695,
+ "step": 900
+ },
+ {
+ "epoch": 1.456,
+ "grad_norm": 0.20154239237308502,
+ "learning_rate": 4.580912863070539e-06,
+ "log_odds_chosen": 0.1900930106639862,
+ "log_odds_ratio": -0.6721845269203186,
+ "logits/chosen": 1.7324788570404053,
+ "logits/rejected": 1.6658681631088257,
+ "logps/chosen": -1.8505785465240479,
+ "logps/rejected": -2.0169787406921387,
+ "loss": 1.6994483947753907,
+ "nll_loss": 1.6322300434112549,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.1850578486919403,
+ "rewards/margins": 0.01664000190794468,
+ "rewards/rejected": -0.20169785618782043,
+ "step": 910
+ },
+ {
+ "epoch": 1.472,
+ "grad_norm": 0.16085605323314667,
+ "learning_rate": 4.5334914048606994e-06,
+ "log_odds_chosen": 0.09199506789445877,
+ "log_odds_ratio": -0.6975706219673157,
+ "logits/chosen": 1.7461503744125366,
+ "logits/rejected": 1.8116916418075562,
+ "logps/chosen": -1.8981988430023193,
+ "logps/rejected": -1.982012391090393,
+ "loss": 1.7591039657592773,
+ "nll_loss": 1.6893469095230103,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.18981988728046417,
+ "rewards/margins": 0.00838138721883297,
+ "rewards/rejected": -0.1982012689113617,
+ "step": 920
+ },
+ {
+ "epoch": 1.488,
+ "grad_norm": 0.15088561177253723,
+ "learning_rate": 4.4860699466508594e-06,
+ "log_odds_chosen": 0.09973030537366867,
+ "log_odds_ratio": -0.6972100138664246,
+ "logits/chosen": 1.8033208847045898,
+ "logits/rejected": 1.7551723718643188,
+ "logps/chosen": -1.8274751901626587,
+ "logps/rejected": -1.9026908874511719,
+ "loss": 1.7895126342773438,
+ "nll_loss": 1.7197917699813843,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.18274752795696259,
+ "rewards/margins": 0.0075215683318674564,
+ "rewards/rejected": -0.1902690976858139,
+ "step": 930
+ },
+ {
+ "epoch": 1.504,
+ "grad_norm": 0.13715308904647827,
+ "learning_rate": 4.4386484884410194e-06,
+ "log_odds_chosen": 0.07654011994600296,
+ "log_odds_ratio": -0.7238441705703735,
+ "logits/chosen": 1.7913706302642822,
+ "logits/rejected": 1.768434762954712,
+ "logps/chosen": -1.8972448110580444,
+ "logps/rejected": -1.952164888381958,
+ "loss": 1.749094581604004,
+ "nll_loss": 1.6767103672027588,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.18972447514533997,
+ "rewards/margins": 0.005491999443620443,
+ "rewards/rejected": -0.19521646201610565,
+ "step": 940
+ },
+ {
+ "epoch": 1.52,
+ "grad_norm": 0.1736515462398529,
+ "learning_rate": 4.3912270302311794e-06,
+ "log_odds_chosen": -0.02450474537909031,
+ "log_odds_ratio": -0.7657513618469238,
+ "logits/chosen": 1.8066694736480713,
+ "logits/rejected": 1.7840650081634521,
+ "logps/chosen": -1.9759057760238647,
+ "logps/rejected": -1.9469897747039795,
+ "loss": 1.7701189041137695,
+ "nll_loss": 1.6935440301895142,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": -0.19759058952331543,
+ "rewards/margins": -0.00289161060936749,
+ "rewards/rejected": -0.1946989744901657,
+ "step": 950
+ },
+ {
+ "epoch": 1.52,
+ "eval_log_odds_chosen": 0.03024761565029621,
+ "eval_log_odds_ratio": -0.7591846585273743,
+ "eval_logits/chosen": 1.7475262880325317,
+ "eval_logits/rejected": 1.718088150024414,
+ "eval_logps/chosen": -2.0002715587615967,
+ "eval_logps/rejected": -2.0316338539123535,
+ "eval_loss": 1.821697473526001,
+ "eval_nll_loss": 1.745779037475586,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.20002716779708862,
+ "eval_rewards/margins": 0.003136243438348174,
+ "eval_rewards/rejected": -0.20316338539123535,
+ "eval_runtime": 53.1737,
+ "eval_samples_per_second": 9.403,
+ "eval_steps_per_second": 4.702,
+ "step": 950
+ },
+ {
+ "epoch": 1.536,
+ "grad_norm": 0.16075001657009125,
+ "learning_rate": 4.343805572021339e-06,
+ "log_odds_chosen": 0.108977772295475,
+ "log_odds_ratio": -0.69707852602005,
+ "logits/chosen": 1.9180431365966797,
+ "logits/rejected": 1.8730928897857666,
+ "logps/chosen": -1.9387962818145752,
+ "logps/rejected": -2.0468382835388184,
+ "loss": 1.8717756271362305,
+ "nll_loss": 1.8020679950714111,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.193879634141922,
+ "rewards/margins": 0.010804187506437302,
+ "rewards/rejected": -0.2046838104724884,
+ "step": 960
+ },
+ {
+ "epoch": 1.552,
+ "grad_norm": 0.21932974457740784,
+ "learning_rate": 4.296384113811499e-06,
+ "log_odds_chosen": 0.0989312157034874,
+ "log_odds_ratio": -0.7125958800315857,
+ "logits/chosen": 1.7110919952392578,
+ "logits/rejected": 1.728533387184143,
+ "logps/chosen": -1.9263118505477905,
+ "logps/rejected": -2.00539231300354,
+ "loss": 1.7788570404052735,
+ "nll_loss": 1.7075974941253662,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.19263119995594025,
+ "rewards/margins": 0.007908042520284653,
+ "rewards/rejected": -0.200539231300354,
+ "step": 970
+ },
+ {
+ "epoch": 1.568,
+ "grad_norm": 0.1886725127696991,
+ "learning_rate": 4.248962655601659e-06,
+ "log_odds_chosen": 0.2252909392118454,
+ "log_odds_ratio": -0.6316526532173157,
+ "logits/chosen": 1.69448983669281,
+ "logits/rejected": 1.6676839590072632,
+ "logps/chosen": -1.8613817691802979,
+ "logps/rejected": -2.057375431060791,
+ "loss": 1.7345258712768554,
+ "nll_loss": 1.6713603734970093,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": -0.18613818287849426,
+ "rewards/margins": 0.01959935761988163,
+ "rewards/rejected": -0.20573754608631134,
+ "step": 980
+ },
+ {
+ "epoch": 1.584,
+ "grad_norm": 0.1785261482000351,
+ "learning_rate": 4.201541197391819e-06,
+ "log_odds_chosen": 0.18465228378772736,
+ "log_odds_ratio": -0.6800971627235413,
+ "logits/chosen": 1.693155288696289,
+ "logits/rejected": 1.6254899501800537,
+ "logps/chosen": -1.922136902809143,
+ "logps/rejected": -2.0776562690734863,
+ "loss": 1.7461135864257813,
+ "nll_loss": 1.6781036853790283,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19221369922161102,
+ "rewards/margins": 0.01555191446095705,
+ "rewards/rejected": -0.2077656239271164,
+ "step": 990
+ },
+ {
+ "epoch": 1.6,
+ "grad_norm": 0.19836267828941345,
+ "learning_rate": 4.15411973918198e-06,
+ "log_odds_chosen": 0.16690519452095032,
+ "log_odds_ratio": -0.6907414793968201,
+ "logits/chosen": 1.78627610206604,
+ "logits/rejected": 1.6995677947998047,
+ "logps/chosen": -1.9538847208023071,
+ "logps/rejected": -2.0953662395477295,
+ "loss": 1.8010309219360352,
+ "nll_loss": 1.7319567203521729,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.19538846611976624,
+ "rewards/margins": 0.014148158952593803,
+ "rewards/rejected": -0.2095366269350052,
+ "step": 1000
+ },
+ {
+ "epoch": 1.6,
+ "eval_log_odds_chosen": 0.027905963361263275,
+ "eval_log_odds_ratio": -0.7601120471954346,
+ "eval_logits/chosen": 1.7237838506698608,
+ "eval_logits/rejected": 1.6969308853149414,
+ "eval_logps/chosen": -1.9980021715164185,
+ "eval_logps/rejected": -2.027127742767334,
+ "eval_loss": 1.8202314376831055,
+ "eval_nll_loss": 1.74422025680542,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": -0.19980023801326752,
+ "eval_rewards/margins": 0.0029125558212399483,
+ "eval_rewards/rejected": -0.2027127742767334,
+ "eval_runtime": 53.17,
+ "eval_samples_per_second": 9.404,
+ "eval_steps_per_second": 4.702,
+ "step": 1000
+ }
+ ],
+ "logging_steps": 10,
+ "max_steps": 1875,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1000/training_args.bin b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..e369ed5ca0fb0f2b706a5adca8ab527baae13e31
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4ecab525cdf207fed7abfb6f0080dd00da9f0e35d23314a97f0bff66fe6176b
+size 5457
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1500/README.md b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..df73bb4ed6d6a8e69c79ba9761e4cdd4272cc1ab
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- lora
+- orpo
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1500/adapter_config.json b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d02ef66b5d06bfc12ca93c16b0b45a216cff2a6a
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "k_proj",
+ "down_proj",
+ "up_proj",
+ "v_proj",
+ "q_proj",
+ "o_proj",
+ "gate_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1500/adapter_model.safetensors b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..be1a44bed1af3f79acf12aead7942344ce4f55c2
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2b0dd2b0585724c05b5fb52242aef8196debbb41a4cfcfe13e42fc6ec80e035
+size 180385008
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1500/chat_template.jinja b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1500/optimizer.pt b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7fc747e3e339234e46959c903e67918243a71726
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99e618cb53693c4f4616c901f340507ae05af09322d1c7b51cab0535060725de
+size 360902475
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1500/rng_state.pth b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..33ab1511c4dbfc6e61dcb7f70233792670b01e19
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:428aed83766f09a6a2d9ccfb86972eeda646225b3b0cd6accec4a90e4f900161
+size 14645
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1500/scaler.pt b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3664bb212147e2acc5cde00bad4860996ad25fe2
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca372268f4fa9335030c0cb7aedb6cdba75f457da50e7a4034abb1a2d0843689
+size 1383
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1500/scheduler.pt b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..98c98bddf33275b2183921ee28f9c8a580ef0a12
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36c8dd2fb40dc4ff6a440b78942ba4a0a0a4218aae6b8e4e0737f15d71a478ec
+size 1465
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1500/tokenizer.json b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1500/tokenizer_config.json b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1500/trainer_state.json b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..04f7d20c1bce91343017ae3358d4bcb7c8c3cbe8
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/trainer_state.json
@@ -0,0 +1,3304 @@
+{
+ "best_global_step": 1500,
+ "best_metric": 1.8059344291687012,
+ "best_model_checkpoint": "output/lora/checkpoint-1500",
+ "epoch": 2.4,
+ "eval_steps": 50,
+ "global_step": 1500,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.016,
+ "grad_norm": 1.373120665550232,
+ "learning_rate": 3.829787234042553e-07,
+ "log_odds_chosen": 0.00716871302574873,
+ "log_odds_ratio": -0.8661719560623169,
+ "logits/chosen": 1.1518588066101074,
+ "logits/rejected": 1.1109380722045898,
+ "logps/chosen": -3.045567512512207,
+ "logps/rejected": -3.052224636077881,
+ "loss": 3.4928749084472654,
+ "nll_loss": 3.4062583446502686,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.3045567572116852,
+ "rewards/margins": 0.000665713450871408,
+ "rewards/rejected": -0.30522245168685913,
+ "step": 10
+ },
+ {
+ "epoch": 0.032,
+ "grad_norm": 1.9124146699905396,
+ "learning_rate": 8.085106382978723e-07,
+ "log_odds_chosen": -0.07713592052459717,
+ "log_odds_ratio": -0.9291092157363892,
+ "logits/chosen": 0.9864249229431152,
+ "logits/rejected": 0.9885759353637695,
+ "logps/chosen": -3.2009873390197754,
+ "logps/rejected": -3.1334564685821533,
+ "loss": 3.693857192993164,
+ "nll_loss": 3.600947141647339,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.32009872794151306,
+ "rewards/margins": -0.0067530544474720955,
+ "rewards/rejected": -0.31334567070007324,
+ "step": 20
+ },
+ {
+ "epoch": 0.048,
+ "grad_norm": 0.8211373686790466,
+ "learning_rate": 1.2340425531914892e-06,
+ "log_odds_chosen": -0.13502930104732513,
+ "log_odds_ratio": -0.9718782305717468,
+ "logits/chosen": 1.1002552509307861,
+ "logits/rejected": 1.1391098499298096,
+ "logps/chosen": -3.2072739601135254,
+ "logps/rejected": -3.0774378776550293,
+ "loss": 3.3957988739013674,
+ "nll_loss": 3.2986106872558594,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.3207274079322815,
+ "rewards/margins": -0.012983566150069237,
+ "rewards/rejected": -0.3077438473701477,
+ "step": 30
+ },
+ {
+ "epoch": 0.064,
+ "grad_norm": 0.768266499042511,
+ "learning_rate": 1.6595744680851064e-06,
+ "log_odds_chosen": 0.06783491373062134,
+ "log_odds_ratio": -0.7825279235839844,
+ "logits/chosen": 1.0638954639434814,
+ "logits/rejected": 1.0289121866226196,
+ "logps/chosen": -2.831758975982666,
+ "logps/rejected": -2.8962886333465576,
+ "loss": 3.35846061706543,
+ "nll_loss": 3.2802085876464844,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.28317588567733765,
+ "rewards/margins": 0.006452961824834347,
+ "rewards/rejected": -0.28962892293930054,
+ "step": 40
+ },
+ {
+ "epoch": 0.08,
+ "grad_norm": 0.686996340751648,
+ "learning_rate": 2.0851063829787235e-06,
+ "log_odds_chosen": -0.1561535745859146,
+ "log_odds_ratio": -0.9219404458999634,
+ "logits/chosen": 1.161010980606079,
+ "logits/rejected": 1.1353846788406372,
+ "logps/chosen": -3.047670841217041,
+ "logps/rejected": -2.895892858505249,
+ "loss": 3.245359420776367,
+ "nll_loss": 3.153165817260742,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.30476707220077515,
+ "rewards/margins": -0.0151777733117342,
+ "rewards/rejected": -0.2895893156528473,
+ "step": 50
+ },
+ {
+ "epoch": 0.08,
+ "eval_log_odds_chosen": 0.13973847031593323,
+ "eval_log_odds_ratio": -0.8127303719520569,
+ "eval_logits/chosen": 1.0891079902648926,
+ "eval_logits/rejected": 1.077158808708191,
+ "eval_logps/chosen": -2.9398701190948486,
+ "eval_logps/rejected": -3.080166816711426,
+ "eval_loss": 3.217437505722046,
+ "eval_nll_loss": 3.1361641883850098,
+ "eval_rewards/accuracies": 0.5460000038146973,
+ "eval_rewards/chosen": -0.2939870059490204,
+ "eval_rewards/margins": 0.01402965560555458,
+ "eval_rewards/rejected": -0.30801668763160706,
+ "eval_runtime": 53.3593,
+ "eval_samples_per_second": 9.37,
+ "eval_steps_per_second": 4.685,
+ "step": 50
+ },
+ {
+ "epoch": 0.096,
+ "grad_norm": 0.5995831489562988,
+ "learning_rate": 2.5106382978723402e-06,
+ "log_odds_chosen": 0.17807230353355408,
+ "log_odds_ratio": -0.7261534333229065,
+ "logits/chosen": 1.1882927417755127,
+ "logits/rejected": 1.1475657224655151,
+ "logps/chosen": -2.6847589015960693,
+ "logps/rejected": -2.866147994995117,
+ "loss": 3.019968795776367,
+ "nll_loss": 2.9473533630371094,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.2684759199619293,
+ "rewards/margins": 0.01813889667391777,
+ "rewards/rejected": -0.2866148054599762,
+ "step": 60
+ },
+ {
+ "epoch": 0.112,
+ "grad_norm": 0.42678770422935486,
+ "learning_rate": 2.9361702127659574e-06,
+ "log_odds_chosen": 0.04040491580963135,
+ "log_odds_ratio": -0.7541126012802124,
+ "logits/chosen": 1.2164686918258667,
+ "logits/rejected": 1.1548993587493896,
+ "logps/chosen": -2.732120990753174,
+ "logps/rejected": -2.7696893215179443,
+ "loss": 2.990768241882324,
+ "nll_loss": 2.915356159210205,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.2732120752334595,
+ "rewards/margins": 0.0037568225525319576,
+ "rewards/rejected": -0.27696892619132996,
+ "step": 70
+ },
+ {
+ "epoch": 0.128,
+ "grad_norm": 0.39134183526039124,
+ "learning_rate": 3.3617021276595745e-06,
+ "log_odds_chosen": 0.2615276277065277,
+ "log_odds_ratio": -0.780361533164978,
+ "logits/chosen": 1.0079171657562256,
+ "logits/rejected": 0.9202032089233398,
+ "logps/chosen": -2.813866138458252,
+ "logps/rejected": -3.050983190536499,
+ "loss": 2.885699653625488,
+ "nll_loss": 2.807663679122925,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.28138667345046997,
+ "rewards/margins": 0.0237116776406765,
+ "rewards/rejected": -0.3050983250141144,
+ "step": 80
+ },
+ {
+ "epoch": 0.144,
+ "grad_norm": 0.3603176474571228,
+ "learning_rate": 3.7872340425531912e-06,
+ "log_odds_chosen": 0.0377628318965435,
+ "log_odds_ratio": -0.8245071172714233,
+ "logits/chosen": 1.0985459089279175,
+ "logits/rejected": 1.0714184045791626,
+ "logps/chosen": -2.8029580116271973,
+ "logps/rejected": -2.8335766792297363,
+ "loss": 2.8207944869995116,
+ "nll_loss": 2.7383437156677246,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.28029578924179077,
+ "rewards/margins": 0.0030618987511843443,
+ "rewards/rejected": -0.2833576798439026,
+ "step": 90
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 0.627916157245636,
+ "learning_rate": 4.212765957446808e-06,
+ "log_odds_chosen": 0.181382954120636,
+ "log_odds_ratio": -0.6789280772209167,
+ "logits/chosen": 1.0331000089645386,
+ "logits/rejected": 1.1242140531539917,
+ "logps/chosen": -2.438321828842163,
+ "logps/rejected": -2.604365825653076,
+ "loss": 2.5767364501953125,
+ "nll_loss": 2.5088438987731934,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.24383218586444855,
+ "rewards/margins": 0.016604404896497726,
+ "rewards/rejected": -0.2604365944862366,
+ "step": 100
+ },
+ {
+ "epoch": 0.16,
+ "eval_log_odds_chosen": 0.07509047538042068,
+ "eval_log_odds_ratio": -0.7700561881065369,
+ "eval_logits/chosen": 1.0837428569793701,
+ "eval_logits/rejected": 1.0448086261749268,
+ "eval_logps/chosen": -2.481499433517456,
+ "eval_logps/rejected": -2.561527967453003,
+ "eval_loss": 2.5757269859313965,
+ "eval_nll_loss": 2.4987213611602783,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": -0.24814994633197784,
+ "eval_rewards/margins": 0.008002860471606255,
+ "eval_rewards/rejected": -0.25615280866622925,
+ "eval_runtime": 53.3273,
+ "eval_samples_per_second": 9.376,
+ "eval_steps_per_second": 4.688,
+ "step": 100
+ },
+ {
+ "epoch": 0.176,
+ "grad_norm": 0.2816033363342285,
+ "learning_rate": 4.638297872340425e-06,
+ "log_odds_chosen": 0.0019359707366675138,
+ "log_odds_ratio": -0.7848639488220215,
+ "logits/chosen": 1.014973759651184,
+ "logits/rejected": 1.0038645267486572,
+ "logps/chosen": -2.474358320236206,
+ "logps/rejected": -2.4702632427215576,
+ "loss": 2.4666120529174806,
+ "nll_loss": 2.3881256580352783,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.24743585288524628,
+ "rewards/margins": -0.0004095211625099182,
+ "rewards/rejected": -0.24702632427215576,
+ "step": 110
+ },
+ {
+ "epoch": 0.192,
+ "grad_norm": 0.3091694712638855,
+ "learning_rate": 5.063829787234042e-06,
+ "log_odds_chosen": 0.18548759818077087,
+ "log_odds_ratio": -0.7040785551071167,
+ "logits/chosen": 1.0326206684112549,
+ "logits/rejected": 1.03751802444458,
+ "logps/chosen": -2.2152860164642334,
+ "logps/rejected": -2.3625495433807373,
+ "loss": 2.2838161468505858,
+ "nll_loss": 2.2134079933166504,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.22152860462665558,
+ "rewards/margins": 0.01472637802362442,
+ "rewards/rejected": -0.2362549751996994,
+ "step": 120
+ },
+ {
+ "epoch": 0.208,
+ "grad_norm": 0.2403600662946701,
+ "learning_rate": 5.489361702127659e-06,
+ "log_odds_chosen": -0.029171090573072433,
+ "log_odds_ratio": -0.7839903831481934,
+ "logits/chosen": 1.124861717224121,
+ "logits/rejected": 1.0936174392700195,
+ "logps/chosen": -2.2851500511169434,
+ "logps/rejected": -2.2685129642486572,
+ "loss": 2.286957359313965,
+ "nll_loss": 2.2085585594177246,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.22851502895355225,
+ "rewards/margins": -0.0016637055668979883,
+ "rewards/rejected": -0.22685131430625916,
+ "step": 130
+ },
+ {
+ "epoch": 0.224,
+ "grad_norm": 0.2264973670244217,
+ "learning_rate": 5.9148936170212766e-06,
+ "log_odds_chosen": 0.008796649985015392,
+ "log_odds_ratio": -0.8179906010627747,
+ "logits/chosen": 1.0210378170013428,
+ "logits/rejected": 1.0923821926116943,
+ "logps/chosen": -2.3308796882629395,
+ "logps/rejected": -2.352290153503418,
+ "loss": 2.1581136703491213,
+ "nll_loss": 2.076314687728882,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.23308798670768738,
+ "rewards/margins": 0.0021410156041383743,
+ "rewards/rejected": -0.2352290153503418,
+ "step": 140
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 0.20365770161151886,
+ "learning_rate": 6.340425531914893e-06,
+ "log_odds_chosen": -0.009427487850189209,
+ "log_odds_ratio": -0.805976390838623,
+ "logits/chosen": 1.1640613079071045,
+ "logits/rejected": 1.076269268989563,
+ "logps/chosen": -2.211876153945923,
+ "logps/rejected": -2.1832690238952637,
+ "loss": 2.04748592376709,
+ "nll_loss": 1.9668887853622437,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.22118762135505676,
+ "rewards/margins": -0.0028607314452528954,
+ "rewards/rejected": -0.2183268964290619,
+ "step": 150
+ },
+ {
+ "epoch": 0.24,
+ "eval_log_odds_chosen": 0.034868109971284866,
+ "eval_log_odds_ratio": -0.7663777470588684,
+ "eval_logits/chosen": 1.1222375631332397,
+ "eval_logits/rejected": 1.0752418041229248,
+ "eval_logps/chosen": -2.245656967163086,
+ "eval_logps/rejected": -2.2865161895751953,
+ "eval_loss": 2.085561990737915,
+ "eval_nll_loss": 2.0089242458343506,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.22456568479537964,
+ "eval_rewards/margins": 0.004085900261998177,
+ "eval_rewards/rejected": -0.22865159809589386,
+ "eval_runtime": 53.2365,
+ "eval_samples_per_second": 9.392,
+ "eval_steps_per_second": 4.696,
+ "step": 150
+ },
+ {
+ "epoch": 0.256,
+ "grad_norm": 0.17429247498512268,
+ "learning_rate": 6.76595744680851e-06,
+ "log_odds_chosen": 0.03718019276857376,
+ "log_odds_ratio": -0.7309927940368652,
+ "logits/chosen": 1.243948221206665,
+ "logits/rejected": 1.155428171157837,
+ "logps/chosen": -2.222162961959839,
+ "logps/rejected": -2.2655863761901855,
+ "loss": 2.079151153564453,
+ "nll_loss": 2.006051778793335,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.22221633791923523,
+ "rewards/margins": 0.004342344123870134,
+ "rewards/rejected": -0.22655868530273438,
+ "step": 160
+ },
+ {
+ "epoch": 0.272,
+ "grad_norm": 0.2044232040643692,
+ "learning_rate": 7.191489361702127e-06,
+ "log_odds_chosen": 0.2378263771533966,
+ "log_odds_ratio": -0.6989302635192871,
+ "logits/chosen": 1.1669152975082397,
+ "logits/rejected": 1.1139662265777588,
+ "logps/chosen": -2.1898856163024902,
+ "logps/rejected": -2.405423641204834,
+ "loss": 2.0023525238037108,
+ "nll_loss": 1.9324592351913452,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.2189885824918747,
+ "rewards/margins": 0.021553806960582733,
+ "rewards/rejected": -0.24054238200187683,
+ "step": 170
+ },
+ {
+ "epoch": 0.288,
+ "grad_norm": 0.17495381832122803,
+ "learning_rate": 7.617021276595744e-06,
+ "log_odds_chosen": -0.06665988266468048,
+ "log_odds_ratio": -0.804401695728302,
+ "logits/chosen": 1.1880234479904175,
+ "logits/rejected": 1.193927526473999,
+ "logps/chosen": -2.1306421756744385,
+ "logps/rejected": -2.082124948501587,
+ "loss": 1.9733341217041016,
+ "nll_loss": 1.8928937911987305,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.21306422352790833,
+ "rewards/margins": -0.004851718433201313,
+ "rewards/rejected": -0.2082124948501587,
+ "step": 180
+ },
+ {
+ "epoch": 0.304,
+ "grad_norm": 0.17830601334571838,
+ "learning_rate": 7.995257854179015e-06,
+ "log_odds_chosen": 0.06894228607416153,
+ "log_odds_ratio": -0.7259312868118286,
+ "logits/chosen": 1.171451449394226,
+ "logits/rejected": 1.1703966856002808,
+ "logps/chosen": -2.0300755500793457,
+ "logps/rejected": -2.1007497310638428,
+ "loss": 1.8905454635620118,
+ "nll_loss": 1.8179525136947632,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.2030075490474701,
+ "rewards/margins": 0.007067441940307617,
+ "rewards/rejected": -0.2100749909877777,
+ "step": 190
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 0.20069865882396698,
+ "learning_rate": 7.947836395969175e-06,
+ "log_odds_chosen": 0.2506960332393646,
+ "log_odds_ratio": -0.6373531222343445,
+ "logits/chosen": 1.1848938465118408,
+ "logits/rejected": 1.1178070306777954,
+ "logps/chosen": -1.992468237876892,
+ "logps/rejected": -2.2120540142059326,
+ "loss": 1.8577205657958984,
+ "nll_loss": 1.79398512840271,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": -0.19924680888652802,
+ "rewards/margins": 0.021958591416478157,
+ "rewards/rejected": -0.2212054282426834,
+ "step": 200
+ },
+ {
+ "epoch": 0.32,
+ "eval_log_odds_chosen": 0.028671210631728172,
+ "eval_log_odds_ratio": -0.7659140825271606,
+ "eval_logits/chosen": 1.267584204673767,
+ "eval_logits/rejected": 1.2251330614089966,
+ "eval_logps/chosen": -2.12668776512146,
+ "eval_logps/rejected": -2.1608970165252686,
+ "eval_loss": 1.9326850175857544,
+ "eval_nll_loss": 1.8560935258865356,
+ "eval_rewards/accuracies": 0.5260000228881836,
+ "eval_rewards/chosen": -0.212668776512146,
+ "eval_rewards/margins": 0.0034209203440696,
+ "eval_rewards/rejected": -0.21608968079090118,
+ "eval_runtime": 53.2553,
+ "eval_samples_per_second": 9.389,
+ "eval_steps_per_second": 4.694,
+ "step": 200
+ },
+ {
+ "epoch": 0.336,
+ "grad_norm": 0.20797842741012573,
+ "learning_rate": 7.900414937759335e-06,
+ "log_odds_chosen": 0.04750821739435196,
+ "log_odds_ratio": -0.702393114566803,
+ "logits/chosen": 1.3723499774932861,
+ "logits/rejected": 1.101571798324585,
+ "logps/chosen": -2.08408784866333,
+ "logps/rejected": -2.1285855770111084,
+ "loss": 1.855466651916504,
+ "nll_loss": 1.7852274179458618,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.20840883255004883,
+ "rewards/margins": 0.004449741914868355,
+ "rewards/rejected": -0.21285855770111084,
+ "step": 210
+ },
+ {
+ "epoch": 0.352,
+ "grad_norm": 0.19125986099243164,
+ "learning_rate": 7.852993479549495e-06,
+ "log_odds_chosen": -0.14522936940193176,
+ "log_odds_ratio": -0.8650233149528503,
+ "logits/chosen": 1.3711931705474854,
+ "logits/rejected": 1.2990773916244507,
+ "logps/chosen": -2.171778440475464,
+ "logps/rejected": -2.039144277572632,
+ "loss": 1.8723089218139648,
+ "nll_loss": 1.7858068943023682,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": -0.2171778380870819,
+ "rewards/margins": -0.013263422064483166,
+ "rewards/rejected": -0.20391443371772766,
+ "step": 220
+ },
+ {
+ "epoch": 0.368,
+ "grad_norm": 0.17002636194229126,
+ "learning_rate": 7.805572021339655e-06,
+ "log_odds_chosen": 0.06036282703280449,
+ "log_odds_ratio": -0.7291742563247681,
+ "logits/chosen": 1.3134567737579346,
+ "logits/rejected": 1.3123853206634521,
+ "logps/chosen": -2.0283656120300293,
+ "logps/rejected": -2.0922353267669678,
+ "loss": 1.8563121795654296,
+ "nll_loss": 1.7833948135375977,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.20283658802509308,
+ "rewards/margins": 0.006386973895132542,
+ "rewards/rejected": -0.20922355353832245,
+ "step": 230
+ },
+ {
+ "epoch": 0.384,
+ "grad_norm": 0.16414593160152435,
+ "learning_rate": 7.758150563129815e-06,
+ "log_odds_chosen": 0.22585150599479675,
+ "log_odds_ratio": -0.6319423913955688,
+ "logits/chosen": 1.317280650138855,
+ "logits/rejected": 1.2409361600875854,
+ "logps/chosen": -1.9960790872573853,
+ "logps/rejected": -2.1858954429626465,
+ "loss": 1.8553081512451173,
+ "nll_loss": 1.792114019393921,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.19960792362689972,
+ "rewards/margins": 0.018981631845235825,
+ "rewards/rejected": -0.21858954429626465,
+ "step": 240
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.14812582731246948,
+ "learning_rate": 7.710729104919975e-06,
+ "log_odds_chosen": -0.09517587721347809,
+ "log_odds_ratio": -0.8104914426803589,
+ "logits/chosen": 1.3900704383850098,
+ "logits/rejected": 1.4599993228912354,
+ "logps/chosen": -2.050788402557373,
+ "logps/rejected": -1.9786239862442017,
+ "loss": 1.8190387725830077,
+ "nll_loss": 1.7379897832870483,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.2050788700580597,
+ "rewards/margins": -0.0072164698503911495,
+ "rewards/rejected": -0.19786237180233002,
+ "step": 250
+ },
+ {
+ "epoch": 0.4,
+ "eval_log_odds_chosen": 0.027970660477876663,
+ "eval_log_odds_ratio": -0.7652882933616638,
+ "eval_logits/chosen": 1.365133285522461,
+ "eval_logits/rejected": 1.3242768049240112,
+ "eval_logps/chosen": -2.095747947692871,
+ "eval_logps/rejected": -2.128530263900757,
+ "eval_loss": 1.9009666442871094,
+ "eval_nll_loss": 1.8244377374649048,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": -0.20957481861114502,
+ "eval_rewards/margins": 0.0032782391645014286,
+ "eval_rewards/rejected": -0.21285304427146912,
+ "eval_runtime": 53.1908,
+ "eval_samples_per_second": 9.4,
+ "eval_steps_per_second": 4.7,
+ "step": 250
+ },
+ {
+ "epoch": 0.416,
+ "grad_norm": 0.18252891302108765,
+ "learning_rate": 7.663307646710137e-06,
+ "log_odds_chosen": 0.07316568493843079,
+ "log_odds_ratio": -0.7166140675544739,
+ "logits/chosen": 1.4400272369384766,
+ "logits/rejected": 1.3449039459228516,
+ "logps/chosen": -2.0099174976348877,
+ "logps/rejected": -2.075176954269409,
+ "loss": 1.8593158721923828,
+ "nll_loss": 1.7876542806625366,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.20099174976348877,
+ "rewards/margins": 0.006525977049022913,
+ "rewards/rejected": -0.20751771330833435,
+ "step": 260
+ },
+ {
+ "epoch": 0.432,
+ "grad_norm": 0.15253905951976776,
+ "learning_rate": 7.6158861885002954e-06,
+ "log_odds_chosen": 0.20866680145263672,
+ "log_odds_ratio": -0.685832142829895,
+ "logits/chosen": 1.3512670993804932,
+ "logits/rejected": 1.3183681964874268,
+ "logps/chosen": -1.9916179180145264,
+ "logps/rejected": -2.1834797859191895,
+ "loss": 1.8131860733032226,
+ "nll_loss": 1.7446027994155884,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.1991618126630783,
+ "rewards/margins": 0.019186144694685936,
+ "rewards/rejected": -0.21834798157215118,
+ "step": 270
+ },
+ {
+ "epoch": 0.448,
+ "grad_norm": 0.3055081069469452,
+ "learning_rate": 7.5684647302904554e-06,
+ "log_odds_chosen": 0.19830426573753357,
+ "log_odds_ratio": -0.6579998135566711,
+ "logits/chosen": 1.3933019638061523,
+ "logits/rejected": 1.433093547821045,
+ "logps/chosen": -1.9509963989257812,
+ "logps/rejected": -2.128086566925049,
+ "loss": 1.8611320495605468,
+ "nll_loss": 1.7953321933746338,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.1950996369123459,
+ "rewards/margins": 0.017709016799926758,
+ "rewards/rejected": -0.21280868351459503,
+ "step": 280
+ },
+ {
+ "epoch": 0.464,
+ "grad_norm": 0.1907416731119156,
+ "learning_rate": 7.521043272080616e-06,
+ "log_odds_chosen": 0.2938005328178406,
+ "log_odds_ratio": -0.6218821406364441,
+ "logits/chosen": 1.454288125038147,
+ "logits/rejected": 1.364547848701477,
+ "logps/chosen": -1.907086968421936,
+ "logps/rejected": -2.168692111968994,
+ "loss": 1.7643163681030274,
+ "nll_loss": 1.7021281719207764,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.1907087117433548,
+ "rewards/margins": 0.026160534471273422,
+ "rewards/rejected": -0.21686923503875732,
+ "step": 290
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 0.20577998459339142,
+ "learning_rate": 7.473621813870776e-06,
+ "log_odds_chosen": 0.0781371220946312,
+ "log_odds_ratio": -0.7277525663375854,
+ "logits/chosen": 1.3441762924194336,
+ "logits/rejected": 1.339400291442871,
+ "logps/chosen": -1.986854910850525,
+ "logps/rejected": -2.0572762489318848,
+ "loss": 1.8164241790771485,
+ "nll_loss": 1.7436487674713135,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.19868549704551697,
+ "rewards/margins": 0.007042132318019867,
+ "rewards/rejected": -0.20572762191295624,
+ "step": 300
+ },
+ {
+ "epoch": 0.48,
+ "eval_log_odds_chosen": 0.027198966592550278,
+ "eval_log_odds_ratio": -0.7654678821563721,
+ "eval_logits/chosen": 1.4181112051010132,
+ "eval_logits/rejected": 1.3766058683395386,
+ "eval_logps/chosen": -2.0740718841552734,
+ "eval_logps/rejected": -2.1053483486175537,
+ "eval_loss": 1.8834025859832764,
+ "eval_nll_loss": 1.8068556785583496,
+ "eval_rewards/accuracies": 0.5260000228881836,
+ "eval_rewards/chosen": -0.2074071615934372,
+ "eval_rewards/margins": 0.0031276745721697807,
+ "eval_rewards/rejected": -0.21053485572338104,
+ "eval_runtime": 53.2972,
+ "eval_samples_per_second": 9.381,
+ "eval_steps_per_second": 4.691,
+ "step": 300
+ },
+ {
+ "epoch": 0.496,
+ "grad_norm": 0.15923626720905304,
+ "learning_rate": 7.426200355660936e-06,
+ "log_odds_chosen": -0.007339484989643097,
+ "log_odds_ratio": -0.7585854530334473,
+ "logits/chosen": 1.4372234344482422,
+ "logits/rejected": 1.3944370746612549,
+ "logps/chosen": -1.9776242971420288,
+ "logps/rejected": -1.9795879125595093,
+ "loss": 1.846565055847168,
+ "nll_loss": 1.7707065343856812,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.19776242971420288,
+ "rewards/margins": 0.00019635623903013766,
+ "rewards/rejected": -0.1979587972164154,
+ "step": 310
+ },
+ {
+ "epoch": 0.512,
+ "grad_norm": 0.2272547334432602,
+ "learning_rate": 7.378778897451097e-06,
+ "log_odds_chosen": 0.06932153552770615,
+ "log_odds_ratio": -0.7089754939079285,
+ "logits/chosen": 1.4551548957824707,
+ "logits/rejected": 1.413223147392273,
+ "logps/chosen": -2.0715298652648926,
+ "logps/rejected": -2.1249260902404785,
+ "loss": 1.8696435928344726,
+ "nll_loss": 1.79874587059021,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.20715300738811493,
+ "rewards/margins": 0.005339615046977997,
+ "rewards/rejected": -0.21249261498451233,
+ "step": 320
+ },
+ {
+ "epoch": 0.528,
+ "grad_norm": 0.17856107652187347,
+ "learning_rate": 7.331357439241256e-06,
+ "log_odds_chosen": 0.01983504556119442,
+ "log_odds_ratio": -0.7833854556083679,
+ "logits/chosen": 1.3731528520584106,
+ "logits/rejected": 1.454303503036499,
+ "logps/chosen": -1.9772266149520874,
+ "logps/rejected": -1.9896529912948608,
+ "loss": 1.7815839767456054,
+ "nll_loss": 1.7032454013824463,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.1977226734161377,
+ "rewards/margins": 0.0012426606845110655,
+ "rewards/rejected": -0.19896534085273743,
+ "step": 330
+ },
+ {
+ "epoch": 0.544,
+ "grad_norm": 0.19845250248908997,
+ "learning_rate": 7.283935981031416e-06,
+ "log_odds_chosen": 0.1632343828678131,
+ "log_odds_ratio": -0.6663784384727478,
+ "logits/chosen": 1.4501641988754272,
+ "logits/rejected": 1.3493249416351318,
+ "logps/chosen": -2.048593521118164,
+ "logps/rejected": -2.186927080154419,
+ "loss": 1.8446569442749023,
+ "nll_loss": 1.7780191898345947,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.20485934615135193,
+ "rewards/margins": 0.013833333738148212,
+ "rewards/rejected": -0.21869266033172607,
+ "step": 340
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 0.21156993508338928,
+ "learning_rate": 7.236514522821576e-06,
+ "log_odds_chosen": 0.15967318415641785,
+ "log_odds_ratio": -0.6973395943641663,
+ "logits/chosen": 1.405096173286438,
+ "logits/rejected": 1.3601795434951782,
+ "logps/chosen": -2.0488533973693848,
+ "logps/rejected": -2.1905434131622314,
+ "loss": 1.8135244369506835,
+ "nll_loss": 1.7437903881072998,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.2048853635787964,
+ "rewards/margins": 0.014168987981975079,
+ "rewards/rejected": -0.21905431151390076,
+ "step": 350
+ },
+ {
+ "epoch": 0.56,
+ "eval_log_odds_chosen": 0.02301771193742752,
+ "eval_log_odds_ratio": -0.7660307288169861,
+ "eval_logits/chosen": 1.4248073101043701,
+ "eval_logits/rejected": 1.3838006258010864,
+ "eval_logps/chosen": -2.0602059364318848,
+ "eval_logps/rejected": -2.0874686241149902,
+ "eval_loss": 1.8709203004837036,
+ "eval_nll_loss": 1.7943171262741089,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.20602059364318848,
+ "eval_rewards/margins": 0.0027262638323009014,
+ "eval_rewards/rejected": -0.20874685049057007,
+ "eval_runtime": 53.1993,
+ "eval_samples_per_second": 9.399,
+ "eval_steps_per_second": 4.699,
+ "step": 350
+ },
+ {
+ "epoch": 0.576,
+ "grad_norm": 0.21077947318553925,
+ "learning_rate": 7.189093064611737e-06,
+ "log_odds_chosen": 0.14307400584220886,
+ "log_odds_ratio": -0.7016319036483765,
+ "logits/chosen": 1.5183265209197998,
+ "logits/rejected": 1.5037349462509155,
+ "logps/chosen": -2.0080947875976562,
+ "logps/rejected": -2.136094570159912,
+ "loss": 1.8600387573242188,
+ "nll_loss": 1.7898753881454468,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.20080947875976562,
+ "rewards/margins": 0.012799995020031929,
+ "rewards/rejected": -0.2136094570159912,
+ "step": 360
+ },
+ {
+ "epoch": 0.592,
+ "grad_norm": 0.21159255504608154,
+ "learning_rate": 7.141671606401897e-06,
+ "log_odds_chosen": 0.05201685428619385,
+ "log_odds_ratio": -0.7168989777565002,
+ "logits/chosen": 1.4038429260253906,
+ "logits/rejected": 1.3609113693237305,
+ "logps/chosen": -2.0137746334075928,
+ "logps/rejected": -2.051905870437622,
+ "loss": 1.864394760131836,
+ "nll_loss": 1.7927048206329346,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.20137743651866913,
+ "rewards/margins": 0.0038131114561110735,
+ "rewards/rejected": -0.20519058406352997,
+ "step": 370
+ },
+ {
+ "epoch": 0.608,
+ "grad_norm": 0.17047084867954254,
+ "learning_rate": 7.094250148192056e-06,
+ "log_odds_chosen": 0.14928987622261047,
+ "log_odds_ratio": -0.7049298286437988,
+ "logits/chosen": 1.4637548923492432,
+ "logits/rejected": 1.5022916793823242,
+ "logps/chosen": -2.015235424041748,
+ "logps/rejected": -2.1330373287200928,
+ "loss": 1.7998531341552735,
+ "nll_loss": 1.7293602228164673,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.2015235424041748,
+ "rewards/margins": 0.011780187487602234,
+ "rewards/rejected": -0.21330371499061584,
+ "step": 380
+ },
+ {
+ "epoch": 0.624,
+ "grad_norm": 0.30234500765800476,
+ "learning_rate": 7.046828689982216e-06,
+ "log_odds_chosen": 0.13443878293037415,
+ "log_odds_ratio": -0.6994689702987671,
+ "logits/chosen": 1.3651323318481445,
+ "logits/rejected": 1.4916176795959473,
+ "logps/chosen": -1.9770710468292236,
+ "logps/rejected": -2.0873372554779053,
+ "loss": 1.8301820755004883,
+ "nll_loss": 1.7602351903915405,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.19770710170269012,
+ "rewards/margins": 0.011026628315448761,
+ "rewards/rejected": -0.20873375236988068,
+ "step": 390
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 0.15642815828323364,
+ "learning_rate": 6.999407231772377e-06,
+ "log_odds_chosen": 0.19918577373027802,
+ "log_odds_ratio": -0.6648043394088745,
+ "logits/chosen": 1.658177137374878,
+ "logits/rejected": 1.6399805545806885,
+ "logps/chosen": -1.924300193786621,
+ "logps/rejected": -2.1058459281921387,
+ "loss": 1.8306264877319336,
+ "nll_loss": 1.764146089553833,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.19243000447750092,
+ "rewards/margins": 0.018154580146074295,
+ "rewards/rejected": -0.2105845957994461,
+ "step": 400
+ },
+ {
+ "epoch": 0.64,
+ "eval_log_odds_chosen": 0.029309768229722977,
+ "eval_log_odds_ratio": -0.7632622718811035,
+ "eval_logits/chosen": 1.523261308670044,
+ "eval_logits/rejected": 1.48293936252594,
+ "eval_logps/chosen": -2.0515761375427246,
+ "eval_logps/rejected": -2.0838050842285156,
+ "eval_loss": 1.8621249198913574,
+ "eval_nll_loss": 1.7857985496520996,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.20515760779380798,
+ "eval_rewards/margins": 0.0032229016069322824,
+ "eval_rewards/rejected": -0.20838052034378052,
+ "eval_runtime": 53.1851,
+ "eval_samples_per_second": 9.401,
+ "eval_steps_per_second": 4.701,
+ "step": 400
+ },
+ {
+ "epoch": 0.656,
+ "grad_norm": 0.1967567503452301,
+ "learning_rate": 6.951985773562537e-06,
+ "log_odds_chosen": 0.1653221845626831,
+ "log_odds_ratio": -0.6918090581893921,
+ "logits/chosen": 1.5250298976898193,
+ "logits/rejected": 1.4470211267471313,
+ "logps/chosen": -2.0086913108825684,
+ "logps/rejected": -2.145521879196167,
+ "loss": 1.8166702270507813,
+ "nll_loss": 1.7474892139434814,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.20086915791034698,
+ "rewards/margins": 0.013683043420314789,
+ "rewards/rejected": -0.21455219388008118,
+ "step": 410
+ },
+ {
+ "epoch": 0.672,
+ "grad_norm": 0.2208651602268219,
+ "learning_rate": 6.904564315352697e-06,
+ "log_odds_chosen": 0.14104491472244263,
+ "log_odds_ratio": -0.7459835410118103,
+ "logits/chosen": 1.605399489402771,
+ "logits/rejected": 1.5775949954986572,
+ "logps/chosen": -1.9965150356292725,
+ "logps/rejected": -2.143871545791626,
+ "loss": 1.749098777770996,
+ "nll_loss": 1.6745002269744873,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.19965147972106934,
+ "rewards/margins": 0.01473566610366106,
+ "rewards/rejected": -0.21438714861869812,
+ "step": 420
+ },
+ {
+ "epoch": 0.688,
+ "grad_norm": 0.2232414036989212,
+ "learning_rate": 6.857142857142856e-06,
+ "log_odds_chosen": 0.10664594173431396,
+ "log_odds_ratio": -0.7230110168457031,
+ "logits/chosen": 1.5467585325241089,
+ "logits/rejected": 1.5005398988723755,
+ "logps/chosen": -1.9454959630966187,
+ "logps/rejected": -2.01902437210083,
+ "loss": 1.771216583251953,
+ "nll_loss": 1.698915719985962,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.1945495903491974,
+ "rewards/margins": 0.0073528350330889225,
+ "rewards/rejected": -0.20190243422985077,
+ "step": 430
+ },
+ {
+ "epoch": 0.704,
+ "grad_norm": 0.19019967317581177,
+ "learning_rate": 6.809721398933017e-06,
+ "log_odds_chosen": -0.022097941488027573,
+ "log_odds_ratio": -0.7597793340682983,
+ "logits/chosen": 1.6924508810043335,
+ "logits/rejected": 1.6551586389541626,
+ "logps/chosen": -1.9923264980316162,
+ "logps/rejected": -1.9681341648101807,
+ "loss": 1.8285696029663085,
+ "nll_loss": 1.7525920867919922,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.19923266768455505,
+ "rewards/margins": -0.002419233787804842,
+ "rewards/rejected": -0.1968134194612503,
+ "step": 440
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 0.15414057672023773,
+ "learning_rate": 6.762299940723177e-06,
+ "log_odds_chosen": -0.07213816046714783,
+ "log_odds_ratio": -0.815600574016571,
+ "logits/chosen": 1.5608292818069458,
+ "logits/rejected": 1.5346075296401978,
+ "logps/chosen": -2.106017589569092,
+ "logps/rejected": -2.0266618728637695,
+ "loss": 1.8748476028442382,
+ "nll_loss": 1.7932875156402588,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": -0.21060176193714142,
+ "rewards/margins": -0.007935596629977226,
+ "rewards/rejected": -0.20266616344451904,
+ "step": 450
+ },
+ {
+ "epoch": 0.72,
+ "eval_log_odds_chosen": 0.02593321166932583,
+ "eval_log_odds_ratio": -0.7637203335762024,
+ "eval_logits/chosen": 1.5849090814590454,
+ "eval_logits/rejected": 1.5462759733200073,
+ "eval_logps/chosen": -2.046966791152954,
+ "eval_logps/rejected": -2.076876401901245,
+ "eval_loss": 1.856321930885315,
+ "eval_nll_loss": 1.7799499034881592,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.2046966552734375,
+ "eval_rewards/margins": 0.0029909671284258366,
+ "eval_rewards/rejected": -0.2076876312494278,
+ "eval_runtime": 53.2289,
+ "eval_samples_per_second": 9.393,
+ "eval_steps_per_second": 4.697,
+ "step": 450
+ },
+ {
+ "epoch": 0.736,
+ "grad_norm": 0.2198966145515442,
+ "learning_rate": 6.714878482513337e-06,
+ "log_odds_chosen": 0.1490866094827652,
+ "log_odds_ratio": -0.6993222832679749,
+ "logits/chosen": 1.530315637588501,
+ "logits/rejected": 1.5981829166412354,
+ "logps/chosen": -2.0311427116394043,
+ "logps/rejected": -2.1558847427368164,
+ "loss": 1.852096939086914,
+ "nll_loss": 1.7821648120880127,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.20311430096626282,
+ "rewards/margins": 0.01247421558946371,
+ "rewards/rejected": -0.21558848023414612,
+ "step": 460
+ },
+ {
+ "epoch": 0.752,
+ "grad_norm": 0.17686985433101654,
+ "learning_rate": 6.667457024303497e-06,
+ "log_odds_chosen": 0.17564800381660461,
+ "log_odds_ratio": -0.6861358880996704,
+ "logits/chosen": 1.559126377105713,
+ "logits/rejected": 1.4911739826202393,
+ "logps/chosen": -1.9338982105255127,
+ "logps/rejected": -2.090195417404175,
+ "loss": 1.7766691207885743,
+ "nll_loss": 1.7080554962158203,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.19338981807231903,
+ "rewards/margins": 0.015629729256033897,
+ "rewards/rejected": -0.20901954174041748,
+ "step": 470
+ },
+ {
+ "epoch": 0.768,
+ "grad_norm": 0.23684880137443542,
+ "learning_rate": 6.620035566093657e-06,
+ "log_odds_chosen": 0.07966659963130951,
+ "log_odds_ratio": -0.7355421781539917,
+ "logits/chosen": 1.610801100730896,
+ "logits/rejected": 1.5581674575805664,
+ "logps/chosen": -1.9333826303482056,
+ "logps/rejected": -2.002697229385376,
+ "loss": 1.8244489669799804,
+ "nll_loss": 1.7508949041366577,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.19333826005458832,
+ "rewards/margins": 0.0069314478896558285,
+ "rewards/rejected": -0.20026972889900208,
+ "step": 480
+ },
+ {
+ "epoch": 0.784,
+ "grad_norm": 0.170082688331604,
+ "learning_rate": 6.572614107883817e-06,
+ "log_odds_chosen": -0.016495058313012123,
+ "log_odds_ratio": -0.7567610740661621,
+ "logits/chosen": 1.4795844554901123,
+ "logits/rejected": 1.5042058229446411,
+ "logps/chosen": -2.035449981689453,
+ "logps/rejected": -2.0238888263702393,
+ "loss": 1.8243209838867187,
+ "nll_loss": 1.7486448287963867,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": -0.2035450041294098,
+ "rewards/margins": -0.001156145939603448,
+ "rewards/rejected": -0.20238885283470154,
+ "step": 490
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 0.1873895525932312,
+ "learning_rate": 6.525192649673977e-06,
+ "log_odds_chosen": 0.1201021671295166,
+ "log_odds_ratio": -0.7277526259422302,
+ "logits/chosen": 1.5447202920913696,
+ "logits/rejected": 1.494986891746521,
+ "logps/chosen": -2.0172603130340576,
+ "logps/rejected": -2.110663890838623,
+ "loss": 1.7948209762573242,
+ "nll_loss": 1.7220455408096313,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": -0.2017260491847992,
+ "rewards/margins": 0.009340338408946991,
+ "rewards/rejected": -0.21106640994548798,
+ "step": 500
+ },
+ {
+ "epoch": 0.8,
+ "eval_log_odds_chosen": 0.02295607700943947,
+ "eval_log_odds_ratio": -0.7634766697883606,
+ "eval_logits/chosen": 1.589399814605713,
+ "eval_logits/rejected": 1.5498627424240112,
+ "eval_logps/chosen": -2.0389089584350586,
+ "eval_logps/rejected": -2.0656278133392334,
+ "eval_loss": 1.851176142692566,
+ "eval_nll_loss": 1.774828314781189,
+ "eval_rewards/accuracies": 0.5379999876022339,
+ "eval_rewards/chosen": -0.20389088988304138,
+ "eval_rewards/margins": 0.0026718771550804377,
+ "eval_rewards/rejected": -0.20656277239322662,
+ "eval_runtime": 53.271,
+ "eval_samples_per_second": 9.386,
+ "eval_steps_per_second": 4.693,
+ "step": 500
+ },
+ {
+ "epoch": 0.816,
+ "grad_norm": 0.18255847692489624,
+ "learning_rate": 6.477771191464137e-06,
+ "log_odds_chosen": 0.0806904286146164,
+ "log_odds_ratio": -0.717610239982605,
+ "logits/chosen": 1.5147536993026733,
+ "logits/rejected": 1.444838285446167,
+ "logps/chosen": -2.0275161266326904,
+ "logps/rejected": -2.100182056427002,
+ "loss": 1.8432020187377929,
+ "nll_loss": 1.771440863609314,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.20275163650512695,
+ "rewards/margins": 0.007266544736921787,
+ "rewards/rejected": -0.21001818776130676,
+ "step": 510
+ },
+ {
+ "epoch": 0.832,
+ "grad_norm": 0.17855268716812134,
+ "learning_rate": 6.430349733254298e-06,
+ "log_odds_chosen": 0.2732783854007721,
+ "log_odds_ratio": -0.6957250237464905,
+ "logits/chosen": 1.557079553604126,
+ "logits/rejected": 1.585305094718933,
+ "logps/chosen": -1.9428716897964478,
+ "logps/rejected": -2.2009921073913574,
+ "loss": 1.809073257446289,
+ "nll_loss": 1.7395007610321045,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.19428716599941254,
+ "rewards/margins": 0.0258120596408844,
+ "rewards/rejected": -0.22009921073913574,
+ "step": 520
+ },
+ {
+ "epoch": 0.848,
+ "grad_norm": 0.27463603019714355,
+ "learning_rate": 6.382928275044458e-06,
+ "log_odds_chosen": -0.053782153874635696,
+ "log_odds_ratio": -0.7889940142631531,
+ "logits/chosen": 1.546355128288269,
+ "logits/rejected": 1.5052711963653564,
+ "logps/chosen": -2.0254311561584473,
+ "logps/rejected": -1.9745910167694092,
+ "loss": 1.8272314071655273,
+ "nll_loss": 1.7483322620391846,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.20254313945770264,
+ "rewards/margins": -0.005084014497697353,
+ "rewards/rejected": -0.1974591165781021,
+ "step": 530
+ },
+ {
+ "epoch": 0.864,
+ "grad_norm": 0.19943900406360626,
+ "learning_rate": 6.335506816834617e-06,
+ "log_odds_chosen": 0.007906230166554451,
+ "log_odds_ratio": -0.7769323587417603,
+ "logits/chosen": 1.686943769454956,
+ "logits/rejected": 1.773566484451294,
+ "logps/chosen": -1.9026868343353271,
+ "logps/rejected": -1.8993861675262451,
+ "loss": 1.8080410003662108,
+ "nll_loss": 1.7303478717803955,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.19026869535446167,
+ "rewards/margins": -0.0003300657554063946,
+ "rewards/rejected": -0.18993863463401794,
+ "step": 540
+ },
+ {
+ "epoch": 0.88,
+ "grad_norm": 0.12966203689575195,
+ "learning_rate": 6.288085358624777e-06,
+ "log_odds_chosen": 0.18394824862480164,
+ "log_odds_ratio": -0.656446099281311,
+ "logits/chosen": 1.578307867050171,
+ "logits/rejected": 1.5473096370697021,
+ "logps/chosen": -1.8790676593780518,
+ "logps/rejected": -2.0375452041625977,
+ "loss": 1.80413818359375,
+ "nll_loss": 1.7384932041168213,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": -0.18790677189826965,
+ "rewards/margins": 0.015847764909267426,
+ "rewards/rejected": -0.20375454425811768,
+ "step": 550
+ },
+ {
+ "epoch": 0.88,
+ "eval_log_odds_chosen": 0.026566974818706512,
+ "eval_log_odds_ratio": -0.7653356790542603,
+ "eval_logits/chosen": 1.5679305791854858,
+ "eval_logits/rejected": 1.532114028930664,
+ "eval_logps/chosen": -2.0328524112701416,
+ "eval_logps/rejected": -2.0620551109313965,
+ "eval_loss": 1.846232533454895,
+ "eval_nll_loss": 1.769698977470398,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": -0.20328520238399506,
+ "eval_rewards/margins": 0.002920305822044611,
+ "eval_rewards/rejected": -0.20620553195476532,
+ "eval_runtime": 53.3132,
+ "eval_samples_per_second": 9.379,
+ "eval_steps_per_second": 4.689,
+ "step": 550
+ },
+ {
+ "epoch": 0.896,
+ "grad_norm": 0.24944448471069336,
+ "learning_rate": 6.240663900414938e-06,
+ "log_odds_chosen": 0.2502843141555786,
+ "log_odds_ratio": -0.6478752493858337,
+ "logits/chosen": 1.4606131315231323,
+ "logits/rejected": 1.3790860176086426,
+ "logps/chosen": -1.9443740844726562,
+ "logps/rejected": -2.1599698066711426,
+ "loss": 1.7577186584472657,
+ "nll_loss": 1.6929309368133545,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.1944374144077301,
+ "rewards/margins": 0.021559562534093857,
+ "rewards/rejected": -0.21599698066711426,
+ "step": 560
+ },
+ {
+ "epoch": 0.912,
+ "grad_norm": 0.1742384284734726,
+ "learning_rate": 6.193242442205098e-06,
+ "log_odds_chosen": 0.0590474009513855,
+ "log_odds_ratio": -0.7506214380264282,
+ "logits/chosen": 1.6640907526016235,
+ "logits/rejected": 1.6049493551254272,
+ "logps/chosen": -1.9722331762313843,
+ "logps/rejected": -2.026829481124878,
+ "loss": 1.7977088928222655,
+ "nll_loss": 1.722646951675415,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19722333550453186,
+ "rewards/margins": 0.005459627602249384,
+ "rewards/rejected": -0.2026829719543457,
+ "step": 570
+ },
+ {
+ "epoch": 0.928,
+ "grad_norm": 0.2213008999824524,
+ "learning_rate": 6.145820983995258e-06,
+ "log_odds_chosen": -0.03383875638246536,
+ "log_odds_ratio": -0.7764871716499329,
+ "logits/chosen": 1.6381727457046509,
+ "logits/rejected": 1.5094597339630127,
+ "logps/chosen": -1.9914861917495728,
+ "logps/rejected": -1.964422583580017,
+ "loss": 1.8107261657714844,
+ "nll_loss": 1.7330776453018188,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.19914861023426056,
+ "rewards/margins": -0.0027063353918492794,
+ "rewards/rejected": -0.19644227623939514,
+ "step": 580
+ },
+ {
+ "epoch": 0.944,
+ "grad_norm": 0.1825234442949295,
+ "learning_rate": 6.098399525785417e-06,
+ "log_odds_chosen": 0.10232333838939667,
+ "log_odds_ratio": -0.7037243843078613,
+ "logits/chosen": 1.5093460083007812,
+ "logits/rejected": 1.4225064516067505,
+ "logps/chosen": -1.9882131814956665,
+ "logps/rejected": -2.0832431316375732,
+ "loss": 1.7983221054077148,
+ "nll_loss": 1.727950096130371,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.1988213062286377,
+ "rewards/margins": 0.009502974338829517,
+ "rewards/rejected": -0.20832428336143494,
+ "step": 590
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 0.17290247976779938,
+ "learning_rate": 6.050978067575578e-06,
+ "log_odds_chosen": 0.06585167348384857,
+ "log_odds_ratio": -0.7220332026481628,
+ "logits/chosen": 1.5872576236724854,
+ "logits/rejected": 1.5169059038162231,
+ "logps/chosen": -1.8601738214492798,
+ "logps/rejected": -1.914232850074768,
+ "loss": 1.7327194213867188,
+ "nll_loss": 1.660516381263733,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.18601737916469574,
+ "rewards/margins": 0.005405900534242392,
+ "rewards/rejected": -0.19142326712608337,
+ "step": 600
+ },
+ {
+ "epoch": 0.96,
+ "eval_log_odds_chosen": 0.031927358359098434,
+ "eval_log_odds_ratio": -0.7613769173622131,
+ "eval_logits/chosen": 1.5999646186828613,
+ "eval_logits/rejected": 1.5608441829681396,
+ "eval_logps/chosen": -2.031277656555176,
+ "eval_logps/rejected": -2.0653114318847656,
+ "eval_loss": 1.842340111732483,
+ "eval_nll_loss": 1.7662022113800049,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.20312777161598206,
+ "eval_rewards/margins": 0.003403406124562025,
+ "eval_rewards/rejected": -0.20653115212917328,
+ "eval_runtime": 53.2205,
+ "eval_samples_per_second": 9.395,
+ "eval_steps_per_second": 4.697,
+ "step": 600
+ },
+ {
+ "epoch": 0.976,
+ "grad_norm": 0.17003707587718964,
+ "learning_rate": 6.003556609365738e-06,
+ "log_odds_chosen": 0.2910720705986023,
+ "log_odds_ratio": -0.6602123975753784,
+ "logits/chosen": 1.622191071510315,
+ "logits/rejected": 1.5799508094787598,
+ "logps/chosen": -1.8309406042099,
+ "logps/rejected": -2.0560221672058105,
+ "loss": 1.74339599609375,
+ "nll_loss": 1.6773748397827148,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": -0.1830940693616867,
+ "rewards/margins": 0.022508159279823303,
+ "rewards/rejected": -0.20560221374034882,
+ "step": 610
+ },
+ {
+ "epoch": 0.992,
+ "grad_norm": 0.23827488720417023,
+ "learning_rate": 5.956135151155898e-06,
+ "log_odds_chosen": 0.09837356209754944,
+ "log_odds_ratio": -0.7269458174705505,
+ "logits/chosen": 1.5235488414764404,
+ "logits/rejected": 1.539660096168518,
+ "logps/chosen": -1.8142712116241455,
+ "logps/rejected": -1.8749595880508423,
+ "loss": 1.7412891387939453,
+ "nll_loss": 1.6685945987701416,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.18142713606357574,
+ "rewards/margins": 0.00606885552406311,
+ "rewards/rejected": -0.18749597668647766,
+ "step": 620
+ },
+ {
+ "epoch": 1.008,
+ "grad_norm": 0.20535889267921448,
+ "learning_rate": 5.908713692946058e-06,
+ "log_odds_chosen": 0.22268417477607727,
+ "log_odds_ratio": -0.7025212049484253,
+ "logits/chosen": 1.5684313774108887,
+ "logits/rejected": 1.452976107597351,
+ "logps/chosen": -1.874678373336792,
+ "logps/rejected": -2.072539806365967,
+ "loss": 1.774791145324707,
+ "nll_loss": 1.7045390605926514,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.18746784329414368,
+ "rewards/margins": 0.019786134362220764,
+ "rewards/rejected": -0.20725397765636444,
+ "step": 630
+ },
+ {
+ "epoch": 1.024,
+ "grad_norm": 0.4164029061794281,
+ "learning_rate": 5.861292234736218e-06,
+ "log_odds_chosen": 0.13269853591918945,
+ "log_odds_ratio": -0.6831094026565552,
+ "logits/chosen": 1.5523912906646729,
+ "logits/rejected": 1.554421067237854,
+ "logps/chosen": -1.9016844034194946,
+ "logps/rejected": -2.0167248249053955,
+ "loss": 1.8452745437622071,
+ "nll_loss": 1.7769638299942017,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.19016845524311066,
+ "rewards/margins": 0.011504052206873894,
+ "rewards/rejected": -0.2016725093126297,
+ "step": 640
+ },
+ {
+ "epoch": 1.04,
+ "grad_norm": 0.33490127325057983,
+ "learning_rate": 5.813870776526378e-06,
+ "log_odds_chosen": -0.07161623239517212,
+ "log_odds_ratio": -0.8163315057754517,
+ "logits/chosen": 1.585946798324585,
+ "logits/rejected": 1.5961542129516602,
+ "logps/chosen": -2.0562708377838135,
+ "logps/rejected": -1.9879754781723022,
+ "loss": 1.7808271408081056,
+ "nll_loss": 1.6991941928863525,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.20562703907489777,
+ "rewards/margins": -0.006829507648944855,
+ "rewards/rejected": -0.1987975537776947,
+ "step": 650
+ },
+ {
+ "epoch": 1.04,
+ "eval_log_odds_chosen": 0.026153452694416046,
+ "eval_log_odds_ratio": -0.7626987099647522,
+ "eval_logits/chosen": 1.5905648469924927,
+ "eval_logits/rejected": 1.5565553903579712,
+ "eval_logps/chosen": -2.0193655490875244,
+ "eval_logps/rejected": -2.047696352005005,
+ "eval_loss": 1.8379414081573486,
+ "eval_nll_loss": 1.7616716623306274,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.20193655788898468,
+ "eval_rewards/margins": 0.002833074890077114,
+ "eval_rewards/rejected": -0.20476962625980377,
+ "eval_runtime": 53.3878,
+ "eval_samples_per_second": 9.365,
+ "eval_steps_per_second": 4.683,
+ "step": 650
+ },
+ {
+ "epoch": 1.056,
+ "grad_norm": 0.21944740414619446,
+ "learning_rate": 5.766449318316538e-06,
+ "log_odds_chosen": 0.09746942669153214,
+ "log_odds_ratio": -0.7083339095115662,
+ "logits/chosen": 1.5755507946014404,
+ "logits/rejected": 1.551966667175293,
+ "logps/chosen": -1.9140907526016235,
+ "logps/rejected": -1.9912245273590088,
+ "loss": 1.7622905731201173,
+ "nll_loss": 1.6914570331573486,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19140908122062683,
+ "rewards/margins": 0.00771337142214179,
+ "rewards/rejected": -0.19912245869636536,
+ "step": 660
+ },
+ {
+ "epoch": 1.072,
+ "grad_norm": 0.20419229567050934,
+ "learning_rate": 5.719027860106698e-06,
+ "log_odds_chosen": 0.10894493013620377,
+ "log_odds_ratio": -0.6956179738044739,
+ "logits/chosen": 1.7187572717666626,
+ "logits/rejected": 1.6571474075317383,
+ "logps/chosen": -1.9920858144760132,
+ "logps/rejected": -2.0984714031219482,
+ "loss": 1.802743148803711,
+ "nll_loss": 1.7331812381744385,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.1992085874080658,
+ "rewards/margins": 0.010638559237122536,
+ "rewards/rejected": -0.20984713733196259,
+ "step": 670
+ },
+ {
+ "epoch": 1.088,
+ "grad_norm": 0.1502259224653244,
+ "learning_rate": 5.671606401896859e-06,
+ "log_odds_chosen": 0.06736402213573456,
+ "log_odds_ratio": -0.7346867322921753,
+ "logits/chosen": 1.662166953086853,
+ "logits/rejected": 1.6347684860229492,
+ "logps/chosen": -1.9538319110870361,
+ "logps/rejected": -2.0279860496520996,
+ "loss": 1.7677694320678712,
+ "nll_loss": 1.694300651550293,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.1953831911087036,
+ "rewards/margins": 0.007415431551635265,
+ "rewards/rejected": -0.20279863476753235,
+ "step": 680
+ },
+ {
+ "epoch": 1.104,
+ "grad_norm": 0.1765543818473816,
+ "learning_rate": 5.624184943687018e-06,
+ "log_odds_chosen": 0.10347869247198105,
+ "log_odds_ratio": -0.7048436403274536,
+ "logits/chosen": 1.7106029987335205,
+ "logits/rejected": 1.6761833429336548,
+ "logps/chosen": -1.9210326671600342,
+ "logps/rejected": -2.0231428146362305,
+ "loss": 1.7810544967651367,
+ "nll_loss": 1.7105700969696045,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.1921032965183258,
+ "rewards/margins": 0.010210996493697166,
+ "rewards/rejected": -0.20231428742408752,
+ "step": 690
+ },
+ {
+ "epoch": 1.12,
+ "grad_norm": 0.26942944526672363,
+ "learning_rate": 5.576763485477178e-06,
+ "log_odds_chosen": 0.2199089527130127,
+ "log_odds_ratio": -0.6759704947471619,
+ "logits/chosen": 1.5313873291015625,
+ "logits/rejected": 1.4286370277404785,
+ "logps/chosen": -1.934305191040039,
+ "logps/rejected": -2.104454517364502,
+ "loss": 1.7423564910888671,
+ "nll_loss": 1.6747591495513916,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19343051314353943,
+ "rewards/margins": 0.01701495423913002,
+ "rewards/rejected": -0.21044547855854034,
+ "step": 700
+ },
+ {
+ "epoch": 1.12,
+ "eval_log_odds_chosen": 0.029196415096521378,
+ "eval_log_odds_ratio": -0.7604461312294006,
+ "eval_logits/chosen": 1.6299145221710205,
+ "eval_logits/rejected": 1.59683358669281,
+ "eval_logps/chosen": -2.016523599624634,
+ "eval_logps/rejected": -2.0469908714294434,
+ "eval_loss": 1.8337444067001343,
+ "eval_nll_loss": 1.757699728012085,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.20165236294269562,
+ "eval_rewards/margins": 0.0030467414762824774,
+ "eval_rewards/rejected": -0.2046990990638733,
+ "eval_runtime": 53.2721,
+ "eval_samples_per_second": 9.386,
+ "eval_steps_per_second": 4.693,
+ "step": 700
+ },
+ {
+ "epoch": 1.1360000000000001,
+ "grad_norm": 0.20357680320739746,
+ "learning_rate": 5.529342027267338e-06,
+ "log_odds_chosen": -0.03970063477754593,
+ "log_odds_ratio": -0.7776111364364624,
+ "logits/chosen": 1.688511848449707,
+ "logits/rejected": 1.5558183193206787,
+ "logps/chosen": -2.083556652069092,
+ "logps/rejected": -2.0501885414123535,
+ "loss": 1.849853515625,
+ "nll_loss": 1.7720924615859985,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.20835569500923157,
+ "rewards/margins": -0.0033368065487593412,
+ "rewards/rejected": -0.20501887798309326,
+ "step": 710
+ },
+ {
+ "epoch": 1.152,
+ "grad_norm": 0.20546825230121613,
+ "learning_rate": 5.481920569057499e-06,
+ "log_odds_chosen": 0.038821566849946976,
+ "log_odds_ratio": -0.7310380935668945,
+ "logits/chosen": 1.694077730178833,
+ "logits/rejected": 1.6808054447174072,
+ "logps/chosen": -1.8798902034759521,
+ "logps/rejected": -1.9120309352874756,
+ "loss": 1.8013761520385743,
+ "nll_loss": 1.728272795677185,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.1879890114068985,
+ "rewards/margins": 0.0032140829134732485,
+ "rewards/rejected": -0.19120310246944427,
+ "step": 720
+ },
+ {
+ "epoch": 1.168,
+ "grad_norm": 0.215702086687088,
+ "learning_rate": 5.434499110847659e-06,
+ "log_odds_chosen": -0.00810262281447649,
+ "log_odds_ratio": -0.7776116132736206,
+ "logits/chosen": 1.7699267864227295,
+ "logits/rejected": 1.8135936260223389,
+ "logps/chosen": -1.9550107717514038,
+ "logps/rejected": -1.9629520177841187,
+ "loss": 1.8207136154174806,
+ "nll_loss": 1.7429521083831787,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.19550108909606934,
+ "rewards/margins": 0.0007941495859995484,
+ "rewards/rejected": -0.19629523158073425,
+ "step": 730
+ },
+ {
+ "epoch": 1.184,
+ "grad_norm": 0.18298740684986115,
+ "learning_rate": 5.387077652637819e-06,
+ "log_odds_chosen": 0.18474040925502777,
+ "log_odds_ratio": -0.672413170337677,
+ "logits/chosen": 1.7683032751083374,
+ "logits/rejected": 1.8081409931182861,
+ "logps/chosen": -1.906714677810669,
+ "logps/rejected": -2.0783538818359375,
+ "loss": 1.7294036865234375,
+ "nll_loss": 1.662162184715271,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.19067144393920898,
+ "rewards/margins": 0.01716393604874611,
+ "rewards/rejected": -0.2078353613615036,
+ "step": 740
+ },
+ {
+ "epoch": 1.2,
+ "grad_norm": 0.13119018077850342,
+ "learning_rate": 5.339656194427978e-06,
+ "log_odds_chosen": 0.3178519606590271,
+ "log_odds_ratio": -0.6739884614944458,
+ "logits/chosen": 1.7732694149017334,
+ "logits/rejected": 1.6666276454925537,
+ "logps/chosen": -1.9240577220916748,
+ "logps/rejected": -2.2177700996398926,
+ "loss": 1.767026710510254,
+ "nll_loss": 1.6996276378631592,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": -0.19240576028823853,
+ "rewards/margins": 0.029371242970228195,
+ "rewards/rejected": -0.2217770367860794,
+ "step": 750
+ },
+ {
+ "epoch": 1.2,
+ "eval_log_odds_chosen": 0.024058617651462555,
+ "eval_log_odds_ratio": -0.763344407081604,
+ "eval_logits/chosen": 1.695021390914917,
+ "eval_logits/rejected": 1.6631685495376587,
+ "eval_logps/chosen": -2.0103158950805664,
+ "eval_logps/rejected": -2.0368571281433105,
+ "eval_loss": 1.8313435316085815,
+ "eval_nll_loss": 1.7550091743469238,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.20103158056735992,
+ "eval_rewards/margins": 0.0026541370898485184,
+ "eval_rewards/rejected": -0.2036857306957245,
+ "eval_runtime": 53.3008,
+ "eval_samples_per_second": 9.381,
+ "eval_steps_per_second": 4.69,
+ "step": 750
+ },
+ {
+ "epoch": 1.216,
+ "grad_norm": 0.1924748718738556,
+ "learning_rate": 5.292234736218139e-06,
+ "log_odds_chosen": 0.07262767851352692,
+ "log_odds_ratio": -0.7266906499862671,
+ "logits/chosen": 1.7665992975234985,
+ "logits/rejected": 1.6683565378189087,
+ "logps/chosen": -1.960656762123108,
+ "logps/rejected": -2.0233612060546875,
+ "loss": 1.7435152053833007,
+ "nll_loss": 1.6708459854125977,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.19606567919254303,
+ "rewards/margins": 0.006270436104387045,
+ "rewards/rejected": -0.20233610272407532,
+ "step": 760
+ },
+ {
+ "epoch": 1.232,
+ "grad_norm": 0.16488949954509735,
+ "learning_rate": 5.244813278008299e-06,
+ "log_odds_chosen": 0.2733311653137207,
+ "log_odds_ratio": -0.6829133629798889,
+ "logits/chosen": 1.707964539527893,
+ "logits/rejected": 1.6122114658355713,
+ "logps/chosen": -1.90310537815094,
+ "logps/rejected": -2.1515767574310303,
+ "loss": 1.7614355087280273,
+ "nll_loss": 1.6931442022323608,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.190310537815094,
+ "rewards/margins": 0.024847140535712242,
+ "rewards/rejected": -0.2151576727628708,
+ "step": 770
+ },
+ {
+ "epoch": 1.248,
+ "grad_norm": 0.1734672635793686,
+ "learning_rate": 5.197391819798459e-06,
+ "log_odds_chosen": 0.13369520008563995,
+ "log_odds_ratio": -0.675721287727356,
+ "logits/chosen": 1.79034423828125,
+ "logits/rejected": 1.811118721961975,
+ "logps/chosen": -1.8553498983383179,
+ "logps/rejected": -1.9695507287979126,
+ "loss": 1.7561630249023437,
+ "nll_loss": 1.6885910034179688,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.1855349838733673,
+ "rewards/margins": 0.011420061811804771,
+ "rewards/rejected": -0.19695505499839783,
+ "step": 780
+ },
+ {
+ "epoch": 1.264,
+ "grad_norm": 0.2151460200548172,
+ "learning_rate": 5.149970361588619e-06,
+ "log_odds_chosen": -0.15310701727867126,
+ "log_odds_ratio": -0.8164137005805969,
+ "logits/chosen": 1.8224090337753296,
+ "logits/rejected": 1.8069593906402588,
+ "logps/chosen": -2.0459067821502686,
+ "logps/rejected": -1.9169931411743164,
+ "loss": 1.8225961685180665,
+ "nll_loss": 1.7409546375274658,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": -0.20459070801734924,
+ "rewards/margins": -0.012891381978988647,
+ "rewards/rejected": -0.1916993111371994,
+ "step": 790
+ },
+ {
+ "epoch": 1.28,
+ "grad_norm": 0.23022110760211945,
+ "learning_rate": 5.102548903378779e-06,
+ "log_odds_chosen": 0.17419373989105225,
+ "log_odds_ratio": -0.665838360786438,
+ "logits/chosen": 1.745001196861267,
+ "logits/rejected": 1.7248725891113281,
+ "logps/chosen": -1.9748042821884155,
+ "logps/rejected": -2.124072313308716,
+ "loss": 1.7632785797119142,
+ "nll_loss": 1.6966947317123413,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.1974804550409317,
+ "rewards/margins": 0.014926811680197716,
+ "rewards/rejected": -0.21240726113319397,
+ "step": 800
+ },
+ {
+ "epoch": 1.28,
+ "eval_log_odds_chosen": 0.0298521239310503,
+ "eval_log_odds_ratio": -0.7603721022605896,
+ "eval_logits/chosen": 1.7159233093261719,
+ "eval_logits/rejected": 1.6853290796279907,
+ "eval_logps/chosen": -2.008664608001709,
+ "eval_logps/rejected": -2.0401124954223633,
+ "eval_loss": 1.8289790153503418,
+ "eval_nll_loss": 1.7529417276382446,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.2008664608001709,
+ "eval_rewards/margins": 0.0031447894871234894,
+ "eval_rewards/rejected": -0.20401126146316528,
+ "eval_runtime": 53.2154,
+ "eval_samples_per_second": 9.396,
+ "eval_steps_per_second": 4.698,
+ "step": 800
+ },
+ {
+ "epoch": 1.296,
+ "grad_norm": 0.1864694058895111,
+ "learning_rate": 5.055127445168939e-06,
+ "log_odds_chosen": -0.02357078157365322,
+ "log_odds_ratio": -0.7400822043418884,
+ "logits/chosen": 1.7981126308441162,
+ "logits/rejected": 1.7508691549301147,
+ "logps/chosen": -2.0388572216033936,
+ "logps/rejected": -2.024317979812622,
+ "loss": 1.8243423461914063,
+ "nll_loss": 1.7503341436386108,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.20388571918010712,
+ "rewards/margins": -0.001453907461836934,
+ "rewards/rejected": -0.20243176817893982,
+ "step": 810
+ },
+ {
+ "epoch": 1.312,
+ "grad_norm": 0.18279023468494415,
+ "learning_rate": 5.007705986959099e-06,
+ "log_odds_chosen": -0.011245409026741982,
+ "log_odds_ratio": -0.8205389976501465,
+ "logits/chosen": 1.726945161819458,
+ "logits/rejected": 1.6804269552230835,
+ "logps/chosen": -2.0612854957580566,
+ "logps/rejected": -2.0304996967315674,
+ "loss": 1.8141677856445313,
+ "nll_loss": 1.7321138381958008,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.2061285525560379,
+ "rewards/margins": -0.00307856360450387,
+ "rewards/rejected": -0.20304997265338898,
+ "step": 820
+ },
+ {
+ "epoch": 1.328,
+ "grad_norm": 0.194142147898674,
+ "learning_rate": 4.960284528749259e-06,
+ "log_odds_chosen": -0.10107870399951935,
+ "log_odds_ratio": -0.8795238733291626,
+ "logits/chosen": 1.6292463541030884,
+ "logits/rejected": 1.6127665042877197,
+ "logps/chosen": -1.9763052463531494,
+ "logps/rejected": -1.872083067893982,
+ "loss": 1.7666927337646485,
+ "nll_loss": 1.67874014377594,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.19763052463531494,
+ "rewards/margins": -0.010422185063362122,
+ "rewards/rejected": -0.18720832467079163,
+ "step": 830
+ },
+ {
+ "epoch": 1.3439999999999999,
+ "grad_norm": 0.18887867033481598,
+ "learning_rate": 4.9128630705394195e-06,
+ "log_odds_chosen": -0.0421462245285511,
+ "log_odds_ratio": -0.7816078662872314,
+ "logits/chosen": 1.6638116836547852,
+ "logits/rejected": 1.5873361825942993,
+ "logps/chosen": -1.980798363685608,
+ "logps/rejected": -1.949537992477417,
+ "loss": 1.793659782409668,
+ "nll_loss": 1.715498924255371,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": -0.19807983934879303,
+ "rewards/margins": -0.003126043127849698,
+ "rewards/rejected": -0.1949537992477417,
+ "step": 840
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "grad_norm": 0.2071056365966797,
+ "learning_rate": 4.865441612329579e-06,
+ "log_odds_chosen": 0.20311307907104492,
+ "log_odds_ratio": -0.6712511777877808,
+ "logits/chosen": 1.5604090690612793,
+ "logits/rejected": 1.6046473979949951,
+ "logps/chosen": -1.8914225101470947,
+ "logps/rejected": -2.0582470893859863,
+ "loss": 1.7601259231567383,
+ "nll_loss": 1.6930010318756104,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.18914227187633514,
+ "rewards/margins": 0.016682462766766548,
+ "rewards/rejected": -0.20582470297813416,
+ "step": 850
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "eval_log_odds_chosen": 0.027712374925613403,
+ "eval_log_odds_ratio": -0.7594860196113586,
+ "eval_logits/chosen": 1.666095495223999,
+ "eval_logits/rejected": 1.633544683456421,
+ "eval_logps/chosen": -2.002187967300415,
+ "eval_logps/rejected": -2.0309674739837646,
+ "eval_loss": 1.824683666229248,
+ "eval_nll_loss": 1.7487353086471558,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": -0.2002188116312027,
+ "eval_rewards/margins": 0.0028779483400285244,
+ "eval_rewards/rejected": -0.20309676229953766,
+ "eval_runtime": 53.1899,
+ "eval_samples_per_second": 9.4,
+ "eval_steps_per_second": 4.7,
+ "step": 850
+ },
+ {
+ "epoch": 1.376,
+ "grad_norm": 0.1590212732553482,
+ "learning_rate": 4.818020154119739e-06,
+ "log_odds_chosen": 0.15208786725997925,
+ "log_odds_ratio": -0.6756098866462708,
+ "logits/chosen": 1.6706883907318115,
+ "logits/rejected": 1.6443389654159546,
+ "logps/chosen": -1.9115455150604248,
+ "logps/rejected": -2.0435924530029297,
+ "loss": 1.7545036315917968,
+ "nll_loss": 1.686942458152771,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.19115455448627472,
+ "rewards/margins": 0.013204696588218212,
+ "rewards/rejected": -0.2043592482805252,
+ "step": 860
+ },
+ {
+ "epoch": 1.392,
+ "grad_norm": 0.15062956511974335,
+ "learning_rate": 4.770598695909899e-06,
+ "log_odds_chosen": 0.2899920344352722,
+ "log_odds_ratio": -0.6323431730270386,
+ "logits/chosen": 1.7012927532196045,
+ "logits/rejected": 1.6665140390396118,
+ "logps/chosen": -1.8794418573379517,
+ "logps/rejected": -2.131627321243286,
+ "loss": 1.817972183227539,
+ "nll_loss": 1.7547378540039062,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": -0.1879442036151886,
+ "rewards/margins": 0.025218535214662552,
+ "rewards/rejected": -0.21316273510456085,
+ "step": 870
+ },
+ {
+ "epoch": 1.408,
+ "grad_norm": 0.23191672563552856,
+ "learning_rate": 4.7231772377000595e-06,
+ "log_odds_chosen": 0.08231823146343231,
+ "log_odds_ratio": -0.7166029810905457,
+ "logits/chosen": 1.739164113998413,
+ "logits/rejected": 1.7106529474258423,
+ "logps/chosen": -1.9510221481323242,
+ "logps/rejected": -2.0197978019714355,
+ "loss": 1.7590656280517578,
+ "nll_loss": 1.6874055862426758,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.1951022446155548,
+ "rewards/margins": 0.006877546664327383,
+ "rewards/rejected": -0.20197978615760803,
+ "step": 880
+ },
+ {
+ "epoch": 1.424,
+ "grad_norm": 0.2074817270040512,
+ "learning_rate": 4.6757557794902194e-06,
+ "log_odds_chosen": 0.09656596183776855,
+ "log_odds_ratio": -0.708441436290741,
+ "logits/chosen": 1.7313311100006104,
+ "logits/rejected": 1.6554332971572876,
+ "logps/chosen": -1.9648983478546143,
+ "logps/rejected": -2.0429916381835938,
+ "loss": 1.77806396484375,
+ "nll_loss": 1.7072200775146484,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.19648981094360352,
+ "rewards/margins": 0.007809334900230169,
+ "rewards/rejected": -0.20429913699626923,
+ "step": 890
+ },
+ {
+ "epoch": 1.44,
+ "grad_norm": 0.17369434237480164,
+ "learning_rate": 4.6283343212803794e-06,
+ "log_odds_chosen": 0.02706352435052395,
+ "log_odds_ratio": -0.7602623701095581,
+ "logits/chosen": 1.6460374593734741,
+ "logits/rejected": 1.6657674312591553,
+ "logps/chosen": -2.0089709758758545,
+ "logps/rejected": -2.0413382053375244,
+ "loss": 1.788330078125,
+ "nll_loss": 1.7123043537139893,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": -0.20089709758758545,
+ "rewards/margins": 0.003236697521060705,
+ "rewards/rejected": -0.2041337937116623,
+ "step": 900
+ },
+ {
+ "epoch": 1.44,
+ "eval_log_odds_chosen": 0.029239021241664886,
+ "eval_log_odds_ratio": -0.7607759237289429,
+ "eval_logits/chosen": 1.7303014993667603,
+ "eval_logits/rejected": 1.7004859447479248,
+ "eval_logps/chosen": -2.001692056655884,
+ "eval_logps/rejected": -2.031625747680664,
+ "eval_loss": 1.8236775398254395,
+ "eval_nll_loss": 1.7475998401641846,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.20016922056674957,
+ "eval_rewards/margins": 0.002993369475007057,
+ "eval_rewards/rejected": -0.20316259562969208,
+ "eval_runtime": 53.2446,
+ "eval_samples_per_second": 9.391,
+ "eval_steps_per_second": 4.695,
+ "step": 900
+ },
+ {
+ "epoch": 1.456,
+ "grad_norm": 0.20154239237308502,
+ "learning_rate": 4.580912863070539e-06,
+ "log_odds_chosen": 0.1900930106639862,
+ "log_odds_ratio": -0.6721845269203186,
+ "logits/chosen": 1.7324788570404053,
+ "logits/rejected": 1.6658681631088257,
+ "logps/chosen": -1.8505785465240479,
+ "logps/rejected": -2.0169787406921387,
+ "loss": 1.6994483947753907,
+ "nll_loss": 1.6322300434112549,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.1850578486919403,
+ "rewards/margins": 0.01664000190794468,
+ "rewards/rejected": -0.20169785618782043,
+ "step": 910
+ },
+ {
+ "epoch": 1.472,
+ "grad_norm": 0.16085605323314667,
+ "learning_rate": 4.5334914048606994e-06,
+ "log_odds_chosen": 0.09199506789445877,
+ "log_odds_ratio": -0.6975706219673157,
+ "logits/chosen": 1.7461503744125366,
+ "logits/rejected": 1.8116916418075562,
+ "logps/chosen": -1.8981988430023193,
+ "logps/rejected": -1.982012391090393,
+ "loss": 1.7591039657592773,
+ "nll_loss": 1.6893469095230103,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.18981988728046417,
+ "rewards/margins": 0.00838138721883297,
+ "rewards/rejected": -0.1982012689113617,
+ "step": 920
+ },
+ {
+ "epoch": 1.488,
+ "grad_norm": 0.15088561177253723,
+ "learning_rate": 4.4860699466508594e-06,
+ "log_odds_chosen": 0.09973030537366867,
+ "log_odds_ratio": -0.6972100138664246,
+ "logits/chosen": 1.8033208847045898,
+ "logits/rejected": 1.7551723718643188,
+ "logps/chosen": -1.8274751901626587,
+ "logps/rejected": -1.9026908874511719,
+ "loss": 1.7895126342773438,
+ "nll_loss": 1.7197917699813843,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.18274752795696259,
+ "rewards/margins": 0.0075215683318674564,
+ "rewards/rejected": -0.1902690976858139,
+ "step": 930
+ },
+ {
+ "epoch": 1.504,
+ "grad_norm": 0.13715308904647827,
+ "learning_rate": 4.4386484884410194e-06,
+ "log_odds_chosen": 0.07654011994600296,
+ "log_odds_ratio": -0.7238441705703735,
+ "logits/chosen": 1.7913706302642822,
+ "logits/rejected": 1.768434762954712,
+ "logps/chosen": -1.8972448110580444,
+ "logps/rejected": -1.952164888381958,
+ "loss": 1.749094581604004,
+ "nll_loss": 1.6767103672027588,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.18972447514533997,
+ "rewards/margins": 0.005491999443620443,
+ "rewards/rejected": -0.19521646201610565,
+ "step": 940
+ },
+ {
+ "epoch": 1.52,
+ "grad_norm": 0.1736515462398529,
+ "learning_rate": 4.3912270302311794e-06,
+ "log_odds_chosen": -0.02450474537909031,
+ "log_odds_ratio": -0.7657513618469238,
+ "logits/chosen": 1.8066694736480713,
+ "logits/rejected": 1.7840650081634521,
+ "logps/chosen": -1.9759057760238647,
+ "logps/rejected": -1.9469897747039795,
+ "loss": 1.7701189041137695,
+ "nll_loss": 1.6935440301895142,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": -0.19759058952331543,
+ "rewards/margins": -0.00289161060936749,
+ "rewards/rejected": -0.1946989744901657,
+ "step": 950
+ },
+ {
+ "epoch": 1.52,
+ "eval_log_odds_chosen": 0.03024761565029621,
+ "eval_log_odds_ratio": -0.7591846585273743,
+ "eval_logits/chosen": 1.7475262880325317,
+ "eval_logits/rejected": 1.718088150024414,
+ "eval_logps/chosen": -2.0002715587615967,
+ "eval_logps/rejected": -2.0316338539123535,
+ "eval_loss": 1.821697473526001,
+ "eval_nll_loss": 1.745779037475586,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.20002716779708862,
+ "eval_rewards/margins": 0.003136243438348174,
+ "eval_rewards/rejected": -0.20316338539123535,
+ "eval_runtime": 53.1737,
+ "eval_samples_per_second": 9.403,
+ "eval_steps_per_second": 4.702,
+ "step": 950
+ },
+ {
+ "epoch": 1.536,
+ "grad_norm": 0.16075001657009125,
+ "learning_rate": 4.343805572021339e-06,
+ "log_odds_chosen": 0.108977772295475,
+ "log_odds_ratio": -0.69707852602005,
+ "logits/chosen": 1.9180431365966797,
+ "logits/rejected": 1.8730928897857666,
+ "logps/chosen": -1.9387962818145752,
+ "logps/rejected": -2.0468382835388184,
+ "loss": 1.8717756271362305,
+ "nll_loss": 1.8020679950714111,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.193879634141922,
+ "rewards/margins": 0.010804187506437302,
+ "rewards/rejected": -0.2046838104724884,
+ "step": 960
+ },
+ {
+ "epoch": 1.552,
+ "grad_norm": 0.21932974457740784,
+ "learning_rate": 4.296384113811499e-06,
+ "log_odds_chosen": 0.0989312157034874,
+ "log_odds_ratio": -0.7125958800315857,
+ "logits/chosen": 1.7110919952392578,
+ "logits/rejected": 1.728533387184143,
+ "logps/chosen": -1.9263118505477905,
+ "logps/rejected": -2.00539231300354,
+ "loss": 1.7788570404052735,
+ "nll_loss": 1.7075974941253662,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.19263119995594025,
+ "rewards/margins": 0.007908042520284653,
+ "rewards/rejected": -0.200539231300354,
+ "step": 970
+ },
+ {
+ "epoch": 1.568,
+ "grad_norm": 0.1886725127696991,
+ "learning_rate": 4.248962655601659e-06,
+ "log_odds_chosen": 0.2252909392118454,
+ "log_odds_ratio": -0.6316526532173157,
+ "logits/chosen": 1.69448983669281,
+ "logits/rejected": 1.6676839590072632,
+ "logps/chosen": -1.8613817691802979,
+ "logps/rejected": -2.057375431060791,
+ "loss": 1.7345258712768554,
+ "nll_loss": 1.6713603734970093,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": -0.18613818287849426,
+ "rewards/margins": 0.01959935761988163,
+ "rewards/rejected": -0.20573754608631134,
+ "step": 980
+ },
+ {
+ "epoch": 1.584,
+ "grad_norm": 0.1785261482000351,
+ "learning_rate": 4.201541197391819e-06,
+ "log_odds_chosen": 0.18465228378772736,
+ "log_odds_ratio": -0.6800971627235413,
+ "logits/chosen": 1.693155288696289,
+ "logits/rejected": 1.6254899501800537,
+ "logps/chosen": -1.922136902809143,
+ "logps/rejected": -2.0776562690734863,
+ "loss": 1.7461135864257813,
+ "nll_loss": 1.6781036853790283,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19221369922161102,
+ "rewards/margins": 0.01555191446095705,
+ "rewards/rejected": -0.2077656239271164,
+ "step": 990
+ },
+ {
+ "epoch": 1.6,
+ "grad_norm": 0.19836267828941345,
+ "learning_rate": 4.15411973918198e-06,
+ "log_odds_chosen": 0.16690519452095032,
+ "log_odds_ratio": -0.6907414793968201,
+ "logits/chosen": 1.78627610206604,
+ "logits/rejected": 1.6995677947998047,
+ "logps/chosen": -1.9538847208023071,
+ "logps/rejected": -2.0953662395477295,
+ "loss": 1.8010309219360352,
+ "nll_loss": 1.7319567203521729,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.19538846611976624,
+ "rewards/margins": 0.014148158952593803,
+ "rewards/rejected": -0.2095366269350052,
+ "step": 1000
+ },
+ {
+ "epoch": 1.6,
+ "eval_log_odds_chosen": 0.027905963361263275,
+ "eval_log_odds_ratio": -0.7601120471954346,
+ "eval_logits/chosen": 1.7237838506698608,
+ "eval_logits/rejected": 1.6969308853149414,
+ "eval_logps/chosen": -1.9980021715164185,
+ "eval_logps/rejected": -2.027127742767334,
+ "eval_loss": 1.8202314376831055,
+ "eval_nll_loss": 1.74422025680542,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": -0.19980023801326752,
+ "eval_rewards/margins": 0.0029125558212399483,
+ "eval_rewards/rejected": -0.2027127742767334,
+ "eval_runtime": 53.17,
+ "eval_samples_per_second": 9.404,
+ "eval_steps_per_second": 4.702,
+ "step": 1000
+ },
+ {
+ "epoch": 1.616,
+ "grad_norm": 0.2467470020055771,
+ "learning_rate": 4.106698280972139e-06,
+ "log_odds_chosen": 0.1805444061756134,
+ "log_odds_ratio": -0.6592761874198914,
+ "logits/chosen": 1.7704235315322876,
+ "logits/rejected": 1.8331016302108765,
+ "logps/chosen": -1.9225962162017822,
+ "logps/rejected": -2.0647215843200684,
+ "loss": 1.7989471435546875,
+ "nll_loss": 1.7330195903778076,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.19225959479808807,
+ "rewards/margins": 0.014212558977305889,
+ "rewards/rejected": -0.20647215843200684,
+ "step": 1010
+ },
+ {
+ "epoch": 1.6320000000000001,
+ "grad_norm": 0.21627788245677948,
+ "learning_rate": 4.059276822762299e-06,
+ "log_odds_chosen": 0.19997625052928925,
+ "log_odds_ratio": -0.6929422616958618,
+ "logits/chosen": 1.7510759830474854,
+ "logits/rejected": 1.7739620208740234,
+ "logps/chosen": -1.8798294067382812,
+ "logps/rejected": -2.0255956649780273,
+ "loss": 1.7694732666015625,
+ "nll_loss": 1.7001785039901733,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.1879829615354538,
+ "rewards/margins": 0.014576609246432781,
+ "rewards/rejected": -0.20255956053733826,
+ "step": 1020
+ },
+ {
+ "epoch": 1.6480000000000001,
+ "grad_norm": 0.14596332609653473,
+ "learning_rate": 4.011855364552459e-06,
+ "log_odds_chosen": 0.08883054554462433,
+ "log_odds_ratio": -0.8015972375869751,
+ "logits/chosen": 1.7602264881134033,
+ "logits/rejected": 1.7082704305648804,
+ "logps/chosen": -1.987823247909546,
+ "logps/rejected": -2.0893337726593018,
+ "loss": 1.768618392944336,
+ "nll_loss": 1.6884586811065674,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": -0.1987823098897934,
+ "rewards/margins": 0.010151032358407974,
+ "rewards/rejected": -0.20893338322639465,
+ "step": 1030
+ },
+ {
+ "epoch": 1.6640000000000001,
+ "grad_norm": 0.21014870703220367,
+ "learning_rate": 3.96443390634262e-06,
+ "log_odds_chosen": 0.20391491055488586,
+ "log_odds_ratio": -0.6863493323326111,
+ "logits/chosen": 1.6720123291015625,
+ "logits/rejected": 1.6457217931747437,
+ "logps/chosen": -1.8646005392074585,
+ "logps/rejected": -2.0223734378814697,
+ "loss": 1.6701059341430664,
+ "nll_loss": 1.601470947265625,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.18646006286144257,
+ "rewards/margins": 0.015777289867401123,
+ "rewards/rejected": -0.2022373378276825,
+ "step": 1040
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "grad_norm": 0.16176322102546692,
+ "learning_rate": 3.917012448132779e-06,
+ "log_odds_chosen": 0.1454552561044693,
+ "log_odds_ratio": -0.7019529342651367,
+ "logits/chosen": 1.7449884414672852,
+ "logits/rejected": 1.7378383874893188,
+ "logps/chosen": -1.9023891687393188,
+ "logps/rejected": -2.027350902557373,
+ "loss": 1.7594617843627929,
+ "nll_loss": 1.6892664432525635,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.19023892283439636,
+ "rewards/margins": 0.012496160343289375,
+ "rewards/rejected": -0.20273509621620178,
+ "step": 1050
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "eval_log_odds_chosen": 0.027047131210565567,
+ "eval_log_odds_ratio": -0.7607759237289429,
+ "eval_logits/chosen": 1.752084493637085,
+ "eval_logits/rejected": 1.726226568222046,
+ "eval_logps/chosen": -1.9921609163284302,
+ "eval_logps/rejected": -2.0205676555633545,
+ "eval_loss": 1.8179807662963867,
+ "eval_nll_loss": 1.7419030666351318,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.1992160975933075,
+ "eval_rewards/margins": 0.002840682165697217,
+ "eval_rewards/rejected": -0.20205678045749664,
+ "eval_runtime": 53.3183,
+ "eval_samples_per_second": 9.378,
+ "eval_steps_per_second": 4.689,
+ "step": 1050
+ },
+ {
+ "epoch": 1.696,
+ "grad_norm": 0.19938912987709045,
+ "learning_rate": 3.86959098992294e-06,
+ "log_odds_chosen": -0.06742805987596512,
+ "log_odds_ratio": -0.8377403020858765,
+ "logits/chosen": 1.626834511756897,
+ "logits/rejected": 1.6696176528930664,
+ "logps/chosen": -2.060955286026001,
+ "logps/rejected": -2.0158073902130127,
+ "loss": 1.7704721450805665,
+ "nll_loss": 1.6866979598999023,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.20609550178050995,
+ "rewards/margins": -0.004514762666076422,
+ "rewards/rejected": -0.2015807330608368,
+ "step": 1060
+ },
+ {
+ "epoch": 1.712,
+ "grad_norm": 0.20868155360221863,
+ "learning_rate": 3.8221695317131e-06,
+ "log_odds_chosen": 0.2089385986328125,
+ "log_odds_ratio": -0.6722566485404968,
+ "logits/chosen": 1.8093836307525635,
+ "logits/rejected": 1.7903034687042236,
+ "logps/chosen": -1.8783435821533203,
+ "logps/rejected": -2.0554146766662598,
+ "loss": 1.7485641479492187,
+ "nll_loss": 1.6813385486602783,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": -0.18783435225486755,
+ "rewards/margins": 0.017707128077745438,
+ "rewards/rejected": -0.2055414617061615,
+ "step": 1070
+ },
+ {
+ "epoch": 1.728,
+ "grad_norm": 0.17801117897033691,
+ "learning_rate": 3.77474807350326e-06,
+ "log_odds_chosen": -0.08966319262981415,
+ "log_odds_ratio": -0.8123272061347961,
+ "logits/chosen": 1.8008207082748413,
+ "logits/rejected": 1.7534077167510986,
+ "logps/chosen": -1.9915469884872437,
+ "logps/rejected": -1.9123618602752686,
+ "loss": 1.743325424194336,
+ "nll_loss": 1.662092924118042,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": -0.19915470480918884,
+ "rewards/margins": -0.007918504998087883,
+ "rewards/rejected": -0.1912361979484558,
+ "step": 1080
+ },
+ {
+ "epoch": 1.744,
+ "grad_norm": 0.12194813042879105,
+ "learning_rate": 3.7273266152934202e-06,
+ "log_odds_chosen": -0.11027821153402328,
+ "log_odds_ratio": -0.8348665237426758,
+ "logits/chosen": 1.8450218439102173,
+ "logits/rejected": 1.8813024759292603,
+ "logps/chosen": -1.9666748046875,
+ "logps/rejected": -1.8820431232452393,
+ "loss": 1.8245243072509765,
+ "nll_loss": 1.7410377264022827,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.19666747748851776,
+ "rewards/margins": -0.008463181555271149,
+ "rewards/rejected": -0.1882043182849884,
+ "step": 1090
+ },
+ {
+ "epoch": 1.76,
+ "grad_norm": 0.16607670485973358,
+ "learning_rate": 3.67990515708358e-06,
+ "log_odds_chosen": 0.16690079867839813,
+ "log_odds_ratio": -0.791414737701416,
+ "logits/chosen": 1.7368566989898682,
+ "logits/rejected": 1.6425182819366455,
+ "logps/chosen": -1.9711978435516357,
+ "logps/rejected": -2.154827356338501,
+ "loss": 1.762497901916504,
+ "nll_loss": 1.6833562850952148,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.197119802236557,
+ "rewards/margins": 0.01836293376982212,
+ "rewards/rejected": -0.2154827117919922,
+ "step": 1100
+ },
+ {
+ "epoch": 1.76,
+ "eval_log_odds_chosen": 0.029554516077041626,
+ "eval_log_odds_ratio": -0.7602362036705017,
+ "eval_logits/chosen": 1.7638286352157593,
+ "eval_logits/rejected": 1.7391140460968018,
+ "eval_logps/chosen": -1.9919315576553345,
+ "eval_logps/rejected": -2.0221877098083496,
+ "eval_loss": 1.815456509590149,
+ "eval_nll_loss": 1.7394328117370605,
+ "eval_rewards/accuracies": 0.5379999876022339,
+ "eval_rewards/chosen": -0.19919316470623016,
+ "eval_rewards/margins": 0.0030256081372499466,
+ "eval_rewards/rejected": -0.20221877098083496,
+ "eval_runtime": 53.2855,
+ "eval_samples_per_second": 9.383,
+ "eval_steps_per_second": 4.692,
+ "step": 1100
+ },
+ {
+ "epoch": 1.776,
+ "grad_norm": 0.18845929205417633,
+ "learning_rate": 3.6324836988737402e-06,
+ "log_odds_chosen": 0.14334020018577576,
+ "log_odds_ratio": -0.6870154142379761,
+ "logits/chosen": 1.7760759592056274,
+ "logits/rejected": 1.7927675247192383,
+ "logps/chosen": -1.8970282077789307,
+ "logps/rejected": -2.0081281661987305,
+ "loss": 1.7522649765014648,
+ "nll_loss": 1.683563232421875,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.1897028237581253,
+ "rewards/margins": 0.011109992861747742,
+ "rewards/rejected": -0.20081281661987305,
+ "step": 1110
+ },
+ {
+ "epoch": 1.792,
+ "grad_norm": 0.16642089188098907,
+ "learning_rate": 3.5850622406639e-06,
+ "log_odds_chosen": 0.12583844363689423,
+ "log_odds_ratio": -0.705106258392334,
+ "logits/chosen": 1.7084674835205078,
+ "logits/rejected": 1.7520864009857178,
+ "logps/chosen": -2.0032966136932373,
+ "logps/rejected": -2.121325969696045,
+ "loss": 1.7604061126708985,
+ "nll_loss": 1.6898952722549438,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.20032963156700134,
+ "rewards/margins": 0.011802928522229195,
+ "rewards/rejected": -0.21213257312774658,
+ "step": 1120
+ },
+ {
+ "epoch": 1.808,
+ "grad_norm": 0.26899227499961853,
+ "learning_rate": 3.53764078245406e-06,
+ "log_odds_chosen": 0.04995861276984215,
+ "log_odds_ratio": -0.725029706954956,
+ "logits/chosen": 1.7520374059677124,
+ "logits/rejected": 1.7503259181976318,
+ "logps/chosen": -1.935824990272522,
+ "logps/rejected": -1.9734337329864502,
+ "loss": 1.7329391479492187,
+ "nll_loss": 1.6604360342025757,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19358250498771667,
+ "rewards/margins": 0.0037608749698847532,
+ "rewards/rejected": -0.1973433792591095,
+ "step": 1130
+ },
+ {
+ "epoch": 1.8239999999999998,
+ "grad_norm": 0.21221907436847687,
+ "learning_rate": 3.49021932424422e-06,
+ "log_odds_chosen": 0.07214214652776718,
+ "log_odds_ratio": -0.7553919553756714,
+ "logits/chosen": 1.669593095779419,
+ "logits/rejected": 1.698304533958435,
+ "logps/chosen": -1.9012906551361084,
+ "logps/rejected": -1.941220998764038,
+ "loss": 1.742647933959961,
+ "nll_loss": 1.6671091318130493,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19012905657291412,
+ "rewards/margins": 0.003993041347712278,
+ "rewards/rejected": -0.1941221058368683,
+ "step": 1140
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "grad_norm": 0.14006629586219788,
+ "learning_rate": 3.4427978660343806e-06,
+ "log_odds_chosen": 0.1731099784374237,
+ "log_odds_ratio": -0.6817640662193298,
+ "logits/chosen": 1.7826522588729858,
+ "logits/rejected": 1.7909168004989624,
+ "logps/chosen": -1.9337961673736572,
+ "logps/rejected": -2.0631959438323975,
+ "loss": 1.7244823455810547,
+ "nll_loss": 1.6563060283660889,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.19337964057922363,
+ "rewards/margins": 0.012939969077706337,
+ "rewards/rejected": -0.20631961524486542,
+ "step": 1150
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "eval_log_odds_chosen": 0.026097064837813377,
+ "eval_log_odds_ratio": -0.7620403170585632,
+ "eval_logits/chosen": 1.757448673248291,
+ "eval_logits/rejected": 1.7327815294265747,
+ "eval_logps/chosen": -1.9885954856872559,
+ "eval_logps/rejected": -2.016005754470825,
+ "eval_loss": 1.8154748678207397,
+ "eval_nll_loss": 1.7392706871032715,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": -0.1988595575094223,
+ "eval_rewards/margins": 0.0027410530019551516,
+ "eval_rewards/rejected": -0.2016005963087082,
+ "eval_runtime": 53.1886,
+ "eval_samples_per_second": 9.401,
+ "eval_steps_per_second": 4.7,
+ "step": 1150
+ },
+ {
+ "epoch": 1.8559999999999999,
+ "grad_norm": 0.2366112917661667,
+ "learning_rate": 3.39537640782454e-06,
+ "log_odds_chosen": 0.16461338102817535,
+ "log_odds_ratio": -0.6795645952224731,
+ "logits/chosen": 1.6777217388153076,
+ "logits/rejected": 1.7064342498779297,
+ "logps/chosen": -1.7678172588348389,
+ "logps/rejected": -1.9083778858184814,
+ "loss": 1.684773063659668,
+ "nll_loss": 1.6168162822723389,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.17678174376487732,
+ "rewards/margins": 0.014056049287319183,
+ "rewards/rejected": -0.1908378005027771,
+ "step": 1160
+ },
+ {
+ "epoch": 1.8719999999999999,
+ "grad_norm": 0.17859044671058655,
+ "learning_rate": 3.3479549496147006e-06,
+ "log_odds_chosen": 0.029232267290353775,
+ "log_odds_ratio": -0.7420104742050171,
+ "logits/chosen": 1.7861093282699585,
+ "logits/rejected": 1.7708505392074585,
+ "logps/chosen": -1.9521045684814453,
+ "logps/rejected": -1.9810863733291626,
+ "loss": 1.8265951156616211,
+ "nll_loss": 1.7523937225341797,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.19521045684814453,
+ "rewards/margins": 0.0028981652576476336,
+ "rewards/rejected": -0.19810864329338074,
+ "step": 1170
+ },
+ {
+ "epoch": 1.888,
+ "grad_norm": 0.20688028633594513,
+ "learning_rate": 3.30053349140486e-06,
+ "log_odds_chosen": 0.04731910675764084,
+ "log_odds_ratio": -0.7147753238677979,
+ "logits/chosen": 1.7652273178100586,
+ "logits/rejected": 1.7344173192977905,
+ "logps/chosen": -1.845867395401001,
+ "logps/rejected": -1.8912805318832397,
+ "loss": 1.6884075164794923,
+ "nll_loss": 1.6169300079345703,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.18458673357963562,
+ "rewards/margins": 0.0045413183979690075,
+ "rewards/rejected": -0.1891280561685562,
+ "step": 1180
+ },
+ {
+ "epoch": 1.904,
+ "grad_norm": 0.15886512398719788,
+ "learning_rate": 3.2531120331950206e-06,
+ "log_odds_chosen": 0.11942672729492188,
+ "log_odds_ratio": -0.6965886354446411,
+ "logits/chosen": 1.8913081884384155,
+ "logits/rejected": 1.8108240365982056,
+ "logps/chosen": -1.9068269729614258,
+ "logps/rejected": -2.0091569423675537,
+ "loss": 1.7399011611938477,
+ "nll_loss": 1.6702420711517334,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.19068269431591034,
+ "rewards/margins": 0.010232994332909584,
+ "rewards/rejected": -0.20091569423675537,
+ "step": 1190
+ },
+ {
+ "epoch": 1.92,
+ "grad_norm": 0.1977105587720871,
+ "learning_rate": 3.2056905749851806e-06,
+ "log_odds_chosen": 0.1517036259174347,
+ "log_odds_ratio": -0.710344672203064,
+ "logits/chosen": 1.7977663278579712,
+ "logits/rejected": 1.7616544961929321,
+ "logps/chosen": -1.8372957706451416,
+ "logps/rejected": -1.9735416173934937,
+ "loss": 1.7458187103271485,
+ "nll_loss": 1.6747843027114868,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.18372957408428192,
+ "rewards/margins": 0.013624586164951324,
+ "rewards/rejected": -0.19735416769981384,
+ "step": 1200
+ },
+ {
+ "epoch": 1.92,
+ "eval_log_odds_chosen": 0.028467997908592224,
+ "eval_log_odds_ratio": -0.7607693076133728,
+ "eval_logits/chosen": 1.7756459712982178,
+ "eval_logits/rejected": 1.74927818775177,
+ "eval_logps/chosen": -1.98622727394104,
+ "eval_logps/rejected": -2.0155580043792725,
+ "eval_loss": 1.8127713203430176,
+ "eval_nll_loss": 1.7366943359375,
+ "eval_rewards/accuracies": 0.5400000214576721,
+ "eval_rewards/chosen": -0.19862274825572968,
+ "eval_rewards/margins": 0.0029331075493246317,
+ "eval_rewards/rejected": -0.20155581831932068,
+ "eval_runtime": 53.2002,
+ "eval_samples_per_second": 9.398,
+ "eval_steps_per_second": 4.699,
+ "step": 1200
+ },
+ {
+ "epoch": 1.936,
+ "grad_norm": 0.179337278008461,
+ "learning_rate": 3.1582691167753406e-06,
+ "log_odds_chosen": 0.07452553510665894,
+ "log_odds_ratio": -0.7306004762649536,
+ "logits/chosen": 1.699223279953003,
+ "logits/rejected": 1.7217528820037842,
+ "logps/chosen": -1.920426607131958,
+ "logps/rejected": -1.985222578048706,
+ "loss": 1.7297235488891602,
+ "nll_loss": 1.6566635370254517,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19204264879226685,
+ "rewards/margins": 0.006479593925178051,
+ "rewards/rejected": -0.19852226972579956,
+ "step": 1210
+ },
+ {
+ "epoch": 1.952,
+ "grad_norm": 0.19489862024784088,
+ "learning_rate": 3.1108476585655006e-06,
+ "log_odds_chosen": -0.06367218494415283,
+ "log_odds_ratio": -0.7718582153320312,
+ "logits/chosen": 1.7101131677627563,
+ "logits/rejected": 1.7847702503204346,
+ "logps/chosen": -2.0594370365142822,
+ "logps/rejected": -2.006824016571045,
+ "loss": 1.8293329238891602,
+ "nll_loss": 1.7521469593048096,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.20594370365142822,
+ "rewards/margins": -0.0052613066509366035,
+ "rewards/rejected": -0.2006824016571045,
+ "step": 1220
+ },
+ {
+ "epoch": 1.968,
+ "grad_norm": 0.1728423684835434,
+ "learning_rate": 3.063426200355661e-06,
+ "log_odds_chosen": 0.1272212713956833,
+ "log_odds_ratio": -0.6985543966293335,
+ "logits/chosen": 1.8437741994857788,
+ "logits/rejected": 1.7424207925796509,
+ "logps/chosen": -1.9680700302124023,
+ "logps/rejected": -2.072535753250122,
+ "loss": 1.7730525970458983,
+ "nll_loss": 1.7031971216201782,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.19680698215961456,
+ "rewards/margins": 0.010446581058204174,
+ "rewards/rejected": -0.20725354552268982,
+ "step": 1230
+ },
+ {
+ "epoch": 1.984,
+ "grad_norm": 0.1812407225370407,
+ "learning_rate": 3.0160047421458206e-06,
+ "log_odds_chosen": 0.07307764887809753,
+ "log_odds_ratio": -0.7265633344650269,
+ "logits/chosen": 1.7150694131851196,
+ "logits/rejected": 1.6220579147338867,
+ "logps/chosen": -1.8851966857910156,
+ "logps/rejected": -1.9514576196670532,
+ "loss": 1.7142372131347656,
+ "nll_loss": 1.6415809392929077,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.1885196715593338,
+ "rewards/margins": 0.006626097019761801,
+ "rewards/rejected": -0.19514577090740204,
+ "step": 1240
+ },
+ {
+ "epoch": 2.0,
+ "grad_norm": 0.21866431832313538,
+ "learning_rate": 2.968583283935981e-06,
+ "log_odds_chosen": 0.09864354133605957,
+ "log_odds_ratio": -0.7356117963790894,
+ "logits/chosen": 1.8142271041870117,
+ "logits/rejected": 1.7990251779556274,
+ "logps/chosen": -2.0075278282165527,
+ "logps/rejected": -2.080134391784668,
+ "loss": 1.826127052307129,
+ "nll_loss": 1.7525659799575806,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.20075277984142303,
+ "rewards/margins": 0.007260681129992008,
+ "rewards/rejected": -0.20801346004009247,
+ "step": 1250
+ },
+ {
+ "epoch": 2.0,
+ "eval_log_odds_chosen": 0.032603029161691666,
+ "eval_log_odds_ratio": -0.7592425346374512,
+ "eval_logits/chosen": 1.7272087335586548,
+ "eval_logits/rejected": 1.701614260673523,
+ "eval_logps/chosen": -1.9889473915100098,
+ "eval_logps/rejected": -2.021564483642578,
+ "eval_loss": 1.8116061687469482,
+ "eval_nll_loss": 1.7356820106506348,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.19889472424983978,
+ "eval_rewards/margins": 0.003261698642745614,
+ "eval_rewards/rejected": -0.2021564543247223,
+ "eval_runtime": 53.2359,
+ "eval_samples_per_second": 9.392,
+ "eval_steps_per_second": 4.696,
+ "step": 1250
+ },
+ {
+ "epoch": 2.016,
+ "grad_norm": 0.20461951196193695,
+ "learning_rate": 2.9211618257261406e-06,
+ "log_odds_chosen": 0.2361210584640503,
+ "log_odds_ratio": -0.6880162954330444,
+ "logits/chosen": 1.7546977996826172,
+ "logits/rejected": 1.6977161169052124,
+ "logps/chosen": -1.8337652683258057,
+ "logps/rejected": -2.0438666343688965,
+ "loss": 1.7133241653442384,
+ "nll_loss": 1.6445229053497314,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.1833765208721161,
+ "rewards/margins": 0.02101016975939274,
+ "rewards/rejected": -0.20438671112060547,
+ "step": 1260
+ },
+ {
+ "epoch": 2.032,
+ "grad_norm": 0.18985678255558014,
+ "learning_rate": 2.873740367516301e-06,
+ "log_odds_chosen": -0.006541083566844463,
+ "log_odds_ratio": -0.7571430802345276,
+ "logits/chosen": 1.7348535060882568,
+ "logits/rejected": 1.7211494445800781,
+ "logps/chosen": -1.9841225147247314,
+ "logps/rejected": -1.9915357828140259,
+ "loss": 1.757932472229004,
+ "nll_loss": 1.682218313217163,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.19841225445270538,
+ "rewards/margins": 0.0007413355633616447,
+ "rewards/rejected": -0.1991535723209381,
+ "step": 1270
+ },
+ {
+ "epoch": 2.048,
+ "grad_norm": 0.1865416169166565,
+ "learning_rate": 2.8263189093064614e-06,
+ "log_odds_chosen": 0.1962173581123352,
+ "log_odds_ratio": -0.6550278663635254,
+ "logits/chosen": 1.8164112567901611,
+ "logits/rejected": 1.8721809387207031,
+ "logps/chosen": -1.8975756168365479,
+ "logps/rejected": -2.0585479736328125,
+ "loss": 1.7741456985473634,
+ "nll_loss": 1.7086429595947266,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": -0.1897575706243515,
+ "rewards/margins": 0.016097230836749077,
+ "rewards/rejected": -0.20585481822490692,
+ "step": 1280
+ },
+ {
+ "epoch": 2.064,
+ "grad_norm": 0.22608967125415802,
+ "learning_rate": 2.778897451096621e-06,
+ "log_odds_chosen": -0.025246893987059593,
+ "log_odds_ratio": -0.7659823298454285,
+ "logits/chosen": 1.784846544265747,
+ "logits/rejected": 1.765973687171936,
+ "logps/chosen": -1.9505832195281982,
+ "logps/rejected": -1.9396007061004639,
+ "loss": 1.7477140426635742,
+ "nll_loss": 1.6711156368255615,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.19505834579467773,
+ "rewards/margins": -0.0010982556268572807,
+ "rewards/rejected": -0.19396010041236877,
+ "step": 1290
+ },
+ {
+ "epoch": 2.08,
+ "grad_norm": 0.17561422288417816,
+ "learning_rate": 2.7314759928867814e-06,
+ "log_odds_chosen": 0.05084152892231941,
+ "log_odds_ratio": -0.7318845391273499,
+ "logits/chosen": 1.7349708080291748,
+ "logits/rejected": 1.7631950378417969,
+ "logps/chosen": -2.0045390129089355,
+ "logps/rejected": -2.0500693321228027,
+ "loss": 1.8187286376953125,
+ "nll_loss": 1.7455400228500366,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.20045390725135803,
+ "rewards/margins": 0.004553015343844891,
+ "rewards/rejected": -0.2050069272518158,
+ "step": 1300
+ },
+ {
+ "epoch": 2.08,
+ "eval_log_odds_chosen": 0.03401584550738335,
+ "eval_log_odds_ratio": -0.7578551769256592,
+ "eval_logits/chosen": 1.7610890865325928,
+ "eval_logits/rejected": 1.7374581098556519,
+ "eval_logps/chosen": -1.9867304563522339,
+ "eval_logps/rejected": -2.020127773284912,
+ "eval_loss": 1.809829592704773,
+ "eval_nll_loss": 1.734044075012207,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": -0.1986730545759201,
+ "eval_rewards/margins": 0.0033397264778614044,
+ "eval_rewards/rejected": -0.2020127773284912,
+ "eval_runtime": 53.3296,
+ "eval_samples_per_second": 9.376,
+ "eval_steps_per_second": 4.688,
+ "step": 1300
+ },
+ {
+ "epoch": 2.096,
+ "grad_norm": 0.1651468127965927,
+ "learning_rate": 2.684054534676941e-06,
+ "log_odds_chosen": 0.09325788170099258,
+ "log_odds_ratio": -0.7075183987617493,
+ "logits/chosen": 1.7508665323257446,
+ "logits/rejected": 1.8297382593154907,
+ "logps/chosen": -1.863616704940796,
+ "logps/rejected": -1.9378494024276733,
+ "loss": 1.7222581863403321,
+ "nll_loss": 1.6515061855316162,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.1863616704940796,
+ "rewards/margins": 0.007423256523907185,
+ "rewards/rejected": -0.1937849372625351,
+ "step": 1310
+ },
+ {
+ "epoch": 2.112,
+ "grad_norm": 0.16485963761806488,
+ "learning_rate": 2.6366330764671014e-06,
+ "log_odds_chosen": 0.0591282956302166,
+ "log_odds_ratio": -0.7139178514480591,
+ "logits/chosen": 1.7827821969985962,
+ "logits/rejected": 1.7864627838134766,
+ "logps/chosen": -1.9632068872451782,
+ "logps/rejected": -2.0267958641052246,
+ "loss": 1.8074531555175781,
+ "nll_loss": 1.7360610961914062,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.19632069766521454,
+ "rewards/margins": 0.006358909420669079,
+ "rewards/rejected": -0.2026796042919159,
+ "step": 1320
+ },
+ {
+ "epoch": 2.128,
+ "grad_norm": 0.16226664185523987,
+ "learning_rate": 2.5892116182572614e-06,
+ "log_odds_chosen": 0.18185576796531677,
+ "log_odds_ratio": -0.6743385791778564,
+ "logits/chosen": 1.95318603515625,
+ "logits/rejected": 1.914075493812561,
+ "logps/chosen": -1.8613958358764648,
+ "logps/rejected": -2.020050048828125,
+ "loss": 1.747477149963379,
+ "nll_loss": 1.6800432205200195,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.18613959848880768,
+ "rewards/margins": 0.015865424647927284,
+ "rewards/rejected": -0.2020050287246704,
+ "step": 1330
+ },
+ {
+ "epoch": 2.144,
+ "grad_norm": 0.18390049040317535,
+ "learning_rate": 2.5417901600474214e-06,
+ "log_odds_chosen": 0.14728814363479614,
+ "log_odds_ratio": -0.6720021367073059,
+ "logits/chosen": 1.7619132995605469,
+ "logits/rejected": 1.7984975576400757,
+ "logps/chosen": -1.774634599685669,
+ "logps/rejected": -1.890054702758789,
+ "loss": 1.7107378005981446,
+ "nll_loss": 1.6435377597808838,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.17746345698833466,
+ "rewards/margins": 0.011542020365595818,
+ "rewards/rejected": -0.18900546431541443,
+ "step": 1340
+ },
+ {
+ "epoch": 2.16,
+ "grad_norm": 0.23228837549686432,
+ "learning_rate": 2.4943687018375814e-06,
+ "log_odds_chosen": -0.015979360789060593,
+ "log_odds_ratio": -0.7752725481987,
+ "logits/chosen": 1.828468680381775,
+ "logits/rejected": 1.7667421102523804,
+ "logps/chosen": -1.8710674047470093,
+ "logps/rejected": -1.8696765899658203,
+ "loss": 1.752641487121582,
+ "nll_loss": 1.6751142740249634,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.18710675835609436,
+ "rewards/margins": -0.00013908911205362529,
+ "rewards/rejected": -0.186967670917511,
+ "step": 1350
+ },
+ {
+ "epoch": 2.16,
+ "eval_log_odds_chosen": 0.03191310539841652,
+ "eval_log_odds_ratio": -0.7588774561882019,
+ "eval_logits/chosen": 1.7761640548706055,
+ "eval_logits/rejected": 1.7524988651275635,
+ "eval_logps/chosen": -1.9843788146972656,
+ "eval_logps/rejected": -2.016181230545044,
+ "eval_loss": 1.8089464902877808,
+ "eval_nll_loss": 1.7330589294433594,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": -0.1984378695487976,
+ "eval_rewards/margins": 0.0031802442390471697,
+ "eval_rewards/rejected": -0.20161812007427216,
+ "eval_runtime": 53.3932,
+ "eval_samples_per_second": 9.364,
+ "eval_steps_per_second": 4.682,
+ "step": 1350
+ },
+ {
+ "epoch": 2.176,
+ "grad_norm": 0.18132326006889343,
+ "learning_rate": 2.446947243627742e-06,
+ "log_odds_chosen": 0.0305715911090374,
+ "log_odds_ratio": -0.7504960894584656,
+ "logits/chosen": 1.7744085788726807,
+ "logits/rejected": 1.7282178401947021,
+ "logps/chosen": -1.9199994802474976,
+ "logps/rejected": -1.9519102573394775,
+ "loss": 1.7735115051269532,
+ "nll_loss": 1.6984617710113525,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.19199995696544647,
+ "rewards/margins": 0.0031910568941384554,
+ "rewards/rejected": -0.19519101083278656,
+ "step": 1360
+ },
+ {
+ "epoch": 2.192,
+ "grad_norm": 0.23645685613155365,
+ "learning_rate": 2.3995257854179014e-06,
+ "log_odds_chosen": 0.14658379554748535,
+ "log_odds_ratio": -0.6893800497055054,
+ "logits/chosen": 1.7729228734970093,
+ "logits/rejected": 1.7737751007080078,
+ "logps/chosen": -1.8243513107299805,
+ "logps/rejected": -1.9623281955718994,
+ "loss": 1.6700077056884766,
+ "nll_loss": 1.601069688796997,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.18243512511253357,
+ "rewards/margins": 0.013797680847346783,
+ "rewards/rejected": -0.19623282551765442,
+ "step": 1370
+ },
+ {
+ "epoch": 2.208,
+ "grad_norm": 0.15323151648044586,
+ "learning_rate": 2.352104327208062e-06,
+ "log_odds_chosen": 0.1115313395857811,
+ "log_odds_ratio": -0.6943892240524292,
+ "logits/chosen": 1.8522207736968994,
+ "logits/rejected": 1.8103185892105103,
+ "logps/chosen": -1.9417545795440674,
+ "logps/rejected": -2.0404393672943115,
+ "loss": 1.8153194427490233,
+ "nll_loss": 1.745880365371704,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.19417545199394226,
+ "rewards/margins": 0.009868469089269638,
+ "rewards/rejected": -0.2040439397096634,
+ "step": 1380
+ },
+ {
+ "epoch": 2.224,
+ "grad_norm": 0.21786808967590332,
+ "learning_rate": 2.3046828689982214e-06,
+ "log_odds_chosen": 0.05500102788209915,
+ "log_odds_ratio": -0.7242174744606018,
+ "logits/chosen": 1.838382363319397,
+ "logits/rejected": 1.8554975986480713,
+ "logps/chosen": -1.99395751953125,
+ "logps/rejected": -2.045518636703491,
+ "loss": 1.7693500518798828,
+ "nll_loss": 1.69692862033844,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.19939574599266052,
+ "rewards/margins": 0.005156124476343393,
+ "rewards/rejected": -0.20455189049243927,
+ "step": 1390
+ },
+ {
+ "epoch": 2.24,
+ "grad_norm": 0.19683599472045898,
+ "learning_rate": 2.257261410788382e-06,
+ "log_odds_chosen": 0.15008209645748138,
+ "log_odds_ratio": -0.6654382944107056,
+ "logits/chosen": 1.7431036233901978,
+ "logits/rejected": 1.731433629989624,
+ "logps/chosen": -1.8933416604995728,
+ "logps/rejected": -2.0205039978027344,
+ "loss": 1.7619529724121095,
+ "nll_loss": 1.6954091787338257,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": -0.18933415412902832,
+ "rewards/margins": 0.012716245837509632,
+ "rewards/rejected": -0.20205040276050568,
+ "step": 1400
+ },
+ {
+ "epoch": 2.24,
+ "eval_log_odds_chosen": 0.032577622681856155,
+ "eval_log_odds_ratio": -0.7580828070640564,
+ "eval_logits/chosen": 1.812838077545166,
+ "eval_logits/rejected": 1.790816068649292,
+ "eval_logps/chosen": -1.9839202165603638,
+ "eval_logps/rejected": -2.0164804458618164,
+ "eval_loss": 1.808388590812683,
+ "eval_nll_loss": 1.732580304145813,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": -0.19839203357696533,
+ "eval_rewards/margins": 0.0032560396939516068,
+ "eval_rewards/rejected": -0.2016480565071106,
+ "eval_runtime": 53.2287,
+ "eval_samples_per_second": 9.393,
+ "eval_steps_per_second": 4.697,
+ "step": 1400
+ },
+ {
+ "epoch": 2.2560000000000002,
+ "grad_norm": 0.2112002670764923,
+ "learning_rate": 2.209839952578542e-06,
+ "log_odds_chosen": 0.2766411304473877,
+ "log_odds_ratio": -0.6827906370162964,
+ "logits/chosen": 1.862683892250061,
+ "logits/rejected": 1.8868215084075928,
+ "logps/chosen": -1.915928840637207,
+ "logps/rejected": -2.172340154647827,
+ "loss": 1.7464689254760741,
+ "nll_loss": 1.6781896352767944,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.19159288704395294,
+ "rewards/margins": 0.025641148909926414,
+ "rewards/rejected": -0.2172340452671051,
+ "step": 1410
+ },
+ {
+ "epoch": 2.2720000000000002,
+ "grad_norm": 0.17736928164958954,
+ "learning_rate": 2.162418494368702e-06,
+ "log_odds_chosen": 0.06808965653181076,
+ "log_odds_ratio": -0.7222734689712524,
+ "logits/chosen": 1.7881934642791748,
+ "logits/rejected": 1.7765629291534424,
+ "logps/chosen": -1.9753538370132446,
+ "logps/rejected": -2.0361990928649902,
+ "loss": 1.7180429458618165,
+ "nll_loss": 1.6458156108856201,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.19753539562225342,
+ "rewards/margins": 0.0060845063999295235,
+ "rewards/rejected": -0.20361991226673126,
+ "step": 1420
+ },
+ {
+ "epoch": 2.288,
+ "grad_norm": 0.17618675529956818,
+ "learning_rate": 2.114997036158862e-06,
+ "log_odds_chosen": 0.08562587201595306,
+ "log_odds_ratio": -0.7272316813468933,
+ "logits/chosen": 1.7778189182281494,
+ "logits/rejected": 1.70413339138031,
+ "logps/chosen": -1.9755465984344482,
+ "logps/rejected": -2.048130512237549,
+ "loss": 1.8506391525268555,
+ "nll_loss": 1.7779159545898438,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.19755466282367706,
+ "rewards/margins": 0.007258380763232708,
+ "rewards/rejected": -0.20481304824352264,
+ "step": 1430
+ },
+ {
+ "epoch": 2.304,
+ "grad_norm": 0.2521407902240753,
+ "learning_rate": 2.067575577949022e-06,
+ "log_odds_chosen": 0.11862572282552719,
+ "log_odds_ratio": -0.7083112001419067,
+ "logits/chosen": 1.8262131214141846,
+ "logits/rejected": 1.8660300970077515,
+ "logps/chosen": -1.885925054550171,
+ "logps/rejected": -1.9840829372406006,
+ "loss": 1.734739112854004,
+ "nll_loss": 1.6639076471328735,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.18859252333641052,
+ "rewards/margins": 0.009815781377255917,
+ "rewards/rejected": -0.19840829074382782,
+ "step": 1440
+ },
+ {
+ "epoch": 2.32,
+ "grad_norm": 0.17723698914051056,
+ "learning_rate": 2.0201541197391818e-06,
+ "log_odds_chosen": 0.12196867167949677,
+ "log_odds_ratio": -0.6998961567878723,
+ "logits/chosen": 1.731156587600708,
+ "logits/rejected": 1.7110185623168945,
+ "logps/chosen": -1.9318151473999023,
+ "logps/rejected": -2.035583972930908,
+ "loss": 1.7764135360717774,
+ "nll_loss": 1.7064237594604492,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.19318152964115143,
+ "rewards/margins": 0.01037687249481678,
+ "rewards/rejected": -0.20355840027332306,
+ "step": 1450
+ },
+ {
+ "epoch": 2.32,
+ "eval_log_odds_chosen": 0.033345215022563934,
+ "eval_log_odds_ratio": -0.7588449716567993,
+ "eval_logits/chosen": 1.7935512065887451,
+ "eval_logits/rejected": 1.7720842361450195,
+ "eval_logps/chosen": -1.9827799797058105,
+ "eval_logps/rejected": -2.0161314010620117,
+ "eval_loss": 1.8074625730514526,
+ "eval_nll_loss": 1.73157799243927,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.19827799499034882,
+ "eval_rewards/margins": 0.0033351616002619267,
+ "eval_rewards/rejected": -0.2016131579875946,
+ "eval_runtime": 53.2316,
+ "eval_samples_per_second": 9.393,
+ "eval_steps_per_second": 4.696,
+ "step": 1450
+ },
+ {
+ "epoch": 2.336,
+ "grad_norm": 0.16808928549289703,
+ "learning_rate": 1.972732661529342e-06,
+ "log_odds_chosen": -0.025525325909256935,
+ "log_odds_ratio": -0.7912946939468384,
+ "logits/chosen": 1.8211339712142944,
+ "logits/rejected": 1.8124920129776,
+ "logps/chosen": -1.856103539466858,
+ "logps/rejected": -1.829656958580017,
+ "loss": 1.7368072509765624,
+ "nll_loss": 1.657678246498108,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": -0.18561036884784698,
+ "rewards/margins": -0.002644673688337207,
+ "rewards/rejected": -0.1829657107591629,
+ "step": 1460
+ },
+ {
+ "epoch": 2.352,
+ "grad_norm": 0.18361903727054596,
+ "learning_rate": 1.925311203319502e-06,
+ "log_odds_chosen": 0.005142314825206995,
+ "log_odds_ratio": -0.8110187649726868,
+ "logits/chosen": 1.825429916381836,
+ "logits/rejected": 1.7501672506332397,
+ "logps/chosen": -2.0082828998565674,
+ "logps/rejected": -2.031491279602051,
+ "loss": 1.7301027297973632,
+ "nll_loss": 1.649000883102417,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.20082826912403107,
+ "rewards/margins": 0.0023208647035062313,
+ "rewards/rejected": -0.20314912497997284,
+ "step": 1470
+ },
+ {
+ "epoch": 2.368,
+ "grad_norm": 0.19813720881938934,
+ "learning_rate": 1.877889745109662e-06,
+ "log_odds_chosen": 0.1795060634613037,
+ "log_odds_ratio": -0.7214691042900085,
+ "logits/chosen": 1.8677520751953125,
+ "logits/rejected": 1.7595751285552979,
+ "logps/chosen": -1.943748116493225,
+ "logps/rejected": -2.1158955097198486,
+ "loss": 1.7257247924804688,
+ "nll_loss": 1.6535780429840088,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.19437481462955475,
+ "rewards/margins": 0.017214741557836533,
+ "rewards/rejected": -0.21158954501152039,
+ "step": 1480
+ },
+ {
+ "epoch": 2.384,
+ "grad_norm": 0.1770639717578888,
+ "learning_rate": 1.830468286899822e-06,
+ "log_odds_chosen": 0.06336086243391037,
+ "log_odds_ratio": -0.7451409101486206,
+ "logits/chosen": 1.8294976949691772,
+ "logits/rejected": 1.835206389427185,
+ "logps/chosen": -1.982630968093872,
+ "logps/rejected": -2.0292210578918457,
+ "loss": 1.777639389038086,
+ "nll_loss": 1.703125238418579,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.19826309382915497,
+ "rewards/margins": 0.004659010097384453,
+ "rewards/rejected": -0.20292210578918457,
+ "step": 1490
+ },
+ {
+ "epoch": 2.4,
+ "grad_norm": 0.22681613266468048,
+ "learning_rate": 1.7830468286899822e-06,
+ "log_odds_chosen": 0.010824749246239662,
+ "log_odds_ratio": -0.7730723023414612,
+ "logits/chosen": 1.7533601522445679,
+ "logits/rejected": 1.706671953201294,
+ "logps/chosen": -2.045074939727783,
+ "logps/rejected": -2.0547096729278564,
+ "loss": 1.7060966491699219,
+ "nll_loss": 1.6287896633148193,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.2045074999332428,
+ "rewards/margins": 0.0009634803864173591,
+ "rewards/rejected": -0.2054709643125534,
+ "step": 1500
+ },
+ {
+ "epoch": 2.4,
+ "eval_log_odds_chosen": 0.03045523911714554,
+ "eval_log_odds_ratio": -0.7589520215988159,
+ "eval_logits/chosen": 1.8010083436965942,
+ "eval_logits/rejected": 1.7791268825531006,
+ "eval_logps/chosen": -1.9805711507797241,
+ "eval_logps/rejected": -2.011178731918335,
+ "eval_loss": 1.8059344291687012,
+ "eval_nll_loss": 1.730039358139038,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": -0.1980571150779724,
+ "eval_rewards/margins": 0.0030607657972723246,
+ "eval_rewards/rejected": -0.2011178731918335,
+ "eval_runtime": 53.2537,
+ "eval_samples_per_second": 9.389,
+ "eval_steps_per_second": 4.695,
+ "step": 1500
+ }
+ ],
+ "logging_steps": 10,
+ "max_steps": 1875,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1500/training_args.bin b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..e369ed5ca0fb0f2b706a5adca8ab527baae13e31
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4ecab525cdf207fed7abfb6f0080dd00da9f0e35d23314a97f0bff66fe6176b
+size 5457
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1875/README.md b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..df73bb4ed6d6a8e69c79ba9761e4cdd4272cc1ab
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- lora
+- orpo
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1875/adapter_config.json b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d02ef66b5d06bfc12ca93c16b0b45a216cff2a6a
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "k_proj",
+ "down_proj",
+ "up_proj",
+ "v_proj",
+ "q_proj",
+ "o_proj",
+ "gate_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1875/adapter_model.safetensors b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ff807925e5efb3b97337f689994123300f25dfe9
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c8e916b496d9da05732b8f3bcea45e366ae8a57f51d054b6910800544b425ce
+size 180385008
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1875/chat_template.jinja b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1875/optimizer.pt b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..325a794f77807fe050a9dbde7337f39571f0bd1a
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:052d3e13f541ac8597a878f373f6a3a208f703042bf47a6dfe79b9377aaa59be
+size 360902475
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1875/rng_state.pth b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..7859360e6e5ad44f3ed47e7c17f4cafb2aebd41a
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50dd579ef1d2d932c3d7726ef0f6ec72d0e25b023ae2fd8b67112ac75e1041b9
+size 14645
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1875/scaler.pt b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..62674612b56e28ca98da704825f72028fdee802e
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0dcc47514d330a99944bf3bc8eb7381f257363691e003bc57a61172f4d6a6cfe
+size 1383
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1875/scheduler.pt b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1f51ff4c44f50f9520c0aa833d388123faad439b
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d4d48109a87523c48c5a3ffd00349d67e823803ed4d5ccf5732c22a8a8c7b5e
+size 1465
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1875/tokenizer.json b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1875/tokenizer_config.json b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1875/trainer_state.json b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..e620619e18480f0787e0d73e71beba29366565ad
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/trainer_state.json
@@ -0,0 +1,4103 @@
+{
+ "best_global_step": 1850,
+ "best_metric": 1.8020251989364624,
+ "best_model_checkpoint": "output/lora/checkpoint-1500",
+ "epoch": 3.0,
+ "eval_steps": 50,
+ "global_step": 1875,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.016,
+ "grad_norm": 1.373120665550232,
+ "learning_rate": 3.829787234042553e-07,
+ "log_odds_chosen": 0.00716871302574873,
+ "log_odds_ratio": -0.8661719560623169,
+ "logits/chosen": 1.1518588066101074,
+ "logits/rejected": 1.1109380722045898,
+ "logps/chosen": -3.045567512512207,
+ "logps/rejected": -3.052224636077881,
+ "loss": 3.4928749084472654,
+ "nll_loss": 3.4062583446502686,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.3045567572116852,
+ "rewards/margins": 0.000665713450871408,
+ "rewards/rejected": -0.30522245168685913,
+ "step": 10
+ },
+ {
+ "epoch": 0.032,
+ "grad_norm": 1.9124146699905396,
+ "learning_rate": 8.085106382978723e-07,
+ "log_odds_chosen": -0.07713592052459717,
+ "log_odds_ratio": -0.9291092157363892,
+ "logits/chosen": 0.9864249229431152,
+ "logits/rejected": 0.9885759353637695,
+ "logps/chosen": -3.2009873390197754,
+ "logps/rejected": -3.1334564685821533,
+ "loss": 3.693857192993164,
+ "nll_loss": 3.600947141647339,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.32009872794151306,
+ "rewards/margins": -0.0067530544474720955,
+ "rewards/rejected": -0.31334567070007324,
+ "step": 20
+ },
+ {
+ "epoch": 0.048,
+ "grad_norm": 0.8211373686790466,
+ "learning_rate": 1.2340425531914892e-06,
+ "log_odds_chosen": -0.13502930104732513,
+ "log_odds_ratio": -0.9718782305717468,
+ "logits/chosen": 1.1002552509307861,
+ "logits/rejected": 1.1391098499298096,
+ "logps/chosen": -3.2072739601135254,
+ "logps/rejected": -3.0774378776550293,
+ "loss": 3.3957988739013674,
+ "nll_loss": 3.2986106872558594,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.3207274079322815,
+ "rewards/margins": -0.012983566150069237,
+ "rewards/rejected": -0.3077438473701477,
+ "step": 30
+ },
+ {
+ "epoch": 0.064,
+ "grad_norm": 0.768266499042511,
+ "learning_rate": 1.6595744680851064e-06,
+ "log_odds_chosen": 0.06783491373062134,
+ "log_odds_ratio": -0.7825279235839844,
+ "logits/chosen": 1.0638954639434814,
+ "logits/rejected": 1.0289121866226196,
+ "logps/chosen": -2.831758975982666,
+ "logps/rejected": -2.8962886333465576,
+ "loss": 3.35846061706543,
+ "nll_loss": 3.2802085876464844,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.28317588567733765,
+ "rewards/margins": 0.006452961824834347,
+ "rewards/rejected": -0.28962892293930054,
+ "step": 40
+ },
+ {
+ "epoch": 0.08,
+ "grad_norm": 0.686996340751648,
+ "learning_rate": 2.0851063829787235e-06,
+ "log_odds_chosen": -0.1561535745859146,
+ "log_odds_ratio": -0.9219404458999634,
+ "logits/chosen": 1.161010980606079,
+ "logits/rejected": 1.1353846788406372,
+ "logps/chosen": -3.047670841217041,
+ "logps/rejected": -2.895892858505249,
+ "loss": 3.245359420776367,
+ "nll_loss": 3.153165817260742,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.30476707220077515,
+ "rewards/margins": -0.0151777733117342,
+ "rewards/rejected": -0.2895893156528473,
+ "step": 50
+ },
+ {
+ "epoch": 0.08,
+ "eval_log_odds_chosen": 0.13973847031593323,
+ "eval_log_odds_ratio": -0.8127303719520569,
+ "eval_logits/chosen": 1.0891079902648926,
+ "eval_logits/rejected": 1.077158808708191,
+ "eval_logps/chosen": -2.9398701190948486,
+ "eval_logps/rejected": -3.080166816711426,
+ "eval_loss": 3.217437505722046,
+ "eval_nll_loss": 3.1361641883850098,
+ "eval_rewards/accuracies": 0.5460000038146973,
+ "eval_rewards/chosen": -0.2939870059490204,
+ "eval_rewards/margins": 0.01402965560555458,
+ "eval_rewards/rejected": -0.30801668763160706,
+ "eval_runtime": 53.3593,
+ "eval_samples_per_second": 9.37,
+ "eval_steps_per_second": 4.685,
+ "step": 50
+ },
+ {
+ "epoch": 0.096,
+ "grad_norm": 0.5995831489562988,
+ "learning_rate": 2.5106382978723402e-06,
+ "log_odds_chosen": 0.17807230353355408,
+ "log_odds_ratio": -0.7261534333229065,
+ "logits/chosen": 1.1882927417755127,
+ "logits/rejected": 1.1475657224655151,
+ "logps/chosen": -2.6847589015960693,
+ "logps/rejected": -2.866147994995117,
+ "loss": 3.019968795776367,
+ "nll_loss": 2.9473533630371094,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.2684759199619293,
+ "rewards/margins": 0.01813889667391777,
+ "rewards/rejected": -0.2866148054599762,
+ "step": 60
+ },
+ {
+ "epoch": 0.112,
+ "grad_norm": 0.42678770422935486,
+ "learning_rate": 2.9361702127659574e-06,
+ "log_odds_chosen": 0.04040491580963135,
+ "log_odds_ratio": -0.7541126012802124,
+ "logits/chosen": 1.2164686918258667,
+ "logits/rejected": 1.1548993587493896,
+ "logps/chosen": -2.732120990753174,
+ "logps/rejected": -2.7696893215179443,
+ "loss": 2.990768241882324,
+ "nll_loss": 2.915356159210205,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.2732120752334595,
+ "rewards/margins": 0.0037568225525319576,
+ "rewards/rejected": -0.27696892619132996,
+ "step": 70
+ },
+ {
+ "epoch": 0.128,
+ "grad_norm": 0.39134183526039124,
+ "learning_rate": 3.3617021276595745e-06,
+ "log_odds_chosen": 0.2615276277065277,
+ "log_odds_ratio": -0.780361533164978,
+ "logits/chosen": 1.0079171657562256,
+ "logits/rejected": 0.9202032089233398,
+ "logps/chosen": -2.813866138458252,
+ "logps/rejected": -3.050983190536499,
+ "loss": 2.885699653625488,
+ "nll_loss": 2.807663679122925,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.28138667345046997,
+ "rewards/margins": 0.0237116776406765,
+ "rewards/rejected": -0.3050983250141144,
+ "step": 80
+ },
+ {
+ "epoch": 0.144,
+ "grad_norm": 0.3603176474571228,
+ "learning_rate": 3.7872340425531912e-06,
+ "log_odds_chosen": 0.0377628318965435,
+ "log_odds_ratio": -0.8245071172714233,
+ "logits/chosen": 1.0985459089279175,
+ "logits/rejected": 1.0714184045791626,
+ "logps/chosen": -2.8029580116271973,
+ "logps/rejected": -2.8335766792297363,
+ "loss": 2.8207944869995116,
+ "nll_loss": 2.7383437156677246,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.28029578924179077,
+ "rewards/margins": 0.0030618987511843443,
+ "rewards/rejected": -0.2833576798439026,
+ "step": 90
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 0.627916157245636,
+ "learning_rate": 4.212765957446808e-06,
+ "log_odds_chosen": 0.181382954120636,
+ "log_odds_ratio": -0.6789280772209167,
+ "logits/chosen": 1.0331000089645386,
+ "logits/rejected": 1.1242140531539917,
+ "logps/chosen": -2.438321828842163,
+ "logps/rejected": -2.604365825653076,
+ "loss": 2.5767364501953125,
+ "nll_loss": 2.5088438987731934,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.24383218586444855,
+ "rewards/margins": 0.016604404896497726,
+ "rewards/rejected": -0.2604365944862366,
+ "step": 100
+ },
+ {
+ "epoch": 0.16,
+ "eval_log_odds_chosen": 0.07509047538042068,
+ "eval_log_odds_ratio": -0.7700561881065369,
+ "eval_logits/chosen": 1.0837428569793701,
+ "eval_logits/rejected": 1.0448086261749268,
+ "eval_logps/chosen": -2.481499433517456,
+ "eval_logps/rejected": -2.561527967453003,
+ "eval_loss": 2.5757269859313965,
+ "eval_nll_loss": 2.4987213611602783,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": -0.24814994633197784,
+ "eval_rewards/margins": 0.008002860471606255,
+ "eval_rewards/rejected": -0.25615280866622925,
+ "eval_runtime": 53.3273,
+ "eval_samples_per_second": 9.376,
+ "eval_steps_per_second": 4.688,
+ "step": 100
+ },
+ {
+ "epoch": 0.176,
+ "grad_norm": 0.2816033363342285,
+ "learning_rate": 4.638297872340425e-06,
+ "log_odds_chosen": 0.0019359707366675138,
+ "log_odds_ratio": -0.7848639488220215,
+ "logits/chosen": 1.014973759651184,
+ "logits/rejected": 1.0038645267486572,
+ "logps/chosen": -2.474358320236206,
+ "logps/rejected": -2.4702632427215576,
+ "loss": 2.4666120529174806,
+ "nll_loss": 2.3881256580352783,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.24743585288524628,
+ "rewards/margins": -0.0004095211625099182,
+ "rewards/rejected": -0.24702632427215576,
+ "step": 110
+ },
+ {
+ "epoch": 0.192,
+ "grad_norm": 0.3091694712638855,
+ "learning_rate": 5.063829787234042e-06,
+ "log_odds_chosen": 0.18548759818077087,
+ "log_odds_ratio": -0.7040785551071167,
+ "logits/chosen": 1.0326206684112549,
+ "logits/rejected": 1.03751802444458,
+ "logps/chosen": -2.2152860164642334,
+ "logps/rejected": -2.3625495433807373,
+ "loss": 2.2838161468505858,
+ "nll_loss": 2.2134079933166504,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.22152860462665558,
+ "rewards/margins": 0.01472637802362442,
+ "rewards/rejected": -0.2362549751996994,
+ "step": 120
+ },
+ {
+ "epoch": 0.208,
+ "grad_norm": 0.2403600662946701,
+ "learning_rate": 5.489361702127659e-06,
+ "log_odds_chosen": -0.029171090573072433,
+ "log_odds_ratio": -0.7839903831481934,
+ "logits/chosen": 1.124861717224121,
+ "logits/rejected": 1.0936174392700195,
+ "logps/chosen": -2.2851500511169434,
+ "logps/rejected": -2.2685129642486572,
+ "loss": 2.286957359313965,
+ "nll_loss": 2.2085585594177246,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.22851502895355225,
+ "rewards/margins": -0.0016637055668979883,
+ "rewards/rejected": -0.22685131430625916,
+ "step": 130
+ },
+ {
+ "epoch": 0.224,
+ "grad_norm": 0.2264973670244217,
+ "learning_rate": 5.9148936170212766e-06,
+ "log_odds_chosen": 0.008796649985015392,
+ "log_odds_ratio": -0.8179906010627747,
+ "logits/chosen": 1.0210378170013428,
+ "logits/rejected": 1.0923821926116943,
+ "logps/chosen": -2.3308796882629395,
+ "logps/rejected": -2.352290153503418,
+ "loss": 2.1581136703491213,
+ "nll_loss": 2.076314687728882,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.23308798670768738,
+ "rewards/margins": 0.0021410156041383743,
+ "rewards/rejected": -0.2352290153503418,
+ "step": 140
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 0.20365770161151886,
+ "learning_rate": 6.340425531914893e-06,
+ "log_odds_chosen": -0.009427487850189209,
+ "log_odds_ratio": -0.805976390838623,
+ "logits/chosen": 1.1640613079071045,
+ "logits/rejected": 1.076269268989563,
+ "logps/chosen": -2.211876153945923,
+ "logps/rejected": -2.1832690238952637,
+ "loss": 2.04748592376709,
+ "nll_loss": 1.9668887853622437,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.22118762135505676,
+ "rewards/margins": -0.0028607314452528954,
+ "rewards/rejected": -0.2183268964290619,
+ "step": 150
+ },
+ {
+ "epoch": 0.24,
+ "eval_log_odds_chosen": 0.034868109971284866,
+ "eval_log_odds_ratio": -0.7663777470588684,
+ "eval_logits/chosen": 1.1222375631332397,
+ "eval_logits/rejected": 1.0752418041229248,
+ "eval_logps/chosen": -2.245656967163086,
+ "eval_logps/rejected": -2.2865161895751953,
+ "eval_loss": 2.085561990737915,
+ "eval_nll_loss": 2.0089242458343506,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.22456568479537964,
+ "eval_rewards/margins": 0.004085900261998177,
+ "eval_rewards/rejected": -0.22865159809589386,
+ "eval_runtime": 53.2365,
+ "eval_samples_per_second": 9.392,
+ "eval_steps_per_second": 4.696,
+ "step": 150
+ },
+ {
+ "epoch": 0.256,
+ "grad_norm": 0.17429247498512268,
+ "learning_rate": 6.76595744680851e-06,
+ "log_odds_chosen": 0.03718019276857376,
+ "log_odds_ratio": -0.7309927940368652,
+ "logits/chosen": 1.243948221206665,
+ "logits/rejected": 1.155428171157837,
+ "logps/chosen": -2.222162961959839,
+ "logps/rejected": -2.2655863761901855,
+ "loss": 2.079151153564453,
+ "nll_loss": 2.006051778793335,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.22221633791923523,
+ "rewards/margins": 0.004342344123870134,
+ "rewards/rejected": -0.22655868530273438,
+ "step": 160
+ },
+ {
+ "epoch": 0.272,
+ "grad_norm": 0.2044232040643692,
+ "learning_rate": 7.191489361702127e-06,
+ "log_odds_chosen": 0.2378263771533966,
+ "log_odds_ratio": -0.6989302635192871,
+ "logits/chosen": 1.1669152975082397,
+ "logits/rejected": 1.1139662265777588,
+ "logps/chosen": -2.1898856163024902,
+ "logps/rejected": -2.405423641204834,
+ "loss": 2.0023525238037108,
+ "nll_loss": 1.9324592351913452,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.2189885824918747,
+ "rewards/margins": 0.021553806960582733,
+ "rewards/rejected": -0.24054238200187683,
+ "step": 170
+ },
+ {
+ "epoch": 0.288,
+ "grad_norm": 0.17495381832122803,
+ "learning_rate": 7.617021276595744e-06,
+ "log_odds_chosen": -0.06665988266468048,
+ "log_odds_ratio": -0.804401695728302,
+ "logits/chosen": 1.1880234479904175,
+ "logits/rejected": 1.193927526473999,
+ "logps/chosen": -2.1306421756744385,
+ "logps/rejected": -2.082124948501587,
+ "loss": 1.9733341217041016,
+ "nll_loss": 1.8928937911987305,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.21306422352790833,
+ "rewards/margins": -0.004851718433201313,
+ "rewards/rejected": -0.2082124948501587,
+ "step": 180
+ },
+ {
+ "epoch": 0.304,
+ "grad_norm": 0.17830601334571838,
+ "learning_rate": 7.995257854179015e-06,
+ "log_odds_chosen": 0.06894228607416153,
+ "log_odds_ratio": -0.7259312868118286,
+ "logits/chosen": 1.171451449394226,
+ "logits/rejected": 1.1703966856002808,
+ "logps/chosen": -2.0300755500793457,
+ "logps/rejected": -2.1007497310638428,
+ "loss": 1.8905454635620118,
+ "nll_loss": 1.8179525136947632,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.2030075490474701,
+ "rewards/margins": 0.007067441940307617,
+ "rewards/rejected": -0.2100749909877777,
+ "step": 190
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 0.20069865882396698,
+ "learning_rate": 7.947836395969175e-06,
+ "log_odds_chosen": 0.2506960332393646,
+ "log_odds_ratio": -0.6373531222343445,
+ "logits/chosen": 1.1848938465118408,
+ "logits/rejected": 1.1178070306777954,
+ "logps/chosen": -1.992468237876892,
+ "logps/rejected": -2.2120540142059326,
+ "loss": 1.8577205657958984,
+ "nll_loss": 1.79398512840271,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": -0.19924680888652802,
+ "rewards/margins": 0.021958591416478157,
+ "rewards/rejected": -0.2212054282426834,
+ "step": 200
+ },
+ {
+ "epoch": 0.32,
+ "eval_log_odds_chosen": 0.028671210631728172,
+ "eval_log_odds_ratio": -0.7659140825271606,
+ "eval_logits/chosen": 1.267584204673767,
+ "eval_logits/rejected": 1.2251330614089966,
+ "eval_logps/chosen": -2.12668776512146,
+ "eval_logps/rejected": -2.1608970165252686,
+ "eval_loss": 1.9326850175857544,
+ "eval_nll_loss": 1.8560935258865356,
+ "eval_rewards/accuracies": 0.5260000228881836,
+ "eval_rewards/chosen": -0.212668776512146,
+ "eval_rewards/margins": 0.0034209203440696,
+ "eval_rewards/rejected": -0.21608968079090118,
+ "eval_runtime": 53.2553,
+ "eval_samples_per_second": 9.389,
+ "eval_steps_per_second": 4.694,
+ "step": 200
+ },
+ {
+ "epoch": 0.336,
+ "grad_norm": 0.20797842741012573,
+ "learning_rate": 7.900414937759335e-06,
+ "log_odds_chosen": 0.04750821739435196,
+ "log_odds_ratio": -0.702393114566803,
+ "logits/chosen": 1.3723499774932861,
+ "logits/rejected": 1.101571798324585,
+ "logps/chosen": -2.08408784866333,
+ "logps/rejected": -2.1285855770111084,
+ "loss": 1.855466651916504,
+ "nll_loss": 1.7852274179458618,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.20840883255004883,
+ "rewards/margins": 0.004449741914868355,
+ "rewards/rejected": -0.21285855770111084,
+ "step": 210
+ },
+ {
+ "epoch": 0.352,
+ "grad_norm": 0.19125986099243164,
+ "learning_rate": 7.852993479549495e-06,
+ "log_odds_chosen": -0.14522936940193176,
+ "log_odds_ratio": -0.8650233149528503,
+ "logits/chosen": 1.3711931705474854,
+ "logits/rejected": 1.2990773916244507,
+ "logps/chosen": -2.171778440475464,
+ "logps/rejected": -2.039144277572632,
+ "loss": 1.8723089218139648,
+ "nll_loss": 1.7858068943023682,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": -0.2171778380870819,
+ "rewards/margins": -0.013263422064483166,
+ "rewards/rejected": -0.20391443371772766,
+ "step": 220
+ },
+ {
+ "epoch": 0.368,
+ "grad_norm": 0.17002636194229126,
+ "learning_rate": 7.805572021339655e-06,
+ "log_odds_chosen": 0.06036282703280449,
+ "log_odds_ratio": -0.7291742563247681,
+ "logits/chosen": 1.3134567737579346,
+ "logits/rejected": 1.3123853206634521,
+ "logps/chosen": -2.0283656120300293,
+ "logps/rejected": -2.0922353267669678,
+ "loss": 1.8563121795654296,
+ "nll_loss": 1.7833948135375977,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.20283658802509308,
+ "rewards/margins": 0.006386973895132542,
+ "rewards/rejected": -0.20922355353832245,
+ "step": 230
+ },
+ {
+ "epoch": 0.384,
+ "grad_norm": 0.16414593160152435,
+ "learning_rate": 7.758150563129815e-06,
+ "log_odds_chosen": 0.22585150599479675,
+ "log_odds_ratio": -0.6319423913955688,
+ "logits/chosen": 1.317280650138855,
+ "logits/rejected": 1.2409361600875854,
+ "logps/chosen": -1.9960790872573853,
+ "logps/rejected": -2.1858954429626465,
+ "loss": 1.8553081512451173,
+ "nll_loss": 1.792114019393921,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.19960792362689972,
+ "rewards/margins": 0.018981631845235825,
+ "rewards/rejected": -0.21858954429626465,
+ "step": 240
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.14812582731246948,
+ "learning_rate": 7.710729104919975e-06,
+ "log_odds_chosen": -0.09517587721347809,
+ "log_odds_ratio": -0.8104914426803589,
+ "logits/chosen": 1.3900704383850098,
+ "logits/rejected": 1.4599993228912354,
+ "logps/chosen": -2.050788402557373,
+ "logps/rejected": -1.9786239862442017,
+ "loss": 1.8190387725830077,
+ "nll_loss": 1.7379897832870483,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.2050788700580597,
+ "rewards/margins": -0.0072164698503911495,
+ "rewards/rejected": -0.19786237180233002,
+ "step": 250
+ },
+ {
+ "epoch": 0.4,
+ "eval_log_odds_chosen": 0.027970660477876663,
+ "eval_log_odds_ratio": -0.7652882933616638,
+ "eval_logits/chosen": 1.365133285522461,
+ "eval_logits/rejected": 1.3242768049240112,
+ "eval_logps/chosen": -2.095747947692871,
+ "eval_logps/rejected": -2.128530263900757,
+ "eval_loss": 1.9009666442871094,
+ "eval_nll_loss": 1.8244377374649048,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": -0.20957481861114502,
+ "eval_rewards/margins": 0.0032782391645014286,
+ "eval_rewards/rejected": -0.21285304427146912,
+ "eval_runtime": 53.1908,
+ "eval_samples_per_second": 9.4,
+ "eval_steps_per_second": 4.7,
+ "step": 250
+ },
+ {
+ "epoch": 0.416,
+ "grad_norm": 0.18252891302108765,
+ "learning_rate": 7.663307646710137e-06,
+ "log_odds_chosen": 0.07316568493843079,
+ "log_odds_ratio": -0.7166140675544739,
+ "logits/chosen": 1.4400272369384766,
+ "logits/rejected": 1.3449039459228516,
+ "logps/chosen": -2.0099174976348877,
+ "logps/rejected": -2.075176954269409,
+ "loss": 1.8593158721923828,
+ "nll_loss": 1.7876542806625366,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.20099174976348877,
+ "rewards/margins": 0.006525977049022913,
+ "rewards/rejected": -0.20751771330833435,
+ "step": 260
+ },
+ {
+ "epoch": 0.432,
+ "grad_norm": 0.15253905951976776,
+ "learning_rate": 7.6158861885002954e-06,
+ "log_odds_chosen": 0.20866680145263672,
+ "log_odds_ratio": -0.685832142829895,
+ "logits/chosen": 1.3512670993804932,
+ "logits/rejected": 1.3183681964874268,
+ "logps/chosen": -1.9916179180145264,
+ "logps/rejected": -2.1834797859191895,
+ "loss": 1.8131860733032226,
+ "nll_loss": 1.7446027994155884,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.1991618126630783,
+ "rewards/margins": 0.019186144694685936,
+ "rewards/rejected": -0.21834798157215118,
+ "step": 270
+ },
+ {
+ "epoch": 0.448,
+ "grad_norm": 0.3055081069469452,
+ "learning_rate": 7.5684647302904554e-06,
+ "log_odds_chosen": 0.19830426573753357,
+ "log_odds_ratio": -0.6579998135566711,
+ "logits/chosen": 1.3933019638061523,
+ "logits/rejected": 1.433093547821045,
+ "logps/chosen": -1.9509963989257812,
+ "logps/rejected": -2.128086566925049,
+ "loss": 1.8611320495605468,
+ "nll_loss": 1.7953321933746338,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.1950996369123459,
+ "rewards/margins": 0.017709016799926758,
+ "rewards/rejected": -0.21280868351459503,
+ "step": 280
+ },
+ {
+ "epoch": 0.464,
+ "grad_norm": 0.1907416731119156,
+ "learning_rate": 7.521043272080616e-06,
+ "log_odds_chosen": 0.2938005328178406,
+ "log_odds_ratio": -0.6218821406364441,
+ "logits/chosen": 1.454288125038147,
+ "logits/rejected": 1.364547848701477,
+ "logps/chosen": -1.907086968421936,
+ "logps/rejected": -2.168692111968994,
+ "loss": 1.7643163681030274,
+ "nll_loss": 1.7021281719207764,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.1907087117433548,
+ "rewards/margins": 0.026160534471273422,
+ "rewards/rejected": -0.21686923503875732,
+ "step": 290
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 0.20577998459339142,
+ "learning_rate": 7.473621813870776e-06,
+ "log_odds_chosen": 0.0781371220946312,
+ "log_odds_ratio": -0.7277525663375854,
+ "logits/chosen": 1.3441762924194336,
+ "logits/rejected": 1.339400291442871,
+ "logps/chosen": -1.986854910850525,
+ "logps/rejected": -2.0572762489318848,
+ "loss": 1.8164241790771485,
+ "nll_loss": 1.7436487674713135,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.19868549704551697,
+ "rewards/margins": 0.007042132318019867,
+ "rewards/rejected": -0.20572762191295624,
+ "step": 300
+ },
+ {
+ "epoch": 0.48,
+ "eval_log_odds_chosen": 0.027198966592550278,
+ "eval_log_odds_ratio": -0.7654678821563721,
+ "eval_logits/chosen": 1.4181112051010132,
+ "eval_logits/rejected": 1.3766058683395386,
+ "eval_logps/chosen": -2.0740718841552734,
+ "eval_logps/rejected": -2.1053483486175537,
+ "eval_loss": 1.8834025859832764,
+ "eval_nll_loss": 1.8068556785583496,
+ "eval_rewards/accuracies": 0.5260000228881836,
+ "eval_rewards/chosen": -0.2074071615934372,
+ "eval_rewards/margins": 0.0031276745721697807,
+ "eval_rewards/rejected": -0.21053485572338104,
+ "eval_runtime": 53.2972,
+ "eval_samples_per_second": 9.381,
+ "eval_steps_per_second": 4.691,
+ "step": 300
+ },
+ {
+ "epoch": 0.496,
+ "grad_norm": 0.15923626720905304,
+ "learning_rate": 7.426200355660936e-06,
+ "log_odds_chosen": -0.007339484989643097,
+ "log_odds_ratio": -0.7585854530334473,
+ "logits/chosen": 1.4372234344482422,
+ "logits/rejected": 1.3944370746612549,
+ "logps/chosen": -1.9776242971420288,
+ "logps/rejected": -1.9795879125595093,
+ "loss": 1.846565055847168,
+ "nll_loss": 1.7707065343856812,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.19776242971420288,
+ "rewards/margins": 0.00019635623903013766,
+ "rewards/rejected": -0.1979587972164154,
+ "step": 310
+ },
+ {
+ "epoch": 0.512,
+ "grad_norm": 0.2272547334432602,
+ "learning_rate": 7.378778897451097e-06,
+ "log_odds_chosen": 0.06932153552770615,
+ "log_odds_ratio": -0.7089754939079285,
+ "logits/chosen": 1.4551548957824707,
+ "logits/rejected": 1.413223147392273,
+ "logps/chosen": -2.0715298652648926,
+ "logps/rejected": -2.1249260902404785,
+ "loss": 1.8696435928344726,
+ "nll_loss": 1.79874587059021,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.20715300738811493,
+ "rewards/margins": 0.005339615046977997,
+ "rewards/rejected": -0.21249261498451233,
+ "step": 320
+ },
+ {
+ "epoch": 0.528,
+ "grad_norm": 0.17856107652187347,
+ "learning_rate": 7.331357439241256e-06,
+ "log_odds_chosen": 0.01983504556119442,
+ "log_odds_ratio": -0.7833854556083679,
+ "logits/chosen": 1.3731528520584106,
+ "logits/rejected": 1.454303503036499,
+ "logps/chosen": -1.9772266149520874,
+ "logps/rejected": -1.9896529912948608,
+ "loss": 1.7815839767456054,
+ "nll_loss": 1.7032454013824463,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.1977226734161377,
+ "rewards/margins": 0.0012426606845110655,
+ "rewards/rejected": -0.19896534085273743,
+ "step": 330
+ },
+ {
+ "epoch": 0.544,
+ "grad_norm": 0.19845250248908997,
+ "learning_rate": 7.283935981031416e-06,
+ "log_odds_chosen": 0.1632343828678131,
+ "log_odds_ratio": -0.6663784384727478,
+ "logits/chosen": 1.4501641988754272,
+ "logits/rejected": 1.3493249416351318,
+ "logps/chosen": -2.048593521118164,
+ "logps/rejected": -2.186927080154419,
+ "loss": 1.8446569442749023,
+ "nll_loss": 1.7780191898345947,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.20485934615135193,
+ "rewards/margins": 0.013833333738148212,
+ "rewards/rejected": -0.21869266033172607,
+ "step": 340
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 0.21156993508338928,
+ "learning_rate": 7.236514522821576e-06,
+ "log_odds_chosen": 0.15967318415641785,
+ "log_odds_ratio": -0.6973395943641663,
+ "logits/chosen": 1.405096173286438,
+ "logits/rejected": 1.3601795434951782,
+ "logps/chosen": -2.0488533973693848,
+ "logps/rejected": -2.1905434131622314,
+ "loss": 1.8135244369506835,
+ "nll_loss": 1.7437903881072998,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.2048853635787964,
+ "rewards/margins": 0.014168987981975079,
+ "rewards/rejected": -0.21905431151390076,
+ "step": 350
+ },
+ {
+ "epoch": 0.56,
+ "eval_log_odds_chosen": 0.02301771193742752,
+ "eval_log_odds_ratio": -0.7660307288169861,
+ "eval_logits/chosen": 1.4248073101043701,
+ "eval_logits/rejected": 1.3838006258010864,
+ "eval_logps/chosen": -2.0602059364318848,
+ "eval_logps/rejected": -2.0874686241149902,
+ "eval_loss": 1.8709203004837036,
+ "eval_nll_loss": 1.7943171262741089,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.20602059364318848,
+ "eval_rewards/margins": 0.0027262638323009014,
+ "eval_rewards/rejected": -0.20874685049057007,
+ "eval_runtime": 53.1993,
+ "eval_samples_per_second": 9.399,
+ "eval_steps_per_second": 4.699,
+ "step": 350
+ },
+ {
+ "epoch": 0.576,
+ "grad_norm": 0.21077947318553925,
+ "learning_rate": 7.189093064611737e-06,
+ "log_odds_chosen": 0.14307400584220886,
+ "log_odds_ratio": -0.7016319036483765,
+ "logits/chosen": 1.5183265209197998,
+ "logits/rejected": 1.5037349462509155,
+ "logps/chosen": -2.0080947875976562,
+ "logps/rejected": -2.136094570159912,
+ "loss": 1.8600387573242188,
+ "nll_loss": 1.7898753881454468,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.20080947875976562,
+ "rewards/margins": 0.012799995020031929,
+ "rewards/rejected": -0.2136094570159912,
+ "step": 360
+ },
+ {
+ "epoch": 0.592,
+ "grad_norm": 0.21159255504608154,
+ "learning_rate": 7.141671606401897e-06,
+ "log_odds_chosen": 0.05201685428619385,
+ "log_odds_ratio": -0.7168989777565002,
+ "logits/chosen": 1.4038429260253906,
+ "logits/rejected": 1.3609113693237305,
+ "logps/chosen": -2.0137746334075928,
+ "logps/rejected": -2.051905870437622,
+ "loss": 1.864394760131836,
+ "nll_loss": 1.7927048206329346,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.20137743651866913,
+ "rewards/margins": 0.0038131114561110735,
+ "rewards/rejected": -0.20519058406352997,
+ "step": 370
+ },
+ {
+ "epoch": 0.608,
+ "grad_norm": 0.17047084867954254,
+ "learning_rate": 7.094250148192056e-06,
+ "log_odds_chosen": 0.14928987622261047,
+ "log_odds_ratio": -0.7049298286437988,
+ "logits/chosen": 1.4637548923492432,
+ "logits/rejected": 1.5022916793823242,
+ "logps/chosen": -2.015235424041748,
+ "logps/rejected": -2.1330373287200928,
+ "loss": 1.7998531341552735,
+ "nll_loss": 1.7293602228164673,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.2015235424041748,
+ "rewards/margins": 0.011780187487602234,
+ "rewards/rejected": -0.21330371499061584,
+ "step": 380
+ },
+ {
+ "epoch": 0.624,
+ "grad_norm": 0.30234500765800476,
+ "learning_rate": 7.046828689982216e-06,
+ "log_odds_chosen": 0.13443878293037415,
+ "log_odds_ratio": -0.6994689702987671,
+ "logits/chosen": 1.3651323318481445,
+ "logits/rejected": 1.4916176795959473,
+ "logps/chosen": -1.9770710468292236,
+ "logps/rejected": -2.0873372554779053,
+ "loss": 1.8301820755004883,
+ "nll_loss": 1.7602351903915405,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.19770710170269012,
+ "rewards/margins": 0.011026628315448761,
+ "rewards/rejected": -0.20873375236988068,
+ "step": 390
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 0.15642815828323364,
+ "learning_rate": 6.999407231772377e-06,
+ "log_odds_chosen": 0.19918577373027802,
+ "log_odds_ratio": -0.6648043394088745,
+ "logits/chosen": 1.658177137374878,
+ "logits/rejected": 1.6399805545806885,
+ "logps/chosen": -1.924300193786621,
+ "logps/rejected": -2.1058459281921387,
+ "loss": 1.8306264877319336,
+ "nll_loss": 1.764146089553833,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.19243000447750092,
+ "rewards/margins": 0.018154580146074295,
+ "rewards/rejected": -0.2105845957994461,
+ "step": 400
+ },
+ {
+ "epoch": 0.64,
+ "eval_log_odds_chosen": 0.029309768229722977,
+ "eval_log_odds_ratio": -0.7632622718811035,
+ "eval_logits/chosen": 1.523261308670044,
+ "eval_logits/rejected": 1.48293936252594,
+ "eval_logps/chosen": -2.0515761375427246,
+ "eval_logps/rejected": -2.0838050842285156,
+ "eval_loss": 1.8621249198913574,
+ "eval_nll_loss": 1.7857985496520996,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.20515760779380798,
+ "eval_rewards/margins": 0.0032229016069322824,
+ "eval_rewards/rejected": -0.20838052034378052,
+ "eval_runtime": 53.1851,
+ "eval_samples_per_second": 9.401,
+ "eval_steps_per_second": 4.701,
+ "step": 400
+ },
+ {
+ "epoch": 0.656,
+ "grad_norm": 0.1967567503452301,
+ "learning_rate": 6.951985773562537e-06,
+ "log_odds_chosen": 0.1653221845626831,
+ "log_odds_ratio": -0.6918090581893921,
+ "logits/chosen": 1.5250298976898193,
+ "logits/rejected": 1.4470211267471313,
+ "logps/chosen": -2.0086913108825684,
+ "logps/rejected": -2.145521879196167,
+ "loss": 1.8166702270507813,
+ "nll_loss": 1.7474892139434814,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.20086915791034698,
+ "rewards/margins": 0.013683043420314789,
+ "rewards/rejected": -0.21455219388008118,
+ "step": 410
+ },
+ {
+ "epoch": 0.672,
+ "grad_norm": 0.2208651602268219,
+ "learning_rate": 6.904564315352697e-06,
+ "log_odds_chosen": 0.14104491472244263,
+ "log_odds_ratio": -0.7459835410118103,
+ "logits/chosen": 1.605399489402771,
+ "logits/rejected": 1.5775949954986572,
+ "logps/chosen": -1.9965150356292725,
+ "logps/rejected": -2.143871545791626,
+ "loss": 1.749098777770996,
+ "nll_loss": 1.6745002269744873,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.19965147972106934,
+ "rewards/margins": 0.01473566610366106,
+ "rewards/rejected": -0.21438714861869812,
+ "step": 420
+ },
+ {
+ "epoch": 0.688,
+ "grad_norm": 0.2232414036989212,
+ "learning_rate": 6.857142857142856e-06,
+ "log_odds_chosen": 0.10664594173431396,
+ "log_odds_ratio": -0.7230110168457031,
+ "logits/chosen": 1.5467585325241089,
+ "logits/rejected": 1.5005398988723755,
+ "logps/chosen": -1.9454959630966187,
+ "logps/rejected": -2.01902437210083,
+ "loss": 1.771216583251953,
+ "nll_loss": 1.698915719985962,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.1945495903491974,
+ "rewards/margins": 0.0073528350330889225,
+ "rewards/rejected": -0.20190243422985077,
+ "step": 430
+ },
+ {
+ "epoch": 0.704,
+ "grad_norm": 0.19019967317581177,
+ "learning_rate": 6.809721398933017e-06,
+ "log_odds_chosen": -0.022097941488027573,
+ "log_odds_ratio": -0.7597793340682983,
+ "logits/chosen": 1.6924508810043335,
+ "logits/rejected": 1.6551586389541626,
+ "logps/chosen": -1.9923264980316162,
+ "logps/rejected": -1.9681341648101807,
+ "loss": 1.8285696029663085,
+ "nll_loss": 1.7525920867919922,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.19923266768455505,
+ "rewards/margins": -0.002419233787804842,
+ "rewards/rejected": -0.1968134194612503,
+ "step": 440
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 0.15414057672023773,
+ "learning_rate": 6.762299940723177e-06,
+ "log_odds_chosen": -0.07213816046714783,
+ "log_odds_ratio": -0.815600574016571,
+ "logits/chosen": 1.5608292818069458,
+ "logits/rejected": 1.5346075296401978,
+ "logps/chosen": -2.106017589569092,
+ "logps/rejected": -2.0266618728637695,
+ "loss": 1.8748476028442382,
+ "nll_loss": 1.7932875156402588,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": -0.21060176193714142,
+ "rewards/margins": -0.007935596629977226,
+ "rewards/rejected": -0.20266616344451904,
+ "step": 450
+ },
+ {
+ "epoch": 0.72,
+ "eval_log_odds_chosen": 0.02593321166932583,
+ "eval_log_odds_ratio": -0.7637203335762024,
+ "eval_logits/chosen": 1.5849090814590454,
+ "eval_logits/rejected": 1.5462759733200073,
+ "eval_logps/chosen": -2.046966791152954,
+ "eval_logps/rejected": -2.076876401901245,
+ "eval_loss": 1.856321930885315,
+ "eval_nll_loss": 1.7799499034881592,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.2046966552734375,
+ "eval_rewards/margins": 0.0029909671284258366,
+ "eval_rewards/rejected": -0.2076876312494278,
+ "eval_runtime": 53.2289,
+ "eval_samples_per_second": 9.393,
+ "eval_steps_per_second": 4.697,
+ "step": 450
+ },
+ {
+ "epoch": 0.736,
+ "grad_norm": 0.2198966145515442,
+ "learning_rate": 6.714878482513337e-06,
+ "log_odds_chosen": 0.1490866094827652,
+ "log_odds_ratio": -0.6993222832679749,
+ "logits/chosen": 1.530315637588501,
+ "logits/rejected": 1.5981829166412354,
+ "logps/chosen": -2.0311427116394043,
+ "logps/rejected": -2.1558847427368164,
+ "loss": 1.852096939086914,
+ "nll_loss": 1.7821648120880127,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.20311430096626282,
+ "rewards/margins": 0.01247421558946371,
+ "rewards/rejected": -0.21558848023414612,
+ "step": 460
+ },
+ {
+ "epoch": 0.752,
+ "grad_norm": 0.17686985433101654,
+ "learning_rate": 6.667457024303497e-06,
+ "log_odds_chosen": 0.17564800381660461,
+ "log_odds_ratio": -0.6861358880996704,
+ "logits/chosen": 1.559126377105713,
+ "logits/rejected": 1.4911739826202393,
+ "logps/chosen": -1.9338982105255127,
+ "logps/rejected": -2.090195417404175,
+ "loss": 1.7766691207885743,
+ "nll_loss": 1.7080554962158203,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.19338981807231903,
+ "rewards/margins": 0.015629729256033897,
+ "rewards/rejected": -0.20901954174041748,
+ "step": 470
+ },
+ {
+ "epoch": 0.768,
+ "grad_norm": 0.23684880137443542,
+ "learning_rate": 6.620035566093657e-06,
+ "log_odds_chosen": 0.07966659963130951,
+ "log_odds_ratio": -0.7355421781539917,
+ "logits/chosen": 1.610801100730896,
+ "logits/rejected": 1.5581674575805664,
+ "logps/chosen": -1.9333826303482056,
+ "logps/rejected": -2.002697229385376,
+ "loss": 1.8244489669799804,
+ "nll_loss": 1.7508949041366577,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.19333826005458832,
+ "rewards/margins": 0.0069314478896558285,
+ "rewards/rejected": -0.20026972889900208,
+ "step": 480
+ },
+ {
+ "epoch": 0.784,
+ "grad_norm": 0.170082688331604,
+ "learning_rate": 6.572614107883817e-06,
+ "log_odds_chosen": -0.016495058313012123,
+ "log_odds_ratio": -0.7567610740661621,
+ "logits/chosen": 1.4795844554901123,
+ "logits/rejected": 1.5042058229446411,
+ "logps/chosen": -2.035449981689453,
+ "logps/rejected": -2.0238888263702393,
+ "loss": 1.8243209838867187,
+ "nll_loss": 1.7486448287963867,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": -0.2035450041294098,
+ "rewards/margins": -0.001156145939603448,
+ "rewards/rejected": -0.20238885283470154,
+ "step": 490
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 0.1873895525932312,
+ "learning_rate": 6.525192649673977e-06,
+ "log_odds_chosen": 0.1201021671295166,
+ "log_odds_ratio": -0.7277526259422302,
+ "logits/chosen": 1.5447202920913696,
+ "logits/rejected": 1.494986891746521,
+ "logps/chosen": -2.0172603130340576,
+ "logps/rejected": -2.110663890838623,
+ "loss": 1.7948209762573242,
+ "nll_loss": 1.7220455408096313,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": -0.2017260491847992,
+ "rewards/margins": 0.009340338408946991,
+ "rewards/rejected": -0.21106640994548798,
+ "step": 500
+ },
+ {
+ "epoch": 0.8,
+ "eval_log_odds_chosen": 0.02295607700943947,
+ "eval_log_odds_ratio": -0.7634766697883606,
+ "eval_logits/chosen": 1.589399814605713,
+ "eval_logits/rejected": 1.5498627424240112,
+ "eval_logps/chosen": -2.0389089584350586,
+ "eval_logps/rejected": -2.0656278133392334,
+ "eval_loss": 1.851176142692566,
+ "eval_nll_loss": 1.774828314781189,
+ "eval_rewards/accuracies": 0.5379999876022339,
+ "eval_rewards/chosen": -0.20389088988304138,
+ "eval_rewards/margins": 0.0026718771550804377,
+ "eval_rewards/rejected": -0.20656277239322662,
+ "eval_runtime": 53.271,
+ "eval_samples_per_second": 9.386,
+ "eval_steps_per_second": 4.693,
+ "step": 500
+ },
+ {
+ "epoch": 0.816,
+ "grad_norm": 0.18255847692489624,
+ "learning_rate": 6.477771191464137e-06,
+ "log_odds_chosen": 0.0806904286146164,
+ "log_odds_ratio": -0.717610239982605,
+ "logits/chosen": 1.5147536993026733,
+ "logits/rejected": 1.444838285446167,
+ "logps/chosen": -2.0275161266326904,
+ "logps/rejected": -2.100182056427002,
+ "loss": 1.8432020187377929,
+ "nll_loss": 1.771440863609314,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.20275163650512695,
+ "rewards/margins": 0.007266544736921787,
+ "rewards/rejected": -0.21001818776130676,
+ "step": 510
+ },
+ {
+ "epoch": 0.832,
+ "grad_norm": 0.17855268716812134,
+ "learning_rate": 6.430349733254298e-06,
+ "log_odds_chosen": 0.2732783854007721,
+ "log_odds_ratio": -0.6957250237464905,
+ "logits/chosen": 1.557079553604126,
+ "logits/rejected": 1.585305094718933,
+ "logps/chosen": -1.9428716897964478,
+ "logps/rejected": -2.2009921073913574,
+ "loss": 1.809073257446289,
+ "nll_loss": 1.7395007610321045,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.19428716599941254,
+ "rewards/margins": 0.0258120596408844,
+ "rewards/rejected": -0.22009921073913574,
+ "step": 520
+ },
+ {
+ "epoch": 0.848,
+ "grad_norm": 0.27463603019714355,
+ "learning_rate": 6.382928275044458e-06,
+ "log_odds_chosen": -0.053782153874635696,
+ "log_odds_ratio": -0.7889940142631531,
+ "logits/chosen": 1.546355128288269,
+ "logits/rejected": 1.5052711963653564,
+ "logps/chosen": -2.0254311561584473,
+ "logps/rejected": -1.9745910167694092,
+ "loss": 1.8272314071655273,
+ "nll_loss": 1.7483322620391846,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.20254313945770264,
+ "rewards/margins": -0.005084014497697353,
+ "rewards/rejected": -0.1974591165781021,
+ "step": 530
+ },
+ {
+ "epoch": 0.864,
+ "grad_norm": 0.19943900406360626,
+ "learning_rate": 6.335506816834617e-06,
+ "log_odds_chosen": 0.007906230166554451,
+ "log_odds_ratio": -0.7769323587417603,
+ "logits/chosen": 1.686943769454956,
+ "logits/rejected": 1.773566484451294,
+ "logps/chosen": -1.9026868343353271,
+ "logps/rejected": -1.8993861675262451,
+ "loss": 1.8080410003662108,
+ "nll_loss": 1.7303478717803955,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.19026869535446167,
+ "rewards/margins": -0.0003300657554063946,
+ "rewards/rejected": -0.18993863463401794,
+ "step": 540
+ },
+ {
+ "epoch": 0.88,
+ "grad_norm": 0.12966203689575195,
+ "learning_rate": 6.288085358624777e-06,
+ "log_odds_chosen": 0.18394824862480164,
+ "log_odds_ratio": -0.656446099281311,
+ "logits/chosen": 1.578307867050171,
+ "logits/rejected": 1.5473096370697021,
+ "logps/chosen": -1.8790676593780518,
+ "logps/rejected": -2.0375452041625977,
+ "loss": 1.80413818359375,
+ "nll_loss": 1.7384932041168213,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": -0.18790677189826965,
+ "rewards/margins": 0.015847764909267426,
+ "rewards/rejected": -0.20375454425811768,
+ "step": 550
+ },
+ {
+ "epoch": 0.88,
+ "eval_log_odds_chosen": 0.026566974818706512,
+ "eval_log_odds_ratio": -0.7653356790542603,
+ "eval_logits/chosen": 1.5679305791854858,
+ "eval_logits/rejected": 1.532114028930664,
+ "eval_logps/chosen": -2.0328524112701416,
+ "eval_logps/rejected": -2.0620551109313965,
+ "eval_loss": 1.846232533454895,
+ "eval_nll_loss": 1.769698977470398,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": -0.20328520238399506,
+ "eval_rewards/margins": 0.002920305822044611,
+ "eval_rewards/rejected": -0.20620553195476532,
+ "eval_runtime": 53.3132,
+ "eval_samples_per_second": 9.379,
+ "eval_steps_per_second": 4.689,
+ "step": 550
+ },
+ {
+ "epoch": 0.896,
+ "grad_norm": 0.24944448471069336,
+ "learning_rate": 6.240663900414938e-06,
+ "log_odds_chosen": 0.2502843141555786,
+ "log_odds_ratio": -0.6478752493858337,
+ "logits/chosen": 1.4606131315231323,
+ "logits/rejected": 1.3790860176086426,
+ "logps/chosen": -1.9443740844726562,
+ "logps/rejected": -2.1599698066711426,
+ "loss": 1.7577186584472657,
+ "nll_loss": 1.6929309368133545,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.1944374144077301,
+ "rewards/margins": 0.021559562534093857,
+ "rewards/rejected": -0.21599698066711426,
+ "step": 560
+ },
+ {
+ "epoch": 0.912,
+ "grad_norm": 0.1742384284734726,
+ "learning_rate": 6.193242442205098e-06,
+ "log_odds_chosen": 0.0590474009513855,
+ "log_odds_ratio": -0.7506214380264282,
+ "logits/chosen": 1.6640907526016235,
+ "logits/rejected": 1.6049493551254272,
+ "logps/chosen": -1.9722331762313843,
+ "logps/rejected": -2.026829481124878,
+ "loss": 1.7977088928222655,
+ "nll_loss": 1.722646951675415,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19722333550453186,
+ "rewards/margins": 0.005459627602249384,
+ "rewards/rejected": -0.2026829719543457,
+ "step": 570
+ },
+ {
+ "epoch": 0.928,
+ "grad_norm": 0.2213008999824524,
+ "learning_rate": 6.145820983995258e-06,
+ "log_odds_chosen": -0.03383875638246536,
+ "log_odds_ratio": -0.7764871716499329,
+ "logits/chosen": 1.6381727457046509,
+ "logits/rejected": 1.5094597339630127,
+ "logps/chosen": -1.9914861917495728,
+ "logps/rejected": -1.964422583580017,
+ "loss": 1.8107261657714844,
+ "nll_loss": 1.7330776453018188,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.19914861023426056,
+ "rewards/margins": -0.0027063353918492794,
+ "rewards/rejected": -0.19644227623939514,
+ "step": 580
+ },
+ {
+ "epoch": 0.944,
+ "grad_norm": 0.1825234442949295,
+ "learning_rate": 6.098399525785417e-06,
+ "log_odds_chosen": 0.10232333838939667,
+ "log_odds_ratio": -0.7037243843078613,
+ "logits/chosen": 1.5093460083007812,
+ "logits/rejected": 1.4225064516067505,
+ "logps/chosen": -1.9882131814956665,
+ "logps/rejected": -2.0832431316375732,
+ "loss": 1.7983221054077148,
+ "nll_loss": 1.727950096130371,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.1988213062286377,
+ "rewards/margins": 0.009502974338829517,
+ "rewards/rejected": -0.20832428336143494,
+ "step": 590
+ },
+ {
+ "epoch": 0.96,
+ "grad_norm": 0.17290247976779938,
+ "learning_rate": 6.050978067575578e-06,
+ "log_odds_chosen": 0.06585167348384857,
+ "log_odds_ratio": -0.7220332026481628,
+ "logits/chosen": 1.5872576236724854,
+ "logits/rejected": 1.5169059038162231,
+ "logps/chosen": -1.8601738214492798,
+ "logps/rejected": -1.914232850074768,
+ "loss": 1.7327194213867188,
+ "nll_loss": 1.660516381263733,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.18601737916469574,
+ "rewards/margins": 0.005405900534242392,
+ "rewards/rejected": -0.19142326712608337,
+ "step": 600
+ },
+ {
+ "epoch": 0.96,
+ "eval_log_odds_chosen": 0.031927358359098434,
+ "eval_log_odds_ratio": -0.7613769173622131,
+ "eval_logits/chosen": 1.5999646186828613,
+ "eval_logits/rejected": 1.5608441829681396,
+ "eval_logps/chosen": -2.031277656555176,
+ "eval_logps/rejected": -2.0653114318847656,
+ "eval_loss": 1.842340111732483,
+ "eval_nll_loss": 1.7662022113800049,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.20312777161598206,
+ "eval_rewards/margins": 0.003403406124562025,
+ "eval_rewards/rejected": -0.20653115212917328,
+ "eval_runtime": 53.2205,
+ "eval_samples_per_second": 9.395,
+ "eval_steps_per_second": 4.697,
+ "step": 600
+ },
+ {
+ "epoch": 0.976,
+ "grad_norm": 0.17003707587718964,
+ "learning_rate": 6.003556609365738e-06,
+ "log_odds_chosen": 0.2910720705986023,
+ "log_odds_ratio": -0.6602123975753784,
+ "logits/chosen": 1.622191071510315,
+ "logits/rejected": 1.5799508094787598,
+ "logps/chosen": -1.8309406042099,
+ "logps/rejected": -2.0560221672058105,
+ "loss": 1.74339599609375,
+ "nll_loss": 1.6773748397827148,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": -0.1830940693616867,
+ "rewards/margins": 0.022508159279823303,
+ "rewards/rejected": -0.20560221374034882,
+ "step": 610
+ },
+ {
+ "epoch": 0.992,
+ "grad_norm": 0.23827488720417023,
+ "learning_rate": 5.956135151155898e-06,
+ "log_odds_chosen": 0.09837356209754944,
+ "log_odds_ratio": -0.7269458174705505,
+ "logits/chosen": 1.5235488414764404,
+ "logits/rejected": 1.539660096168518,
+ "logps/chosen": -1.8142712116241455,
+ "logps/rejected": -1.8749595880508423,
+ "loss": 1.7412891387939453,
+ "nll_loss": 1.6685945987701416,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.18142713606357574,
+ "rewards/margins": 0.00606885552406311,
+ "rewards/rejected": -0.18749597668647766,
+ "step": 620
+ },
+ {
+ "epoch": 1.008,
+ "grad_norm": 0.20535889267921448,
+ "learning_rate": 5.908713692946058e-06,
+ "log_odds_chosen": 0.22268417477607727,
+ "log_odds_ratio": -0.7025212049484253,
+ "logits/chosen": 1.5684313774108887,
+ "logits/rejected": 1.452976107597351,
+ "logps/chosen": -1.874678373336792,
+ "logps/rejected": -2.072539806365967,
+ "loss": 1.774791145324707,
+ "nll_loss": 1.7045390605926514,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.18746784329414368,
+ "rewards/margins": 0.019786134362220764,
+ "rewards/rejected": -0.20725397765636444,
+ "step": 630
+ },
+ {
+ "epoch": 1.024,
+ "grad_norm": 0.4164029061794281,
+ "learning_rate": 5.861292234736218e-06,
+ "log_odds_chosen": 0.13269853591918945,
+ "log_odds_ratio": -0.6831094026565552,
+ "logits/chosen": 1.5523912906646729,
+ "logits/rejected": 1.554421067237854,
+ "logps/chosen": -1.9016844034194946,
+ "logps/rejected": -2.0167248249053955,
+ "loss": 1.8452745437622071,
+ "nll_loss": 1.7769638299942017,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.19016845524311066,
+ "rewards/margins": 0.011504052206873894,
+ "rewards/rejected": -0.2016725093126297,
+ "step": 640
+ },
+ {
+ "epoch": 1.04,
+ "grad_norm": 0.33490127325057983,
+ "learning_rate": 5.813870776526378e-06,
+ "log_odds_chosen": -0.07161623239517212,
+ "log_odds_ratio": -0.8163315057754517,
+ "logits/chosen": 1.585946798324585,
+ "logits/rejected": 1.5961542129516602,
+ "logps/chosen": -2.0562708377838135,
+ "logps/rejected": -1.9879754781723022,
+ "loss": 1.7808271408081056,
+ "nll_loss": 1.6991941928863525,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.20562703907489777,
+ "rewards/margins": -0.006829507648944855,
+ "rewards/rejected": -0.1987975537776947,
+ "step": 650
+ },
+ {
+ "epoch": 1.04,
+ "eval_log_odds_chosen": 0.026153452694416046,
+ "eval_log_odds_ratio": -0.7626987099647522,
+ "eval_logits/chosen": 1.5905648469924927,
+ "eval_logits/rejected": 1.5565553903579712,
+ "eval_logps/chosen": -2.0193655490875244,
+ "eval_logps/rejected": -2.047696352005005,
+ "eval_loss": 1.8379414081573486,
+ "eval_nll_loss": 1.7616716623306274,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.20193655788898468,
+ "eval_rewards/margins": 0.002833074890077114,
+ "eval_rewards/rejected": -0.20476962625980377,
+ "eval_runtime": 53.3878,
+ "eval_samples_per_second": 9.365,
+ "eval_steps_per_second": 4.683,
+ "step": 650
+ },
+ {
+ "epoch": 1.056,
+ "grad_norm": 0.21944740414619446,
+ "learning_rate": 5.766449318316538e-06,
+ "log_odds_chosen": 0.09746942669153214,
+ "log_odds_ratio": -0.7083339095115662,
+ "logits/chosen": 1.5755507946014404,
+ "logits/rejected": 1.551966667175293,
+ "logps/chosen": -1.9140907526016235,
+ "logps/rejected": -1.9912245273590088,
+ "loss": 1.7622905731201173,
+ "nll_loss": 1.6914570331573486,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19140908122062683,
+ "rewards/margins": 0.00771337142214179,
+ "rewards/rejected": -0.19912245869636536,
+ "step": 660
+ },
+ {
+ "epoch": 1.072,
+ "grad_norm": 0.20419229567050934,
+ "learning_rate": 5.719027860106698e-06,
+ "log_odds_chosen": 0.10894493013620377,
+ "log_odds_ratio": -0.6956179738044739,
+ "logits/chosen": 1.7187572717666626,
+ "logits/rejected": 1.6571474075317383,
+ "logps/chosen": -1.9920858144760132,
+ "logps/rejected": -2.0984714031219482,
+ "loss": 1.802743148803711,
+ "nll_loss": 1.7331812381744385,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.1992085874080658,
+ "rewards/margins": 0.010638559237122536,
+ "rewards/rejected": -0.20984713733196259,
+ "step": 670
+ },
+ {
+ "epoch": 1.088,
+ "grad_norm": 0.1502259224653244,
+ "learning_rate": 5.671606401896859e-06,
+ "log_odds_chosen": 0.06736402213573456,
+ "log_odds_ratio": -0.7346867322921753,
+ "logits/chosen": 1.662166953086853,
+ "logits/rejected": 1.6347684860229492,
+ "logps/chosen": -1.9538319110870361,
+ "logps/rejected": -2.0279860496520996,
+ "loss": 1.7677694320678712,
+ "nll_loss": 1.694300651550293,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.1953831911087036,
+ "rewards/margins": 0.007415431551635265,
+ "rewards/rejected": -0.20279863476753235,
+ "step": 680
+ },
+ {
+ "epoch": 1.104,
+ "grad_norm": 0.1765543818473816,
+ "learning_rate": 5.624184943687018e-06,
+ "log_odds_chosen": 0.10347869247198105,
+ "log_odds_ratio": -0.7048436403274536,
+ "logits/chosen": 1.7106029987335205,
+ "logits/rejected": 1.6761833429336548,
+ "logps/chosen": -1.9210326671600342,
+ "logps/rejected": -2.0231428146362305,
+ "loss": 1.7810544967651367,
+ "nll_loss": 1.7105700969696045,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.1921032965183258,
+ "rewards/margins": 0.010210996493697166,
+ "rewards/rejected": -0.20231428742408752,
+ "step": 690
+ },
+ {
+ "epoch": 1.12,
+ "grad_norm": 0.26942944526672363,
+ "learning_rate": 5.576763485477178e-06,
+ "log_odds_chosen": 0.2199089527130127,
+ "log_odds_ratio": -0.6759704947471619,
+ "logits/chosen": 1.5313873291015625,
+ "logits/rejected": 1.4286370277404785,
+ "logps/chosen": -1.934305191040039,
+ "logps/rejected": -2.104454517364502,
+ "loss": 1.7423564910888671,
+ "nll_loss": 1.6747591495513916,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19343051314353943,
+ "rewards/margins": 0.01701495423913002,
+ "rewards/rejected": -0.21044547855854034,
+ "step": 700
+ },
+ {
+ "epoch": 1.12,
+ "eval_log_odds_chosen": 0.029196415096521378,
+ "eval_log_odds_ratio": -0.7604461312294006,
+ "eval_logits/chosen": 1.6299145221710205,
+ "eval_logits/rejected": 1.59683358669281,
+ "eval_logps/chosen": -2.016523599624634,
+ "eval_logps/rejected": -2.0469908714294434,
+ "eval_loss": 1.8337444067001343,
+ "eval_nll_loss": 1.757699728012085,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.20165236294269562,
+ "eval_rewards/margins": 0.0030467414762824774,
+ "eval_rewards/rejected": -0.2046990990638733,
+ "eval_runtime": 53.2721,
+ "eval_samples_per_second": 9.386,
+ "eval_steps_per_second": 4.693,
+ "step": 700
+ },
+ {
+ "epoch": 1.1360000000000001,
+ "grad_norm": 0.20357680320739746,
+ "learning_rate": 5.529342027267338e-06,
+ "log_odds_chosen": -0.03970063477754593,
+ "log_odds_ratio": -0.7776111364364624,
+ "logits/chosen": 1.688511848449707,
+ "logits/rejected": 1.5558183193206787,
+ "logps/chosen": -2.083556652069092,
+ "logps/rejected": -2.0501885414123535,
+ "loss": 1.849853515625,
+ "nll_loss": 1.7720924615859985,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.20835569500923157,
+ "rewards/margins": -0.0033368065487593412,
+ "rewards/rejected": -0.20501887798309326,
+ "step": 710
+ },
+ {
+ "epoch": 1.152,
+ "grad_norm": 0.20546825230121613,
+ "learning_rate": 5.481920569057499e-06,
+ "log_odds_chosen": 0.038821566849946976,
+ "log_odds_ratio": -0.7310380935668945,
+ "logits/chosen": 1.694077730178833,
+ "logits/rejected": 1.6808054447174072,
+ "logps/chosen": -1.8798902034759521,
+ "logps/rejected": -1.9120309352874756,
+ "loss": 1.8013761520385743,
+ "nll_loss": 1.728272795677185,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.1879890114068985,
+ "rewards/margins": 0.0032140829134732485,
+ "rewards/rejected": -0.19120310246944427,
+ "step": 720
+ },
+ {
+ "epoch": 1.168,
+ "grad_norm": 0.215702086687088,
+ "learning_rate": 5.434499110847659e-06,
+ "log_odds_chosen": -0.00810262281447649,
+ "log_odds_ratio": -0.7776116132736206,
+ "logits/chosen": 1.7699267864227295,
+ "logits/rejected": 1.8135936260223389,
+ "logps/chosen": -1.9550107717514038,
+ "logps/rejected": -1.9629520177841187,
+ "loss": 1.8207136154174806,
+ "nll_loss": 1.7429521083831787,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.19550108909606934,
+ "rewards/margins": 0.0007941495859995484,
+ "rewards/rejected": -0.19629523158073425,
+ "step": 730
+ },
+ {
+ "epoch": 1.184,
+ "grad_norm": 0.18298740684986115,
+ "learning_rate": 5.387077652637819e-06,
+ "log_odds_chosen": 0.18474040925502777,
+ "log_odds_ratio": -0.672413170337677,
+ "logits/chosen": 1.7683032751083374,
+ "logits/rejected": 1.8081409931182861,
+ "logps/chosen": -1.906714677810669,
+ "logps/rejected": -2.0783538818359375,
+ "loss": 1.7294036865234375,
+ "nll_loss": 1.662162184715271,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.19067144393920898,
+ "rewards/margins": 0.01716393604874611,
+ "rewards/rejected": -0.2078353613615036,
+ "step": 740
+ },
+ {
+ "epoch": 1.2,
+ "grad_norm": 0.13119018077850342,
+ "learning_rate": 5.339656194427978e-06,
+ "log_odds_chosen": 0.3178519606590271,
+ "log_odds_ratio": -0.6739884614944458,
+ "logits/chosen": 1.7732694149017334,
+ "logits/rejected": 1.6666276454925537,
+ "logps/chosen": -1.9240577220916748,
+ "logps/rejected": -2.2177700996398926,
+ "loss": 1.767026710510254,
+ "nll_loss": 1.6996276378631592,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": -0.19240576028823853,
+ "rewards/margins": 0.029371242970228195,
+ "rewards/rejected": -0.2217770367860794,
+ "step": 750
+ },
+ {
+ "epoch": 1.2,
+ "eval_log_odds_chosen": 0.024058617651462555,
+ "eval_log_odds_ratio": -0.763344407081604,
+ "eval_logits/chosen": 1.695021390914917,
+ "eval_logits/rejected": 1.6631685495376587,
+ "eval_logps/chosen": -2.0103158950805664,
+ "eval_logps/rejected": -2.0368571281433105,
+ "eval_loss": 1.8313435316085815,
+ "eval_nll_loss": 1.7550091743469238,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.20103158056735992,
+ "eval_rewards/margins": 0.0026541370898485184,
+ "eval_rewards/rejected": -0.2036857306957245,
+ "eval_runtime": 53.3008,
+ "eval_samples_per_second": 9.381,
+ "eval_steps_per_second": 4.69,
+ "step": 750
+ },
+ {
+ "epoch": 1.216,
+ "grad_norm": 0.1924748718738556,
+ "learning_rate": 5.292234736218139e-06,
+ "log_odds_chosen": 0.07262767851352692,
+ "log_odds_ratio": -0.7266906499862671,
+ "logits/chosen": 1.7665992975234985,
+ "logits/rejected": 1.6683565378189087,
+ "logps/chosen": -1.960656762123108,
+ "logps/rejected": -2.0233612060546875,
+ "loss": 1.7435152053833007,
+ "nll_loss": 1.6708459854125977,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.19606567919254303,
+ "rewards/margins": 0.006270436104387045,
+ "rewards/rejected": -0.20233610272407532,
+ "step": 760
+ },
+ {
+ "epoch": 1.232,
+ "grad_norm": 0.16488949954509735,
+ "learning_rate": 5.244813278008299e-06,
+ "log_odds_chosen": 0.2733311653137207,
+ "log_odds_ratio": -0.6829133629798889,
+ "logits/chosen": 1.707964539527893,
+ "logits/rejected": 1.6122114658355713,
+ "logps/chosen": -1.90310537815094,
+ "logps/rejected": -2.1515767574310303,
+ "loss": 1.7614355087280273,
+ "nll_loss": 1.6931442022323608,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.190310537815094,
+ "rewards/margins": 0.024847140535712242,
+ "rewards/rejected": -0.2151576727628708,
+ "step": 770
+ },
+ {
+ "epoch": 1.248,
+ "grad_norm": 0.1734672635793686,
+ "learning_rate": 5.197391819798459e-06,
+ "log_odds_chosen": 0.13369520008563995,
+ "log_odds_ratio": -0.675721287727356,
+ "logits/chosen": 1.79034423828125,
+ "logits/rejected": 1.811118721961975,
+ "logps/chosen": -1.8553498983383179,
+ "logps/rejected": -1.9695507287979126,
+ "loss": 1.7561630249023437,
+ "nll_loss": 1.6885910034179688,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.1855349838733673,
+ "rewards/margins": 0.011420061811804771,
+ "rewards/rejected": -0.19695505499839783,
+ "step": 780
+ },
+ {
+ "epoch": 1.264,
+ "grad_norm": 0.2151460200548172,
+ "learning_rate": 5.149970361588619e-06,
+ "log_odds_chosen": -0.15310701727867126,
+ "log_odds_ratio": -0.8164137005805969,
+ "logits/chosen": 1.8224090337753296,
+ "logits/rejected": 1.8069593906402588,
+ "logps/chosen": -2.0459067821502686,
+ "logps/rejected": -1.9169931411743164,
+ "loss": 1.8225961685180665,
+ "nll_loss": 1.7409546375274658,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": -0.20459070801734924,
+ "rewards/margins": -0.012891381978988647,
+ "rewards/rejected": -0.1916993111371994,
+ "step": 790
+ },
+ {
+ "epoch": 1.28,
+ "grad_norm": 0.23022110760211945,
+ "learning_rate": 5.102548903378779e-06,
+ "log_odds_chosen": 0.17419373989105225,
+ "log_odds_ratio": -0.665838360786438,
+ "logits/chosen": 1.745001196861267,
+ "logits/rejected": 1.7248725891113281,
+ "logps/chosen": -1.9748042821884155,
+ "logps/rejected": -2.124072313308716,
+ "loss": 1.7632785797119142,
+ "nll_loss": 1.6966947317123413,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.1974804550409317,
+ "rewards/margins": 0.014926811680197716,
+ "rewards/rejected": -0.21240726113319397,
+ "step": 800
+ },
+ {
+ "epoch": 1.28,
+ "eval_log_odds_chosen": 0.0298521239310503,
+ "eval_log_odds_ratio": -0.7603721022605896,
+ "eval_logits/chosen": 1.7159233093261719,
+ "eval_logits/rejected": 1.6853290796279907,
+ "eval_logps/chosen": -2.008664608001709,
+ "eval_logps/rejected": -2.0401124954223633,
+ "eval_loss": 1.8289790153503418,
+ "eval_nll_loss": 1.7529417276382446,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.2008664608001709,
+ "eval_rewards/margins": 0.0031447894871234894,
+ "eval_rewards/rejected": -0.20401126146316528,
+ "eval_runtime": 53.2154,
+ "eval_samples_per_second": 9.396,
+ "eval_steps_per_second": 4.698,
+ "step": 800
+ },
+ {
+ "epoch": 1.296,
+ "grad_norm": 0.1864694058895111,
+ "learning_rate": 5.055127445168939e-06,
+ "log_odds_chosen": -0.02357078157365322,
+ "log_odds_ratio": -0.7400822043418884,
+ "logits/chosen": 1.7981126308441162,
+ "logits/rejected": 1.7508691549301147,
+ "logps/chosen": -2.0388572216033936,
+ "logps/rejected": -2.024317979812622,
+ "loss": 1.8243423461914063,
+ "nll_loss": 1.7503341436386108,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.20388571918010712,
+ "rewards/margins": -0.001453907461836934,
+ "rewards/rejected": -0.20243176817893982,
+ "step": 810
+ },
+ {
+ "epoch": 1.312,
+ "grad_norm": 0.18279023468494415,
+ "learning_rate": 5.007705986959099e-06,
+ "log_odds_chosen": -0.011245409026741982,
+ "log_odds_ratio": -0.8205389976501465,
+ "logits/chosen": 1.726945161819458,
+ "logits/rejected": 1.6804269552230835,
+ "logps/chosen": -2.0612854957580566,
+ "logps/rejected": -2.0304996967315674,
+ "loss": 1.8141677856445313,
+ "nll_loss": 1.7321138381958008,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.2061285525560379,
+ "rewards/margins": -0.00307856360450387,
+ "rewards/rejected": -0.20304997265338898,
+ "step": 820
+ },
+ {
+ "epoch": 1.328,
+ "grad_norm": 0.194142147898674,
+ "learning_rate": 4.960284528749259e-06,
+ "log_odds_chosen": -0.10107870399951935,
+ "log_odds_ratio": -0.8795238733291626,
+ "logits/chosen": 1.6292463541030884,
+ "logits/rejected": 1.6127665042877197,
+ "logps/chosen": -1.9763052463531494,
+ "logps/rejected": -1.872083067893982,
+ "loss": 1.7666927337646485,
+ "nll_loss": 1.67874014377594,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.19763052463531494,
+ "rewards/margins": -0.010422185063362122,
+ "rewards/rejected": -0.18720832467079163,
+ "step": 830
+ },
+ {
+ "epoch": 1.3439999999999999,
+ "grad_norm": 0.18887867033481598,
+ "learning_rate": 4.9128630705394195e-06,
+ "log_odds_chosen": -0.0421462245285511,
+ "log_odds_ratio": -0.7816078662872314,
+ "logits/chosen": 1.6638116836547852,
+ "logits/rejected": 1.5873361825942993,
+ "logps/chosen": -1.980798363685608,
+ "logps/rejected": -1.949537992477417,
+ "loss": 1.793659782409668,
+ "nll_loss": 1.715498924255371,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": -0.19807983934879303,
+ "rewards/margins": -0.003126043127849698,
+ "rewards/rejected": -0.1949537992477417,
+ "step": 840
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "grad_norm": 0.2071056365966797,
+ "learning_rate": 4.865441612329579e-06,
+ "log_odds_chosen": 0.20311307907104492,
+ "log_odds_ratio": -0.6712511777877808,
+ "logits/chosen": 1.5604090690612793,
+ "logits/rejected": 1.6046473979949951,
+ "logps/chosen": -1.8914225101470947,
+ "logps/rejected": -2.0582470893859863,
+ "loss": 1.7601259231567383,
+ "nll_loss": 1.6930010318756104,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.18914227187633514,
+ "rewards/margins": 0.016682462766766548,
+ "rewards/rejected": -0.20582470297813416,
+ "step": 850
+ },
+ {
+ "epoch": 1.3599999999999999,
+ "eval_log_odds_chosen": 0.027712374925613403,
+ "eval_log_odds_ratio": -0.7594860196113586,
+ "eval_logits/chosen": 1.666095495223999,
+ "eval_logits/rejected": 1.633544683456421,
+ "eval_logps/chosen": -2.002187967300415,
+ "eval_logps/rejected": -2.0309674739837646,
+ "eval_loss": 1.824683666229248,
+ "eval_nll_loss": 1.7487353086471558,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": -0.2002188116312027,
+ "eval_rewards/margins": 0.0028779483400285244,
+ "eval_rewards/rejected": -0.20309676229953766,
+ "eval_runtime": 53.1899,
+ "eval_samples_per_second": 9.4,
+ "eval_steps_per_second": 4.7,
+ "step": 850
+ },
+ {
+ "epoch": 1.376,
+ "grad_norm": 0.1590212732553482,
+ "learning_rate": 4.818020154119739e-06,
+ "log_odds_chosen": 0.15208786725997925,
+ "log_odds_ratio": -0.6756098866462708,
+ "logits/chosen": 1.6706883907318115,
+ "logits/rejected": 1.6443389654159546,
+ "logps/chosen": -1.9115455150604248,
+ "logps/rejected": -2.0435924530029297,
+ "loss": 1.7545036315917968,
+ "nll_loss": 1.686942458152771,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.19115455448627472,
+ "rewards/margins": 0.013204696588218212,
+ "rewards/rejected": -0.2043592482805252,
+ "step": 860
+ },
+ {
+ "epoch": 1.392,
+ "grad_norm": 0.15062956511974335,
+ "learning_rate": 4.770598695909899e-06,
+ "log_odds_chosen": 0.2899920344352722,
+ "log_odds_ratio": -0.6323431730270386,
+ "logits/chosen": 1.7012927532196045,
+ "logits/rejected": 1.6665140390396118,
+ "logps/chosen": -1.8794418573379517,
+ "logps/rejected": -2.131627321243286,
+ "loss": 1.817972183227539,
+ "nll_loss": 1.7547378540039062,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": -0.1879442036151886,
+ "rewards/margins": 0.025218535214662552,
+ "rewards/rejected": -0.21316273510456085,
+ "step": 870
+ },
+ {
+ "epoch": 1.408,
+ "grad_norm": 0.23191672563552856,
+ "learning_rate": 4.7231772377000595e-06,
+ "log_odds_chosen": 0.08231823146343231,
+ "log_odds_ratio": -0.7166029810905457,
+ "logits/chosen": 1.739164113998413,
+ "logits/rejected": 1.7106529474258423,
+ "logps/chosen": -1.9510221481323242,
+ "logps/rejected": -2.0197978019714355,
+ "loss": 1.7590656280517578,
+ "nll_loss": 1.6874055862426758,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.1951022446155548,
+ "rewards/margins": 0.006877546664327383,
+ "rewards/rejected": -0.20197978615760803,
+ "step": 880
+ },
+ {
+ "epoch": 1.424,
+ "grad_norm": 0.2074817270040512,
+ "learning_rate": 4.6757557794902194e-06,
+ "log_odds_chosen": 0.09656596183776855,
+ "log_odds_ratio": -0.708441436290741,
+ "logits/chosen": 1.7313311100006104,
+ "logits/rejected": 1.6554332971572876,
+ "logps/chosen": -1.9648983478546143,
+ "logps/rejected": -2.0429916381835938,
+ "loss": 1.77806396484375,
+ "nll_loss": 1.7072200775146484,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.19648981094360352,
+ "rewards/margins": 0.007809334900230169,
+ "rewards/rejected": -0.20429913699626923,
+ "step": 890
+ },
+ {
+ "epoch": 1.44,
+ "grad_norm": 0.17369434237480164,
+ "learning_rate": 4.6283343212803794e-06,
+ "log_odds_chosen": 0.02706352435052395,
+ "log_odds_ratio": -0.7602623701095581,
+ "logits/chosen": 1.6460374593734741,
+ "logits/rejected": 1.6657674312591553,
+ "logps/chosen": -2.0089709758758545,
+ "logps/rejected": -2.0413382053375244,
+ "loss": 1.788330078125,
+ "nll_loss": 1.7123043537139893,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": -0.20089709758758545,
+ "rewards/margins": 0.003236697521060705,
+ "rewards/rejected": -0.2041337937116623,
+ "step": 900
+ },
+ {
+ "epoch": 1.44,
+ "eval_log_odds_chosen": 0.029239021241664886,
+ "eval_log_odds_ratio": -0.7607759237289429,
+ "eval_logits/chosen": 1.7303014993667603,
+ "eval_logits/rejected": 1.7004859447479248,
+ "eval_logps/chosen": -2.001692056655884,
+ "eval_logps/rejected": -2.031625747680664,
+ "eval_loss": 1.8236775398254395,
+ "eval_nll_loss": 1.7475998401641846,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.20016922056674957,
+ "eval_rewards/margins": 0.002993369475007057,
+ "eval_rewards/rejected": -0.20316259562969208,
+ "eval_runtime": 53.2446,
+ "eval_samples_per_second": 9.391,
+ "eval_steps_per_second": 4.695,
+ "step": 900
+ },
+ {
+ "epoch": 1.456,
+ "grad_norm": 0.20154239237308502,
+ "learning_rate": 4.580912863070539e-06,
+ "log_odds_chosen": 0.1900930106639862,
+ "log_odds_ratio": -0.6721845269203186,
+ "logits/chosen": 1.7324788570404053,
+ "logits/rejected": 1.6658681631088257,
+ "logps/chosen": -1.8505785465240479,
+ "logps/rejected": -2.0169787406921387,
+ "loss": 1.6994483947753907,
+ "nll_loss": 1.6322300434112549,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.1850578486919403,
+ "rewards/margins": 0.01664000190794468,
+ "rewards/rejected": -0.20169785618782043,
+ "step": 910
+ },
+ {
+ "epoch": 1.472,
+ "grad_norm": 0.16085605323314667,
+ "learning_rate": 4.5334914048606994e-06,
+ "log_odds_chosen": 0.09199506789445877,
+ "log_odds_ratio": -0.6975706219673157,
+ "logits/chosen": 1.7461503744125366,
+ "logits/rejected": 1.8116916418075562,
+ "logps/chosen": -1.8981988430023193,
+ "logps/rejected": -1.982012391090393,
+ "loss": 1.7591039657592773,
+ "nll_loss": 1.6893469095230103,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.18981988728046417,
+ "rewards/margins": 0.00838138721883297,
+ "rewards/rejected": -0.1982012689113617,
+ "step": 920
+ },
+ {
+ "epoch": 1.488,
+ "grad_norm": 0.15088561177253723,
+ "learning_rate": 4.4860699466508594e-06,
+ "log_odds_chosen": 0.09973030537366867,
+ "log_odds_ratio": -0.6972100138664246,
+ "logits/chosen": 1.8033208847045898,
+ "logits/rejected": 1.7551723718643188,
+ "logps/chosen": -1.8274751901626587,
+ "logps/rejected": -1.9026908874511719,
+ "loss": 1.7895126342773438,
+ "nll_loss": 1.7197917699813843,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.18274752795696259,
+ "rewards/margins": 0.0075215683318674564,
+ "rewards/rejected": -0.1902690976858139,
+ "step": 930
+ },
+ {
+ "epoch": 1.504,
+ "grad_norm": 0.13715308904647827,
+ "learning_rate": 4.4386484884410194e-06,
+ "log_odds_chosen": 0.07654011994600296,
+ "log_odds_ratio": -0.7238441705703735,
+ "logits/chosen": 1.7913706302642822,
+ "logits/rejected": 1.768434762954712,
+ "logps/chosen": -1.8972448110580444,
+ "logps/rejected": -1.952164888381958,
+ "loss": 1.749094581604004,
+ "nll_loss": 1.6767103672027588,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.18972447514533997,
+ "rewards/margins": 0.005491999443620443,
+ "rewards/rejected": -0.19521646201610565,
+ "step": 940
+ },
+ {
+ "epoch": 1.52,
+ "grad_norm": 0.1736515462398529,
+ "learning_rate": 4.3912270302311794e-06,
+ "log_odds_chosen": -0.02450474537909031,
+ "log_odds_ratio": -0.7657513618469238,
+ "logits/chosen": 1.8066694736480713,
+ "logits/rejected": 1.7840650081634521,
+ "logps/chosen": -1.9759057760238647,
+ "logps/rejected": -1.9469897747039795,
+ "loss": 1.7701189041137695,
+ "nll_loss": 1.6935440301895142,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": -0.19759058952331543,
+ "rewards/margins": -0.00289161060936749,
+ "rewards/rejected": -0.1946989744901657,
+ "step": 950
+ },
+ {
+ "epoch": 1.52,
+ "eval_log_odds_chosen": 0.03024761565029621,
+ "eval_log_odds_ratio": -0.7591846585273743,
+ "eval_logits/chosen": 1.7475262880325317,
+ "eval_logits/rejected": 1.718088150024414,
+ "eval_logps/chosen": -2.0002715587615967,
+ "eval_logps/rejected": -2.0316338539123535,
+ "eval_loss": 1.821697473526001,
+ "eval_nll_loss": 1.745779037475586,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.20002716779708862,
+ "eval_rewards/margins": 0.003136243438348174,
+ "eval_rewards/rejected": -0.20316338539123535,
+ "eval_runtime": 53.1737,
+ "eval_samples_per_second": 9.403,
+ "eval_steps_per_second": 4.702,
+ "step": 950
+ },
+ {
+ "epoch": 1.536,
+ "grad_norm": 0.16075001657009125,
+ "learning_rate": 4.343805572021339e-06,
+ "log_odds_chosen": 0.108977772295475,
+ "log_odds_ratio": -0.69707852602005,
+ "logits/chosen": 1.9180431365966797,
+ "logits/rejected": 1.8730928897857666,
+ "logps/chosen": -1.9387962818145752,
+ "logps/rejected": -2.0468382835388184,
+ "loss": 1.8717756271362305,
+ "nll_loss": 1.8020679950714111,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.193879634141922,
+ "rewards/margins": 0.010804187506437302,
+ "rewards/rejected": -0.2046838104724884,
+ "step": 960
+ },
+ {
+ "epoch": 1.552,
+ "grad_norm": 0.21932974457740784,
+ "learning_rate": 4.296384113811499e-06,
+ "log_odds_chosen": 0.0989312157034874,
+ "log_odds_ratio": -0.7125958800315857,
+ "logits/chosen": 1.7110919952392578,
+ "logits/rejected": 1.728533387184143,
+ "logps/chosen": -1.9263118505477905,
+ "logps/rejected": -2.00539231300354,
+ "loss": 1.7788570404052735,
+ "nll_loss": 1.7075974941253662,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.19263119995594025,
+ "rewards/margins": 0.007908042520284653,
+ "rewards/rejected": -0.200539231300354,
+ "step": 970
+ },
+ {
+ "epoch": 1.568,
+ "grad_norm": 0.1886725127696991,
+ "learning_rate": 4.248962655601659e-06,
+ "log_odds_chosen": 0.2252909392118454,
+ "log_odds_ratio": -0.6316526532173157,
+ "logits/chosen": 1.69448983669281,
+ "logits/rejected": 1.6676839590072632,
+ "logps/chosen": -1.8613817691802979,
+ "logps/rejected": -2.057375431060791,
+ "loss": 1.7345258712768554,
+ "nll_loss": 1.6713603734970093,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": -0.18613818287849426,
+ "rewards/margins": 0.01959935761988163,
+ "rewards/rejected": -0.20573754608631134,
+ "step": 980
+ },
+ {
+ "epoch": 1.584,
+ "grad_norm": 0.1785261482000351,
+ "learning_rate": 4.201541197391819e-06,
+ "log_odds_chosen": 0.18465228378772736,
+ "log_odds_ratio": -0.6800971627235413,
+ "logits/chosen": 1.693155288696289,
+ "logits/rejected": 1.6254899501800537,
+ "logps/chosen": -1.922136902809143,
+ "logps/rejected": -2.0776562690734863,
+ "loss": 1.7461135864257813,
+ "nll_loss": 1.6781036853790283,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19221369922161102,
+ "rewards/margins": 0.01555191446095705,
+ "rewards/rejected": -0.2077656239271164,
+ "step": 990
+ },
+ {
+ "epoch": 1.6,
+ "grad_norm": 0.19836267828941345,
+ "learning_rate": 4.15411973918198e-06,
+ "log_odds_chosen": 0.16690519452095032,
+ "log_odds_ratio": -0.6907414793968201,
+ "logits/chosen": 1.78627610206604,
+ "logits/rejected": 1.6995677947998047,
+ "logps/chosen": -1.9538847208023071,
+ "logps/rejected": -2.0953662395477295,
+ "loss": 1.8010309219360352,
+ "nll_loss": 1.7319567203521729,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.19538846611976624,
+ "rewards/margins": 0.014148158952593803,
+ "rewards/rejected": -0.2095366269350052,
+ "step": 1000
+ },
+ {
+ "epoch": 1.6,
+ "eval_log_odds_chosen": 0.027905963361263275,
+ "eval_log_odds_ratio": -0.7601120471954346,
+ "eval_logits/chosen": 1.7237838506698608,
+ "eval_logits/rejected": 1.6969308853149414,
+ "eval_logps/chosen": -1.9980021715164185,
+ "eval_logps/rejected": -2.027127742767334,
+ "eval_loss": 1.8202314376831055,
+ "eval_nll_loss": 1.74422025680542,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": -0.19980023801326752,
+ "eval_rewards/margins": 0.0029125558212399483,
+ "eval_rewards/rejected": -0.2027127742767334,
+ "eval_runtime": 53.17,
+ "eval_samples_per_second": 9.404,
+ "eval_steps_per_second": 4.702,
+ "step": 1000
+ },
+ {
+ "epoch": 1.616,
+ "grad_norm": 0.2467470020055771,
+ "learning_rate": 4.106698280972139e-06,
+ "log_odds_chosen": 0.1805444061756134,
+ "log_odds_ratio": -0.6592761874198914,
+ "logits/chosen": 1.7704235315322876,
+ "logits/rejected": 1.8331016302108765,
+ "logps/chosen": -1.9225962162017822,
+ "logps/rejected": -2.0647215843200684,
+ "loss": 1.7989471435546875,
+ "nll_loss": 1.7330195903778076,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.19225959479808807,
+ "rewards/margins": 0.014212558977305889,
+ "rewards/rejected": -0.20647215843200684,
+ "step": 1010
+ },
+ {
+ "epoch": 1.6320000000000001,
+ "grad_norm": 0.21627788245677948,
+ "learning_rate": 4.059276822762299e-06,
+ "log_odds_chosen": 0.19997625052928925,
+ "log_odds_ratio": -0.6929422616958618,
+ "logits/chosen": 1.7510759830474854,
+ "logits/rejected": 1.7739620208740234,
+ "logps/chosen": -1.8798294067382812,
+ "logps/rejected": -2.0255956649780273,
+ "loss": 1.7694732666015625,
+ "nll_loss": 1.7001785039901733,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.1879829615354538,
+ "rewards/margins": 0.014576609246432781,
+ "rewards/rejected": -0.20255956053733826,
+ "step": 1020
+ },
+ {
+ "epoch": 1.6480000000000001,
+ "grad_norm": 0.14596332609653473,
+ "learning_rate": 4.011855364552459e-06,
+ "log_odds_chosen": 0.08883054554462433,
+ "log_odds_ratio": -0.8015972375869751,
+ "logits/chosen": 1.7602264881134033,
+ "logits/rejected": 1.7082704305648804,
+ "logps/chosen": -1.987823247909546,
+ "logps/rejected": -2.0893337726593018,
+ "loss": 1.768618392944336,
+ "nll_loss": 1.6884586811065674,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": -0.1987823098897934,
+ "rewards/margins": 0.010151032358407974,
+ "rewards/rejected": -0.20893338322639465,
+ "step": 1030
+ },
+ {
+ "epoch": 1.6640000000000001,
+ "grad_norm": 0.21014870703220367,
+ "learning_rate": 3.96443390634262e-06,
+ "log_odds_chosen": 0.20391491055488586,
+ "log_odds_ratio": -0.6863493323326111,
+ "logits/chosen": 1.6720123291015625,
+ "logits/rejected": 1.6457217931747437,
+ "logps/chosen": -1.8646005392074585,
+ "logps/rejected": -2.0223734378814697,
+ "loss": 1.6701059341430664,
+ "nll_loss": 1.601470947265625,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.18646006286144257,
+ "rewards/margins": 0.015777289867401123,
+ "rewards/rejected": -0.2022373378276825,
+ "step": 1040
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "grad_norm": 0.16176322102546692,
+ "learning_rate": 3.917012448132779e-06,
+ "log_odds_chosen": 0.1454552561044693,
+ "log_odds_ratio": -0.7019529342651367,
+ "logits/chosen": 1.7449884414672852,
+ "logits/rejected": 1.7378383874893188,
+ "logps/chosen": -1.9023891687393188,
+ "logps/rejected": -2.027350902557373,
+ "loss": 1.7594617843627929,
+ "nll_loss": 1.6892664432525635,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.19023892283439636,
+ "rewards/margins": 0.012496160343289375,
+ "rewards/rejected": -0.20273509621620178,
+ "step": 1050
+ },
+ {
+ "epoch": 1.6800000000000002,
+ "eval_log_odds_chosen": 0.027047131210565567,
+ "eval_log_odds_ratio": -0.7607759237289429,
+ "eval_logits/chosen": 1.752084493637085,
+ "eval_logits/rejected": 1.726226568222046,
+ "eval_logps/chosen": -1.9921609163284302,
+ "eval_logps/rejected": -2.0205676555633545,
+ "eval_loss": 1.8179807662963867,
+ "eval_nll_loss": 1.7419030666351318,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.1992160975933075,
+ "eval_rewards/margins": 0.002840682165697217,
+ "eval_rewards/rejected": -0.20205678045749664,
+ "eval_runtime": 53.3183,
+ "eval_samples_per_second": 9.378,
+ "eval_steps_per_second": 4.689,
+ "step": 1050
+ },
+ {
+ "epoch": 1.696,
+ "grad_norm": 0.19938912987709045,
+ "learning_rate": 3.86959098992294e-06,
+ "log_odds_chosen": -0.06742805987596512,
+ "log_odds_ratio": -0.8377403020858765,
+ "logits/chosen": 1.626834511756897,
+ "logits/rejected": 1.6696176528930664,
+ "logps/chosen": -2.060955286026001,
+ "logps/rejected": -2.0158073902130127,
+ "loss": 1.7704721450805665,
+ "nll_loss": 1.6866979598999023,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.20609550178050995,
+ "rewards/margins": -0.004514762666076422,
+ "rewards/rejected": -0.2015807330608368,
+ "step": 1060
+ },
+ {
+ "epoch": 1.712,
+ "grad_norm": 0.20868155360221863,
+ "learning_rate": 3.8221695317131e-06,
+ "log_odds_chosen": 0.2089385986328125,
+ "log_odds_ratio": -0.6722566485404968,
+ "logits/chosen": 1.8093836307525635,
+ "logits/rejected": 1.7903034687042236,
+ "logps/chosen": -1.8783435821533203,
+ "logps/rejected": -2.0554146766662598,
+ "loss": 1.7485641479492187,
+ "nll_loss": 1.6813385486602783,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": -0.18783435225486755,
+ "rewards/margins": 0.017707128077745438,
+ "rewards/rejected": -0.2055414617061615,
+ "step": 1070
+ },
+ {
+ "epoch": 1.728,
+ "grad_norm": 0.17801117897033691,
+ "learning_rate": 3.77474807350326e-06,
+ "log_odds_chosen": -0.08966319262981415,
+ "log_odds_ratio": -0.8123272061347961,
+ "logits/chosen": 1.8008207082748413,
+ "logits/rejected": 1.7534077167510986,
+ "logps/chosen": -1.9915469884872437,
+ "logps/rejected": -1.9123618602752686,
+ "loss": 1.743325424194336,
+ "nll_loss": 1.662092924118042,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": -0.19915470480918884,
+ "rewards/margins": -0.007918504998087883,
+ "rewards/rejected": -0.1912361979484558,
+ "step": 1080
+ },
+ {
+ "epoch": 1.744,
+ "grad_norm": 0.12194813042879105,
+ "learning_rate": 3.7273266152934202e-06,
+ "log_odds_chosen": -0.11027821153402328,
+ "log_odds_ratio": -0.8348665237426758,
+ "logits/chosen": 1.8450218439102173,
+ "logits/rejected": 1.8813024759292603,
+ "logps/chosen": -1.9666748046875,
+ "logps/rejected": -1.8820431232452393,
+ "loss": 1.8245243072509765,
+ "nll_loss": 1.7410377264022827,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.19666747748851776,
+ "rewards/margins": -0.008463181555271149,
+ "rewards/rejected": -0.1882043182849884,
+ "step": 1090
+ },
+ {
+ "epoch": 1.76,
+ "grad_norm": 0.16607670485973358,
+ "learning_rate": 3.67990515708358e-06,
+ "log_odds_chosen": 0.16690079867839813,
+ "log_odds_ratio": -0.791414737701416,
+ "logits/chosen": 1.7368566989898682,
+ "logits/rejected": 1.6425182819366455,
+ "logps/chosen": -1.9711978435516357,
+ "logps/rejected": -2.154827356338501,
+ "loss": 1.762497901916504,
+ "nll_loss": 1.6833562850952148,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.197119802236557,
+ "rewards/margins": 0.01836293376982212,
+ "rewards/rejected": -0.2154827117919922,
+ "step": 1100
+ },
+ {
+ "epoch": 1.76,
+ "eval_log_odds_chosen": 0.029554516077041626,
+ "eval_log_odds_ratio": -0.7602362036705017,
+ "eval_logits/chosen": 1.7638286352157593,
+ "eval_logits/rejected": 1.7391140460968018,
+ "eval_logps/chosen": -1.9919315576553345,
+ "eval_logps/rejected": -2.0221877098083496,
+ "eval_loss": 1.815456509590149,
+ "eval_nll_loss": 1.7394328117370605,
+ "eval_rewards/accuracies": 0.5379999876022339,
+ "eval_rewards/chosen": -0.19919316470623016,
+ "eval_rewards/margins": 0.0030256081372499466,
+ "eval_rewards/rejected": -0.20221877098083496,
+ "eval_runtime": 53.2855,
+ "eval_samples_per_second": 9.383,
+ "eval_steps_per_second": 4.692,
+ "step": 1100
+ },
+ {
+ "epoch": 1.776,
+ "grad_norm": 0.18845929205417633,
+ "learning_rate": 3.6324836988737402e-06,
+ "log_odds_chosen": 0.14334020018577576,
+ "log_odds_ratio": -0.6870154142379761,
+ "logits/chosen": 1.7760759592056274,
+ "logits/rejected": 1.7927675247192383,
+ "logps/chosen": -1.8970282077789307,
+ "logps/rejected": -2.0081281661987305,
+ "loss": 1.7522649765014648,
+ "nll_loss": 1.683563232421875,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.1897028237581253,
+ "rewards/margins": 0.011109992861747742,
+ "rewards/rejected": -0.20081281661987305,
+ "step": 1110
+ },
+ {
+ "epoch": 1.792,
+ "grad_norm": 0.16642089188098907,
+ "learning_rate": 3.5850622406639e-06,
+ "log_odds_chosen": 0.12583844363689423,
+ "log_odds_ratio": -0.705106258392334,
+ "logits/chosen": 1.7084674835205078,
+ "logits/rejected": 1.7520864009857178,
+ "logps/chosen": -2.0032966136932373,
+ "logps/rejected": -2.121325969696045,
+ "loss": 1.7604061126708985,
+ "nll_loss": 1.6898952722549438,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.20032963156700134,
+ "rewards/margins": 0.011802928522229195,
+ "rewards/rejected": -0.21213257312774658,
+ "step": 1120
+ },
+ {
+ "epoch": 1.808,
+ "grad_norm": 0.26899227499961853,
+ "learning_rate": 3.53764078245406e-06,
+ "log_odds_chosen": 0.04995861276984215,
+ "log_odds_ratio": -0.725029706954956,
+ "logits/chosen": 1.7520374059677124,
+ "logits/rejected": 1.7503259181976318,
+ "logps/chosen": -1.935824990272522,
+ "logps/rejected": -1.9734337329864502,
+ "loss": 1.7329391479492187,
+ "nll_loss": 1.6604360342025757,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19358250498771667,
+ "rewards/margins": 0.0037608749698847532,
+ "rewards/rejected": -0.1973433792591095,
+ "step": 1130
+ },
+ {
+ "epoch": 1.8239999999999998,
+ "grad_norm": 0.21221907436847687,
+ "learning_rate": 3.49021932424422e-06,
+ "log_odds_chosen": 0.07214214652776718,
+ "log_odds_ratio": -0.7553919553756714,
+ "logits/chosen": 1.669593095779419,
+ "logits/rejected": 1.698304533958435,
+ "logps/chosen": -1.9012906551361084,
+ "logps/rejected": -1.941220998764038,
+ "loss": 1.742647933959961,
+ "nll_loss": 1.6671091318130493,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19012905657291412,
+ "rewards/margins": 0.003993041347712278,
+ "rewards/rejected": -0.1941221058368683,
+ "step": 1140
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "grad_norm": 0.14006629586219788,
+ "learning_rate": 3.4427978660343806e-06,
+ "log_odds_chosen": 0.1731099784374237,
+ "log_odds_ratio": -0.6817640662193298,
+ "logits/chosen": 1.7826522588729858,
+ "logits/rejected": 1.7909168004989624,
+ "logps/chosen": -1.9337961673736572,
+ "logps/rejected": -2.0631959438323975,
+ "loss": 1.7244823455810547,
+ "nll_loss": 1.6563060283660889,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.19337964057922363,
+ "rewards/margins": 0.012939969077706337,
+ "rewards/rejected": -0.20631961524486542,
+ "step": 1150
+ },
+ {
+ "epoch": 1.8399999999999999,
+ "eval_log_odds_chosen": 0.026097064837813377,
+ "eval_log_odds_ratio": -0.7620403170585632,
+ "eval_logits/chosen": 1.757448673248291,
+ "eval_logits/rejected": 1.7327815294265747,
+ "eval_logps/chosen": -1.9885954856872559,
+ "eval_logps/rejected": -2.016005754470825,
+ "eval_loss": 1.8154748678207397,
+ "eval_nll_loss": 1.7392706871032715,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": -0.1988595575094223,
+ "eval_rewards/margins": 0.0027410530019551516,
+ "eval_rewards/rejected": -0.2016005963087082,
+ "eval_runtime": 53.1886,
+ "eval_samples_per_second": 9.401,
+ "eval_steps_per_second": 4.7,
+ "step": 1150
+ },
+ {
+ "epoch": 1.8559999999999999,
+ "grad_norm": 0.2366112917661667,
+ "learning_rate": 3.39537640782454e-06,
+ "log_odds_chosen": 0.16461338102817535,
+ "log_odds_ratio": -0.6795645952224731,
+ "logits/chosen": 1.6777217388153076,
+ "logits/rejected": 1.7064342498779297,
+ "logps/chosen": -1.7678172588348389,
+ "logps/rejected": -1.9083778858184814,
+ "loss": 1.684773063659668,
+ "nll_loss": 1.6168162822723389,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.17678174376487732,
+ "rewards/margins": 0.014056049287319183,
+ "rewards/rejected": -0.1908378005027771,
+ "step": 1160
+ },
+ {
+ "epoch": 1.8719999999999999,
+ "grad_norm": 0.17859044671058655,
+ "learning_rate": 3.3479549496147006e-06,
+ "log_odds_chosen": 0.029232267290353775,
+ "log_odds_ratio": -0.7420104742050171,
+ "logits/chosen": 1.7861093282699585,
+ "logits/rejected": 1.7708505392074585,
+ "logps/chosen": -1.9521045684814453,
+ "logps/rejected": -1.9810863733291626,
+ "loss": 1.8265951156616211,
+ "nll_loss": 1.7523937225341797,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.19521045684814453,
+ "rewards/margins": 0.0028981652576476336,
+ "rewards/rejected": -0.19810864329338074,
+ "step": 1170
+ },
+ {
+ "epoch": 1.888,
+ "grad_norm": 0.20688028633594513,
+ "learning_rate": 3.30053349140486e-06,
+ "log_odds_chosen": 0.04731910675764084,
+ "log_odds_ratio": -0.7147753238677979,
+ "logits/chosen": 1.7652273178100586,
+ "logits/rejected": 1.7344173192977905,
+ "logps/chosen": -1.845867395401001,
+ "logps/rejected": -1.8912805318832397,
+ "loss": 1.6884075164794923,
+ "nll_loss": 1.6169300079345703,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.18458673357963562,
+ "rewards/margins": 0.0045413183979690075,
+ "rewards/rejected": -0.1891280561685562,
+ "step": 1180
+ },
+ {
+ "epoch": 1.904,
+ "grad_norm": 0.15886512398719788,
+ "learning_rate": 3.2531120331950206e-06,
+ "log_odds_chosen": 0.11942672729492188,
+ "log_odds_ratio": -0.6965886354446411,
+ "logits/chosen": 1.8913081884384155,
+ "logits/rejected": 1.8108240365982056,
+ "logps/chosen": -1.9068269729614258,
+ "logps/rejected": -2.0091569423675537,
+ "loss": 1.7399011611938477,
+ "nll_loss": 1.6702420711517334,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.19068269431591034,
+ "rewards/margins": 0.010232994332909584,
+ "rewards/rejected": -0.20091569423675537,
+ "step": 1190
+ },
+ {
+ "epoch": 1.92,
+ "grad_norm": 0.1977105587720871,
+ "learning_rate": 3.2056905749851806e-06,
+ "log_odds_chosen": 0.1517036259174347,
+ "log_odds_ratio": -0.710344672203064,
+ "logits/chosen": 1.7977663278579712,
+ "logits/rejected": 1.7616544961929321,
+ "logps/chosen": -1.8372957706451416,
+ "logps/rejected": -1.9735416173934937,
+ "loss": 1.7458187103271485,
+ "nll_loss": 1.6747843027114868,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.18372957408428192,
+ "rewards/margins": 0.013624586164951324,
+ "rewards/rejected": -0.19735416769981384,
+ "step": 1200
+ },
+ {
+ "epoch": 1.92,
+ "eval_log_odds_chosen": 0.028467997908592224,
+ "eval_log_odds_ratio": -0.7607693076133728,
+ "eval_logits/chosen": 1.7756459712982178,
+ "eval_logits/rejected": 1.74927818775177,
+ "eval_logps/chosen": -1.98622727394104,
+ "eval_logps/rejected": -2.0155580043792725,
+ "eval_loss": 1.8127713203430176,
+ "eval_nll_loss": 1.7366943359375,
+ "eval_rewards/accuracies": 0.5400000214576721,
+ "eval_rewards/chosen": -0.19862274825572968,
+ "eval_rewards/margins": 0.0029331075493246317,
+ "eval_rewards/rejected": -0.20155581831932068,
+ "eval_runtime": 53.2002,
+ "eval_samples_per_second": 9.398,
+ "eval_steps_per_second": 4.699,
+ "step": 1200
+ },
+ {
+ "epoch": 1.936,
+ "grad_norm": 0.179337278008461,
+ "learning_rate": 3.1582691167753406e-06,
+ "log_odds_chosen": 0.07452553510665894,
+ "log_odds_ratio": -0.7306004762649536,
+ "logits/chosen": 1.699223279953003,
+ "logits/rejected": 1.7217528820037842,
+ "logps/chosen": -1.920426607131958,
+ "logps/rejected": -1.985222578048706,
+ "loss": 1.7297235488891602,
+ "nll_loss": 1.6566635370254517,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19204264879226685,
+ "rewards/margins": 0.006479593925178051,
+ "rewards/rejected": -0.19852226972579956,
+ "step": 1210
+ },
+ {
+ "epoch": 1.952,
+ "grad_norm": 0.19489862024784088,
+ "learning_rate": 3.1108476585655006e-06,
+ "log_odds_chosen": -0.06367218494415283,
+ "log_odds_ratio": -0.7718582153320312,
+ "logits/chosen": 1.7101131677627563,
+ "logits/rejected": 1.7847702503204346,
+ "logps/chosen": -2.0594370365142822,
+ "logps/rejected": -2.006824016571045,
+ "loss": 1.8293329238891602,
+ "nll_loss": 1.7521469593048096,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.20594370365142822,
+ "rewards/margins": -0.0052613066509366035,
+ "rewards/rejected": -0.2006824016571045,
+ "step": 1220
+ },
+ {
+ "epoch": 1.968,
+ "grad_norm": 0.1728423684835434,
+ "learning_rate": 3.063426200355661e-06,
+ "log_odds_chosen": 0.1272212713956833,
+ "log_odds_ratio": -0.6985543966293335,
+ "logits/chosen": 1.8437741994857788,
+ "logits/rejected": 1.7424207925796509,
+ "logps/chosen": -1.9680700302124023,
+ "logps/rejected": -2.072535753250122,
+ "loss": 1.7730525970458983,
+ "nll_loss": 1.7031971216201782,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.19680698215961456,
+ "rewards/margins": 0.010446581058204174,
+ "rewards/rejected": -0.20725354552268982,
+ "step": 1230
+ },
+ {
+ "epoch": 1.984,
+ "grad_norm": 0.1812407225370407,
+ "learning_rate": 3.0160047421458206e-06,
+ "log_odds_chosen": 0.07307764887809753,
+ "log_odds_ratio": -0.7265633344650269,
+ "logits/chosen": 1.7150694131851196,
+ "logits/rejected": 1.6220579147338867,
+ "logps/chosen": -1.8851966857910156,
+ "logps/rejected": -1.9514576196670532,
+ "loss": 1.7142372131347656,
+ "nll_loss": 1.6415809392929077,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.1885196715593338,
+ "rewards/margins": 0.006626097019761801,
+ "rewards/rejected": -0.19514577090740204,
+ "step": 1240
+ },
+ {
+ "epoch": 2.0,
+ "grad_norm": 0.21866431832313538,
+ "learning_rate": 2.968583283935981e-06,
+ "log_odds_chosen": 0.09864354133605957,
+ "log_odds_ratio": -0.7356117963790894,
+ "logits/chosen": 1.8142271041870117,
+ "logits/rejected": 1.7990251779556274,
+ "logps/chosen": -2.0075278282165527,
+ "logps/rejected": -2.080134391784668,
+ "loss": 1.826127052307129,
+ "nll_loss": 1.7525659799575806,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.20075277984142303,
+ "rewards/margins": 0.007260681129992008,
+ "rewards/rejected": -0.20801346004009247,
+ "step": 1250
+ },
+ {
+ "epoch": 2.0,
+ "eval_log_odds_chosen": 0.032603029161691666,
+ "eval_log_odds_ratio": -0.7592425346374512,
+ "eval_logits/chosen": 1.7272087335586548,
+ "eval_logits/rejected": 1.701614260673523,
+ "eval_logps/chosen": -1.9889473915100098,
+ "eval_logps/rejected": -2.021564483642578,
+ "eval_loss": 1.8116061687469482,
+ "eval_nll_loss": 1.7356820106506348,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.19889472424983978,
+ "eval_rewards/margins": 0.003261698642745614,
+ "eval_rewards/rejected": -0.2021564543247223,
+ "eval_runtime": 53.2359,
+ "eval_samples_per_second": 9.392,
+ "eval_steps_per_second": 4.696,
+ "step": 1250
+ },
+ {
+ "epoch": 2.016,
+ "grad_norm": 0.20461951196193695,
+ "learning_rate": 2.9211618257261406e-06,
+ "log_odds_chosen": 0.2361210584640503,
+ "log_odds_ratio": -0.6880162954330444,
+ "logits/chosen": 1.7546977996826172,
+ "logits/rejected": 1.6977161169052124,
+ "logps/chosen": -1.8337652683258057,
+ "logps/rejected": -2.0438666343688965,
+ "loss": 1.7133241653442384,
+ "nll_loss": 1.6445229053497314,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.1833765208721161,
+ "rewards/margins": 0.02101016975939274,
+ "rewards/rejected": -0.20438671112060547,
+ "step": 1260
+ },
+ {
+ "epoch": 2.032,
+ "grad_norm": 0.18985678255558014,
+ "learning_rate": 2.873740367516301e-06,
+ "log_odds_chosen": -0.006541083566844463,
+ "log_odds_ratio": -0.7571430802345276,
+ "logits/chosen": 1.7348535060882568,
+ "logits/rejected": 1.7211494445800781,
+ "logps/chosen": -1.9841225147247314,
+ "logps/rejected": -1.9915357828140259,
+ "loss": 1.757932472229004,
+ "nll_loss": 1.682218313217163,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.19841225445270538,
+ "rewards/margins": 0.0007413355633616447,
+ "rewards/rejected": -0.1991535723209381,
+ "step": 1270
+ },
+ {
+ "epoch": 2.048,
+ "grad_norm": 0.1865416169166565,
+ "learning_rate": 2.8263189093064614e-06,
+ "log_odds_chosen": 0.1962173581123352,
+ "log_odds_ratio": -0.6550278663635254,
+ "logits/chosen": 1.8164112567901611,
+ "logits/rejected": 1.8721809387207031,
+ "logps/chosen": -1.8975756168365479,
+ "logps/rejected": -2.0585479736328125,
+ "loss": 1.7741456985473634,
+ "nll_loss": 1.7086429595947266,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": -0.1897575706243515,
+ "rewards/margins": 0.016097230836749077,
+ "rewards/rejected": -0.20585481822490692,
+ "step": 1280
+ },
+ {
+ "epoch": 2.064,
+ "grad_norm": 0.22608967125415802,
+ "learning_rate": 2.778897451096621e-06,
+ "log_odds_chosen": -0.025246893987059593,
+ "log_odds_ratio": -0.7659823298454285,
+ "logits/chosen": 1.784846544265747,
+ "logits/rejected": 1.765973687171936,
+ "logps/chosen": -1.9505832195281982,
+ "logps/rejected": -1.9396007061004639,
+ "loss": 1.7477140426635742,
+ "nll_loss": 1.6711156368255615,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.19505834579467773,
+ "rewards/margins": -0.0010982556268572807,
+ "rewards/rejected": -0.19396010041236877,
+ "step": 1290
+ },
+ {
+ "epoch": 2.08,
+ "grad_norm": 0.17561422288417816,
+ "learning_rate": 2.7314759928867814e-06,
+ "log_odds_chosen": 0.05084152892231941,
+ "log_odds_ratio": -0.7318845391273499,
+ "logits/chosen": 1.7349708080291748,
+ "logits/rejected": 1.7631950378417969,
+ "logps/chosen": -2.0045390129089355,
+ "logps/rejected": -2.0500693321228027,
+ "loss": 1.8187286376953125,
+ "nll_loss": 1.7455400228500366,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.20045390725135803,
+ "rewards/margins": 0.004553015343844891,
+ "rewards/rejected": -0.2050069272518158,
+ "step": 1300
+ },
+ {
+ "epoch": 2.08,
+ "eval_log_odds_chosen": 0.03401584550738335,
+ "eval_log_odds_ratio": -0.7578551769256592,
+ "eval_logits/chosen": 1.7610890865325928,
+ "eval_logits/rejected": 1.7374581098556519,
+ "eval_logps/chosen": -1.9867304563522339,
+ "eval_logps/rejected": -2.020127773284912,
+ "eval_loss": 1.809829592704773,
+ "eval_nll_loss": 1.734044075012207,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": -0.1986730545759201,
+ "eval_rewards/margins": 0.0033397264778614044,
+ "eval_rewards/rejected": -0.2020127773284912,
+ "eval_runtime": 53.3296,
+ "eval_samples_per_second": 9.376,
+ "eval_steps_per_second": 4.688,
+ "step": 1300
+ },
+ {
+ "epoch": 2.096,
+ "grad_norm": 0.1651468127965927,
+ "learning_rate": 2.684054534676941e-06,
+ "log_odds_chosen": 0.09325788170099258,
+ "log_odds_ratio": -0.7075183987617493,
+ "logits/chosen": 1.7508665323257446,
+ "logits/rejected": 1.8297382593154907,
+ "logps/chosen": -1.863616704940796,
+ "logps/rejected": -1.9378494024276733,
+ "loss": 1.7222581863403321,
+ "nll_loss": 1.6515061855316162,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.1863616704940796,
+ "rewards/margins": 0.007423256523907185,
+ "rewards/rejected": -0.1937849372625351,
+ "step": 1310
+ },
+ {
+ "epoch": 2.112,
+ "grad_norm": 0.16485963761806488,
+ "learning_rate": 2.6366330764671014e-06,
+ "log_odds_chosen": 0.0591282956302166,
+ "log_odds_ratio": -0.7139178514480591,
+ "logits/chosen": 1.7827821969985962,
+ "logits/rejected": 1.7864627838134766,
+ "logps/chosen": -1.9632068872451782,
+ "logps/rejected": -2.0267958641052246,
+ "loss": 1.8074531555175781,
+ "nll_loss": 1.7360610961914062,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.19632069766521454,
+ "rewards/margins": 0.006358909420669079,
+ "rewards/rejected": -0.2026796042919159,
+ "step": 1320
+ },
+ {
+ "epoch": 2.128,
+ "grad_norm": 0.16226664185523987,
+ "learning_rate": 2.5892116182572614e-06,
+ "log_odds_chosen": 0.18185576796531677,
+ "log_odds_ratio": -0.6743385791778564,
+ "logits/chosen": 1.95318603515625,
+ "logits/rejected": 1.914075493812561,
+ "logps/chosen": -1.8613958358764648,
+ "logps/rejected": -2.020050048828125,
+ "loss": 1.747477149963379,
+ "nll_loss": 1.6800432205200195,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.18613959848880768,
+ "rewards/margins": 0.015865424647927284,
+ "rewards/rejected": -0.2020050287246704,
+ "step": 1330
+ },
+ {
+ "epoch": 2.144,
+ "grad_norm": 0.18390049040317535,
+ "learning_rate": 2.5417901600474214e-06,
+ "log_odds_chosen": 0.14728814363479614,
+ "log_odds_ratio": -0.6720021367073059,
+ "logits/chosen": 1.7619132995605469,
+ "logits/rejected": 1.7984975576400757,
+ "logps/chosen": -1.774634599685669,
+ "logps/rejected": -1.890054702758789,
+ "loss": 1.7107378005981446,
+ "nll_loss": 1.6435377597808838,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.17746345698833466,
+ "rewards/margins": 0.011542020365595818,
+ "rewards/rejected": -0.18900546431541443,
+ "step": 1340
+ },
+ {
+ "epoch": 2.16,
+ "grad_norm": 0.23228837549686432,
+ "learning_rate": 2.4943687018375814e-06,
+ "log_odds_chosen": -0.015979360789060593,
+ "log_odds_ratio": -0.7752725481987,
+ "logits/chosen": 1.828468680381775,
+ "logits/rejected": 1.7667421102523804,
+ "logps/chosen": -1.8710674047470093,
+ "logps/rejected": -1.8696765899658203,
+ "loss": 1.752641487121582,
+ "nll_loss": 1.6751142740249634,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.18710675835609436,
+ "rewards/margins": -0.00013908911205362529,
+ "rewards/rejected": -0.186967670917511,
+ "step": 1350
+ },
+ {
+ "epoch": 2.16,
+ "eval_log_odds_chosen": 0.03191310539841652,
+ "eval_log_odds_ratio": -0.7588774561882019,
+ "eval_logits/chosen": 1.7761640548706055,
+ "eval_logits/rejected": 1.7524988651275635,
+ "eval_logps/chosen": -1.9843788146972656,
+ "eval_logps/rejected": -2.016181230545044,
+ "eval_loss": 1.8089464902877808,
+ "eval_nll_loss": 1.7330589294433594,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": -0.1984378695487976,
+ "eval_rewards/margins": 0.0031802442390471697,
+ "eval_rewards/rejected": -0.20161812007427216,
+ "eval_runtime": 53.3932,
+ "eval_samples_per_second": 9.364,
+ "eval_steps_per_second": 4.682,
+ "step": 1350
+ },
+ {
+ "epoch": 2.176,
+ "grad_norm": 0.18132326006889343,
+ "learning_rate": 2.446947243627742e-06,
+ "log_odds_chosen": 0.0305715911090374,
+ "log_odds_ratio": -0.7504960894584656,
+ "logits/chosen": 1.7744085788726807,
+ "logits/rejected": 1.7282178401947021,
+ "logps/chosen": -1.9199994802474976,
+ "logps/rejected": -1.9519102573394775,
+ "loss": 1.7735115051269532,
+ "nll_loss": 1.6984617710113525,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.19199995696544647,
+ "rewards/margins": 0.0031910568941384554,
+ "rewards/rejected": -0.19519101083278656,
+ "step": 1360
+ },
+ {
+ "epoch": 2.192,
+ "grad_norm": 0.23645685613155365,
+ "learning_rate": 2.3995257854179014e-06,
+ "log_odds_chosen": 0.14658379554748535,
+ "log_odds_ratio": -0.6893800497055054,
+ "logits/chosen": 1.7729228734970093,
+ "logits/rejected": 1.7737751007080078,
+ "logps/chosen": -1.8243513107299805,
+ "logps/rejected": -1.9623281955718994,
+ "loss": 1.6700077056884766,
+ "nll_loss": 1.601069688796997,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.18243512511253357,
+ "rewards/margins": 0.013797680847346783,
+ "rewards/rejected": -0.19623282551765442,
+ "step": 1370
+ },
+ {
+ "epoch": 2.208,
+ "grad_norm": 0.15323151648044586,
+ "learning_rate": 2.352104327208062e-06,
+ "log_odds_chosen": 0.1115313395857811,
+ "log_odds_ratio": -0.6943892240524292,
+ "logits/chosen": 1.8522207736968994,
+ "logits/rejected": 1.8103185892105103,
+ "logps/chosen": -1.9417545795440674,
+ "logps/rejected": -2.0404393672943115,
+ "loss": 1.8153194427490233,
+ "nll_loss": 1.745880365371704,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.19417545199394226,
+ "rewards/margins": 0.009868469089269638,
+ "rewards/rejected": -0.2040439397096634,
+ "step": 1380
+ },
+ {
+ "epoch": 2.224,
+ "grad_norm": 0.21786808967590332,
+ "learning_rate": 2.3046828689982214e-06,
+ "log_odds_chosen": 0.05500102788209915,
+ "log_odds_ratio": -0.7242174744606018,
+ "logits/chosen": 1.838382363319397,
+ "logits/rejected": 1.8554975986480713,
+ "logps/chosen": -1.99395751953125,
+ "logps/rejected": -2.045518636703491,
+ "loss": 1.7693500518798828,
+ "nll_loss": 1.69692862033844,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.19939574599266052,
+ "rewards/margins": 0.005156124476343393,
+ "rewards/rejected": -0.20455189049243927,
+ "step": 1390
+ },
+ {
+ "epoch": 2.24,
+ "grad_norm": 0.19683599472045898,
+ "learning_rate": 2.257261410788382e-06,
+ "log_odds_chosen": 0.15008209645748138,
+ "log_odds_ratio": -0.6654382944107056,
+ "logits/chosen": 1.7431036233901978,
+ "logits/rejected": 1.731433629989624,
+ "logps/chosen": -1.8933416604995728,
+ "logps/rejected": -2.0205039978027344,
+ "loss": 1.7619529724121095,
+ "nll_loss": 1.6954091787338257,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": -0.18933415412902832,
+ "rewards/margins": 0.012716245837509632,
+ "rewards/rejected": -0.20205040276050568,
+ "step": 1400
+ },
+ {
+ "epoch": 2.24,
+ "eval_log_odds_chosen": 0.032577622681856155,
+ "eval_log_odds_ratio": -0.7580828070640564,
+ "eval_logits/chosen": 1.812838077545166,
+ "eval_logits/rejected": 1.790816068649292,
+ "eval_logps/chosen": -1.9839202165603638,
+ "eval_logps/rejected": -2.0164804458618164,
+ "eval_loss": 1.808388590812683,
+ "eval_nll_loss": 1.732580304145813,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": -0.19839203357696533,
+ "eval_rewards/margins": 0.0032560396939516068,
+ "eval_rewards/rejected": -0.2016480565071106,
+ "eval_runtime": 53.2287,
+ "eval_samples_per_second": 9.393,
+ "eval_steps_per_second": 4.697,
+ "step": 1400
+ },
+ {
+ "epoch": 2.2560000000000002,
+ "grad_norm": 0.2112002670764923,
+ "learning_rate": 2.209839952578542e-06,
+ "log_odds_chosen": 0.2766411304473877,
+ "log_odds_ratio": -0.6827906370162964,
+ "logits/chosen": 1.862683892250061,
+ "logits/rejected": 1.8868215084075928,
+ "logps/chosen": -1.915928840637207,
+ "logps/rejected": -2.172340154647827,
+ "loss": 1.7464689254760741,
+ "nll_loss": 1.6781896352767944,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.19159288704395294,
+ "rewards/margins": 0.025641148909926414,
+ "rewards/rejected": -0.2172340452671051,
+ "step": 1410
+ },
+ {
+ "epoch": 2.2720000000000002,
+ "grad_norm": 0.17736928164958954,
+ "learning_rate": 2.162418494368702e-06,
+ "log_odds_chosen": 0.06808965653181076,
+ "log_odds_ratio": -0.7222734689712524,
+ "logits/chosen": 1.7881934642791748,
+ "logits/rejected": 1.7765629291534424,
+ "logps/chosen": -1.9753538370132446,
+ "logps/rejected": -2.0361990928649902,
+ "loss": 1.7180429458618165,
+ "nll_loss": 1.6458156108856201,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.19753539562225342,
+ "rewards/margins": 0.0060845063999295235,
+ "rewards/rejected": -0.20361991226673126,
+ "step": 1420
+ },
+ {
+ "epoch": 2.288,
+ "grad_norm": 0.17618675529956818,
+ "learning_rate": 2.114997036158862e-06,
+ "log_odds_chosen": 0.08562587201595306,
+ "log_odds_ratio": -0.7272316813468933,
+ "logits/chosen": 1.7778189182281494,
+ "logits/rejected": 1.70413339138031,
+ "logps/chosen": -1.9755465984344482,
+ "logps/rejected": -2.048130512237549,
+ "loss": 1.8506391525268555,
+ "nll_loss": 1.7779159545898438,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.19755466282367706,
+ "rewards/margins": 0.007258380763232708,
+ "rewards/rejected": -0.20481304824352264,
+ "step": 1430
+ },
+ {
+ "epoch": 2.304,
+ "grad_norm": 0.2521407902240753,
+ "learning_rate": 2.067575577949022e-06,
+ "log_odds_chosen": 0.11862572282552719,
+ "log_odds_ratio": -0.7083112001419067,
+ "logits/chosen": 1.8262131214141846,
+ "logits/rejected": 1.8660300970077515,
+ "logps/chosen": -1.885925054550171,
+ "logps/rejected": -1.9840829372406006,
+ "loss": 1.734739112854004,
+ "nll_loss": 1.6639076471328735,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.18859252333641052,
+ "rewards/margins": 0.009815781377255917,
+ "rewards/rejected": -0.19840829074382782,
+ "step": 1440
+ },
+ {
+ "epoch": 2.32,
+ "grad_norm": 0.17723698914051056,
+ "learning_rate": 2.0201541197391818e-06,
+ "log_odds_chosen": 0.12196867167949677,
+ "log_odds_ratio": -0.6998961567878723,
+ "logits/chosen": 1.731156587600708,
+ "logits/rejected": 1.7110185623168945,
+ "logps/chosen": -1.9318151473999023,
+ "logps/rejected": -2.035583972930908,
+ "loss": 1.7764135360717774,
+ "nll_loss": 1.7064237594604492,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.19318152964115143,
+ "rewards/margins": 0.01037687249481678,
+ "rewards/rejected": -0.20355840027332306,
+ "step": 1450
+ },
+ {
+ "epoch": 2.32,
+ "eval_log_odds_chosen": 0.033345215022563934,
+ "eval_log_odds_ratio": -0.7588449716567993,
+ "eval_logits/chosen": 1.7935512065887451,
+ "eval_logits/rejected": 1.7720842361450195,
+ "eval_logps/chosen": -1.9827799797058105,
+ "eval_logps/rejected": -2.0161314010620117,
+ "eval_loss": 1.8074625730514526,
+ "eval_nll_loss": 1.73157799243927,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.19827799499034882,
+ "eval_rewards/margins": 0.0033351616002619267,
+ "eval_rewards/rejected": -0.2016131579875946,
+ "eval_runtime": 53.2316,
+ "eval_samples_per_second": 9.393,
+ "eval_steps_per_second": 4.696,
+ "step": 1450
+ },
+ {
+ "epoch": 2.336,
+ "grad_norm": 0.16808928549289703,
+ "learning_rate": 1.972732661529342e-06,
+ "log_odds_chosen": -0.025525325909256935,
+ "log_odds_ratio": -0.7912946939468384,
+ "logits/chosen": 1.8211339712142944,
+ "logits/rejected": 1.8124920129776,
+ "logps/chosen": -1.856103539466858,
+ "logps/rejected": -1.829656958580017,
+ "loss": 1.7368072509765624,
+ "nll_loss": 1.657678246498108,
+ "rewards/accuracies": 0.4375,
+ "rewards/chosen": -0.18561036884784698,
+ "rewards/margins": -0.002644673688337207,
+ "rewards/rejected": -0.1829657107591629,
+ "step": 1460
+ },
+ {
+ "epoch": 2.352,
+ "grad_norm": 0.18361903727054596,
+ "learning_rate": 1.925311203319502e-06,
+ "log_odds_chosen": 0.005142314825206995,
+ "log_odds_ratio": -0.8110187649726868,
+ "logits/chosen": 1.825429916381836,
+ "logits/rejected": 1.7501672506332397,
+ "logps/chosen": -2.0082828998565674,
+ "logps/rejected": -2.031491279602051,
+ "loss": 1.7301027297973632,
+ "nll_loss": 1.649000883102417,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.20082826912403107,
+ "rewards/margins": 0.0023208647035062313,
+ "rewards/rejected": -0.20314912497997284,
+ "step": 1470
+ },
+ {
+ "epoch": 2.368,
+ "grad_norm": 0.19813720881938934,
+ "learning_rate": 1.877889745109662e-06,
+ "log_odds_chosen": 0.1795060634613037,
+ "log_odds_ratio": -0.7214691042900085,
+ "logits/chosen": 1.8677520751953125,
+ "logits/rejected": 1.7595751285552979,
+ "logps/chosen": -1.943748116493225,
+ "logps/rejected": -2.1158955097198486,
+ "loss": 1.7257247924804688,
+ "nll_loss": 1.6535780429840088,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.19437481462955475,
+ "rewards/margins": 0.017214741557836533,
+ "rewards/rejected": -0.21158954501152039,
+ "step": 1480
+ },
+ {
+ "epoch": 2.384,
+ "grad_norm": 0.1770639717578888,
+ "learning_rate": 1.830468286899822e-06,
+ "log_odds_chosen": 0.06336086243391037,
+ "log_odds_ratio": -0.7451409101486206,
+ "logits/chosen": 1.8294976949691772,
+ "logits/rejected": 1.835206389427185,
+ "logps/chosen": -1.982630968093872,
+ "logps/rejected": -2.0292210578918457,
+ "loss": 1.777639389038086,
+ "nll_loss": 1.703125238418579,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.19826309382915497,
+ "rewards/margins": 0.004659010097384453,
+ "rewards/rejected": -0.20292210578918457,
+ "step": 1490
+ },
+ {
+ "epoch": 2.4,
+ "grad_norm": 0.22681613266468048,
+ "learning_rate": 1.7830468286899822e-06,
+ "log_odds_chosen": 0.010824749246239662,
+ "log_odds_ratio": -0.7730723023414612,
+ "logits/chosen": 1.7533601522445679,
+ "logits/rejected": 1.706671953201294,
+ "logps/chosen": -2.045074939727783,
+ "logps/rejected": -2.0547096729278564,
+ "loss": 1.7060966491699219,
+ "nll_loss": 1.6287896633148193,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.2045074999332428,
+ "rewards/margins": 0.0009634803864173591,
+ "rewards/rejected": -0.2054709643125534,
+ "step": 1500
+ },
+ {
+ "epoch": 2.4,
+ "eval_log_odds_chosen": 0.03045523911714554,
+ "eval_log_odds_ratio": -0.7589520215988159,
+ "eval_logits/chosen": 1.8010083436965942,
+ "eval_logits/rejected": 1.7791268825531006,
+ "eval_logps/chosen": -1.9805711507797241,
+ "eval_logps/rejected": -2.011178731918335,
+ "eval_loss": 1.8059344291687012,
+ "eval_nll_loss": 1.730039358139038,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": -0.1980571150779724,
+ "eval_rewards/margins": 0.0030607657972723246,
+ "eval_rewards/rejected": -0.2011178731918335,
+ "eval_runtime": 53.2537,
+ "eval_samples_per_second": 9.389,
+ "eval_steps_per_second": 4.695,
+ "step": 1500
+ },
+ {
+ "epoch": 2.416,
+ "grad_norm": 0.20802171528339386,
+ "learning_rate": 1.7356253704801422e-06,
+ "log_odds_chosen": 0.1136036291718483,
+ "log_odds_ratio": -0.706071138381958,
+ "logits/chosen": 1.731825828552246,
+ "logits/rejected": 1.7013626098632812,
+ "logps/chosen": -1.938544511795044,
+ "logps/rejected": -2.011859178543091,
+ "loss": 1.7333692550659179,
+ "nll_loss": 1.6627620458602905,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.1938544511795044,
+ "rewards/margins": 0.0073314448818564415,
+ "rewards/rejected": -0.2011858969926834,
+ "step": 1510
+ },
+ {
+ "epoch": 2.432,
+ "grad_norm": 0.20122499763965607,
+ "learning_rate": 1.6882039122703022e-06,
+ "log_odds_chosen": -0.014995841309428215,
+ "log_odds_ratio": -0.797789454460144,
+ "logits/chosen": 1.764818787574768,
+ "logits/rejected": 1.819456696510315,
+ "logps/chosen": -1.9951610565185547,
+ "logps/rejected": -1.9713351726531982,
+ "loss": 1.778700065612793,
+ "nll_loss": 1.6989209651947021,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.19951608777046204,
+ "rewards/margins": -0.0023825743701308966,
+ "rewards/rejected": -0.19713351130485535,
+ "step": 1520
+ },
+ {
+ "epoch": 2.448,
+ "grad_norm": 0.22172124683856964,
+ "learning_rate": 1.6407824540604622e-06,
+ "log_odds_chosen": -0.010859908536076546,
+ "log_odds_ratio": -0.7840942144393921,
+ "logits/chosen": 1.803431749343872,
+ "logits/rejected": 1.80246901512146,
+ "logps/chosen": -1.905480146408081,
+ "logps/rejected": -1.9286388158798218,
+ "loss": 1.7533109664916993,
+ "nll_loss": 1.6749016046524048,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19054800271987915,
+ "rewards/margins": 0.0023158867843449116,
+ "rewards/rejected": -0.19286389648914337,
+ "step": 1530
+ },
+ {
+ "epoch": 2.464,
+ "grad_norm": 0.18010367453098297,
+ "learning_rate": 1.5933609958506222e-06,
+ "log_odds_chosen": 0.16830947995185852,
+ "log_odds_ratio": -0.6711615920066833,
+ "logits/chosen": 1.7893339395523071,
+ "logits/rejected": 1.7532241344451904,
+ "logps/chosen": -1.8904199600219727,
+ "logps/rejected": -2.019512176513672,
+ "loss": 1.698394775390625,
+ "nll_loss": 1.6312782764434814,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.18904200196266174,
+ "rewards/margins": 0.01290925033390522,
+ "rewards/rejected": -0.20195123553276062,
+ "step": 1540
+ },
+ {
+ "epoch": 2.48,
+ "grad_norm": 0.21497370302677155,
+ "learning_rate": 1.5459395376407824e-06,
+ "log_odds_chosen": 0.02481182850897312,
+ "log_odds_ratio": -0.7431084513664246,
+ "logits/chosen": 1.7848838567733765,
+ "logits/rejected": 1.7826042175292969,
+ "logps/chosen": -1.9306681156158447,
+ "logps/rejected": -1.939648985862732,
+ "loss": 1.762909507751465,
+ "nll_loss": 1.6885986328125,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.1930668205022812,
+ "rewards/margins": 0.0008980954298749566,
+ "rewards/rejected": -0.19396492838859558,
+ "step": 1550
+ },
+ {
+ "epoch": 2.48,
+ "eval_log_odds_chosen": 0.03250642493367195,
+ "eval_log_odds_ratio": -0.758705198764801,
+ "eval_logits/chosen": 1.7901184558868408,
+ "eval_logits/rejected": 1.7681894302368164,
+ "eval_logps/chosen": -1.9813108444213867,
+ "eval_logps/rejected": -2.0135128498077393,
+ "eval_loss": 1.8048373460769653,
+ "eval_nll_loss": 1.7289668321609497,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": -0.19813106954097748,
+ "eval_rewards/margins": 0.0032202075235545635,
+ "eval_rewards/rejected": -0.20135128498077393,
+ "eval_runtime": 53.2861,
+ "eval_samples_per_second": 9.383,
+ "eval_steps_per_second": 4.692,
+ "step": 1550
+ },
+ {
+ "epoch": 2.496,
+ "grad_norm": 0.18746241927146912,
+ "learning_rate": 1.4985180794309424e-06,
+ "log_odds_chosen": -0.03534708544611931,
+ "log_odds_ratio": -0.7781793475151062,
+ "logits/chosen": 1.7682994604110718,
+ "logits/rejected": 1.7273156642913818,
+ "logps/chosen": -1.9587968587875366,
+ "logps/rejected": -1.939034104347229,
+ "loss": 1.7451881408691405,
+ "nll_loss": 1.6673702001571655,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.19587968289852142,
+ "rewards/margins": -0.001976292347535491,
+ "rewards/rejected": -0.19390340149402618,
+ "step": 1560
+ },
+ {
+ "epoch": 2.512,
+ "grad_norm": 0.22451210021972656,
+ "learning_rate": 1.4510966212211024e-06,
+ "log_odds_chosen": 0.22642970085144043,
+ "log_odds_ratio": -0.6601076126098633,
+ "logits/chosen": 1.8025572299957275,
+ "logits/rejected": 1.753969430923462,
+ "logps/chosen": -1.8275161981582642,
+ "logps/rejected": -2.014209270477295,
+ "loss": 1.7348913192749023,
+ "nll_loss": 1.6688802242279053,
+ "rewards/accuracies": 0.6625000238418579,
+ "rewards/chosen": -0.1827516257762909,
+ "rewards/margins": 0.01866934448480606,
+ "rewards/rejected": -0.20142097771167755,
+ "step": 1570
+ },
+ {
+ "epoch": 2.528,
+ "grad_norm": 0.13330276310443878,
+ "learning_rate": 1.4036751630112624e-06,
+ "log_odds_chosen": 0.13351204991340637,
+ "log_odds_ratio": -0.6781107187271118,
+ "logits/chosen": 1.8947384357452393,
+ "logits/rejected": 1.9055246114730835,
+ "logps/chosen": -1.8429749011993408,
+ "logps/rejected": -1.9477179050445557,
+ "loss": 1.7747116088867188,
+ "nll_loss": 1.7069003582000732,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.18429750204086304,
+ "rewards/margins": 0.010474300011992455,
+ "rewards/rejected": -0.19477181136608124,
+ "step": 1580
+ },
+ {
+ "epoch": 2.544,
+ "grad_norm": 0.1976797878742218,
+ "learning_rate": 1.3562537048014226e-06,
+ "log_odds_chosen": 0.2522992491722107,
+ "log_odds_ratio": -0.6384642124176025,
+ "logits/chosen": 1.9045484066009521,
+ "logits/rejected": 1.8206974267959595,
+ "logps/chosen": -1.824180245399475,
+ "logps/rejected": -2.0245985984802246,
+ "loss": 1.7275480270385741,
+ "nll_loss": 1.6637012958526611,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.18241801857948303,
+ "rewards/margins": 0.02004183456301689,
+ "rewards/rejected": -0.20245984196662903,
+ "step": 1590
+ },
+ {
+ "epoch": 2.56,
+ "grad_norm": 0.2890682518482208,
+ "learning_rate": 1.3088322465915826e-06,
+ "log_odds_chosen": 0.12986546754837036,
+ "log_odds_ratio": -0.7010930776596069,
+ "logits/chosen": 1.8248602151870728,
+ "logits/rejected": 1.7738860845565796,
+ "logps/chosen": -1.8727095127105713,
+ "logps/rejected": -1.9714171886444092,
+ "loss": 1.7130207061767577,
+ "nll_loss": 1.6429115533828735,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.18727093935012817,
+ "rewards/margins": 0.009870782494544983,
+ "rewards/rejected": -0.19714173674583435,
+ "step": 1600
+ },
+ {
+ "epoch": 2.56,
+ "eval_log_odds_chosen": 0.03216110169887543,
+ "eval_log_odds_ratio": -0.759286642074585,
+ "eval_logits/chosen": 1.818787693977356,
+ "eval_logits/rejected": 1.7976728677749634,
+ "eval_logps/chosen": -1.9809330701828003,
+ "eval_logps/rejected": -2.0131301879882812,
+ "eval_loss": 1.8043718338012695,
+ "eval_nll_loss": 1.7284431457519531,
+ "eval_rewards/accuracies": 0.5299999713897705,
+ "eval_rewards/chosen": -0.19809335470199585,
+ "eval_rewards/margins": 0.003219697857275605,
+ "eval_rewards/rejected": -0.20131301879882812,
+ "eval_runtime": 53.2228,
+ "eval_samples_per_second": 9.394,
+ "eval_steps_per_second": 4.697,
+ "step": 1600
+ },
+ {
+ "epoch": 2.576,
+ "grad_norm": 0.21253572404384613,
+ "learning_rate": 1.2614107883817426e-06,
+ "log_odds_chosen": 0.1767193078994751,
+ "log_odds_ratio": -0.6832129955291748,
+ "logits/chosen": 1.8714525699615479,
+ "logits/rejected": 1.879424810409546,
+ "logps/chosen": -1.9070253372192383,
+ "logps/rejected": -2.0514094829559326,
+ "loss": 1.765305519104004,
+ "nll_loss": 1.696984052658081,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19070252776145935,
+ "rewards/margins": 0.014438417740166187,
+ "rewards/rejected": -0.20514094829559326,
+ "step": 1610
+ },
+ {
+ "epoch": 2.592,
+ "grad_norm": 0.2688898742198944,
+ "learning_rate": 1.2139893301719026e-06,
+ "log_odds_chosen": -0.11991031467914581,
+ "log_odds_ratio": -0.8393659591674805,
+ "logits/chosen": 1.8714869022369385,
+ "logits/rejected": 1.8464893102645874,
+ "logps/chosen": -1.923912763595581,
+ "logps/rejected": -1.86312997341156,
+ "loss": 1.7707178115844726,
+ "nll_loss": 1.686781644821167,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.1923912763595581,
+ "rewards/margins": -0.0060782707296311855,
+ "rewards/rejected": -0.18631300330162048,
+ "step": 1620
+ },
+ {
+ "epoch": 2.608,
+ "grad_norm": 0.15975551307201385,
+ "learning_rate": 1.1665678719620628e-06,
+ "log_odds_chosen": 0.24418707191944122,
+ "log_odds_ratio": -0.6369563341140747,
+ "logits/chosen": 1.7819315195083618,
+ "logits/rejected": 1.7395622730255127,
+ "logps/chosen": -1.856393814086914,
+ "logps/rejected": -2.055032730102539,
+ "loss": 1.7171976089477539,
+ "nll_loss": 1.6535018682479858,
+ "rewards/accuracies": 0.625,
+ "rewards/chosen": -0.1856393814086914,
+ "rewards/margins": 0.019863907247781754,
+ "rewards/rejected": -0.20550329983234406,
+ "step": 1630
+ },
+ {
+ "epoch": 2.624,
+ "grad_norm": 0.279604971408844,
+ "learning_rate": 1.1191464137522228e-06,
+ "log_odds_chosen": 0.10676462948322296,
+ "log_odds_ratio": -0.7044284343719482,
+ "logits/chosen": 1.9067113399505615,
+ "logits/rejected": 1.8337434530258179,
+ "logps/chosen": -1.8776077032089233,
+ "logps/rejected": -1.9718830585479736,
+ "loss": 1.722538948059082,
+ "nll_loss": 1.652096152305603,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.18776077032089233,
+ "rewards/margins": 0.009427549317479134,
+ "rewards/rejected": -0.19718831777572632,
+ "step": 1640
+ },
+ {
+ "epoch": 2.64,
+ "grad_norm": 0.1679636538028717,
+ "learning_rate": 1.0717249555423828e-06,
+ "log_odds_chosen": -0.02650543488562107,
+ "log_odds_ratio": -0.7986313700675964,
+ "logits/chosen": 1.8376400470733643,
+ "logits/rejected": 1.8333361148834229,
+ "logps/chosen": -1.958984375,
+ "logps/rejected": -1.9452766180038452,
+ "loss": 1.7615869522094727,
+ "nll_loss": 1.681724190711975,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.19589844346046448,
+ "rewards/margins": -0.001370773883536458,
+ "rewards/rejected": -0.19452767074108124,
+ "step": 1650
+ },
+ {
+ "epoch": 2.64,
+ "eval_log_odds_chosen": 0.03181964531540871,
+ "eval_log_odds_ratio": -0.7588424682617188,
+ "eval_logits/chosen": 1.8106437921524048,
+ "eval_logits/rejected": 1.790005087852478,
+ "eval_logps/chosen": -1.9805381298065186,
+ "eval_logps/rejected": -2.0122361183166504,
+ "eval_loss": 1.803385853767395,
+ "eval_nll_loss": 1.7275017499923706,
+ "eval_rewards/accuracies": 0.5260000228881836,
+ "eval_rewards/chosen": -0.19805380702018738,
+ "eval_rewards/margins": 0.0031698308885097504,
+ "eval_rewards/rejected": -0.20122362673282623,
+ "eval_runtime": 53.2402,
+ "eval_samples_per_second": 9.391,
+ "eval_steps_per_second": 4.696,
+ "step": 1650
+ },
+ {
+ "epoch": 2.656,
+ "grad_norm": 0.2292976677417755,
+ "learning_rate": 1.0243034973325428e-06,
+ "log_odds_chosen": 0.056896090507507324,
+ "log_odds_ratio": -0.7550673484802246,
+ "logits/chosen": 1.7225093841552734,
+ "logits/rejected": 1.6621208190917969,
+ "logps/chosen": -2.003185987472534,
+ "logps/rejected": -2.0620152950286865,
+ "loss": 1.789383888244629,
+ "nll_loss": 1.7138774394989014,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.2003186047077179,
+ "rewards/margins": 0.005882947705686092,
+ "rewards/rejected": -0.20620155334472656,
+ "step": 1660
+ },
+ {
+ "epoch": 2.672,
+ "grad_norm": 0.19387373328208923,
+ "learning_rate": 9.76882039122703e-07,
+ "log_odds_chosen": 0.07780691236257553,
+ "log_odds_ratio": -0.7319976091384888,
+ "logits/chosen": 1.8635317087173462,
+ "logits/rejected": 1.8434396982192993,
+ "logps/chosen": -1.881768822669983,
+ "logps/rejected": -1.9483654499053955,
+ "loss": 1.7327621459960938,
+ "nll_loss": 1.6595627069473267,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.18817688524723053,
+ "rewards/margins": 0.006659689359366894,
+ "rewards/rejected": -0.1948365718126297,
+ "step": 1670
+ },
+ {
+ "epoch": 2.6879999999999997,
+ "grad_norm": 0.145895317196846,
+ "learning_rate": 9.294605809128631e-07,
+ "log_odds_chosen": -0.008556941524147987,
+ "log_odds_ratio": -0.7830072641372681,
+ "logits/chosen": 1.8769394159317017,
+ "logits/rejected": 1.8084793090820312,
+ "logps/chosen": -1.9570331573486328,
+ "logps/rejected": -1.9466066360473633,
+ "loss": 1.758425521850586,
+ "nll_loss": 1.6801248788833618,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.19570334255695343,
+ "rewards/margins": -0.00104265077970922,
+ "rewards/rejected": -0.19466067850589752,
+ "step": 1680
+ },
+ {
+ "epoch": 2.7039999999999997,
+ "grad_norm": 0.17203278839588165,
+ "learning_rate": 8.820391227030231e-07,
+ "log_odds_chosen": 0.160418301820755,
+ "log_odds_ratio": -0.6872135400772095,
+ "logits/chosen": 1.8363721370697021,
+ "logits/rejected": 1.8728405237197876,
+ "logps/chosen": -1.8802515268325806,
+ "logps/rejected": -2.018533229827881,
+ "loss": 1.733551025390625,
+ "nll_loss": 1.6648292541503906,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.18802514672279358,
+ "rewards/margins": 0.013828190043568611,
+ "rewards/rejected": -0.20185334980487823,
+ "step": 1690
+ },
+ {
+ "epoch": 2.7199999999999998,
+ "grad_norm": 0.21808065474033356,
+ "learning_rate": 8.346176644931832e-07,
+ "log_odds_chosen": 0.09612124413251877,
+ "log_odds_ratio": -0.7150260210037231,
+ "logits/chosen": 1.8535995483398438,
+ "logits/rejected": 1.7809932231903076,
+ "logps/chosen": -1.8756662607192993,
+ "logps/rejected": -1.9498493671417236,
+ "loss": 1.7483797073364258,
+ "nll_loss": 1.6768772602081299,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.18756665289402008,
+ "rewards/margins": 0.007418305613100529,
+ "rewards/rejected": -0.19498494267463684,
+ "step": 1700
+ },
+ {
+ "epoch": 2.7199999999999998,
+ "eval_log_odds_chosen": 0.03287385776638985,
+ "eval_log_odds_ratio": -0.7584671378135681,
+ "eval_logits/chosen": 1.815947413444519,
+ "eval_logits/rejected": 1.795128583908081,
+ "eval_logps/chosen": -1.9797779321670532,
+ "eval_logps/rejected": -2.0125272274017334,
+ "eval_loss": 1.8027502298355103,
+ "eval_nll_loss": 1.7269035577774048,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": -0.19797779619693756,
+ "eval_rewards/margins": 0.003274941351264715,
+ "eval_rewards/rejected": -0.201252743601799,
+ "eval_runtime": 53.3086,
+ "eval_samples_per_second": 9.379,
+ "eval_steps_per_second": 4.69,
+ "step": 1700
+ },
+ {
+ "epoch": 2.7359999999999998,
+ "grad_norm": 0.2510288655757904,
+ "learning_rate": 7.871962062833432e-07,
+ "log_odds_chosen": 0.09704980999231339,
+ "log_odds_ratio": -0.723429799079895,
+ "logits/chosen": 1.7908484935760498,
+ "logits/rejected": 1.8160388469696045,
+ "logps/chosen": -1.8884010314941406,
+ "logps/rejected": -1.969869613647461,
+ "loss": 1.6972316741943358,
+ "nll_loss": 1.6248886585235596,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": -0.1888400763273239,
+ "rewards/margins": 0.008146891370415688,
+ "rewards/rejected": -0.19698697328567505,
+ "step": 1710
+ },
+ {
+ "epoch": 2.752,
+ "grad_norm": 0.22549042105674744,
+ "learning_rate": 7.397747480735033e-07,
+ "log_odds_chosen": 0.00916313100606203,
+ "log_odds_ratio": -0.8096240162849426,
+ "logits/chosen": 1.9111210107803345,
+ "logits/rejected": 1.9511184692382812,
+ "logps/chosen": -1.9315268993377686,
+ "logps/rejected": -1.9021342992782593,
+ "loss": 1.7812232971191406,
+ "nll_loss": 1.7002607583999634,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19315269589424133,
+ "rewards/margins": -0.0029392438009381294,
+ "rewards/rejected": -0.19021344184875488,
+ "step": 1720
+ },
+ {
+ "epoch": 2.768,
+ "grad_norm": 0.2215452641248703,
+ "learning_rate": 6.923532898636633e-07,
+ "log_odds_chosen": 0.0815252810716629,
+ "log_odds_ratio": -0.7426193356513977,
+ "logits/chosen": 1.8194036483764648,
+ "logits/rejected": 1.7964801788330078,
+ "logps/chosen": -1.938783884048462,
+ "logps/rejected": -2.0175678730010986,
+ "loss": 1.7191705703735352,
+ "nll_loss": 1.6449085474014282,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": -0.1938783973455429,
+ "rewards/margins": 0.007878366857767105,
+ "rewards/rejected": -0.20175674557685852,
+ "step": 1730
+ },
+ {
+ "epoch": 2.784,
+ "grad_norm": 0.1890450268983841,
+ "learning_rate": 6.449318316538234e-07,
+ "log_odds_chosen": 0.21359562873840332,
+ "log_odds_ratio": -0.6950697302818298,
+ "logits/chosen": 1.7666311264038086,
+ "logits/rejected": 1.725801706314087,
+ "logps/chosen": -1.8865991830825806,
+ "logps/rejected": -2.0709402561187744,
+ "loss": 1.7412654876708984,
+ "nll_loss": 1.6717584133148193,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.1886599361896515,
+ "rewards/margins": 0.018434103578329086,
+ "rewards/rejected": -0.20709402859210968,
+ "step": 1740
+ },
+ {
+ "epoch": 2.8,
+ "grad_norm": 0.1758573055267334,
+ "learning_rate": 5.975103734439834e-07,
+ "log_odds_chosen": 0.18862012028694153,
+ "log_odds_ratio": -0.6649672389030457,
+ "logits/chosen": 1.7861156463623047,
+ "logits/rejected": 1.8578274250030518,
+ "logps/chosen": -1.8445810079574585,
+ "logps/rejected": -1.9980175495147705,
+ "loss": 1.6906953811645509,
+ "nll_loss": 1.62419855594635,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.18445810675621033,
+ "rewards/margins": 0.015343655832111835,
+ "rewards/rejected": -0.1998017579317093,
+ "step": 1750
+ },
+ {
+ "epoch": 2.8,
+ "eval_log_odds_chosen": 0.0315312035381794,
+ "eval_log_odds_ratio": -0.7593578696250916,
+ "eval_logits/chosen": 1.8106321096420288,
+ "eval_logits/rejected": 1.7899255752563477,
+ "eval_logps/chosen": -1.9789124727249146,
+ "eval_logps/rejected": -2.0103111267089844,
+ "eval_loss": 1.802466630935669,
+ "eval_nll_loss": 1.7265310287475586,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.1978912353515625,
+ "eval_rewards/margins": 0.003139869309961796,
+ "eval_rewards/rejected": -0.20103110373020172,
+ "eval_runtime": 53.31,
+ "eval_samples_per_second": 9.379,
+ "eval_steps_per_second": 4.69,
+ "step": 1750
+ },
+ {
+ "epoch": 2.816,
+ "grad_norm": 0.1975722759962082,
+ "learning_rate": 5.500889152341435e-07,
+ "log_odds_chosen": 0.08768250793218613,
+ "log_odds_ratio": -0.7138137817382812,
+ "logits/chosen": 1.8659217357635498,
+ "logits/rejected": 1.8679332733154297,
+ "logps/chosen": -1.9060866832733154,
+ "logps/rejected": -1.9737317562103271,
+ "loss": 1.7690763473510742,
+ "nll_loss": 1.6976947784423828,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.1906086653470993,
+ "rewards/margins": 0.006764502264559269,
+ "rewards/rejected": -0.197373166680336,
+ "step": 1760
+ },
+ {
+ "epoch": 2.832,
+ "grad_norm": 0.21242299675941467,
+ "learning_rate": 5.026674570243035e-07,
+ "log_odds_chosen": 0.22116176784038544,
+ "log_odds_ratio": -0.650606632232666,
+ "logits/chosen": 1.8105604648590088,
+ "logits/rejected": 1.8452847003936768,
+ "logps/chosen": -1.8956661224365234,
+ "logps/rejected": -2.0801289081573486,
+ "loss": 1.7528053283691407,
+ "nll_loss": 1.6877444982528687,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.18956659734249115,
+ "rewards/margins": 0.018446290865540504,
+ "rewards/rejected": -0.20801286399364471,
+ "step": 1770
+ },
+ {
+ "epoch": 2.848,
+ "grad_norm": 0.16772675514221191,
+ "learning_rate": 4.552459988144635e-07,
+ "log_odds_chosen": 0.18548445403575897,
+ "log_odds_ratio": -0.6828368902206421,
+ "logits/chosen": 1.7234306335449219,
+ "logits/rejected": 1.671616554260254,
+ "logps/chosen": -1.9167991876602173,
+ "logps/rejected": -2.065403461456299,
+ "loss": 1.7305257797241211,
+ "nll_loss": 1.6622416973114014,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.1916799247264862,
+ "rewards/margins": 0.014860411174595356,
+ "rewards/rejected": -0.2065403163433075,
+ "step": 1780
+ },
+ {
+ "epoch": 2.864,
+ "grad_norm": 0.1570485532283783,
+ "learning_rate": 4.0782454060462357e-07,
+ "log_odds_chosen": 0.1770729124546051,
+ "log_odds_ratio": -0.7017732858657837,
+ "logits/chosen": 1.8892648220062256,
+ "logits/rejected": 1.874248743057251,
+ "logps/chosen": -1.94317626953125,
+ "logps/rejected": -2.0713143348693848,
+ "loss": 1.7437744140625,
+ "nll_loss": 1.673596978187561,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.19431765377521515,
+ "rewards/margins": 0.012813791632652283,
+ "rewards/rejected": -0.20713143050670624,
+ "step": 1790
+ },
+ {
+ "epoch": 2.88,
+ "grad_norm": 0.2150043249130249,
+ "learning_rate": 3.604030823947836e-07,
+ "log_odds_chosen": 0.22974467277526855,
+ "log_odds_ratio": -0.6771994829177856,
+ "logits/chosen": 1.8209041357040405,
+ "logits/rejected": 1.7521514892578125,
+ "logps/chosen": -1.888583779335022,
+ "logps/rejected": -2.0834875106811523,
+ "loss": 1.8009231567382813,
+ "nll_loss": 1.7332035303115845,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.18885838985443115,
+ "rewards/margins": 0.019490379840135574,
+ "rewards/rejected": -0.20834875106811523,
+ "step": 1800
+ },
+ {
+ "epoch": 2.88,
+ "eval_log_odds_chosen": 0.033036090433597565,
+ "eval_log_odds_ratio": -0.7587888240814209,
+ "eval_logits/chosen": 1.8102370500564575,
+ "eval_logits/rejected": 1.7895140647888184,
+ "eval_logps/chosen": -1.9799045324325562,
+ "eval_logps/rejected": -2.0127370357513428,
+ "eval_loss": 1.8021843433380127,
+ "eval_nll_loss": 1.726305365562439,
+ "eval_rewards/accuracies": 0.5379999876022339,
+ "eval_rewards/chosen": -0.19799046218395233,
+ "eval_rewards/margins": 0.0032832410652190447,
+ "eval_rewards/rejected": -0.20127369463443756,
+ "eval_runtime": 53.3683,
+ "eval_samples_per_second": 9.369,
+ "eval_steps_per_second": 4.684,
+ "step": 1800
+ },
+ {
+ "epoch": 2.896,
+ "grad_norm": 0.20905476808547974,
+ "learning_rate": 3.1298162418494366e-07,
+ "log_odds_chosen": 0.1070052832365036,
+ "log_odds_ratio": -0.7236612439155579,
+ "logits/chosen": 1.8355512619018555,
+ "logits/rejected": 1.782315969467163,
+ "logps/chosen": -1.8968623876571655,
+ "logps/rejected": -1.991522192955017,
+ "loss": 1.7685279846191406,
+ "nll_loss": 1.6961616277694702,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.18968622386455536,
+ "rewards/margins": 0.009466013871133327,
+ "rewards/rejected": -0.19915224611759186,
+ "step": 1810
+ },
+ {
+ "epoch": 2.912,
+ "grad_norm": 0.21905435621738434,
+ "learning_rate": 2.655601659751037e-07,
+ "log_odds_chosen": 0.199043869972229,
+ "log_odds_ratio": -0.6736449003219604,
+ "logits/chosen": 1.7317460775375366,
+ "logits/rejected": 1.7045551538467407,
+ "logps/chosen": -1.8811628818511963,
+ "logps/rejected": -2.03849720954895,
+ "loss": 1.6910211563110351,
+ "nll_loss": 1.6236566305160522,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.18811628222465515,
+ "rewards/margins": 0.015733417123556137,
+ "rewards/rejected": -0.20384971797466278,
+ "step": 1820
+ },
+ {
+ "epoch": 2.928,
+ "grad_norm": 0.2071811854839325,
+ "learning_rate": 2.1813870776526376e-07,
+ "log_odds_chosen": 0.023101763799786568,
+ "log_odds_ratio": -0.730807900428772,
+ "logits/chosen": 1.7936124801635742,
+ "logits/rejected": 1.794028639793396,
+ "logps/chosen": -1.982717514038086,
+ "logps/rejected": -2.007256269454956,
+ "loss": 1.7140695571899414,
+ "nll_loss": 1.6409885883331299,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.19827178120613098,
+ "rewards/margins": 0.0024538785219192505,
+ "rewards/rejected": -0.20072564482688904,
+ "step": 1830
+ },
+ {
+ "epoch": 2.944,
+ "grad_norm": 0.32137688994407654,
+ "learning_rate": 1.7071724955542384e-07,
+ "log_odds_chosen": 0.029927318915724754,
+ "log_odds_ratio": -0.7750201225280762,
+ "logits/chosen": 1.8755356073379517,
+ "logits/rejected": 1.8843739032745361,
+ "logps/chosen": -1.912776231765747,
+ "logps/rejected": -1.935145616531372,
+ "loss": 1.7645767211914063,
+ "nll_loss": 1.6870746612548828,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.1912776231765747,
+ "rewards/margins": 0.002236946951597929,
+ "rewards/rejected": -0.19351457059383392,
+ "step": 1840
+ },
+ {
+ "epoch": 2.96,
+ "grad_norm": 0.2819410264492035,
+ "learning_rate": 1.232957913455839e-07,
+ "log_odds_chosen": 0.3163163363933563,
+ "log_odds_ratio": -0.6233562231063843,
+ "logits/chosen": 1.7968562841415405,
+ "logits/rejected": 1.81418776512146,
+ "logps/chosen": -1.8932796716690063,
+ "logps/rejected": -2.1572272777557373,
+ "loss": 1.7400293350219727,
+ "nll_loss": 1.6776933670043945,
+ "rewards/accuracies": 0.6499999761581421,
+ "rewards/chosen": -0.18932797014713287,
+ "rewards/margins": 0.02639475092291832,
+ "rewards/rejected": -0.2157227098941803,
+ "step": 1850
+ },
+ {
+ "epoch": 2.96,
+ "eval_log_odds_chosen": 0.0331137478351593,
+ "eval_log_odds_ratio": -0.7587754726409912,
+ "eval_logits/chosen": 1.8180500268936157,
+ "eval_logits/rejected": 1.7974404096603394,
+ "eval_logps/chosen": -1.9795879125595093,
+ "eval_logps/rejected": -2.0124764442443848,
+ "eval_loss": 1.8020251989364624,
+ "eval_nll_loss": 1.7261475324630737,
+ "eval_rewards/accuracies": 0.5360000133514404,
+ "eval_rewards/chosen": -0.19795876741409302,
+ "eval_rewards/margins": 0.0032888641580939293,
+ "eval_rewards/rejected": -0.20124764740467072,
+ "eval_runtime": 53.218,
+ "eval_samples_per_second": 9.395,
+ "eval_steps_per_second": 4.698,
+ "step": 1850
+ },
+ {
+ "epoch": 2.976,
+ "grad_norm": 0.20896051824092865,
+ "learning_rate": 7.587433313574391e-08,
+ "log_odds_chosen": 0.048571012914180756,
+ "log_odds_ratio": -0.7379616498947144,
+ "logits/chosen": 1.9031015634536743,
+ "logits/rejected": 1.7852680683135986,
+ "logps/chosen": -1.9041398763656616,
+ "logps/rejected": -1.941368818283081,
+ "loss": 1.7335838317871093,
+ "nll_loss": 1.659787893295288,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.19041398167610168,
+ "rewards/margins": 0.00372290494851768,
+ "rewards/rejected": -0.19413690268993378,
+ "step": 1860
+ },
+ {
+ "epoch": 2.992,
+ "grad_norm": 0.21155616641044617,
+ "learning_rate": 2.845287492590397e-08,
+ "log_odds_chosen": 0.20803546905517578,
+ "log_odds_ratio": -0.6666355729103088,
+ "logits/chosen": 1.8990240097045898,
+ "logits/rejected": 1.8817529678344727,
+ "logps/chosen": -1.8441791534423828,
+ "logps/rejected": -2.02185320854187,
+ "loss": 1.7164598464965821,
+ "nll_loss": 1.6497962474822998,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.18441791832447052,
+ "rewards/margins": 0.017767416313290596,
+ "rewards/rejected": -0.20218534767627716,
+ "step": 1870
+ }
+ ],
+ "logging_steps": 10,
+ "max_steps": 1875,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": true
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-1875/training_args.bin b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..e369ed5ca0fb0f2b706a5adca8ab527baae13e31
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-1875/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4ecab525cdf207fed7abfb6f0080dd00da9f0e35d23314a97f0bff66fe6176b
+size 5457
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-500/README.md b/v3/ORPO/ORPO_5k/lora/checkpoint-500/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..df73bb4ed6d6a8e69c79ba9761e4cdd4272cc1ab
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-500/README.md
@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- lora
+- orpo
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+
+
+
+
+## Model Details
+
+### Model Description
+
+
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+
+
+### Direct Use
+
+
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+
+
+[More Information Needed]
+
+### Recommendations
+
+
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+
+
+[More Information Needed]
+
+### Training Procedure
+
+
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed]
+
+#### Speeds, Sizes, Times [optional]
+
+
+
+[More Information Needed]
+
+## Evaluation
+
+
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+
+
+[More Information Needed]
+
+#### Factors
+
+
+
+[More Information Needed]
+
+#### Metrics
+
+
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+
+
+[More Information Needed]
+
+## Environmental Impact
+
+
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1
\ No newline at end of file
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-500/adapter_config.json b/v3/ORPO/ORPO_5k/lora/checkpoint-500/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d02ef66b5d06bfc12ca93c16b0b45a216cff2a6a
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-500/adapter_config.json
@@ -0,0 +1,46 @@
+{
+ "alora_invocation_tokens": null,
+ "alpha_pattern": {},
+ "arrow_config": null,
+ "auto_mapping": null,
+ "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+ "bias": "none",
+ "corda_config": null,
+ "ensure_weight_tying": false,
+ "eva_config": null,
+ "exclude_modules": null,
+ "fan_in_fan_out": false,
+ "inference_mode": true,
+ "init_lora_weights": true,
+ "layer_replication": null,
+ "layers_pattern": null,
+ "layers_to_transform": null,
+ "loftq_config": {},
+ "lora_alpha": 32,
+ "lora_bias": false,
+ "lora_dropout": 0.1,
+ "megatron_config": null,
+ "megatron_core": "megatron.core",
+ "modules_to_save": null,
+ "peft_type": "LORA",
+ "peft_version": "0.18.1",
+ "qalora_group_size": 16,
+ "r": 64,
+ "rank_pattern": {},
+ "revision": null,
+ "target_modules": [
+ "k_proj",
+ "down_proj",
+ "up_proj",
+ "v_proj",
+ "q_proj",
+ "o_proj",
+ "gate_proj"
+ ],
+ "target_parameters": null,
+ "task_type": "CAUSAL_LM",
+ "trainable_token_indices": null,
+ "use_dora": false,
+ "use_qalora": false,
+ "use_rslora": false
+}
\ No newline at end of file
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-500/adapter_model.safetensors b/v3/ORPO/ORPO_5k/lora/checkpoint-500/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5373ca2f7e7ae4c16fe1305c9f386950e2f43409
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-500/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab207c04501affa59ceea1ccf651d97466f7d4eecaced9b7be388886a45094ed
+size 180385008
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-500/chat_template.jinja b/v3/ORPO/ORPO_5k/lora/checkpoint-500/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..1bad6a0f648dccdbec523ca79ba90fbcfc806af0
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-500/chat_template.jinja
@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+ {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+ {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+ {%- if strftime_now is defined %}
+ {%- set date_string = strftime_now("%d %b %Y") %}
+ {%- else %}
+ {%- set date_string = "26 Jul 2024" %}
+ {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+ {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+ {%- set system_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+{%- else %}
+ {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+ {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+ {#- Extract the first user message so we can plug it in here #}
+ {%- if messages | length != 0 %}
+ {%- set first_user_message = messages[0]['content']|trim %}
+ {%- set messages = messages[1:] %}
+ {%- else %}
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+ {{- "Do not use variables.\n\n" }}
+ {%- for t in tools %}
+ {{- t | tojson(indent=4) }}
+ {{- "\n\n" }}
+ {%- endfor %}
+ {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+ {%- elif 'tool_calls' in message %}
+ {%- if not message.tool_calls|length == 1 %}
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
+ {%- endif %}
+ {%- set tool_call = message.tool_calls[0].function %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+ {{- '{"name": "' + tool_call.name + '", ' }}
+ {{- '"parameters": ' }}
+ {{- tool_call.arguments | tojson }}
+ {{- "}" }}
+ {{- "<|eot_id|>" }}
+ {%- elif message.role == "tool" or message.role == "ipython" %}
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+ {%- if message.content is mapping or message.content is iterable %}
+ {{- message.content | tojson }}
+ {%- else %}
+ {{- message.content }}
+ {%- endif %}
+ {{- "<|eot_id|>" }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-500/optimizer.pt b/v3/ORPO/ORPO_5k/lora/checkpoint-500/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3da04d52f3ddbdd352eef242dfd4e07d726ddb35
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-500/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:379c9223944578605c40aa208f9121285d0c7c1a0e0d9d79fceef4677c8091d9
+size 360902475
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-500/rng_state.pth b/v3/ORPO/ORPO_5k/lora/checkpoint-500/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..d5e20c9b1725549f13f103bd76fa989eb82478a1
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-500/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea11996454b5587fcf33ae0ab5cf14b2031bf5f53f8c2ed5a48e87de31e29c84
+size 14645
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-500/scaler.pt b/v3/ORPO/ORPO_5k/lora/checkpoint-500/scaler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..86c857e8514e5db52a765434b135695dac4c9c36
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-500/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f77569c2e850b04af982cc8c1389f1430851448915c593b69e5da36ce05b71d7
+size 1383
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-500/scheduler.pt b/v3/ORPO/ORPO_5k/lora/checkpoint-500/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2755db22053be32baa12eaafcd19d42f7d23e22c
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-500/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4c40d8affd7cb2d132e483355486b190f2e4fd6a9220570282b7586aa1122c8
+size 1465
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-500/tokenizer.json b/v3/ORPO/ORPO_5k/lora/checkpoint-500/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-500/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-500/tokenizer_config.json b/v3/ORPO/ORPO_5k/lora/checkpoint-500/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0c736827e07464a3cdd67acebb3699bfd0a38d1
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-500/tokenizer_config.json
@@ -0,0 +1,14 @@
+{
+ "backend": "tokenizers",
+ "bos_token": "<|begin_of_text|>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|eot_id|>",
+ "is_local": false,
+ "model_input_names": [
+ "input_ids",
+ "attention_mask"
+ ],
+ "model_max_length": 131072,
+ "pad_token": "<|eot_id|>",
+ "tokenizer_class": "TokenizersBackend"
+}
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-500/trainer_state.json b/v3/ORPO/ORPO_5k/lora/checkpoint-500/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..fe20316b42e22693c74a6b6ba75d573cc3d8fb45
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-500/trainer_state.json
@@ -0,0 +1,1124 @@
+{
+ "best_global_step": 500,
+ "best_metric": 1.851176142692566,
+ "best_model_checkpoint": "output/lora/checkpoint-500",
+ "epoch": 0.8,
+ "eval_steps": 50,
+ "global_step": 500,
+ "is_hyper_param_search": false,
+ "is_local_process_zero": true,
+ "is_world_process_zero": true,
+ "log_history": [
+ {
+ "epoch": 0.016,
+ "grad_norm": 1.373120665550232,
+ "learning_rate": 3.829787234042553e-07,
+ "log_odds_chosen": 0.00716871302574873,
+ "log_odds_ratio": -0.8661719560623169,
+ "logits/chosen": 1.1518588066101074,
+ "logits/rejected": 1.1109380722045898,
+ "logps/chosen": -3.045567512512207,
+ "logps/rejected": -3.052224636077881,
+ "loss": 3.4928749084472654,
+ "nll_loss": 3.4062583446502686,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.3045567572116852,
+ "rewards/margins": 0.000665713450871408,
+ "rewards/rejected": -0.30522245168685913,
+ "step": 10
+ },
+ {
+ "epoch": 0.032,
+ "grad_norm": 1.9124146699905396,
+ "learning_rate": 8.085106382978723e-07,
+ "log_odds_chosen": -0.07713592052459717,
+ "log_odds_ratio": -0.9291092157363892,
+ "logits/chosen": 0.9864249229431152,
+ "logits/rejected": 0.9885759353637695,
+ "logps/chosen": -3.2009873390197754,
+ "logps/rejected": -3.1334564685821533,
+ "loss": 3.693857192993164,
+ "nll_loss": 3.600947141647339,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.32009872794151306,
+ "rewards/margins": -0.0067530544474720955,
+ "rewards/rejected": -0.31334567070007324,
+ "step": 20
+ },
+ {
+ "epoch": 0.048,
+ "grad_norm": 0.8211373686790466,
+ "learning_rate": 1.2340425531914892e-06,
+ "log_odds_chosen": -0.13502930104732513,
+ "log_odds_ratio": -0.9718782305717468,
+ "logits/chosen": 1.1002552509307861,
+ "logits/rejected": 1.1391098499298096,
+ "logps/chosen": -3.2072739601135254,
+ "logps/rejected": -3.0774378776550293,
+ "loss": 3.3957988739013674,
+ "nll_loss": 3.2986106872558594,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.3207274079322815,
+ "rewards/margins": -0.012983566150069237,
+ "rewards/rejected": -0.3077438473701477,
+ "step": 30
+ },
+ {
+ "epoch": 0.064,
+ "grad_norm": 0.768266499042511,
+ "learning_rate": 1.6595744680851064e-06,
+ "log_odds_chosen": 0.06783491373062134,
+ "log_odds_ratio": -0.7825279235839844,
+ "logits/chosen": 1.0638954639434814,
+ "logits/rejected": 1.0289121866226196,
+ "logps/chosen": -2.831758975982666,
+ "logps/rejected": -2.8962886333465576,
+ "loss": 3.35846061706543,
+ "nll_loss": 3.2802085876464844,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.28317588567733765,
+ "rewards/margins": 0.006452961824834347,
+ "rewards/rejected": -0.28962892293930054,
+ "step": 40
+ },
+ {
+ "epoch": 0.08,
+ "grad_norm": 0.686996340751648,
+ "learning_rate": 2.0851063829787235e-06,
+ "log_odds_chosen": -0.1561535745859146,
+ "log_odds_ratio": -0.9219404458999634,
+ "logits/chosen": 1.161010980606079,
+ "logits/rejected": 1.1353846788406372,
+ "logps/chosen": -3.047670841217041,
+ "logps/rejected": -2.895892858505249,
+ "loss": 3.245359420776367,
+ "nll_loss": 3.153165817260742,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.30476707220077515,
+ "rewards/margins": -0.0151777733117342,
+ "rewards/rejected": -0.2895893156528473,
+ "step": 50
+ },
+ {
+ "epoch": 0.08,
+ "eval_log_odds_chosen": 0.13973847031593323,
+ "eval_log_odds_ratio": -0.8127303719520569,
+ "eval_logits/chosen": 1.0891079902648926,
+ "eval_logits/rejected": 1.077158808708191,
+ "eval_logps/chosen": -2.9398701190948486,
+ "eval_logps/rejected": -3.080166816711426,
+ "eval_loss": 3.217437505722046,
+ "eval_nll_loss": 3.1361641883850098,
+ "eval_rewards/accuracies": 0.5460000038146973,
+ "eval_rewards/chosen": -0.2939870059490204,
+ "eval_rewards/margins": 0.01402965560555458,
+ "eval_rewards/rejected": -0.30801668763160706,
+ "eval_runtime": 53.3593,
+ "eval_samples_per_second": 9.37,
+ "eval_steps_per_second": 4.685,
+ "step": 50
+ },
+ {
+ "epoch": 0.096,
+ "grad_norm": 0.5995831489562988,
+ "learning_rate": 2.5106382978723402e-06,
+ "log_odds_chosen": 0.17807230353355408,
+ "log_odds_ratio": -0.7261534333229065,
+ "logits/chosen": 1.1882927417755127,
+ "logits/rejected": 1.1475657224655151,
+ "logps/chosen": -2.6847589015960693,
+ "logps/rejected": -2.866147994995117,
+ "loss": 3.019968795776367,
+ "nll_loss": 2.9473533630371094,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.2684759199619293,
+ "rewards/margins": 0.01813889667391777,
+ "rewards/rejected": -0.2866148054599762,
+ "step": 60
+ },
+ {
+ "epoch": 0.112,
+ "grad_norm": 0.42678770422935486,
+ "learning_rate": 2.9361702127659574e-06,
+ "log_odds_chosen": 0.04040491580963135,
+ "log_odds_ratio": -0.7541126012802124,
+ "logits/chosen": 1.2164686918258667,
+ "logits/rejected": 1.1548993587493896,
+ "logps/chosen": -2.732120990753174,
+ "logps/rejected": -2.7696893215179443,
+ "loss": 2.990768241882324,
+ "nll_loss": 2.915356159210205,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.2732120752334595,
+ "rewards/margins": 0.0037568225525319576,
+ "rewards/rejected": -0.27696892619132996,
+ "step": 70
+ },
+ {
+ "epoch": 0.128,
+ "grad_norm": 0.39134183526039124,
+ "learning_rate": 3.3617021276595745e-06,
+ "log_odds_chosen": 0.2615276277065277,
+ "log_odds_ratio": -0.780361533164978,
+ "logits/chosen": 1.0079171657562256,
+ "logits/rejected": 0.9202032089233398,
+ "logps/chosen": -2.813866138458252,
+ "logps/rejected": -3.050983190536499,
+ "loss": 2.885699653625488,
+ "nll_loss": 2.807663679122925,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.28138667345046997,
+ "rewards/margins": 0.0237116776406765,
+ "rewards/rejected": -0.3050983250141144,
+ "step": 80
+ },
+ {
+ "epoch": 0.144,
+ "grad_norm": 0.3603176474571228,
+ "learning_rate": 3.7872340425531912e-06,
+ "log_odds_chosen": 0.0377628318965435,
+ "log_odds_ratio": -0.8245071172714233,
+ "logits/chosen": 1.0985459089279175,
+ "logits/rejected": 1.0714184045791626,
+ "logps/chosen": -2.8029580116271973,
+ "logps/rejected": -2.8335766792297363,
+ "loss": 2.8207944869995116,
+ "nll_loss": 2.7383437156677246,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.28029578924179077,
+ "rewards/margins": 0.0030618987511843443,
+ "rewards/rejected": -0.2833576798439026,
+ "step": 90
+ },
+ {
+ "epoch": 0.16,
+ "grad_norm": 0.627916157245636,
+ "learning_rate": 4.212765957446808e-06,
+ "log_odds_chosen": 0.181382954120636,
+ "log_odds_ratio": -0.6789280772209167,
+ "logits/chosen": 1.0331000089645386,
+ "logits/rejected": 1.1242140531539917,
+ "logps/chosen": -2.438321828842163,
+ "logps/rejected": -2.604365825653076,
+ "loss": 2.5767364501953125,
+ "nll_loss": 2.5088438987731934,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.24383218586444855,
+ "rewards/margins": 0.016604404896497726,
+ "rewards/rejected": -0.2604365944862366,
+ "step": 100
+ },
+ {
+ "epoch": 0.16,
+ "eval_log_odds_chosen": 0.07509047538042068,
+ "eval_log_odds_ratio": -0.7700561881065369,
+ "eval_logits/chosen": 1.0837428569793701,
+ "eval_logits/rejected": 1.0448086261749268,
+ "eval_logps/chosen": -2.481499433517456,
+ "eval_logps/rejected": -2.561527967453003,
+ "eval_loss": 2.5757269859313965,
+ "eval_nll_loss": 2.4987213611602783,
+ "eval_rewards/accuracies": 0.550000011920929,
+ "eval_rewards/chosen": -0.24814994633197784,
+ "eval_rewards/margins": 0.008002860471606255,
+ "eval_rewards/rejected": -0.25615280866622925,
+ "eval_runtime": 53.3273,
+ "eval_samples_per_second": 9.376,
+ "eval_steps_per_second": 4.688,
+ "step": 100
+ },
+ {
+ "epoch": 0.176,
+ "grad_norm": 0.2816033363342285,
+ "learning_rate": 4.638297872340425e-06,
+ "log_odds_chosen": 0.0019359707366675138,
+ "log_odds_ratio": -0.7848639488220215,
+ "logits/chosen": 1.014973759651184,
+ "logits/rejected": 1.0038645267486572,
+ "logps/chosen": -2.474358320236206,
+ "logps/rejected": -2.4702632427215576,
+ "loss": 2.4666120529174806,
+ "nll_loss": 2.3881256580352783,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.24743585288524628,
+ "rewards/margins": -0.0004095211625099182,
+ "rewards/rejected": -0.24702632427215576,
+ "step": 110
+ },
+ {
+ "epoch": 0.192,
+ "grad_norm": 0.3091694712638855,
+ "learning_rate": 5.063829787234042e-06,
+ "log_odds_chosen": 0.18548759818077087,
+ "log_odds_ratio": -0.7040785551071167,
+ "logits/chosen": 1.0326206684112549,
+ "logits/rejected": 1.03751802444458,
+ "logps/chosen": -2.2152860164642334,
+ "logps/rejected": -2.3625495433807373,
+ "loss": 2.2838161468505858,
+ "nll_loss": 2.2134079933166504,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.22152860462665558,
+ "rewards/margins": 0.01472637802362442,
+ "rewards/rejected": -0.2362549751996994,
+ "step": 120
+ },
+ {
+ "epoch": 0.208,
+ "grad_norm": 0.2403600662946701,
+ "learning_rate": 5.489361702127659e-06,
+ "log_odds_chosen": -0.029171090573072433,
+ "log_odds_ratio": -0.7839903831481934,
+ "logits/chosen": 1.124861717224121,
+ "logits/rejected": 1.0936174392700195,
+ "logps/chosen": -2.2851500511169434,
+ "logps/rejected": -2.2685129642486572,
+ "loss": 2.286957359313965,
+ "nll_loss": 2.2085585594177246,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.22851502895355225,
+ "rewards/margins": -0.0016637055668979883,
+ "rewards/rejected": -0.22685131430625916,
+ "step": 130
+ },
+ {
+ "epoch": 0.224,
+ "grad_norm": 0.2264973670244217,
+ "learning_rate": 5.9148936170212766e-06,
+ "log_odds_chosen": 0.008796649985015392,
+ "log_odds_ratio": -0.8179906010627747,
+ "logits/chosen": 1.0210378170013428,
+ "logits/rejected": 1.0923821926116943,
+ "logps/chosen": -2.3308796882629395,
+ "logps/rejected": -2.352290153503418,
+ "loss": 2.1581136703491213,
+ "nll_loss": 2.076314687728882,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.23308798670768738,
+ "rewards/margins": 0.0021410156041383743,
+ "rewards/rejected": -0.2352290153503418,
+ "step": 140
+ },
+ {
+ "epoch": 0.24,
+ "grad_norm": 0.20365770161151886,
+ "learning_rate": 6.340425531914893e-06,
+ "log_odds_chosen": -0.009427487850189209,
+ "log_odds_ratio": -0.805976390838623,
+ "logits/chosen": 1.1640613079071045,
+ "logits/rejected": 1.076269268989563,
+ "logps/chosen": -2.211876153945923,
+ "logps/rejected": -2.1832690238952637,
+ "loss": 2.04748592376709,
+ "nll_loss": 1.9668887853622437,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.22118762135505676,
+ "rewards/margins": -0.0028607314452528954,
+ "rewards/rejected": -0.2183268964290619,
+ "step": 150
+ },
+ {
+ "epoch": 0.24,
+ "eval_log_odds_chosen": 0.034868109971284866,
+ "eval_log_odds_ratio": -0.7663777470588684,
+ "eval_logits/chosen": 1.1222375631332397,
+ "eval_logits/rejected": 1.0752418041229248,
+ "eval_logps/chosen": -2.245656967163086,
+ "eval_logps/rejected": -2.2865161895751953,
+ "eval_loss": 2.085561990737915,
+ "eval_nll_loss": 2.0089242458343506,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.22456568479537964,
+ "eval_rewards/margins": 0.004085900261998177,
+ "eval_rewards/rejected": -0.22865159809589386,
+ "eval_runtime": 53.2365,
+ "eval_samples_per_second": 9.392,
+ "eval_steps_per_second": 4.696,
+ "step": 150
+ },
+ {
+ "epoch": 0.256,
+ "grad_norm": 0.17429247498512268,
+ "learning_rate": 6.76595744680851e-06,
+ "log_odds_chosen": 0.03718019276857376,
+ "log_odds_ratio": -0.7309927940368652,
+ "logits/chosen": 1.243948221206665,
+ "logits/rejected": 1.155428171157837,
+ "logps/chosen": -2.222162961959839,
+ "logps/rejected": -2.2655863761901855,
+ "loss": 2.079151153564453,
+ "nll_loss": 2.006051778793335,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.22221633791923523,
+ "rewards/margins": 0.004342344123870134,
+ "rewards/rejected": -0.22655868530273438,
+ "step": 160
+ },
+ {
+ "epoch": 0.272,
+ "grad_norm": 0.2044232040643692,
+ "learning_rate": 7.191489361702127e-06,
+ "log_odds_chosen": 0.2378263771533966,
+ "log_odds_ratio": -0.6989302635192871,
+ "logits/chosen": 1.1669152975082397,
+ "logits/rejected": 1.1139662265777588,
+ "logps/chosen": -2.1898856163024902,
+ "logps/rejected": -2.405423641204834,
+ "loss": 2.0023525238037108,
+ "nll_loss": 1.9324592351913452,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.2189885824918747,
+ "rewards/margins": 0.021553806960582733,
+ "rewards/rejected": -0.24054238200187683,
+ "step": 170
+ },
+ {
+ "epoch": 0.288,
+ "grad_norm": 0.17495381832122803,
+ "learning_rate": 7.617021276595744e-06,
+ "log_odds_chosen": -0.06665988266468048,
+ "log_odds_ratio": -0.804401695728302,
+ "logits/chosen": 1.1880234479904175,
+ "logits/rejected": 1.193927526473999,
+ "logps/chosen": -2.1306421756744385,
+ "logps/rejected": -2.082124948501587,
+ "loss": 1.9733341217041016,
+ "nll_loss": 1.8928937911987305,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.21306422352790833,
+ "rewards/margins": -0.004851718433201313,
+ "rewards/rejected": -0.2082124948501587,
+ "step": 180
+ },
+ {
+ "epoch": 0.304,
+ "grad_norm": 0.17830601334571838,
+ "learning_rate": 7.995257854179015e-06,
+ "log_odds_chosen": 0.06894228607416153,
+ "log_odds_ratio": -0.7259312868118286,
+ "logits/chosen": 1.171451449394226,
+ "logits/rejected": 1.1703966856002808,
+ "logps/chosen": -2.0300755500793457,
+ "logps/rejected": -2.1007497310638428,
+ "loss": 1.8905454635620118,
+ "nll_loss": 1.8179525136947632,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.2030075490474701,
+ "rewards/margins": 0.007067441940307617,
+ "rewards/rejected": -0.2100749909877777,
+ "step": 190
+ },
+ {
+ "epoch": 0.32,
+ "grad_norm": 0.20069865882396698,
+ "learning_rate": 7.947836395969175e-06,
+ "log_odds_chosen": 0.2506960332393646,
+ "log_odds_ratio": -0.6373531222343445,
+ "logits/chosen": 1.1848938465118408,
+ "logits/rejected": 1.1178070306777954,
+ "logps/chosen": -1.992468237876892,
+ "logps/rejected": -2.2120540142059326,
+ "loss": 1.8577205657958984,
+ "nll_loss": 1.79398512840271,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": -0.19924680888652802,
+ "rewards/margins": 0.021958591416478157,
+ "rewards/rejected": -0.2212054282426834,
+ "step": 200
+ },
+ {
+ "epoch": 0.32,
+ "eval_log_odds_chosen": 0.028671210631728172,
+ "eval_log_odds_ratio": -0.7659140825271606,
+ "eval_logits/chosen": 1.267584204673767,
+ "eval_logits/rejected": 1.2251330614089966,
+ "eval_logps/chosen": -2.12668776512146,
+ "eval_logps/rejected": -2.1608970165252686,
+ "eval_loss": 1.9326850175857544,
+ "eval_nll_loss": 1.8560935258865356,
+ "eval_rewards/accuracies": 0.5260000228881836,
+ "eval_rewards/chosen": -0.212668776512146,
+ "eval_rewards/margins": 0.0034209203440696,
+ "eval_rewards/rejected": -0.21608968079090118,
+ "eval_runtime": 53.2553,
+ "eval_samples_per_second": 9.389,
+ "eval_steps_per_second": 4.694,
+ "step": 200
+ },
+ {
+ "epoch": 0.336,
+ "grad_norm": 0.20797842741012573,
+ "learning_rate": 7.900414937759335e-06,
+ "log_odds_chosen": 0.04750821739435196,
+ "log_odds_ratio": -0.702393114566803,
+ "logits/chosen": 1.3723499774932861,
+ "logits/rejected": 1.101571798324585,
+ "logps/chosen": -2.08408784866333,
+ "logps/rejected": -2.1285855770111084,
+ "loss": 1.855466651916504,
+ "nll_loss": 1.7852274179458618,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.20840883255004883,
+ "rewards/margins": 0.004449741914868355,
+ "rewards/rejected": -0.21285855770111084,
+ "step": 210
+ },
+ {
+ "epoch": 0.352,
+ "grad_norm": 0.19125986099243164,
+ "learning_rate": 7.852993479549495e-06,
+ "log_odds_chosen": -0.14522936940193176,
+ "log_odds_ratio": -0.8650233149528503,
+ "logits/chosen": 1.3711931705474854,
+ "logits/rejected": 1.2990773916244507,
+ "logps/chosen": -2.171778440475464,
+ "logps/rejected": -2.039144277572632,
+ "loss": 1.8723089218139648,
+ "nll_loss": 1.7858068943023682,
+ "rewards/accuracies": 0.42500001192092896,
+ "rewards/chosen": -0.2171778380870819,
+ "rewards/margins": -0.013263422064483166,
+ "rewards/rejected": -0.20391443371772766,
+ "step": 220
+ },
+ {
+ "epoch": 0.368,
+ "grad_norm": 0.17002636194229126,
+ "learning_rate": 7.805572021339655e-06,
+ "log_odds_chosen": 0.06036282703280449,
+ "log_odds_ratio": -0.7291742563247681,
+ "logits/chosen": 1.3134567737579346,
+ "logits/rejected": 1.3123853206634521,
+ "logps/chosen": -2.0283656120300293,
+ "logps/rejected": -2.0922353267669678,
+ "loss": 1.8563121795654296,
+ "nll_loss": 1.7833948135375977,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.20283658802509308,
+ "rewards/margins": 0.006386973895132542,
+ "rewards/rejected": -0.20922355353832245,
+ "step": 230
+ },
+ {
+ "epoch": 0.384,
+ "grad_norm": 0.16414593160152435,
+ "learning_rate": 7.758150563129815e-06,
+ "log_odds_chosen": 0.22585150599479675,
+ "log_odds_ratio": -0.6319423913955688,
+ "logits/chosen": 1.317280650138855,
+ "logits/rejected": 1.2409361600875854,
+ "logps/chosen": -1.9960790872573853,
+ "logps/rejected": -2.1858954429626465,
+ "loss": 1.8553081512451173,
+ "nll_loss": 1.792114019393921,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.19960792362689972,
+ "rewards/margins": 0.018981631845235825,
+ "rewards/rejected": -0.21858954429626465,
+ "step": 240
+ },
+ {
+ "epoch": 0.4,
+ "grad_norm": 0.14812582731246948,
+ "learning_rate": 7.710729104919975e-06,
+ "log_odds_chosen": -0.09517587721347809,
+ "log_odds_ratio": -0.8104914426803589,
+ "logits/chosen": 1.3900704383850098,
+ "logits/rejected": 1.4599993228912354,
+ "logps/chosen": -2.050788402557373,
+ "logps/rejected": -1.9786239862442017,
+ "loss": 1.8190387725830077,
+ "nll_loss": 1.7379897832870483,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.2050788700580597,
+ "rewards/margins": -0.0072164698503911495,
+ "rewards/rejected": -0.19786237180233002,
+ "step": 250
+ },
+ {
+ "epoch": 0.4,
+ "eval_log_odds_chosen": 0.027970660477876663,
+ "eval_log_odds_ratio": -0.7652882933616638,
+ "eval_logits/chosen": 1.365133285522461,
+ "eval_logits/rejected": 1.3242768049240112,
+ "eval_logps/chosen": -2.095747947692871,
+ "eval_logps/rejected": -2.128530263900757,
+ "eval_loss": 1.9009666442871094,
+ "eval_nll_loss": 1.8244377374649048,
+ "eval_rewards/accuracies": 0.5320000052452087,
+ "eval_rewards/chosen": -0.20957481861114502,
+ "eval_rewards/margins": 0.0032782391645014286,
+ "eval_rewards/rejected": -0.21285304427146912,
+ "eval_runtime": 53.1908,
+ "eval_samples_per_second": 9.4,
+ "eval_steps_per_second": 4.7,
+ "step": 250
+ },
+ {
+ "epoch": 0.416,
+ "grad_norm": 0.18252891302108765,
+ "learning_rate": 7.663307646710137e-06,
+ "log_odds_chosen": 0.07316568493843079,
+ "log_odds_ratio": -0.7166140675544739,
+ "logits/chosen": 1.4400272369384766,
+ "logits/rejected": 1.3449039459228516,
+ "logps/chosen": -2.0099174976348877,
+ "logps/rejected": -2.075176954269409,
+ "loss": 1.8593158721923828,
+ "nll_loss": 1.7876542806625366,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.20099174976348877,
+ "rewards/margins": 0.006525977049022913,
+ "rewards/rejected": -0.20751771330833435,
+ "step": 260
+ },
+ {
+ "epoch": 0.432,
+ "grad_norm": 0.15253905951976776,
+ "learning_rate": 7.6158861885002954e-06,
+ "log_odds_chosen": 0.20866680145263672,
+ "log_odds_ratio": -0.685832142829895,
+ "logits/chosen": 1.3512670993804932,
+ "logits/rejected": 1.3183681964874268,
+ "logps/chosen": -1.9916179180145264,
+ "logps/rejected": -2.1834797859191895,
+ "loss": 1.8131860733032226,
+ "nll_loss": 1.7446027994155884,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.1991618126630783,
+ "rewards/margins": 0.019186144694685936,
+ "rewards/rejected": -0.21834798157215118,
+ "step": 270
+ },
+ {
+ "epoch": 0.448,
+ "grad_norm": 0.3055081069469452,
+ "learning_rate": 7.5684647302904554e-06,
+ "log_odds_chosen": 0.19830426573753357,
+ "log_odds_ratio": -0.6579998135566711,
+ "logits/chosen": 1.3933019638061523,
+ "logits/rejected": 1.433093547821045,
+ "logps/chosen": -1.9509963989257812,
+ "logps/rejected": -2.128086566925049,
+ "loss": 1.8611320495605468,
+ "nll_loss": 1.7953321933746338,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.1950996369123459,
+ "rewards/margins": 0.017709016799926758,
+ "rewards/rejected": -0.21280868351459503,
+ "step": 280
+ },
+ {
+ "epoch": 0.464,
+ "grad_norm": 0.1907416731119156,
+ "learning_rate": 7.521043272080616e-06,
+ "log_odds_chosen": 0.2938005328178406,
+ "log_odds_ratio": -0.6218821406364441,
+ "logits/chosen": 1.454288125038147,
+ "logits/rejected": 1.364547848701477,
+ "logps/chosen": -1.907086968421936,
+ "logps/rejected": -2.168692111968994,
+ "loss": 1.7643163681030274,
+ "nll_loss": 1.7021281719207764,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.1907087117433548,
+ "rewards/margins": 0.026160534471273422,
+ "rewards/rejected": -0.21686923503875732,
+ "step": 290
+ },
+ {
+ "epoch": 0.48,
+ "grad_norm": 0.20577998459339142,
+ "learning_rate": 7.473621813870776e-06,
+ "log_odds_chosen": 0.0781371220946312,
+ "log_odds_ratio": -0.7277525663375854,
+ "logits/chosen": 1.3441762924194336,
+ "logits/rejected": 1.339400291442871,
+ "logps/chosen": -1.986854910850525,
+ "logps/rejected": -2.0572762489318848,
+ "loss": 1.8164241790771485,
+ "nll_loss": 1.7436487674713135,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.19868549704551697,
+ "rewards/margins": 0.007042132318019867,
+ "rewards/rejected": -0.20572762191295624,
+ "step": 300
+ },
+ {
+ "epoch": 0.48,
+ "eval_log_odds_chosen": 0.027198966592550278,
+ "eval_log_odds_ratio": -0.7654678821563721,
+ "eval_logits/chosen": 1.4181112051010132,
+ "eval_logits/rejected": 1.3766058683395386,
+ "eval_logps/chosen": -2.0740718841552734,
+ "eval_logps/rejected": -2.1053483486175537,
+ "eval_loss": 1.8834025859832764,
+ "eval_nll_loss": 1.8068556785583496,
+ "eval_rewards/accuracies": 0.5260000228881836,
+ "eval_rewards/chosen": -0.2074071615934372,
+ "eval_rewards/margins": 0.0031276745721697807,
+ "eval_rewards/rejected": -0.21053485572338104,
+ "eval_runtime": 53.2972,
+ "eval_samples_per_second": 9.381,
+ "eval_steps_per_second": 4.691,
+ "step": 300
+ },
+ {
+ "epoch": 0.496,
+ "grad_norm": 0.15923626720905304,
+ "learning_rate": 7.426200355660936e-06,
+ "log_odds_chosen": -0.007339484989643097,
+ "log_odds_ratio": -0.7585854530334473,
+ "logits/chosen": 1.4372234344482422,
+ "logits/rejected": 1.3944370746612549,
+ "logps/chosen": -1.9776242971420288,
+ "logps/rejected": -1.9795879125595093,
+ "loss": 1.846565055847168,
+ "nll_loss": 1.7707065343856812,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.19776242971420288,
+ "rewards/margins": 0.00019635623903013766,
+ "rewards/rejected": -0.1979587972164154,
+ "step": 310
+ },
+ {
+ "epoch": 0.512,
+ "grad_norm": 0.2272547334432602,
+ "learning_rate": 7.378778897451097e-06,
+ "log_odds_chosen": 0.06932153552770615,
+ "log_odds_ratio": -0.7089754939079285,
+ "logits/chosen": 1.4551548957824707,
+ "logits/rejected": 1.413223147392273,
+ "logps/chosen": -2.0715298652648926,
+ "logps/rejected": -2.1249260902404785,
+ "loss": 1.8696435928344726,
+ "nll_loss": 1.79874587059021,
+ "rewards/accuracies": 0.5375000238418579,
+ "rewards/chosen": -0.20715300738811493,
+ "rewards/margins": 0.005339615046977997,
+ "rewards/rejected": -0.21249261498451233,
+ "step": 320
+ },
+ {
+ "epoch": 0.528,
+ "grad_norm": 0.17856107652187347,
+ "learning_rate": 7.331357439241256e-06,
+ "log_odds_chosen": 0.01983504556119442,
+ "log_odds_ratio": -0.7833854556083679,
+ "logits/chosen": 1.3731528520584106,
+ "logits/rejected": 1.454303503036499,
+ "logps/chosen": -1.9772266149520874,
+ "logps/rejected": -1.9896529912948608,
+ "loss": 1.7815839767456054,
+ "nll_loss": 1.7032454013824463,
+ "rewards/accuracies": 0.5,
+ "rewards/chosen": -0.1977226734161377,
+ "rewards/margins": 0.0012426606845110655,
+ "rewards/rejected": -0.19896534085273743,
+ "step": 330
+ },
+ {
+ "epoch": 0.544,
+ "grad_norm": 0.19845250248908997,
+ "learning_rate": 7.283935981031416e-06,
+ "log_odds_chosen": 0.1632343828678131,
+ "log_odds_ratio": -0.6663784384727478,
+ "logits/chosen": 1.4501641988754272,
+ "logits/rejected": 1.3493249416351318,
+ "logps/chosen": -2.048593521118164,
+ "logps/rejected": -2.186927080154419,
+ "loss": 1.8446569442749023,
+ "nll_loss": 1.7780191898345947,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": -0.20485934615135193,
+ "rewards/margins": 0.013833333738148212,
+ "rewards/rejected": -0.21869266033172607,
+ "step": 340
+ },
+ {
+ "epoch": 0.56,
+ "grad_norm": 0.21156993508338928,
+ "learning_rate": 7.236514522821576e-06,
+ "log_odds_chosen": 0.15967318415641785,
+ "log_odds_ratio": -0.6973395943641663,
+ "logits/chosen": 1.405096173286438,
+ "logits/rejected": 1.3601795434951782,
+ "logps/chosen": -2.0488533973693848,
+ "logps/rejected": -2.1905434131622314,
+ "loss": 1.8135244369506835,
+ "nll_loss": 1.7437903881072998,
+ "rewards/accuracies": 0.512499988079071,
+ "rewards/chosen": -0.2048853635787964,
+ "rewards/margins": 0.014168987981975079,
+ "rewards/rejected": -0.21905431151390076,
+ "step": 350
+ },
+ {
+ "epoch": 0.56,
+ "eval_log_odds_chosen": 0.02301771193742752,
+ "eval_log_odds_ratio": -0.7660307288169861,
+ "eval_logits/chosen": 1.4248073101043701,
+ "eval_logits/rejected": 1.3838006258010864,
+ "eval_logps/chosen": -2.0602059364318848,
+ "eval_logps/rejected": -2.0874686241149902,
+ "eval_loss": 1.8709203004837036,
+ "eval_nll_loss": 1.7943171262741089,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.20602059364318848,
+ "eval_rewards/margins": 0.0027262638323009014,
+ "eval_rewards/rejected": -0.20874685049057007,
+ "eval_runtime": 53.1993,
+ "eval_samples_per_second": 9.399,
+ "eval_steps_per_second": 4.699,
+ "step": 350
+ },
+ {
+ "epoch": 0.576,
+ "grad_norm": 0.21077947318553925,
+ "learning_rate": 7.189093064611737e-06,
+ "log_odds_chosen": 0.14307400584220886,
+ "log_odds_ratio": -0.7016319036483765,
+ "logits/chosen": 1.5183265209197998,
+ "logits/rejected": 1.5037349462509155,
+ "logps/chosen": -2.0080947875976562,
+ "logps/rejected": -2.136094570159912,
+ "loss": 1.8600387573242188,
+ "nll_loss": 1.7898753881454468,
+ "rewards/accuracies": 0.574999988079071,
+ "rewards/chosen": -0.20080947875976562,
+ "rewards/margins": 0.012799995020031929,
+ "rewards/rejected": -0.2136094570159912,
+ "step": 360
+ },
+ {
+ "epoch": 0.592,
+ "grad_norm": 0.21159255504608154,
+ "learning_rate": 7.141671606401897e-06,
+ "log_odds_chosen": 0.05201685428619385,
+ "log_odds_ratio": -0.7168989777565002,
+ "logits/chosen": 1.4038429260253906,
+ "logits/rejected": 1.3609113693237305,
+ "logps/chosen": -2.0137746334075928,
+ "logps/rejected": -2.051905870437622,
+ "loss": 1.864394760131836,
+ "nll_loss": 1.7927048206329346,
+ "rewards/accuracies": 0.48750001192092896,
+ "rewards/chosen": -0.20137743651866913,
+ "rewards/margins": 0.0038131114561110735,
+ "rewards/rejected": -0.20519058406352997,
+ "step": 370
+ },
+ {
+ "epoch": 0.608,
+ "grad_norm": 0.17047084867954254,
+ "learning_rate": 7.094250148192056e-06,
+ "log_odds_chosen": 0.14928987622261047,
+ "log_odds_ratio": -0.7049298286437988,
+ "logits/chosen": 1.4637548923492432,
+ "logits/rejected": 1.5022916793823242,
+ "logps/chosen": -2.015235424041748,
+ "logps/rejected": -2.1330373287200928,
+ "loss": 1.7998531341552735,
+ "nll_loss": 1.7293602228164673,
+ "rewards/accuracies": 0.5874999761581421,
+ "rewards/chosen": -0.2015235424041748,
+ "rewards/margins": 0.011780187487602234,
+ "rewards/rejected": -0.21330371499061584,
+ "step": 380
+ },
+ {
+ "epoch": 0.624,
+ "grad_norm": 0.30234500765800476,
+ "learning_rate": 7.046828689982216e-06,
+ "log_odds_chosen": 0.13443878293037415,
+ "log_odds_ratio": -0.6994689702987671,
+ "logits/chosen": 1.3651323318481445,
+ "logits/rejected": 1.4916176795959473,
+ "logps/chosen": -1.9770710468292236,
+ "logps/rejected": -2.0873372554779053,
+ "loss": 1.8301820755004883,
+ "nll_loss": 1.7602351903915405,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.19770710170269012,
+ "rewards/margins": 0.011026628315448761,
+ "rewards/rejected": -0.20873375236988068,
+ "step": 390
+ },
+ {
+ "epoch": 0.64,
+ "grad_norm": 0.15642815828323364,
+ "learning_rate": 6.999407231772377e-06,
+ "log_odds_chosen": 0.19918577373027802,
+ "log_odds_ratio": -0.6648043394088745,
+ "logits/chosen": 1.658177137374878,
+ "logits/rejected": 1.6399805545806885,
+ "logps/chosen": -1.924300193786621,
+ "logps/rejected": -2.1058459281921387,
+ "loss": 1.8306264877319336,
+ "nll_loss": 1.764146089553833,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.19243000447750092,
+ "rewards/margins": 0.018154580146074295,
+ "rewards/rejected": -0.2105845957994461,
+ "step": 400
+ },
+ {
+ "epoch": 0.64,
+ "eval_log_odds_chosen": 0.029309768229722977,
+ "eval_log_odds_ratio": -0.7632622718811035,
+ "eval_logits/chosen": 1.523261308670044,
+ "eval_logits/rejected": 1.48293936252594,
+ "eval_logps/chosen": -2.0515761375427246,
+ "eval_logps/rejected": -2.0838050842285156,
+ "eval_loss": 1.8621249198913574,
+ "eval_nll_loss": 1.7857985496520996,
+ "eval_rewards/accuracies": 0.527999997138977,
+ "eval_rewards/chosen": -0.20515760779380798,
+ "eval_rewards/margins": 0.0032229016069322824,
+ "eval_rewards/rejected": -0.20838052034378052,
+ "eval_runtime": 53.1851,
+ "eval_samples_per_second": 9.401,
+ "eval_steps_per_second": 4.701,
+ "step": 400
+ },
+ {
+ "epoch": 0.656,
+ "grad_norm": 0.1967567503452301,
+ "learning_rate": 6.951985773562537e-06,
+ "log_odds_chosen": 0.1653221845626831,
+ "log_odds_ratio": -0.6918090581893921,
+ "logits/chosen": 1.5250298976898193,
+ "logits/rejected": 1.4470211267471313,
+ "logps/chosen": -2.0086913108825684,
+ "logps/rejected": -2.145521879196167,
+ "loss": 1.8166702270507813,
+ "nll_loss": 1.7474892139434814,
+ "rewards/accuracies": 0.550000011920929,
+ "rewards/chosen": -0.20086915791034698,
+ "rewards/margins": 0.013683043420314789,
+ "rewards/rejected": -0.21455219388008118,
+ "step": 410
+ },
+ {
+ "epoch": 0.672,
+ "grad_norm": 0.2208651602268219,
+ "learning_rate": 6.904564315352697e-06,
+ "log_odds_chosen": 0.14104491472244263,
+ "log_odds_ratio": -0.7459835410118103,
+ "logits/chosen": 1.605399489402771,
+ "logits/rejected": 1.5775949954986572,
+ "logps/chosen": -1.9965150356292725,
+ "logps/rejected": -2.143871545791626,
+ "loss": 1.749098777770996,
+ "nll_loss": 1.6745002269744873,
+ "rewards/accuracies": 0.5249999761581421,
+ "rewards/chosen": -0.19965147972106934,
+ "rewards/margins": 0.01473566610366106,
+ "rewards/rejected": -0.21438714861869812,
+ "step": 420
+ },
+ {
+ "epoch": 0.688,
+ "grad_norm": 0.2232414036989212,
+ "learning_rate": 6.857142857142856e-06,
+ "log_odds_chosen": 0.10664594173431396,
+ "log_odds_ratio": -0.7230110168457031,
+ "logits/chosen": 1.5467585325241089,
+ "logits/rejected": 1.5005398988723755,
+ "logps/chosen": -1.9454959630966187,
+ "logps/rejected": -2.01902437210083,
+ "loss": 1.771216583251953,
+ "nll_loss": 1.698915719985962,
+ "rewards/accuracies": 0.4749999940395355,
+ "rewards/chosen": -0.1945495903491974,
+ "rewards/margins": 0.0073528350330889225,
+ "rewards/rejected": -0.20190243422985077,
+ "step": 430
+ },
+ {
+ "epoch": 0.704,
+ "grad_norm": 0.19019967317581177,
+ "learning_rate": 6.809721398933017e-06,
+ "log_odds_chosen": -0.022097941488027573,
+ "log_odds_ratio": -0.7597793340682983,
+ "logits/chosen": 1.6924508810043335,
+ "logits/rejected": 1.6551586389541626,
+ "logps/chosen": -1.9923264980316162,
+ "logps/rejected": -1.9681341648101807,
+ "loss": 1.8285696029663085,
+ "nll_loss": 1.7525920867919922,
+ "rewards/accuracies": 0.4625000059604645,
+ "rewards/chosen": -0.19923266768455505,
+ "rewards/margins": -0.002419233787804842,
+ "rewards/rejected": -0.1968134194612503,
+ "step": 440
+ },
+ {
+ "epoch": 0.72,
+ "grad_norm": 0.15414057672023773,
+ "learning_rate": 6.762299940723177e-06,
+ "log_odds_chosen": -0.07213816046714783,
+ "log_odds_ratio": -0.815600574016571,
+ "logits/chosen": 1.5608292818069458,
+ "logits/rejected": 1.5346075296401978,
+ "logps/chosen": -2.106017589569092,
+ "logps/rejected": -2.0266618728637695,
+ "loss": 1.8748476028442382,
+ "nll_loss": 1.7932875156402588,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": -0.21060176193714142,
+ "rewards/margins": -0.007935596629977226,
+ "rewards/rejected": -0.20266616344451904,
+ "step": 450
+ },
+ {
+ "epoch": 0.72,
+ "eval_log_odds_chosen": 0.02593321166932583,
+ "eval_log_odds_ratio": -0.7637203335762024,
+ "eval_logits/chosen": 1.5849090814590454,
+ "eval_logits/rejected": 1.5462759733200073,
+ "eval_logps/chosen": -2.046966791152954,
+ "eval_logps/rejected": -2.076876401901245,
+ "eval_loss": 1.856321930885315,
+ "eval_nll_loss": 1.7799499034881592,
+ "eval_rewards/accuracies": 0.5339999794960022,
+ "eval_rewards/chosen": -0.2046966552734375,
+ "eval_rewards/margins": 0.0029909671284258366,
+ "eval_rewards/rejected": -0.2076876312494278,
+ "eval_runtime": 53.2289,
+ "eval_samples_per_second": 9.393,
+ "eval_steps_per_second": 4.697,
+ "step": 450
+ },
+ {
+ "epoch": 0.736,
+ "grad_norm": 0.2198966145515442,
+ "learning_rate": 6.714878482513337e-06,
+ "log_odds_chosen": 0.1490866094827652,
+ "log_odds_ratio": -0.6993222832679749,
+ "logits/chosen": 1.530315637588501,
+ "logits/rejected": 1.5981829166412354,
+ "logps/chosen": -2.0311427116394043,
+ "logps/rejected": -2.1558847427368164,
+ "loss": 1.852096939086914,
+ "nll_loss": 1.7821648120880127,
+ "rewards/accuracies": 0.5625,
+ "rewards/chosen": -0.20311430096626282,
+ "rewards/margins": 0.01247421558946371,
+ "rewards/rejected": -0.21558848023414612,
+ "step": 460
+ },
+ {
+ "epoch": 0.752,
+ "grad_norm": 0.17686985433101654,
+ "learning_rate": 6.667457024303497e-06,
+ "log_odds_chosen": 0.17564800381660461,
+ "log_odds_ratio": -0.6861358880996704,
+ "logits/chosen": 1.559126377105713,
+ "logits/rejected": 1.4911739826202393,
+ "logps/chosen": -1.9338982105255127,
+ "logps/rejected": -2.090195417404175,
+ "loss": 1.7766691207885743,
+ "nll_loss": 1.7080554962158203,
+ "rewards/accuracies": 0.612500011920929,
+ "rewards/chosen": -0.19338981807231903,
+ "rewards/margins": 0.015629729256033897,
+ "rewards/rejected": -0.20901954174041748,
+ "step": 470
+ },
+ {
+ "epoch": 0.768,
+ "grad_norm": 0.23684880137443542,
+ "learning_rate": 6.620035566093657e-06,
+ "log_odds_chosen": 0.07966659963130951,
+ "log_odds_ratio": -0.7355421781539917,
+ "logits/chosen": 1.610801100730896,
+ "logits/rejected": 1.5581674575805664,
+ "logps/chosen": -1.9333826303482056,
+ "logps/rejected": -2.002697229385376,
+ "loss": 1.8244489669799804,
+ "nll_loss": 1.7508949041366577,
+ "rewards/accuracies": 0.637499988079071,
+ "rewards/chosen": -0.19333826005458832,
+ "rewards/margins": 0.0069314478896558285,
+ "rewards/rejected": -0.20026972889900208,
+ "step": 480
+ },
+ {
+ "epoch": 0.784,
+ "grad_norm": 0.170082688331604,
+ "learning_rate": 6.572614107883817e-06,
+ "log_odds_chosen": -0.016495058313012123,
+ "log_odds_ratio": -0.7567610740661621,
+ "logits/chosen": 1.4795844554901123,
+ "logits/rejected": 1.5042058229446411,
+ "logps/chosen": -2.035449981689453,
+ "logps/rejected": -2.0238888263702393,
+ "loss": 1.8243209838867187,
+ "nll_loss": 1.7486448287963867,
+ "rewards/accuracies": 0.44999998807907104,
+ "rewards/chosen": -0.2035450041294098,
+ "rewards/margins": -0.001156145939603448,
+ "rewards/rejected": -0.20238885283470154,
+ "step": 490
+ },
+ {
+ "epoch": 0.8,
+ "grad_norm": 0.1873895525932312,
+ "learning_rate": 6.525192649673977e-06,
+ "log_odds_chosen": 0.1201021671295166,
+ "log_odds_ratio": -0.7277526259422302,
+ "logits/chosen": 1.5447202920913696,
+ "logits/rejected": 1.494986891746521,
+ "logps/chosen": -2.0172603130340576,
+ "logps/rejected": -2.110663890838623,
+ "loss": 1.7948209762573242,
+ "nll_loss": 1.7220455408096313,
+ "rewards/accuracies": 0.675000011920929,
+ "rewards/chosen": -0.2017260491847992,
+ "rewards/margins": 0.009340338408946991,
+ "rewards/rejected": -0.21106640994548798,
+ "step": 500
+ },
+ {
+ "epoch": 0.8,
+ "eval_log_odds_chosen": 0.02295607700943947,
+ "eval_log_odds_ratio": -0.7634766697883606,
+ "eval_logits/chosen": 1.589399814605713,
+ "eval_logits/rejected": 1.5498627424240112,
+ "eval_logps/chosen": -2.0389089584350586,
+ "eval_logps/rejected": -2.0656278133392334,
+ "eval_loss": 1.851176142692566,
+ "eval_nll_loss": 1.774828314781189,
+ "eval_rewards/accuracies": 0.5379999876022339,
+ "eval_rewards/chosen": -0.20389088988304138,
+ "eval_rewards/margins": 0.0026718771550804377,
+ "eval_rewards/rejected": -0.20656277239322662,
+ "eval_runtime": 53.271,
+ "eval_samples_per_second": 9.386,
+ "eval_steps_per_second": 4.693,
+ "step": 500
+ }
+ ],
+ "logging_steps": 10,
+ "max_steps": 1875,
+ "num_input_tokens_seen": 0,
+ "num_train_epochs": 3,
+ "save_steps": 500,
+ "stateful_callbacks": {
+ "TrainerControl": {
+ "args": {
+ "should_epoch_stop": false,
+ "should_evaluate": false,
+ "should_log": false,
+ "should_save": true,
+ "should_training_stop": false
+ },
+ "attributes": {}
+ }
+ },
+ "total_flos": 0.0,
+ "train_batch_size": 2,
+ "trial_name": null,
+ "trial_params": null
+}
diff --git a/v3/ORPO/ORPO_5k/lora/checkpoint-500/training_args.bin b/v3/ORPO/ORPO_5k/lora/checkpoint-500/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..e369ed5ca0fb0f2b706a5adca8ab527baae13e31
--- /dev/null
+++ b/v3/ORPO/ORPO_5k/lora/checkpoint-500/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4ecab525cdf207fed7abfb6f0080dd00da9f0e35d23314a97f0bff66fe6176b
+size 5457
diff --git a/v3/ORPO/gen-output/ORPO_1k/data-00000-of-00001.arrow b/v3/ORPO/gen-output/ORPO_1k/data-00000-of-00001.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..9d066f72fad7dd9240332bd2cce3a08bd316c9ed
--- /dev/null
+++ b/v3/ORPO/gen-output/ORPO_1k/data-00000-of-00001.arrow
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53cfd324859bb069ff69dc17f714b3e387c82b14c278ceaadd1e57ba0eb4495a
+size 773672
diff --git a/v3/ORPO/gen-output/ORPO_1k/dataset_info.json b/v3/ORPO/gen-output/ORPO_1k/dataset_info.json
new file mode 100644
index 0000000000000000000000000000000000000000..523b881cd388b7ceb2401ad4759de06c0653cbef
--- /dev/null
+++ b/v3/ORPO/gen-output/ORPO_1k/dataset_info.json
@@ -0,0 +1,34 @@
+{
+ "citation": "",
+ "description": "",
+ "features": {
+ "prompt": {
+ "feature": {
+ "content": {
+ "dtype": "string",
+ "_type": "Value"
+ },
+ "role": {
+ "dtype": "string",
+ "_type": "Value"
+ }
+ },
+ "_type": "List"
+ },
+ "generated_text": {
+ "feature": {
+ "content": {
+ "dtype": "string",
+ "_type": "Value"
+ },
+ "role": {
+ "dtype": "string",
+ "_type": "Value"
+ }
+ },
+ "_type": "List"
+ }
+ },
+ "homepage": "",
+ "license": ""
+}
\ No newline at end of file
diff --git a/v3/ORPO/gen-output/ORPO_1k/state.json b/v3/ORPO/gen-output/ORPO_1k/state.json
new file mode 100644
index 0000000000000000000000000000000000000000..d0acb447db05826eec3f4b61ddd988531e30e37a
--- /dev/null
+++ b/v3/ORPO/gen-output/ORPO_1k/state.json
@@ -0,0 +1,13 @@
+{
+ "_data_files": [
+ {
+ "filename": "data-00000-of-00001.arrow"
+ }
+ ],
+ "_fingerprint": "b3364226e4aa5609",
+ "_format_columns": null,
+ "_format_kwargs": {},
+ "_format_type": null,
+ "_output_all_columns": false,
+ "_split": null
+}
\ No newline at end of file
diff --git a/v3/ORPO/gen-output/ORPO_5k/data-00000-of-00001.arrow b/v3/ORPO/gen-output/ORPO_5k/data-00000-of-00001.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..a24c170138aa7acec7d9eec477bb474775fbed78
--- /dev/null
+++ b/v3/ORPO/gen-output/ORPO_5k/data-00000-of-00001.arrow
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7eb14dfcbb5b8ec6abfd369bd9042bc79b7f4bfa0b289bfb4d5522c51917879
+size 711952
diff --git a/v3/ORPO/gen-output/ORPO_5k/dataset_info.json b/v3/ORPO/gen-output/ORPO_5k/dataset_info.json
new file mode 100644
index 0000000000000000000000000000000000000000..523b881cd388b7ceb2401ad4759de06c0653cbef
--- /dev/null
+++ b/v3/ORPO/gen-output/ORPO_5k/dataset_info.json
@@ -0,0 +1,34 @@
+{
+ "citation": "",
+ "description": "",
+ "features": {
+ "prompt": {
+ "feature": {
+ "content": {
+ "dtype": "string",
+ "_type": "Value"
+ },
+ "role": {
+ "dtype": "string",
+ "_type": "Value"
+ }
+ },
+ "_type": "List"
+ },
+ "generated_text": {
+ "feature": {
+ "content": {
+ "dtype": "string",
+ "_type": "Value"
+ },
+ "role": {
+ "dtype": "string",
+ "_type": "Value"
+ }
+ },
+ "_type": "List"
+ }
+ },
+ "homepage": "",
+ "license": ""
+}
\ No newline at end of file
diff --git a/v3/ORPO/gen-output/ORPO_5k/state.json b/v3/ORPO/gen-output/ORPO_5k/state.json
new file mode 100644
index 0000000000000000000000000000000000000000..01791f1f2d04c136d0bec692df65f96cdf14accb
--- /dev/null
+++ b/v3/ORPO/gen-output/ORPO_5k/state.json
@@ -0,0 +1,13 @@
+{
+ "_data_files": [
+ {
+ "filename": "data-00000-of-00001.arrow"
+ }
+ ],
+ "_fingerprint": "9b693287817e0b0d",
+ "_format_columns": null,
+ "_format_kwargs": {},
+ "_format_type": null,
+ "_output_all_columns": false,
+ "_split": null
+}
\ No newline at end of file