Upload 15 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,209 @@
+---
+base_model: DungND1107/sft3tend2end
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:DungND1107/sft3tend2end
+- cpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.17.1

adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "DungND1107/sft3tend2end",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "o_proj",
+    "v_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01132d1d1543a2e9c05e4f2d44583f0237863d69b68301471b531c7242403a1b
+size 102790608

added_tokens.json

@@ -0,0 +1,28 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.jinja

@@ -0,0 +1,85 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set content = message.content %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is defined and message.reasoning_content is not none %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in message.content %}
+                {%- set content = message.content.split('</think>')[-1].lstrip('\n') %}
+                {%- set reasoning_content = message.content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5c8ec2dbd1b4e8060fb382f9483c708fb60b833c5c9a864ab805d3e125988fe
+size 205713355

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:098b29492211804ab324a36f37466821d948280bb74fce4ba895c03f13ecd878
+size 14645

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78fad3d351a41d21dbe1957a34d6a9e1b4a073e802fb274f780ee4e848ede40c
+size 1465

special_tokens_map.json

@@ -0,0 +1,25 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
+size 11422654

tokenizer_config.json

@@ -0,0 +1,239 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,898 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.2100026888948643,
+  "eval_steps": 500,
+  "global_step": 27000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.022407457201756745,
+      "grad_norm": 8.506103515625,
+      "learning_rate": 2.2356630824372757e-07,
+      "logits/chosen": 6.388164043426514,
+      "logits/rejected": 6.283551216125488,
+      "logps/chosen": -150.98416137695312,
+      "logps/rejected": -164.53150939941406,
+      "loss": 4.7093,
+      "nll_loss": 1.7058002948760986,
+      "rewards/accuracies": 0.5630000233650208,
+      "rewards/chosen": -15.098416328430176,
+      "rewards/margins": 1.3547368049621582,
+      "rewards/rejected": -16.45315170288086,
+      "step": 500
+    },
+    {
+      "epoch": 0.04481491440351349,
+      "grad_norm": 4.922695159912109,
+      "learning_rate": 4.475806451612903e-07,
+      "logits/chosen": 6.306144714355469,
+      "logits/rejected": 6.381274700164795,
+      "logps/chosen": -152.2466278076172,
+      "logps/rejected": -161.44313049316406,
+      "loss": 4.8038,
+      "nll_loss": 1.6513851881027222,
+      "rewards/accuracies": 0.5414999723434448,
+      "rewards/chosen": -15.22465991973877,
+      "rewards/margins": 0.9196523427963257,
+      "rewards/rejected": -16.14431381225586,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06722237160527024,
+      "grad_norm": 9.9428129196167,
+      "learning_rate": 6.71594982078853e-07,
+      "logits/chosen": 6.642213821411133,
+      "logits/rejected": 6.576653957366943,
+      "logps/chosen": -142.99542236328125,
+      "logps/rejected": -156.31275939941406,
+      "loss": 4.4727,
+      "nll_loss": 1.5767009258270264,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -14.299542427062988,
+      "rewards/margins": 1.3317352533340454,
+      "rewards/rejected": -15.63127613067627,
+      "step": 1500
+    },
+    {
+      "epoch": 0.08962982880702698,
+      "grad_norm": 20.546499252319336,
+      "learning_rate": 8.956093189964156e-07,
+      "logits/chosen": 6.944845676422119,
+      "logits/rejected": 6.904578685760498,
+      "logps/chosen": -132.52967834472656,
+      "logps/rejected": -145.89498901367188,
+      "loss": 4.3756,
+      "nll_loss": 1.464522361755371,
+      "rewards/accuracies": 0.5665000081062317,
+      "rewards/chosen": -13.25296688079834,
+      "rewards/margins": 1.336530327796936,
+      "rewards/rejected": -14.589498519897461,
+      "step": 2000
+    },
+    {
+      "epoch": 0.11203728600878372,
+      "grad_norm": 14.785259246826172,
+      "learning_rate": 9.937022360600055e-07,
+      "logits/chosen": 7.011036396026611,
+      "logits/rejected": 7.072568893432617,
+      "logps/chosen": -134.64288330078125,
+      "logps/rejected": -143.9438018798828,
+      "loss": 4.5835,
+      "nll_loss": 1.351072072982788,
+      "rewards/accuracies": 0.578000009059906,
+      "rewards/chosen": -13.464289665222168,
+      "rewards/margins": 0.9300898313522339,
+      "rewards/rejected": -14.394380569458008,
+      "step": 2500
+    },
+    {
+      "epoch": 0.13444474321054048,
+      "grad_norm": 5.165164470672607,
+      "learning_rate": 9.819086706293046e-07,
+      "logits/chosen": 7.212893486022949,
+      "logits/rejected": 7.211627006530762,
+      "logps/chosen": -128.81529235839844,
+      "logps/rejected": -141.94216918945312,
+      "loss": 4.2479,
+      "nll_loss": 1.3148466348648071,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -12.881529808044434,
+      "rewards/margins": 1.3126877546310425,
+      "rewards/rejected": -14.194218635559082,
+      "step": 3000
+    },
+    {
+      "epoch": 0.15685220041229722,
+      "grad_norm": 22.26495933532715,
+      "learning_rate": 9.701151051986037e-07,
+      "logits/chosen": 7.118086338043213,
+      "logits/rejected": 7.057218074798584,
+      "logps/chosen": -125.0680160522461,
+      "logps/rejected": -140.8050994873047,
+      "loss": 4.0929,
+      "nll_loss": 1.2322442531585693,
+      "rewards/accuracies": 0.6004999876022339,
+      "rewards/chosen": -12.506800651550293,
+      "rewards/margins": 1.573709487915039,
+      "rewards/rejected": -14.080512046813965,
+      "step": 3500
+    },
+    {
+      "epoch": 0.17925965761405396,
+      "grad_norm": 21.61150360107422,
+      "learning_rate": 9.583215397679026e-07,
+      "logits/chosen": 7.041640758514404,
+      "logits/rejected": 7.064163684844971,
+      "logps/chosen": -124.78068542480469,
+      "logps/rejected": -141.89564514160156,
+      "loss": 4.0565,
+      "nll_loss": 1.2375086545944214,
+      "rewards/accuracies": 0.5824999809265137,
+      "rewards/chosen": -12.478069305419922,
+      "rewards/margins": 1.7114943265914917,
+      "rewards/rejected": -14.18956470489502,
+      "step": 4000
+    },
+    {
+      "epoch": 0.2016671148158107,
+      "grad_norm": 26.76652717590332,
+      "learning_rate": 9.465279743372016e-07,
+      "logits/chosen": 6.971283912658691,
+      "logits/rejected": 6.967947959899902,
+      "logps/chosen": -123.69017791748047,
+      "logps/rejected": -140.5256805419922,
+      "loss": 4.1844,
+      "nll_loss": 1.235988974571228,
+      "rewards/accuracies": 0.593500018119812,
+      "rewards/chosen": -12.369016647338867,
+      "rewards/margins": 1.6835527420043945,
+      "rewards/rejected": -14.052570343017578,
+      "step": 4500
+    },
+    {
+      "epoch": 0.22407457201756745,
+      "grad_norm": 21.370220184326172,
+      "learning_rate": 9.347344089065006e-07,
+      "logits/chosen": 6.8366312980651855,
+      "logits/rejected": 6.852107048034668,
+      "logps/chosen": -120.40402221679688,
+      "logps/rejected": -143.4335479736328,
+      "loss": 3.9117,
+      "nll_loss": 1.2122446298599243,
+      "rewards/accuracies": 0.6075000166893005,
+      "rewards/chosen": -12.040403366088867,
+      "rewards/margins": 2.302950859069824,
+      "rewards/rejected": -14.343354225158691,
+      "step": 5000
+    },
+    {
+      "epoch": 0.2464820292193242,
+      "grad_norm": 22.27083396911621,
+      "learning_rate": 9.229408434757996e-07,
+      "logits/chosen": 6.642422676086426,
+      "logits/rejected": 6.819465637207031,
+      "logps/chosen": -125.4427490234375,
+      "logps/rejected": -141.2194366455078,
+      "loss": 4.3143,
+      "nll_loss": 1.2294857501983643,
+      "rewards/accuracies": 0.5755000114440918,
+      "rewards/chosen": -12.544275283813477,
+      "rewards/margins": 1.5776691436767578,
+      "rewards/rejected": -14.12194538116455,
+      "step": 5500
+    },
+    {
+      "epoch": 0.26888948642108096,
+      "grad_norm": 6.539377689361572,
+      "learning_rate": 9.111472780450985e-07,
+      "logits/chosen": 6.640513896942139,
+      "logits/rejected": 6.69844913482666,
+      "logps/chosen": -122.97315979003906,
+      "logps/rejected": -147.83261108398438,
+      "loss": 3.9476,
+      "nll_loss": 1.2082951068878174,
+      "rewards/accuracies": 0.628000020980835,
+      "rewards/chosen": -12.297317504882812,
+      "rewards/margins": 2.4859437942504883,
+      "rewards/rejected": -14.783259391784668,
+      "step": 6000
+    },
+    {
+      "epoch": 0.2912969436228377,
+      "grad_norm": 33.63134765625,
+      "learning_rate": 8.993537126143975e-07,
+      "logits/chosen": 6.650857925415039,
+      "logits/rejected": 6.703997611999512,
+      "logps/chosen": -118.37783813476562,
+      "logps/rejected": -146.44586181640625,
+      "loss": 3.6968,
+      "nll_loss": 1.2175222635269165,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -11.837782859802246,
+      "rewards/margins": 2.8068015575408936,
+      "rewards/rejected": -14.644585609436035,
+      "step": 6500
+    },
+    {
+      "epoch": 0.31370440082459444,
+      "grad_norm": 9.306456565856934,
+      "learning_rate": 8.875601471836965e-07,
+      "logits/chosen": 6.396566390991211,
+      "logits/rejected": 6.541921615600586,
+      "logps/chosen": -126.1952133178711,
+      "logps/rejected": -156.82826232910156,
+      "loss": 3.8978,
+      "nll_loss": 1.2233952283859253,
+      "rewards/accuracies": 0.6215000152587891,
+      "rewards/chosen": -12.619522094726562,
+      "rewards/margins": 3.0633037090301514,
+      "rewards/rejected": -15.68282699584961,
+      "step": 7000
+    },
+    {
+      "epoch": 0.3361118580263512,
+      "grad_norm": 52.533870697021484,
+      "learning_rate": 8.757665817529955e-07,
+      "logits/chosen": 6.345678329467773,
+      "logits/rejected": 6.36190128326416,
+      "logps/chosen": -126.47830963134766,
+      "logps/rejected": -172.89540100097656,
+      "loss": 3.5842,
+      "nll_loss": 1.21086585521698,
+      "rewards/accuracies": 0.6635000109672546,
+      "rewards/chosen": -12.64783000946045,
+      "rewards/margins": 4.6417107582092285,
+      "rewards/rejected": -17.289539337158203,
+      "step": 7500
+    },
+    {
+      "epoch": 0.3585193152281079,
+      "grad_norm": 5.680696964263916,
+      "learning_rate": 8.639730163222945e-07,
+      "logits/chosen": 6.292329788208008,
+      "logits/rejected": 6.211752891540527,
+      "logps/chosen": -125.1585922241211,
+      "logps/rejected": -183.92459106445312,
+      "loss": 3.5147,
+      "nll_loss": 1.2218888998031616,
+      "rewards/accuracies": 0.6790000200271606,
+      "rewards/chosen": -12.515860557556152,
+      "rewards/margins": 5.876598358154297,
+      "rewards/rejected": -18.392457962036133,
+      "step": 8000
+    },
+    {
+      "epoch": 0.38092677242986467,
+      "grad_norm": 229.6942138671875,
+      "learning_rate": 8.521794508915935e-07,
+      "logits/chosen": 6.256952285766602,
+      "logits/rejected": 6.1186699867248535,
+      "logps/chosen": -121.80457305908203,
+      "logps/rejected": -195.48875427246094,
+      "loss": 3.1057,
+      "nll_loss": 1.2142161130905151,
+      "rewards/accuracies": 0.6834999918937683,
+      "rewards/chosen": -12.18045711517334,
+      "rewards/margins": 7.368417739868164,
+      "rewards/rejected": -19.54887580871582,
+      "step": 8500
+    },
+    {
+      "epoch": 0.4033342296316214,
+      "grad_norm": 38.856204986572266,
+      "learning_rate": 8.403858854608925e-07,
+      "logits/chosen": 6.191353797912598,
+      "logits/rejected": 6.074046611785889,
+      "logps/chosen": -123.83197784423828,
+      "logps/rejected": -199.24847412109375,
+      "loss": 3.1358,
+      "nll_loss": 1.2347651720046997,
+      "rewards/accuracies": 0.6940000057220459,
+      "rewards/chosen": -12.383196830749512,
+      "rewards/margins": 7.54164981842041,
+      "rewards/rejected": -19.924846649169922,
+      "step": 9000
+    },
+    {
+      "epoch": 0.42574168683337815,
+      "grad_norm": 7.541586875915527,
+      "learning_rate": 8.285923200301915e-07,
+      "logits/chosen": 6.239284992218018,
+      "logits/rejected": 5.987191677093506,
+      "logps/chosen": -124.85488891601562,
+      "logps/rejected": -211.95565795898438,
+      "loss": 3.2063,
+      "nll_loss": 1.2126648426055908,
+      "rewards/accuracies": 0.7059999704360962,
+      "rewards/chosen": -12.485487937927246,
+      "rewards/margins": 8.710078239440918,
+      "rewards/rejected": -21.19556427001953,
+      "step": 9500
+    },
+    {
+      "epoch": 0.4481491440351349,
+      "grad_norm": 203.79039001464844,
+      "learning_rate": 8.167987545994904e-07,
+      "logits/chosen": 6.260392665863037,
+      "logits/rejected": 5.989898204803467,
+      "logps/chosen": -125.32071685791016,
+      "logps/rejected": -219.6128387451172,
+      "loss": 3.0664,
+      "nll_loss": 1.2339407205581665,
+      "rewards/accuracies": 0.6859999895095825,
+      "rewards/chosen": -12.532071113586426,
+      "rewards/margins": 9.429213523864746,
+      "rewards/rejected": -21.961284637451172,
+      "step": 10000
+    },
+    {
+      "epoch": 0.47055660123689164,
+      "grad_norm": 47.179386138916016,
+      "learning_rate": 8.050051891687894e-07,
+      "logits/chosen": 6.154437065124512,
+      "logits/rejected": 5.911861419677734,
+      "logps/chosen": -124.98944091796875,
+      "logps/rejected": -221.49061584472656,
+      "loss": 3.0071,
+      "nll_loss": 1.2235456705093384,
+      "rewards/accuracies": 0.7020000219345093,
+      "rewards/chosen": -12.498944282531738,
+      "rewards/margins": 9.650117874145508,
+      "rewards/rejected": -22.14906120300293,
+      "step": 10500
+    },
+    {
+      "epoch": 0.4929640584386484,
+      "grad_norm": 47.614830017089844,
+      "learning_rate": 7.932116237380885e-07,
+      "logits/chosen": 6.229808807373047,
+      "logits/rejected": 5.8305230140686035,
+      "logps/chosen": -120.84117126464844,
+      "logps/rejected": -226.11627197265625,
+      "loss": 2.8767,
+      "nll_loss": 1.2143878936767578,
+      "rewards/accuracies": 0.7164999842643738,
+      "rewards/chosen": -12.084115982055664,
+      "rewards/margins": 10.527509689331055,
+      "rewards/rejected": -22.61162757873535,
+      "step": 11000
+    },
+    {
+      "epoch": 0.5153715156404052,
+      "grad_norm": 4.079768180847168,
+      "learning_rate": 7.814180583073875e-07,
+      "logits/chosen": 6.222982406616211,
+      "logits/rejected": 5.716063499450684,
+      "logps/chosen": -115.15640258789062,
+      "logps/rejected": -238.39898681640625,
+      "loss": 2.6547,
+      "nll_loss": 1.185829758644104,
+      "rewards/accuracies": 0.7310000061988831,
+      "rewards/chosen": -11.515642166137695,
+      "rewards/margins": 12.324254989624023,
+      "rewards/rejected": -23.839895248413086,
+      "step": 11500
+    },
+    {
+      "epoch": 0.5377789728421619,
+      "grad_norm": 7.2531538009643555,
+      "learning_rate": 7.696244928766864e-07,
+      "logits/chosen": 5.979344367980957,
+      "logits/rejected": 5.487608909606934,
+      "logps/chosen": -123.36662292480469,
+      "logps/rejected": -242.37765502929688,
+      "loss": 2.9167,
+      "nll_loss": 1.171997308731079,
+      "rewards/accuracies": 0.7264999747276306,
+      "rewards/chosen": -12.336663246154785,
+      "rewards/margins": 11.901101112365723,
+      "rewards/rejected": -24.23776626586914,
+      "step": 12000
+    },
+    {
+      "epoch": 0.5601864300439187,
+      "grad_norm": 10.61841106414795,
+      "learning_rate": 7.578309274459854e-07,
+      "logits/chosen": 5.975569248199463,
+      "logits/rejected": 5.519693374633789,
+      "logps/chosen": -121.61878204345703,
+      "logps/rejected": -242.49684143066406,
+      "loss": 2.8891,
+      "nll_loss": 1.1807277202606201,
+      "rewards/accuracies": 0.7315000295639038,
+      "rewards/chosen": -12.16187858581543,
+      "rewards/margins": 12.087806701660156,
+      "rewards/rejected": -24.249683380126953,
+      "step": 12500
+    },
+    {
+      "epoch": 0.5825938872456754,
+      "grad_norm": 7.33936071395874,
+      "learning_rate": 7.460373620152844e-07,
+      "logits/chosen": 5.970071315765381,
+      "logits/rejected": 5.4041032791137695,
+      "logps/chosen": -122.3097152709961,
+      "logps/rejected": -258.98699951171875,
+      "loss": 2.8075,
+      "nll_loss": 1.163968563079834,
+      "rewards/accuracies": 0.7360000014305115,
+      "rewards/chosen": -12.230973243713379,
+      "rewards/margins": 13.667726516723633,
+      "rewards/rejected": -25.898698806762695,
+      "step": 13000
+    },
+    {
+      "epoch": 0.6050013444474321,
+      "grad_norm": 10.452156066894531,
+      "learning_rate": 7.342437965845834e-07,
+      "logits/chosen": 6.058469295501709,
+      "logits/rejected": 5.49982213973999,
+      "logps/chosen": -122.52383422851562,
+      "logps/rejected": -261.96868896484375,
+      "loss": 2.8721,
+      "nll_loss": 1.190604567527771,
+      "rewards/accuracies": 0.7279999852180481,
+      "rewards/chosen": -12.252385139465332,
+      "rewards/margins": 13.944486618041992,
+      "rewards/rejected": -26.196870803833008,
+      "step": 13500
+    },
+    {
+      "epoch": 0.6274088016491889,
+      "grad_norm": 11.139547348022461,
+      "learning_rate": 7.224502311538823e-07,
+      "logits/chosen": 6.06943416595459,
+      "logits/rejected": 5.572029113769531,
+      "logps/chosen": -120.20881652832031,
+      "logps/rejected": -256.650146484375,
+      "loss": 2.7815,
+      "nll_loss": 1.1917482614517212,
+      "rewards/accuracies": 0.7149999737739563,
+      "rewards/chosen": -12.020880699157715,
+      "rewards/margins": 13.64413070678711,
+      "rewards/rejected": -25.665016174316406,
+      "step": 14000
+    },
+    {
+      "epoch": 0.6498162588509456,
+      "grad_norm": 47.280433654785156,
+      "learning_rate": 7.106566657231814e-07,
+      "logits/chosen": 5.949710845947266,
+      "logits/rejected": 5.425146102905273,
+      "logps/chosen": -124.7117691040039,
+      "logps/rejected": -269.748291015625,
+      "loss": 3.11,
+      "nll_loss": 1.1818426847457886,
+      "rewards/accuracies": 0.7260000109672546,
+      "rewards/chosen": -12.471176147460938,
+      "rewards/margins": 14.503654479980469,
+      "rewards/rejected": -26.974828720092773,
+      "step": 14500
+    },
+    {
+      "epoch": 0.6722237160527024,
+      "grad_norm": 887.6209106445312,
+      "learning_rate": 6.988631002924804e-07,
+      "logits/chosen": 5.897467136383057,
+      "logits/rejected": 5.365013122558594,
+      "logps/chosen": -121.7276611328125,
+      "logps/rejected": -270.69476318359375,
+      "loss": 2.7853,
+      "nll_loss": 1.1876649856567383,
+      "rewards/accuracies": 0.7315000295639038,
+      "rewards/chosen": -12.172765731811523,
+      "rewards/margins": 14.896709442138672,
+      "rewards/rejected": -27.069477081298828,
+      "step": 15000
+    },
+    {
+      "epoch": 0.6946311732544591,
+      "grad_norm": 101.6743392944336,
+      "learning_rate": 6.870695348617794e-07,
+      "logits/chosen": 6.049013614654541,
+      "logits/rejected": 5.464139461517334,
+      "logps/chosen": -113.61859130859375,
+      "logps/rejected": -255.9214630126953,
+      "loss": 2.7905,
+      "nll_loss": 1.15791654586792,
+      "rewards/accuracies": 0.7434999942779541,
+      "rewards/chosen": -11.361859321594238,
+      "rewards/margins": 14.2302885055542,
+      "rewards/rejected": -25.592147827148438,
+      "step": 15500
+    },
+    {
+      "epoch": 0.7170386304562159,
+      "grad_norm": 6.648136615753174,
+      "learning_rate": 6.752759694310783e-07,
+      "logits/chosen": 5.878724575042725,
+      "logits/rejected": 5.278157711029053,
+      "logps/chosen": -119.80939483642578,
+      "logps/rejected": -268.7624816894531,
+      "loss": 2.6896,
+      "nll_loss": 1.1567949056625366,
+      "rewards/accuracies": 0.746999979019165,
+      "rewards/chosen": -11.980939865112305,
+      "rewards/margins": 14.895305633544922,
+      "rewards/rejected": -26.876245498657227,
+      "step": 16000
+    },
+    {
+      "epoch": 0.7394460876579726,
+      "grad_norm": 695.8414306640625,
+      "learning_rate": 6.634824040003773e-07,
+      "logits/chosen": 5.902768135070801,
+      "logits/rejected": 5.250386714935303,
+      "logps/chosen": -118.48967742919922,
+      "logps/rejected": -269.3214416503906,
+      "loss": 2.8065,
+      "nll_loss": 1.1784237623214722,
+      "rewards/accuracies": 0.7419999837875366,
+      "rewards/chosen": -11.848968505859375,
+      "rewards/margins": 15.083173751831055,
+      "rewards/rejected": -26.93214225769043,
+      "step": 16500
+    },
+    {
+      "epoch": 0.7618535448597293,
+      "grad_norm": 3.3734772205352783,
+      "learning_rate": 6.516888385696764e-07,
+      "logits/chosen": 5.808507919311523,
+      "logits/rejected": 5.211460590362549,
+      "logps/chosen": -119.68141174316406,
+      "logps/rejected": -278.1335754394531,
+      "loss": 2.6728,
+      "nll_loss": 1.149533987045288,
+      "rewards/accuracies": 0.7584999799728394,
+      "rewards/chosen": -11.968141555786133,
+      "rewards/margins": 15.84521484375,
+      "rewards/rejected": -27.813356399536133,
+      "step": 17000
+    },
+    {
+      "epoch": 0.7842610020614861,
+      "grad_norm": 11.741183280944824,
+      "learning_rate": 6.398952731389754e-07,
+      "logits/chosen": 5.954285144805908,
+      "logits/rejected": 5.2305779457092285,
+      "logps/chosen": -118.8548812866211,
+      "logps/rejected": -274.6248779296875,
+      "loss": 2.782,
+      "nll_loss": 1.1827383041381836,
+      "rewards/accuracies": 0.7519999742507935,
+      "rewards/chosen": -11.885488510131836,
+      "rewards/margins": 15.577001571655273,
+      "rewards/rejected": -27.46249008178711,
+      "step": 17500
+    },
+    {
+      "epoch": 0.8066684592632428,
+      "grad_norm": 6.003170967102051,
+      "learning_rate": 6.281017077082744e-07,
+      "logits/chosen": 5.736708164215088,
+      "logits/rejected": 5.108031749725342,
+      "logps/chosen": -119.36869049072266,
+      "logps/rejected": -283.8721008300781,
+      "loss": 2.5192,
+      "nll_loss": 1.1537575721740723,
+      "rewards/accuracies": 0.746999979019165,
+      "rewards/chosen": -11.936868667602539,
+      "rewards/margins": 16.450340270996094,
+      "rewards/rejected": -28.38720703125,
+      "step": 18000
+    },
+    {
+      "epoch": 0.8290759164649996,
+      "grad_norm": 6.127528190612793,
+      "learning_rate": 6.163081422775734e-07,
+      "logits/chosen": 5.798559665679932,
+      "logits/rejected": 5.145273208618164,
+      "logps/chosen": -116.72356414794922,
+      "logps/rejected": -271.2331848144531,
+      "loss": 2.6768,
+      "nll_loss": 1.162487268447876,
+      "rewards/accuracies": 0.7419999837875366,
+      "rewards/chosen": -11.672357559204102,
+      "rewards/margins": 15.450961112976074,
+      "rewards/rejected": -27.123315811157227,
+      "step": 18500
+    },
+    {
+      "epoch": 0.8514833736667563,
+      "grad_norm": 10.272464752197266,
+      "learning_rate": 6.045145768468724e-07,
+      "logits/chosen": 5.804078102111816,
+      "logits/rejected": 5.157005310058594,
+      "logps/chosen": -125.4914321899414,
+      "logps/rejected": -290.3161315917969,
+      "loss": 2.8775,
+      "nll_loss": 1.1908812522888184,
+      "rewards/accuracies": 0.7505000233650208,
+      "rewards/chosen": -12.54914379119873,
+      "rewards/margins": 16.482467651367188,
+      "rewards/rejected": -29.0316104888916,
+      "step": 19000
+    },
+    {
+      "epoch": 0.873890830868513,
+      "grad_norm": 8.924664497375488,
+      "learning_rate": 5.927210114161713e-07,
+      "logits/chosen": 5.890047550201416,
+      "logits/rejected": 5.312379360198975,
+      "logps/chosen": -117.87967681884766,
+      "logps/rejected": -270.41766357421875,
+      "loss": 2.8344,
+      "nll_loss": 1.1640441417694092,
+      "rewards/accuracies": 0.7595000267028809,
+      "rewards/chosen": -11.78796672821045,
+      "rewards/margins": 15.253799438476562,
+      "rewards/rejected": -27.041765213012695,
+      "step": 19500
+    },
+    {
+      "epoch": 0.8962982880702698,
+      "grad_norm": 84.9556884765625,
+      "learning_rate": 5.809274459854703e-07,
+      "logits/chosen": 5.801331996917725,
+      "logits/rejected": 5.0717902183532715,
+      "logps/chosen": -121.90699768066406,
+      "logps/rejected": -299.4970397949219,
+      "loss": 2.6789,
+      "nll_loss": 1.175416111946106,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -12.19070053100586,
+      "rewards/margins": 17.759004592895508,
+      "rewards/rejected": -29.949705123901367,
+      "step": 20000
+    },
+    {
+      "epoch": 0.9187057452720265,
+      "grad_norm": 9.391518592834473,
+      "learning_rate": 5.691338805547693e-07,
+      "logits/chosen": 5.781292915344238,
+      "logits/rejected": 4.962510585784912,
+      "logps/chosen": -121.11032104492188,
+      "logps/rejected": -308.0752868652344,
+      "loss": 2.5481,
+      "nll_loss": 1.1432969570159912,
+      "rewards/accuracies": 0.7634999752044678,
+      "rewards/chosen": -12.111032485961914,
+      "rewards/margins": 18.69649887084961,
+      "rewards/rejected": -30.807527542114258,
+      "step": 20500
+    },
+    {
+      "epoch": 0.9411132024737833,
+      "grad_norm": 20.431760787963867,
+      "learning_rate": 5.573403151240683e-07,
+      "logits/chosen": 5.772525787353516,
+      "logits/rejected": 5.1370391845703125,
+      "logps/chosen": -119.0662841796875,
+      "logps/rejected": -288.95611572265625,
+      "loss": 2.6056,
+      "nll_loss": 1.155178427696228,
+      "rewards/accuracies": 0.7505000233650208,
+      "rewards/chosen": -11.906628608703613,
+      "rewards/margins": 16.988988876342773,
+      "rewards/rejected": -28.89561653137207,
+      "step": 21000
+    },
+    {
+      "epoch": 0.96352065967554,
+      "grad_norm": 9.091734886169434,
+      "learning_rate": 5.455467496933673e-07,
+      "logits/chosen": 5.795172214508057,
+      "logits/rejected": 5.024606227874756,
+      "logps/chosen": -115.77607727050781,
+      "logps/rejected": -290.7126770019531,
+      "loss": 2.6734,
+      "nll_loss": 1.1651384830474854,
+      "rewards/accuracies": 0.7735000252723694,
+      "rewards/chosen": -11.577609062194824,
+      "rewards/margins": 17.4936580657959,
+      "rewards/rejected": -29.071266174316406,
+      "step": 21500
+    },
+    {
+      "epoch": 0.9859281168772968,
+      "grad_norm": 56.138492584228516,
+      "learning_rate": 5.337531842626663e-07,
+      "logits/chosen": 5.808271408081055,
+      "logits/rejected": 5.124356269836426,
+      "logps/chosen": -114.89234161376953,
+      "logps/rejected": -281.4605407714844,
+      "loss": 2.4982,
+      "nll_loss": 1.1335924863815308,
+      "rewards/accuracies": 0.7540000081062317,
+      "rewards/chosen": -11.489234924316406,
+      "rewards/margins": 16.65682029724121,
+      "rewards/rejected": -28.146055221557617,
+      "step": 22000
+    },
+    {
+      "epoch": 1.0083355740790536,
+      "grad_norm": 3.862569570541382,
+      "learning_rate": 5.219596188319653e-07,
+      "logits/chosen": 5.7207255363464355,
+      "logits/rejected": 5.00246000289917,
+      "logps/chosen": -123.77666473388672,
+      "logps/rejected": -305.8909912109375,
+      "loss": 2.7238,
+      "nll_loss": 1.1860337257385254,
+      "rewards/accuracies": 0.7705000042915344,
+      "rewards/chosen": -12.377666473388672,
+      "rewards/margins": 18.21143341064453,
+      "rewards/rejected": -30.589099884033203,
+      "step": 22500
+    },
+    {
+      "epoch": 1.0307430312808104,
+      "grad_norm": 383.5379638671875,
+      "learning_rate": 5.101660534012643e-07,
+      "logits/chosen": 5.727014541625977,
+      "logits/rejected": 5.060346603393555,
+      "logps/chosen": -116.65225219726562,
+      "logps/rejected": -282.9936828613281,
+      "loss": 2.6371,
+      "nll_loss": 1.171337604522705,
+      "rewards/accuracies": 0.7505000233650208,
+      "rewards/chosen": -11.665225982666016,
+      "rewards/margins": 16.634140014648438,
+      "rewards/rejected": -28.29936981201172,
+      "step": 23000
+    },
+    {
+      "epoch": 1.053150488482567,
+      "grad_norm": 9.49420166015625,
+      "learning_rate": 4.983724879705633e-07,
+      "logits/chosen": 5.759654998779297,
+      "logits/rejected": 4.950366973876953,
+      "logps/chosen": -121.17098999023438,
+      "logps/rejected": -312.89959716796875,
+      "loss": 2.5624,
+      "nll_loss": 1.1499173641204834,
+      "rewards/accuracies": 0.7695000171661377,
+      "rewards/chosen": -12.117101669311523,
+      "rewards/margins": 19.172861099243164,
+      "rewards/rejected": -31.289958953857422,
+      "step": 23500
+    },
+    {
+      "epoch": 1.0755579456843238,
+      "grad_norm": 19.929637908935547,
+      "learning_rate": 4.865789225398622e-07,
+      "logits/chosen": 5.611916542053223,
+      "logits/rejected": 4.861335277557373,
+      "logps/chosen": -119.94727325439453,
+      "logps/rejected": -299.9891052246094,
+      "loss": 2.4789,
+      "nll_loss": 1.1652500629425049,
+      "rewards/accuracies": 0.7724999785423279,
+      "rewards/chosen": -11.994726181030273,
+      "rewards/margins": 18.004182815551758,
+      "rewards/rejected": -29.998910903930664,
+      "step": 24000
+    },
+    {
+      "epoch": 1.0979654028860806,
+      "grad_norm": 23.77342987060547,
+      "learning_rate": 4.7478535710916124e-07,
+      "logits/chosen": 5.708724498748779,
+      "logits/rejected": 4.925475597381592,
+      "logps/chosen": -121.61338806152344,
+      "logps/rejected": -305.72137451171875,
+      "loss": 2.7961,
+      "nll_loss": 1.142238974571228,
+      "rewards/accuracies": 0.7735000252723694,
+      "rewards/chosen": -12.161338806152344,
+      "rewards/margins": 18.410797119140625,
+      "rewards/rejected": -30.57213592529297,
+      "step": 24500
+    },
+    {
+      "epoch": 1.1203728600878373,
+      "grad_norm": 204.37399291992188,
+      "learning_rate": 4.629917916784602e-07,
+      "logits/chosen": 5.644859790802002,
+      "logits/rejected": 4.847799777984619,
+      "logps/chosen": -122.57526397705078,
+      "logps/rejected": -313.4942321777344,
+      "loss": 2.4683,
+      "nll_loss": 1.1514092683792114,
+      "rewards/accuracies": 0.7985000014305115,
+      "rewards/chosen": -12.257527351379395,
+      "rewards/margins": 19.091896057128906,
+      "rewards/rejected": -31.349422454833984,
+      "step": 25000
+    },
+    {
+      "epoch": 1.142780317289594,
+      "grad_norm": 3.609483480453491,
+      "learning_rate": 4.5119822624775924e-07,
+      "logits/chosen": 5.685230255126953,
+      "logits/rejected": 4.9433393478393555,
+      "logps/chosen": -115.14147186279297,
+      "logps/rejected": -300.2004699707031,
+      "loss": 2.3462,
+      "nll_loss": 1.1510778665542603,
+      "rewards/accuracies": 0.7894999980926514,
+      "rewards/chosen": -11.514147758483887,
+      "rewards/margins": 18.505897521972656,
+      "rewards/rejected": -30.02004623413086,
+      "step": 25500
+    },
+    {
+      "epoch": 1.1651877744913508,
+      "grad_norm": 4.68535852432251,
+      "learning_rate": 4.394046608170582e-07,
+      "logits/chosen": 5.764662265777588,
+      "logits/rejected": 4.978487491607666,
+      "logps/chosen": -117.07064819335938,
+      "logps/rejected": -287.5929260253906,
+      "loss": 2.5315,
+      "nll_loss": 1.1573420763015747,
+      "rewards/accuracies": 0.7639999985694885,
+      "rewards/chosen": -11.707064628601074,
+      "rewards/margins": 17.052227020263672,
+      "rewards/rejected": -28.759292602539062,
+      "step": 26000
+    },
+    {
+      "epoch": 1.1875952316931075,
+      "grad_norm": 330.53668212890625,
+      "learning_rate": 4.276110953863572e-07,
+      "logits/chosen": 5.670849800109863,
+      "logits/rejected": 4.9159650802612305,
+      "logps/chosen": -120.54727935791016,
+      "logps/rejected": -312.1987609863281,
+      "loss": 2.6538,
+      "nll_loss": 1.1695470809936523,
+      "rewards/accuracies": 0.7760000228881836,
+      "rewards/chosen": -12.054729461669922,
+      "rewards/margins": 19.165143966674805,
+      "rewards/rejected": -31.219871520996094,
+      "step": 26500
+    },
+    {
+      "epoch": 1.2100026888948643,
+      "grad_norm": 16.44524383544922,
+      "learning_rate": 4.1581752995565614e-07,
+      "logits/chosen": 5.748635768890381,
+      "logits/rejected": 4.906609058380127,
+      "logps/chosen": -113.723876953125,
+      "logps/rejected": -296.6240234375,
+      "loss": 2.3728,
+      "nll_loss": 1.1420414447784424,
+      "rewards/accuracies": 0.784500002861023,
+      "rewards/chosen": -11.372386932373047,
+      "rewards/margins": 18.290016174316406,
+      "rewards/rejected": -29.662403106689453,
+      "step": 27000
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 44628,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a453265bfb7b619bfcd95c17eb719309193110433b90f46208d091906fc3e444
+size 6161