Upload checkpoint with token

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,202 @@
+---
+base_model: hai2131/sailor2_1b_mt_sft
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.15.2

adapter_config.json

@@ -0,0 +1,43 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": {
+    "base_model_class": "Qwen2ForCausalLM",
+    "parent_library": "transformers.models.qwen2.modeling_qwen2",
+    "unsloth_fixed": true
+  },
+  "base_model_name_or_path": "hai2131/sailor2_1b_mt_sft",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "up_proj",
+    "o_proj",
+    "v_proj",
+    "q_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b85280e0ed74c9882d21a5b538b98c48cbe403809e9fc9ed3d1bd7aa913a49f1
+size 140860656

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.jinja

@@ -0,0 +1,6 @@
+{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are an AI assistant named Sailor2, created by Sea AI Lab. As an AI assistant, you can answer questions in English, Chinese, and Southeast Asian languages such as Burmese, Cebuano, Ilocano, Indonesian, Javanese, Khmer, Lao, Malay, Sundanese, Tagalog, Thai, Vietnamese, and Waray. Your responses should be friendly, unbiased, informative, detailed, and faithful.<|im_end|>
+' }}{% endif %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:965e74ce900db6f9d199af61739816aff53fe39e1768d57639ceaaaff2cd9af7
+size 72272293

rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cc3307602102ef5b4699c5d0d0877f4df4e9a8c6ed16159e0eb5f7c205b7a82
+size 15365

rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a564fd29fdaea985856b48f282cabacb0dffa1dd1a79d98c9e6177e1de774cf
+size 15365

rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c1fbc115b0e5dd3df1a482bef8a39fc8b8fc0f501ef419324e60d082c1133bc
+size 15365

rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:476ddfa1fd95c624fc018bdacac3fd79e42329b309ca4076c70c58da0694461a
+size 15365

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc3bfdba43e140aa8e5cd5fc947cfb23559f9daad7f487c0e1a19d5a1110e00e
+size 1465

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

tokenizer_config.json

@@ -0,0 +1,212 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "max_length": 3848,
+  "model_max_length": 4096,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "stride": 0,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,1234 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 3750,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.04,
+      "grad_norm": 24.803234100341797,
+      "learning_rate": 1.3066666666666667e-06,
+      "logits/chosen": 2.3097052574157715,
+      "logits/rejected": 2.3757848739624023,
+      "logps/chosen": -67.16219329833984,
+      "logps/rejected": -80.8543930053711,
+      "loss": 1.0777,
+      "nll_loss": 0.41877487301826477,
+      "rewards/accuracies": 0.7237499952316284,
+      "rewards/chosen": -6.716219425201416,
+      "rewards/margins": 1.3692210912704468,
+      "rewards/rejected": -8.085439682006836,
+      "step": 50
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 12.049943923950195,
+      "learning_rate": 2.64e-06,
+      "logits/chosen": 2.2844557762145996,
+      "logits/rejected": 2.364481210708618,
+      "logps/chosen": -61.06942367553711,
+      "logps/rejected": -75.95172119140625,
+      "loss": 1.0254,
+      "nll_loss": 0.38137227296829224,
+      "rewards/accuracies": 0.7174999713897705,
+      "rewards/chosen": -6.106942176818848,
+      "rewards/margins": 1.4882298707962036,
+      "rewards/rejected": -7.595171928405762,
+      "step": 100
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 19.429025650024414,
+      "learning_rate": 3.973333333333333e-06,
+      "logits/chosen": 2.2412264347076416,
+      "logits/rejected": 2.2957522869110107,
+      "logps/chosen": -56.071502685546875,
+      "logps/rejected": -67.70003509521484,
+      "loss": 1.0618,
+      "nll_loss": 0.3372814655303955,
+      "rewards/accuracies": 0.6650000214576721,
+      "rewards/chosen": -5.607150077819824,
+      "rewards/margins": 1.1628532409667969,
+      "rewards/rejected": -6.770003795623779,
+      "step": 150
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 14.493091583251953,
+      "learning_rate": 5.306666666666667e-06,
+      "logits/chosen": 2.1775848865509033,
+      "logits/rejected": 2.25844144821167,
+      "logps/chosen": -50.50484848022461,
+      "logps/rejected": -64.37080383300781,
+      "loss": 0.9298,
+      "nll_loss": 0.3056701719760895,
+      "rewards/accuracies": 0.7237499952316284,
+      "rewards/chosen": -5.050484657287598,
+      "rewards/margins": 1.3865959644317627,
+      "rewards/rejected": -6.437079906463623,
+      "step": 200
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 17.336822509765625,
+      "learning_rate": 6.640000000000001e-06,
+      "logits/chosen": 2.203662872314453,
+      "logits/rejected": 2.28377103805542,
+      "logps/chosen": -51.80202102661133,
+      "logps/rejected": -65.30438232421875,
+      "loss": 0.9291,
+      "nll_loss": 0.2998887896537781,
+      "rewards/accuracies": 0.7149999737739563,
+      "rewards/chosen": -5.180202007293701,
+      "rewards/margins": 1.3502373695373535,
+      "rewards/rejected": -6.530439376831055,
+      "step": 250
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 25.542375564575195,
+      "learning_rate": 7.973333333333334e-06,
+      "logits/chosen": 2.2555084228515625,
+      "logits/rejected": 2.3327550888061523,
+      "logps/chosen": -49.52461242675781,
+      "logps/rejected": -64.92198181152344,
+      "loss": 0.8288,
+      "nll_loss": 0.29204535484313965,
+      "rewards/accuracies": 0.7674999833106995,
+      "rewards/chosen": -4.9524617195129395,
+      "rewards/margins": 1.5397369861602783,
+      "rewards/rejected": -6.492198467254639,
+      "step": 300
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 10.780914306640625,
+      "learning_rate": 9.306666666666667e-06,
+      "logits/chosen": 2.2995264530181885,
+      "logits/rejected": 2.3702523708343506,
+      "logps/chosen": -49.0943489074707,
+      "logps/rejected": -62.10791015625,
+      "loss": 0.9434,
+      "nll_loss": 0.29649874567985535,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.909435272216797,
+      "rewards/margins": 1.3013556003570557,
+      "rewards/rejected": -6.210791110992432,
+      "step": 350
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 11.368802070617676,
+      "learning_rate": 9.998752338940612e-06,
+      "logits/chosen": 2.4777705669403076,
+      "logits/rejected": 2.5543289184570312,
+      "logps/chosen": -46.3427848815918,
+      "logps/rejected": -58.78908157348633,
+      "loss": 0.8661,
+      "nll_loss": 0.27519530057907104,
+      "rewards/accuracies": 0.7137500047683716,
+      "rewards/chosen": -4.634278297424316,
+      "rewards/margins": 1.2446298599243164,
+      "rewards/rejected": -5.878908634185791,
+      "step": 400
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 13.878002166748047,
+      "learning_rate": 9.988142751731797e-06,
+      "logits/chosen": 2.4681615829467773,
+      "logits/rejected": 2.540001153945923,
+      "logps/chosen": -46.92790985107422,
+      "logps/rejected": -63.343467712402344,
+      "loss": 0.8081,
+      "nll_loss": 0.2872486710548401,
+      "rewards/accuracies": 0.7412499785423279,
+      "rewards/chosen": -4.692790985107422,
+      "rewards/margins": 1.6415550708770752,
+      "rewards/rejected": -6.334346294403076,
+      "step": 450
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 14.774677276611328,
+      "learning_rate": 9.966729958067638e-06,
+      "logits/chosen": 2.509334087371826,
+      "logits/rejected": 2.5635790824890137,
+      "logps/chosen": -48.37749481201172,
+      "logps/rejected": -64.00433349609375,
+      "loss": 0.873,
+      "nll_loss": 0.2907007336616516,
+      "rewards/accuracies": 0.7200000286102295,
+      "rewards/chosen": -4.837749481201172,
+      "rewards/margins": 1.5626837015151978,
+      "rewards/rejected": -6.400432586669922,
+      "step": 500
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 18.610092163085938,
+      "learning_rate": 9.934560333291077e-06,
+      "logits/chosen": 2.5441677570343018,
+      "logits/rejected": 2.613149404525757,
+      "logps/chosen": -48.265201568603516,
+      "logps/rejected": -63.49806213378906,
+      "loss": 0.8581,
+      "nll_loss": 0.2854183316230774,
+      "rewards/accuracies": 0.7287499904632568,
+      "rewards/chosen": -4.826520919799805,
+      "rewards/margins": 1.5232864618301392,
+      "rewards/rejected": -6.349807262420654,
+      "step": 550
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 13.354905128479004,
+      "learning_rate": 9.89170354964594e-06,
+      "logits/chosen": 2.646564245223999,
+      "logits/rejected": 2.700756549835205,
+      "logps/chosen": -45.61009216308594,
+      "logps/rejected": -60.2086067199707,
+      "loss": 0.8363,
+      "nll_loss": 0.27716541290283203,
+      "rewards/accuracies": 0.7300000190734863,
+      "rewards/chosen": -4.561009883880615,
+      "rewards/margins": 1.4598517417907715,
+      "rewards/rejected": -6.0208611488342285,
+      "step": 600
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 5.81559419631958,
+      "learning_rate": 9.83825242538238e-06,
+      "logits/chosen": 2.519411087036133,
+      "logits/rejected": 2.5865354537963867,
+      "logps/chosen": -46.41543960571289,
+      "logps/rejected": -63.667991638183594,
+      "loss": 0.7762,
+      "nll_loss": 0.27636775374412537,
+      "rewards/accuracies": 0.7587500214576721,
+      "rewards/chosen": -4.641544342041016,
+      "rewards/margins": 1.7252554893493652,
+      "rewards/rejected": -6.366799831390381,
+      "step": 650
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 9.440147399902344,
+      "learning_rate": 9.774322723733216e-06,
+      "logits/chosen": 2.4503791332244873,
+      "logits/rejected": 2.5173845291137695,
+      "logps/chosen": -44.43014144897461,
+      "logps/rejected": -62.06157302856445,
+      "loss": 0.7447,
+      "nll_loss": 0.2715470492839813,
+      "rewards/accuracies": 0.7724999785423279,
+      "rewards/chosen": -4.443014621734619,
+      "rewards/margins": 1.7631423473358154,
+      "rewards/rejected": -6.206157207489014,
+      "step": 700
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 9.054909706115723,
+      "learning_rate": 9.700052902196541e-06,
+      "logits/chosen": 2.3488733768463135,
+      "logits/rejected": 2.4224016666412354,
+      "logps/chosen": -45.02783203125,
+      "logps/rejected": -63.32554626464844,
+      "loss": 0.7256,
+      "nll_loss": 0.27446305751800537,
+      "rewards/accuracies": 0.8075000047683716,
+      "rewards/chosen": -4.502783298492432,
+      "rewards/margins": 1.8297713994979858,
+      "rewards/rejected": -6.332554817199707,
+      "step": 750
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 16.773876190185547,
+      "learning_rate": 9.615603812667618e-06,
+      "logits/chosen": 2.4701356887817383,
+      "logits/rejected": 2.5356545448303223,
+      "logps/chosen": -44.635948181152344,
+      "logps/rejected": -62.55714416503906,
+      "loss": 0.7836,
+      "nll_loss": 0.2690128684043884,
+      "rewards/accuracies": 0.7524999976158142,
+      "rewards/chosen": -4.463594913482666,
+      "rewards/margins": 1.7921198606491089,
+      "rewards/rejected": -6.2557148933410645,
+      "step": 800
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 12.325571060180664,
+      "learning_rate": 9.521158353069494e-06,
+      "logits/chosen": 2.450385332107544,
+      "logits/rejected": 2.5140254497528076,
+      "logps/chosen": -47.131587982177734,
+      "logps/rejected": -65.35118865966797,
+      "loss": 0.7507,
+      "nll_loss": 0.27970045804977417,
+      "rewards/accuracies": 0.7762500047683716,
+      "rewards/chosen": -4.713159561157227,
+      "rewards/margins": 1.8219600915908813,
+      "rewards/rejected": -6.535119533538818,
+      "step": 850
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 18.231292724609375,
+      "learning_rate": 9.416921071236821e-06,
+      "logits/chosen": 2.4923791885375977,
+      "logits/rejected": 2.5595548152923584,
+      "logps/chosen": -46.485965728759766,
+      "logps/rejected": -63.454063415527344,
+      "loss": 0.8048,
+      "nll_loss": 0.27803030610084534,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.648596286773682,
+      "rewards/margins": 1.6968092918395996,
+      "rewards/rejected": -6.345406532287598,
+      "step": 900
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 18.762359619140625,
+      "learning_rate": 9.303117721910801e-06,
+      "logits/chosen": 2.571411371231079,
+      "logits/rejected": 2.6245839595794678,
+      "logps/chosen": -44.32810974121094,
+      "logps/rejected": -61.370662689208984,
+      "loss": 0.7592,
+      "nll_loss": 0.2649703919887543,
+      "rewards/accuracies": 0.7662500143051147,
+      "rewards/chosen": -4.4328107833862305,
+      "rewards/margins": 1.7042557001113892,
+      "rewards/rejected": -6.137066841125488,
+      "step": 950
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 13.24950122833252,
+      "learning_rate": 9.179994777804677e-06,
+      "logits/chosen": 2.5039403438568115,
+      "logits/rejected": 2.5774261951446533,
+      "logps/chosen": -45.20160675048828,
+      "logps/rejected": -62.881683349609375,
+      "loss": 0.7717,
+      "nll_loss": 0.2698582708835602,
+      "rewards/accuracies": 0.7787500023841858,
+      "rewards/chosen": -4.520160675048828,
+      "rewards/margins": 1.7680076360702515,
+      "rewards/rejected": -6.288168430328369,
+      "step": 1000
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 12.429143905639648,
+      "learning_rate": 9.04781889579873e-06,
+      "logits/chosen": 2.5070714950561523,
+      "logits/rejected": 2.5675339698791504,
+      "logps/chosen": -45.09945297241211,
+      "logps/rejected": -64.37616729736328,
+      "loss": 0.7095,
+      "nll_loss": 0.2686724364757538,
+      "rewards/accuracies": 0.7912499904632568,
+      "rewards/chosen": -4.509944915771484,
+      "rewards/margins": 1.9276715517044067,
+      "rewards/rejected": -6.437616348266602,
+      "step": 1050
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 13.30483341217041,
+      "learning_rate": 8.906876339420851e-06,
+      "logits/chosen": 2.388920783996582,
+      "logits/rejected": 2.4621076583862305,
+      "logps/chosen": -42.8404541015625,
+      "logps/rejected": -62.490779876708984,
+      "loss": 0.6913,
+      "nll_loss": 0.25781476497650146,
+      "rewards/accuracies": 0.8112499713897705,
+      "rewards/chosen": -4.284045219421387,
+      "rewards/margins": 1.9650330543518066,
+      "rewards/rejected": -6.249078750610352,
+      "step": 1100
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 13.460766792297363,
+      "learning_rate": 8.757472358863481e-06,
+      "logits/chosen": 2.4224061965942383,
+      "logits/rejected": 2.5014472007751465,
+      "logps/chosen": -45.438575744628906,
+      "logps/rejected": -63.852115631103516,
+      "loss": 0.7312,
+      "nll_loss": 0.26547786593437195,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.543857574462891,
+      "rewards/margins": 1.841354250907898,
+      "rewards/rejected": -6.385211944580078,
+      "step": 1150
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 9.995412826538086,
+      "learning_rate": 8.599930529879669e-06,
+      "logits/chosen": 2.289586067199707,
+      "logits/rejected": 2.3660268783569336,
+      "logps/chosen": -43.745357513427734,
+      "logps/rejected": -62.561038970947266,
+      "loss": 0.6825,
+      "nll_loss": 0.2639794647693634,
+      "rewards/accuracies": 0.8100000023841858,
+      "rewards/chosen": -4.37453556060791,
+      "rewards/margins": 1.8815685510635376,
+      "rewards/rejected": -6.256104946136475,
+      "step": 1200
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 9.7256498336792,
+      "learning_rate": 8.434592052990044e-06,
+      "logits/chosen": 2.1411895751953125,
+      "logits/rejected": 2.216475248336792,
+      "logps/chosen": -41.174503326416016,
+      "logps/rejected": -59.75578308105469,
+      "loss": 0.687,
+      "nll_loss": 0.25248458981513977,
+      "rewards/accuracies": 0.7912499904632568,
+      "rewards/chosen": -4.11745023727417,
+      "rewards/margins": 1.8581281900405884,
+      "rewards/rejected": -5.9755778312683105,
+      "step": 1250
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 5.8602166175842285,
+      "learning_rate": 8.261815014518465e-06,
+      "logits/chosen": 2.094317674636841,
+      "logits/rejected": 2.1707794666290283,
+      "logps/chosen": -42.47966003417969,
+      "logps/rejected": -67.64102172851562,
+      "loss": 0.5271,
+      "nll_loss": 0.25495222210884094,
+      "rewards/accuracies": 0.8837500214576721,
+      "rewards/chosen": -4.247966289520264,
+      "rewards/margins": 2.516136884689331,
+      "rewards/rejected": -6.764101982116699,
+      "step": 1300
+    },
+    {
+      "epoch": 1.08,
+      "grad_norm": 7.209846019744873,
+      "learning_rate": 8.081973611056784e-06,
+      "logits/chosen": 1.899792194366455,
+      "logits/rejected": 1.9656882286071777,
+      "logps/chosen": -41.67923355102539,
+      "logps/rejected": -67.45558166503906,
+      "loss": 0.5335,
+      "nll_loss": 0.24939106404781342,
+      "rewards/accuracies": 0.8774999976158142,
+      "rewards/chosen": -4.167923450469971,
+      "rewards/margins": 2.577634811401367,
+      "rewards/rejected": -6.745559215545654,
+      "step": 1350
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 12.841861724853516,
+      "learning_rate": 7.89545733903834e-06,
+      "logits/chosen": 1.8675593137741089,
+      "logits/rejected": 1.9421709775924683,
+      "logps/chosen": -43.389644622802734,
+      "logps/rejected": -68.71646881103516,
+      "loss": 0.5398,
+      "nll_loss": 0.2613371014595032,
+      "rewards/accuracies": 0.8974999785423279,
+      "rewards/chosen": -4.338963985443115,
+      "rewards/margins": 2.5326828956604004,
+      "rewards/rejected": -6.871647834777832,
+      "step": 1400
+    },
+    {
+      "epoch": 1.16,
+      "grad_norm": 13.226041793823242,
+      "learning_rate": 7.702670151175435e-06,
+      "logits/chosen": 1.8003557920455933,
+      "logits/rejected": 1.877779245376587,
+      "logps/chosen": -42.74818801879883,
+      "logps/rejected": -67.07624053955078,
+      "loss": 0.5569,
+      "nll_loss": 0.25505152344703674,
+      "rewards/accuracies": 0.8899999856948853,
+      "rewards/chosen": -4.274819374084473,
+      "rewards/margins": 2.4328057765960693,
+      "rewards/rejected": -6.707624435424805,
+      "step": 1450
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 6.528555393218994,
+      "learning_rate": 7.5040295815877e-06,
+      "logits/chosen": 1.8589718341827393,
+      "logits/rejected": 1.9315310716629028,
+      "logps/chosen": -42.12541198730469,
+      "logps/rejected": -68.34070587158203,
+      "loss": 0.5316,
+      "nll_loss": 0.25116249918937683,
+      "rewards/accuracies": 0.8849999904632568,
+      "rewards/chosen": -4.212540626525879,
+      "rewards/margins": 2.621530055999756,
+      "rewards/rejected": -6.834071636199951,
+      "step": 1500
+    },
+    {
+      "epoch": 1.24,
+      "grad_norm": 8.806782722473145,
+      "learning_rate": 7.299965841516164e-06,
+      "logits/chosen": 1.896395206451416,
+      "logits/rejected": 1.9738116264343262,
+      "logps/chosen": -42.76396560668945,
+      "logps/rejected": -68.06053161621094,
+      "loss": 0.5404,
+      "nll_loss": 0.2575341463088989,
+      "rewards/accuracies": 0.8799999952316284,
+      "rewards/chosen": -4.27639627456665,
+      "rewards/margins": 2.5296568870544434,
+      "rewards/rejected": -6.8060526847839355,
+      "step": 1550
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 13.923650741577148,
+      "learning_rate": 7.090920887581507e-06,
+      "logits/chosen": 1.881974220275879,
+      "logits/rejected": 1.9411964416503906,
+      "logps/chosen": -41.719696044921875,
+      "logps/rejected": -66.24463653564453,
+      "loss": 0.5483,
+      "nll_loss": 0.24970975518226624,
+      "rewards/accuracies": 0.8762500286102295,
+      "rewards/chosen": -4.171969413757324,
+      "rewards/margins": 2.452493667602539,
+      "rewards/rejected": -6.62446403503418,
+      "step": 1600
+    },
+    {
+      "epoch": 1.32,
+      "grad_norm": 11.483983993530273,
+      "learning_rate": 6.877347464604446e-06,
+      "logits/chosen": 1.8503919839859009,
+      "logits/rejected": 1.9176833629608154,
+      "logps/chosen": -42.25181579589844,
+      "logps/rejected": -67.97564697265625,
+      "loss": 0.541,
+      "nll_loss": 0.25359034538269043,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -4.225182056427002,
+      "rewards/margins": 2.572382926940918,
+      "rewards/rejected": -6.797564506530762,
+      "step": 1650
+    },
+    {
+      "epoch": 1.3599999999999999,
+      "grad_norm": 9.980610847473145,
+      "learning_rate": 6.659708125061242e-06,
+      "logits/chosen": 1.7879056930541992,
+      "logits/rejected": 1.8512917757034302,
+      "logps/chosen": -41.724449157714844,
+      "logps/rejected": -68.18392181396484,
+      "loss": 0.5127,
+      "nll_loss": 0.2496194839477539,
+      "rewards/accuracies": 0.9075000286102295,
+      "rewards/chosen": -4.172445297241211,
+      "rewards/margins": 2.645946741104126,
+      "rewards/rejected": -6.818392276763916,
+      "step": 1700
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 8.7877779006958,
+      "learning_rate": 6.438474227298065e-06,
+      "logits/chosen": 1.813609004020691,
+      "logits/rejected": 1.8792080879211426,
+      "logps/chosen": -42.19902420043945,
+      "logps/rejected": -67.10396575927734,
+      "loss": 0.5475,
+      "nll_loss": 0.2563365399837494,
+      "rewards/accuracies": 0.8887500166893005,
+      "rewards/chosen": -4.219902038574219,
+      "rewards/margins": 2.4904944896698,
+      "rewards/rejected": -6.710396766662598,
+      "step": 1750
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 9.004395484924316,
+      "learning_rate": 6.2141249146737545e-06,
+      "logits/chosen": 1.8060498237609863,
+      "logits/rejected": 1.8766037225723267,
+      "logps/chosen": -40.13544845581055,
+      "logps/rejected": -64.01081085205078,
+      "loss": 0.5417,
+      "nll_loss": 0.23975929617881775,
+      "rewards/accuracies": 0.8712499737739563,
+      "rewards/chosen": -4.013545036315918,
+      "rewards/margins": 2.3875370025634766,
+      "rewards/rejected": -6.401081562042236,
+      "step": 1800
+    },
+    {
+      "epoch": 1.48,
+      "grad_norm": 14.659512519836426,
+      "learning_rate": 5.987146077842015e-06,
+      "logits/chosen": 1.7913141250610352,
+      "logits/rejected": 1.8706274032592773,
+      "logps/chosen": -41.8923454284668,
+      "logps/rejected": -68.07263946533203,
+      "loss": 0.5291,
+      "nll_loss": 0.24965395033359528,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -4.189234733581543,
+      "rewards/margins": 2.6180291175842285,
+      "rewards/rejected": -6.8072638511657715,
+      "step": 1850
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 6.758883953094482,
+      "learning_rate": 5.7580293024204455e-06,
+      "logits/chosen": 1.7456320524215698,
+      "logits/rejected": 1.8188166618347168,
+      "logps/chosen": -40.59235763549805,
+      "logps/rejected": -67.43293762207031,
+      "loss": 0.4978,
+      "nll_loss": 0.2416204810142517,
+      "rewards/accuracies": 0.8924999833106995,
+      "rewards/chosen": -4.0592360496521,
+      "rewards/margins": 2.6840572357177734,
+      "rewards/rejected": -6.743292808532715,
+      "step": 1900
+    },
+    {
+      "epoch": 1.56,
+      "grad_norm": 11.839475631713867,
+      "learning_rate": 5.5272708043255605e-06,
+      "logits/chosen": 1.771327257156372,
+      "logits/rejected": 1.8334521055221558,
+      "logps/chosen": -40.4869499206543,
+      "logps/rejected": -64.32723999023438,
+      "loss": 0.5586,
+      "nll_loss": 0.24806198477745056,
+      "rewards/accuracies": 0.8675000071525574,
+      "rewards/chosen": -4.048694610595703,
+      "rewards/margins": 2.3840293884277344,
+      "rewards/rejected": -6.4327239990234375,
+      "step": 1950
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 12.160818099975586,
+      "learning_rate": 5.295370355079615e-06,
+      "logits/chosen": 1.7191228866577148,
+      "logits/rejected": 1.7949659824371338,
+      "logps/chosen": -40.17674255371094,
+      "logps/rejected": -67.11283874511719,
+      "loss": 0.5123,
+      "nll_loss": 0.24404479563236237,
+      "rewards/accuracies": 0.8774999976158142,
+      "rewards/chosen": -4.017674446105957,
+      "rewards/margins": 2.6936092376708984,
+      "rewards/rejected": -6.711284160614014,
+      "step": 2000
+    },
+    {
+      "epoch": 1.6400000000000001,
+      "grad_norm": 12.399029731750488,
+      "learning_rate": 5.062830199416764e-06,
+      "logits/chosen": 1.7645864486694336,
+      "logits/rejected": 1.8290934562683105,
+      "logps/chosen": -41.4688720703125,
+      "logps/rejected": -67.27618408203125,
+      "loss": 0.5257,
+      "nll_loss": 0.24901245534420013,
+      "rewards/accuracies": 0.8949999809265137,
+      "rewards/chosen": -4.146886825561523,
+      "rewards/margins": 2.580731153488159,
+      "rewards/rejected": -6.727618217468262,
+      "step": 2050
+    },
+    {
+      "epoch": 1.6800000000000002,
+      "grad_norm": 13.637982368469238,
+      "learning_rate": 4.8301539675328205e-06,
+      "logits/chosen": 1.6826503276824951,
+      "logits/rejected": 1.7500274181365967,
+      "logps/chosen": -40.96113586425781,
+      "logps/rejected": -67.78974914550781,
+      "loss": 0.513,
+      "nll_loss": 0.2408064305782318,
+      "rewards/accuracies": 0.8987500071525574,
+      "rewards/chosen": -4.096114158630371,
+      "rewards/margins": 2.6828606128692627,
+      "rewards/rejected": -6.778974533081055,
+      "step": 2100
+    },
+    {
+      "epoch": 1.72,
+      "grad_norm": 5.442174911499023,
+      "learning_rate": 4.597845584334387e-06,
+      "logits/chosen": 1.6728637218475342,
+      "logits/rejected": 1.7252342700958252,
+      "logps/chosen": -42.251564025878906,
+      "logps/rejected": -69.24467468261719,
+      "loss": 0.5304,
+      "nll_loss": 0.25177592039108276,
+      "rewards/accuracies": 0.8899999856948853,
+      "rewards/chosen": -4.225156307220459,
+      "rewards/margins": 2.699312210083008,
+      "rewards/rejected": -6.92446756362915,
+      "step": 2150
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 10.488734245300293,
+      "learning_rate": 4.366408178049728e-06,
+      "logits/chosen": 1.6670646667480469,
+      "logits/rejected": 1.7333457469940186,
+      "logps/chosen": -39.98529052734375,
+      "logps/rejected": -65.87380981445312,
+      "loss": 0.5198,
+      "nll_loss": 0.2439391314983368,
+      "rewards/accuracies": 0.8762500286102295,
+      "rewards/chosen": -3.9985289573669434,
+      "rewards/margins": 2.588852643966675,
+      "rewards/rejected": -6.587381362915039,
+      "step": 2200
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 11.035846710205078,
+      "learning_rate": 4.136342990565055e-06,
+      "logits/chosen": 1.6644293069839478,
+      "logits/rejected": 1.7320383787155151,
+      "logps/chosen": -40.47782897949219,
+      "logps/rejected": -69.36231994628906,
+      "loss": 0.5059,
+      "nll_loss": 0.2478472888469696,
+      "rewards/accuracies": 0.8949999809265137,
+      "rewards/chosen": -4.047782897949219,
+      "rewards/margins": 2.888448476791382,
+      "rewards/rejected": -6.93623161315918,
+      "step": 2250
+    },
+    {
+      "epoch": 1.8399999999999999,
+      "grad_norm": 14.5468168258667,
+      "learning_rate": 3.908148291846225e-06,
+      "logits/chosen": 1.6440070867538452,
+      "logits/rejected": 1.6980115175247192,
+      "logps/chosen": -37.658931732177734,
+      "logps/rejected": -64.18938446044922,
+      "loss": 0.4975,
+      "nll_loss": 0.23423399031162262,
+      "rewards/accuracies": 0.8899999856948853,
+      "rewards/chosen": -3.7658934593200684,
+      "rewards/margins": 2.6530449390411377,
+      "rewards/rejected": -6.418938636779785,
+      "step": 2300
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 15.06531810760498,
+      "learning_rate": 3.6823183007969375e-06,
+      "logits/chosen": 1.633602261543274,
+      "logits/rejected": 1.69327974319458,
+      "logps/chosen": -38.722007751464844,
+      "logps/rejected": -65.98956298828125,
+      "loss": 0.512,
+      "nll_loss": 0.23453904688358307,
+      "rewards/accuracies": 0.8837500214576721,
+      "rewards/chosen": -3.8722009658813477,
+      "rewards/margins": 2.726755380630493,
+      "rewards/rejected": -6.598956108093262,
+      "step": 2350
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 9.41374397277832,
+      "learning_rate": 3.4593421148906523e-06,
+      "logits/chosen": 1.6280794143676758,
+      "logits/rejected": 1.694667935371399,
+      "logps/chosen": -41.9329833984375,
+      "logps/rejected": -68.24996948242188,
+      "loss": 0.5314,
+      "nll_loss": 0.24245284497737885,
+      "rewards/accuracies": 0.8887500166893005,
+      "rewards/chosen": -4.19329833984375,
+      "rewards/margins": 2.6316983699798584,
+      "rewards/rejected": -6.8249969482421875,
+      "step": 2400
+    },
+    {
+      "epoch": 1.96,
+      "grad_norm": 10.686447143554688,
+      "learning_rate": 3.239702650894364e-06,
+      "logits/chosen": 1.648103952407837,
+      "logits/rejected": 1.7120572328567505,
+      "logps/chosen": -39.89656066894531,
+      "logps/rejected": -67.84294128417969,
+      "loss": 0.5132,
+      "nll_loss": 0.23985882103443146,
+      "rewards/accuracies": 0.8799999952316284,
+      "rewards/chosen": -3.989656448364258,
+      "rewards/margins": 2.794638156890869,
+      "rewards/rejected": -6.784294605255127,
+      "step": 2450
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 6.338383197784424,
+      "learning_rate": 3.023875598978419e-06,
+      "logits/chosen": 1.6715092658996582,
+      "logits/rejected": 1.7344988584518433,
+      "logps/chosen": -39.51519012451172,
+      "logps/rejected": -68.46276092529297,
+      "loss": 0.4834,
+      "nll_loss": 0.24019798636436462,
+      "rewards/accuracies": 0.9112499952316284,
+      "rewards/chosen": -3.951519250869751,
+      "rewards/margins": 2.8947577476501465,
+      "rewards/rejected": -6.846276760101318,
+      "step": 2500
+    },
+    {
+      "epoch": 2.04,
+      "grad_norm": 5.144768238067627,
+      "learning_rate": 2.812328392477536e-06,
+      "logits/chosen": 1.5945950746536255,
+      "logits/rejected": 1.6544932126998901,
+      "logps/chosen": -37.963443756103516,
+      "logps/rejected": -69.92556762695312,
+      "loss": 0.3913,
+      "nll_loss": 0.22887782752513885,
+      "rewards/accuracies": 0.9712499976158142,
+      "rewards/chosen": -3.796344041824341,
+      "rewards/margins": 3.196213722229004,
+      "rewards/rejected": -6.992558002471924,
+      "step": 2550
+    },
+    {
+      "epoch": 2.08,
+      "grad_norm": 5.904083251953125,
+      "learning_rate": 2.6055191955342886e-06,
+      "logits/chosen": 1.5594576597213745,
+      "logits/rejected": 1.6052813529968262,
+      "logps/chosen": -39.600093841552734,
+      "logps/rejected": -73.07125854492188,
+      "loss": 0.398,
+      "nll_loss": 0.23794788122177124,
+      "rewards/accuracies": 0.9662500023841858,
+      "rewards/chosen": -3.960009813308716,
+      "rewards/margins": 3.3471157550811768,
+      "rewards/rejected": -7.307126045227051,
+      "step": 2600
+    },
+    {
+      "epoch": 2.12,
+      "grad_norm": 5.723136901855469,
+      "learning_rate": 2.403895910817593e-06,
+      "logits/chosen": 1.532933235168457,
+      "logits/rejected": 1.5803064107894897,
+      "logps/chosen": -38.91565704345703,
+      "logps/rejected": -70.781982421875,
+      "loss": 0.4101,
+      "nll_loss": 0.23784016072750092,
+      "rewards/accuracies": 0.9574999809265137,
+      "rewards/chosen": -3.8915653228759766,
+      "rewards/margins": 3.186633348464966,
+      "rewards/rejected": -7.07819938659668,
+      "step": 2650
+    },
+    {
+      "epoch": 2.16,
+      "grad_norm": 7.6983561515808105,
+      "learning_rate": 2.2078952094652705e-06,
+      "logits/chosen": 1.4740697145462036,
+      "logits/rejected": 1.5456255674362183,
+      "logps/chosen": -39.75840377807617,
+      "logps/rejected": -75.84871673583984,
+      "loss": 0.3908,
+      "nll_loss": 0.23151804506778717,
+      "rewards/accuracies": 0.9612500071525574,
+      "rewards/chosen": -3.9758400917053223,
+      "rewards/margins": 3.6090316772460938,
+      "rewards/rejected": -7.584871292114258,
+      "step": 2700
+    },
+    {
+      "epoch": 2.2,
+      "grad_norm": 7.529278755187988,
+      "learning_rate": 2.017941585351591e-06,
+      "logits/chosen": 1.495976209640503,
+      "logits/rejected": 1.5389071702957153,
+      "logps/chosen": -39.2768440246582,
+      "logps/rejected": -71.7361831665039,
+      "loss": 0.4138,
+      "nll_loss": 0.24059143662452698,
+      "rewards/accuracies": 0.9574999809265137,
+      "rewards/chosen": -3.9276845455169678,
+      "rewards/margins": 3.245933771133423,
+      "rewards/rejected": -7.173618793487549,
+      "step": 2750
+    },
+    {
+      "epoch": 2.24,
+      "grad_norm": 6.076731204986572,
+      "learning_rate": 1.8344464357280722e-06,
+      "logits/chosen": 1.4751191139221191,
+      "logits/rejected": 1.5268608331680298,
+      "logps/chosen": -38.4973258972168,
+      "logps/rejected": -71.8733901977539,
+      "loss": 0.3943,
+      "nll_loss": 0.22699041664600372,
+      "rewards/accuracies": 0.9700000286102295,
+      "rewards/chosen": -3.8497328758239746,
+      "rewards/margins": 3.3376071453094482,
+      "rewards/rejected": -7.187338829040527,
+      "step": 2800
+    },
+    {
+      "epoch": 2.2800000000000002,
+      "grad_norm": 5.921169757843018,
+      "learning_rate": 1.6578071702286396e-06,
+      "logits/chosen": 1.4693129062652588,
+      "logits/rejected": 1.5106064081192017,
+      "logps/chosen": -38.8640022277832,
+      "logps/rejected": -72.8025131225586,
+      "loss": 0.4103,
+      "nll_loss": 0.2377665936946869,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -3.8864006996154785,
+      "rewards/margins": 3.393850088119507,
+      "rewards/rejected": -7.280250072479248,
+      "step": 2850
+    },
+    {
+      "epoch": 2.32,
+      "grad_norm": 5.996314525604248,
+      "learning_rate": 1.4884063501688539e-06,
+      "logits/chosen": 1.4711755514144897,
+      "logits/rejected": 1.537732720375061,
+      "logps/chosen": -38.81364440917969,
+      "logps/rejected": -73.80380249023438,
+      "loss": 0.3908,
+      "nll_loss": 0.22893303632736206,
+      "rewards/accuracies": 0.9737499952316284,
+      "rewards/chosen": -3.881364345550537,
+      "rewards/margins": 3.499016523361206,
+      "rewards/rejected": -7.380380153656006,
+      "step": 2900
+    },
+    {
+      "epoch": 2.36,
+      "grad_norm": 10.235599517822266,
+      "learning_rate": 1.3266108600032928e-06,
+      "logits/chosen": 1.469245195388794,
+      "logits/rejected": 1.5233672857284546,
+      "logps/chosen": -39.08128356933594,
+      "logps/rejected": -70.8739242553711,
+      "loss": 0.4124,
+      "nll_loss": 0.2312445044517517,
+      "rewards/accuracies": 0.9512500166893005,
+      "rewards/chosen": -3.9081289768218994,
+      "rewards/margins": 3.1792635917663574,
+      "rewards/rejected": -7.087392807006836,
+      "step": 2950
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 8.700451850891113,
+      "learning_rate": 1.1727711127355118e-06,
+      "logits/chosen": 1.445876955986023,
+      "logits/rejected": 1.494391679763794,
+      "logps/chosen": -38.116580963134766,
+      "logps/rejected": -70.0914535522461,
+      "loss": 0.3982,
+      "nll_loss": 0.22633756697177887,
+      "rewards/accuracies": 0.9574999809265137,
+      "rewards/chosen": -3.8116586208343506,
+      "rewards/margins": 3.1974875926971436,
+      "rewards/rejected": -7.009146213531494,
+      "step": 3000
+    },
+    {
+      "epoch": 2.44,
+      "grad_norm": 9.24150562286377,
+      "learning_rate": 1.0272202910015083e-06,
+      "logits/chosen": 1.4509037733078003,
+      "logits/rejected": 1.5117911100387573,
+      "logps/chosen": -39.70808410644531,
+      "logps/rejected": -74.1922836303711,
+      "loss": 0.4038,
+      "nll_loss": 0.23606754839420319,
+      "rewards/accuracies": 0.9662500023841858,
+      "rewards/chosen": -3.970808744430542,
+      "rewards/margins": 3.4484190940856934,
+      "rewards/rejected": -7.419227123260498,
+      "step": 3050
+    },
+    {
+      "epoch": 2.48,
+      "grad_norm": 10.763381958007812,
+      "learning_rate": 8.902736254703347e-07,
+      "logits/chosen": 1.4675711393356323,
+      "logits/rejected": 1.5065741539001465,
+      "logps/chosen": -36.786865234375,
+      "logps/rejected": -67.48006439208984,
+      "loss": 0.405,
+      "nll_loss": 0.22343981266021729,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -3.6786866188049316,
+      "rewards/margins": 3.0693204402923584,
+      "rewards/rejected": -6.748007297515869,
+      "step": 3100
+    },
+    {
+      "epoch": 2.52,
+      "grad_norm": 5.758388996124268,
+      "learning_rate": 7.622277121246513e-07,
+      "logits/chosen": 1.483445405960083,
+      "logits/rejected": 1.5389055013656616,
+      "logps/chosen": -38.62077331542969,
+      "logps/rejected": -71.79267883300781,
+      "loss": 0.4121,
+      "nll_loss": 0.23168590664863586,
+      "rewards/accuracies": 0.9587500095367432,
+      "rewards/chosen": -3.8620777130126953,
+      "rewards/margins": 3.3171918392181396,
+      "rewards/rejected": -7.179268836975098,
+      "step": 3150
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 7.307446002960205,
+      "learning_rate": 6.433598698998766e-07,
+      "logits/chosen": 1.4688174724578857,
+      "logits/rejected": 1.5306994915008545,
+      "logps/chosen": -39.013912200927734,
+      "logps/rejected": -71.06527709960938,
+      "loss": 0.416,
+      "nll_loss": 0.23500658571720123,
+      "rewards/accuracies": 0.9424999952316284,
+      "rewards/chosen": -3.9013912677764893,
+      "rewards/margins": 3.2051360607147217,
+      "rewards/rejected": -7.106527328491211,
+      "step": 3200
+    },
+    {
+      "epoch": 2.6,
+      "grad_norm": 6.810940742492676,
+      "learning_rate": 5.339275400731331e-07,
+      "logits/chosen": 1.4731531143188477,
+      "logits/rejected": 1.5175625085830688,
+      "logps/chosen": -39.05414581298828,
+      "logps/rejected": -69.38569641113281,
+      "loss": 0.4207,
+      "nll_loss": 0.2375660389661789,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -3.9054150581359863,
+      "rewards/margins": 3.0331552028656006,
+      "rewards/rejected": -6.93856954574585,
+      "step": 3250
+    },
+    {
+      "epoch": 2.64,
+      "grad_norm": 5.935739517211914,
+      "learning_rate": 4.3416772870275295e-07,
+      "logits/chosen": 1.447837233543396,
+      "logits/rejected": 1.5085737705230713,
+      "logps/chosen": -38.072574615478516,
+      "logps/rejected": -71.93220520019531,
+      "loss": 0.3981,
+      "nll_loss": 0.22540073096752167,
+      "rewards/accuracies": 0.9537500143051147,
+      "rewards/chosen": -3.807257652282715,
+      "rewards/margins": 3.385963201522827,
+      "rewards/rejected": -7.193220138549805,
+      "step": 3300
+    },
+    {
+      "epoch": 2.68,
+      "grad_norm": 6.746799468994141,
+      "learning_rate": 3.442964933259474e-07,
+      "logits/chosen": 1.4588629007339478,
+      "logits/rejected": 1.517850637435913,
+      "logps/chosen": -38.37602615356445,
+      "logps/rejected": -72.57811737060547,
+      "loss": 0.3998,
+      "nll_loss": 0.23044270277023315,
+      "rewards/accuracies": 0.9587500095367432,
+      "rewards/chosen": -3.8376026153564453,
+      "rewards/margins": 3.4202094078063965,
+      "rewards/rejected": -7.257812023162842,
+      "step": 3350
+    },
+    {
+      "epoch": 2.7199999999999998,
+      "grad_norm": 7.958431243896484,
+      "learning_rate": 2.6450847502627883e-07,
+      "logits/chosen": 1.4416861534118652,
+      "logits/rejected": 1.5023283958435059,
+      "logps/chosen": -39.2740364074707,
+      "logps/rejected": -73.73572540283203,
+      "loss": 0.4036,
+      "nll_loss": 0.22917690873146057,
+      "rewards/accuracies": 0.9662500023841858,
+      "rewards/chosen": -3.927403450012207,
+      "rewards/margins": 3.446169376373291,
+      "rewards/rejected": -7.373574256896973,
+      "step": 3400
+    },
+    {
+      "epoch": 2.76,
+      "grad_norm": 8.157621383666992,
+      "learning_rate": 1.9497647688442478e-07,
+      "logits/chosen": 1.4702584743499756,
+      "logits/rejected": 1.5254310369491577,
+      "logps/chosen": -38.0818977355957,
+      "logps/rejected": -70.6051254272461,
+      "loss": 0.4055,
+      "nll_loss": 0.23146916925907135,
+      "rewards/accuracies": 0.9549999833106995,
+      "rewards/chosen": -3.808190107345581,
+      "rewards/margins": 3.252322196960449,
+      "rewards/rejected": -7.060512065887451,
+      "step": 3450
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 6.671606540679932,
+      "learning_rate": 1.358510897251808e-07,
+      "logits/chosen": 1.4628223180770874,
+      "logits/rejected": 1.525000810623169,
+      "logps/chosen": -37.097007751464844,
+      "logps/rejected": -72.08160400390625,
+      "loss": 0.3906,
+      "nll_loss": 0.22892680764198303,
+      "rewards/accuracies": 0.9700000286102295,
+      "rewards/chosen": -3.7097012996673584,
+      "rewards/margins": 3.498459577560425,
+      "rewards/rejected": -7.208160400390625,
+      "step": 3500
+    },
+    {
+      "epoch": 2.84,
+      "grad_norm": 8.076554298400879,
+      "learning_rate": 8.726036597126619e-08,
+      "logits/chosen": 1.470629096031189,
+      "logits/rejected": 1.535922646522522,
+      "logps/chosen": -36.950157165527344,
+      "logps/rejected": -68.89533996582031,
+      "loss": 0.395,
+      "nll_loss": 0.22603529691696167,
+      "rewards/accuracies": 0.9549999833106995,
+      "rewards/chosen": -3.6950161457061768,
+      "rewards/margins": 3.1945183277130127,
+      "rewards/rejected": -6.8895344734191895,
+      "step": 3550
+    },
+    {
+      "epoch": 2.88,
+      "grad_norm": 6.550802707672119,
+      "learning_rate": 4.93095423102935e-08,
+      "logits/chosen": 1.4719560146331787,
+      "logits/rejected": 1.5248959064483643,
+      "logps/chosen": -40.710941314697266,
+      "logps/rejected": -74.42230987548828,
+      "loss": 0.4132,
+      "nll_loss": 0.24109821021556854,
+      "rewards/accuracies": 0.9524999856948853,
+      "rewards/chosen": -4.071094512939453,
+      "rewards/margins": 3.371136426925659,
+      "rewards/rejected": -7.442230701446533,
+      "step": 3600
+    },
+    {
+      "epoch": 2.92,
+      "grad_norm": 6.743546485900879,
+      "learning_rate": 2.2080811775535006e-08,
+      "logits/chosen": 1.4744410514831543,
+      "logits/rejected": 1.5339540243148804,
+      "logps/chosen": -39.543033599853516,
+      "logps/rejected": -74.6586685180664,
+      "loss": 0.3964,
+      "nll_loss": 0.23712339997291565,
+      "rewards/accuracies": 0.9674999713897705,
+      "rewards/chosen": -3.954303026199341,
+      "rewards/margins": 3.511564254760742,
+      "rewards/rejected": -7.46586799621582,
+      "step": 3650
+    },
+    {
+      "epoch": 2.96,
+      "grad_norm": 11.845396041870117,
+      "learning_rate": 5.633145734114665e-09,
+      "logits/chosen": 1.4899775981903076,
+      "logits/rejected": 1.533370018005371,
+      "logps/chosen": -38.53838348388672,
+      "logps/rejected": -71.89134979248047,
+      "loss": 0.4058,
+      "nll_loss": 0.23659667372703552,
+      "rewards/accuracies": 0.9674999713897705,
+      "rewards/chosen": -3.853839159011841,
+      "rewards/margins": 3.3352959156036377,
+      "rewards/rejected": -7.18913459777832,
+      "step": 3700
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 9.659139633178711,
+      "learning_rate": 2.1661681620654963e-12,
+      "logits/chosen": 1.4696215391159058,
+      "logits/rejected": 1.526814579963684,
+      "logps/chosen": -38.22508239746094,
+      "logps/rejected": -72.54460144042969,
+      "loss": 0.3938,
+      "nll_loss": 0.22828102111816406,
+      "rewards/accuracies": 0.9637500047683716,
+      "rewards/chosen": -3.8225083351135254,
+      "rewards/margins": 3.4319519996643066,
+      "rewards/rejected": -7.254461288452148,
+      "step": 3750
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 3750,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdc46d7f4e510fac8857457512e9b21ca1bcc57fcd0eb82dff37e7aa86a9dc68
+size 6289