DPO 574ded42 final

README.md

@@ -0,0 +1,209 @@
+---
+base_model: bigcode/starcoder2-3b
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:bigcode/starcoder2-3b
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

adapter_config.json

@@ -0,0 +1,45 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "bigcode/starcoder2-3b",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 256,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 128,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "c_proj",
+    "k_proj",
+    "c_fc",
+    "v_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:919e2e22af4857b4504fa237aac0f276067fd01ed4d6e10b797ed0b60573f4b2
+size 762887704

loss.txt

@@ -0,0 +1 @@
+308,9.752377081895247e-06

special_tokens_map.json

@@ -0,0 +1,64 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<fim_pad>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,358 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<fim_prefix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<fim_middle>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<fim_suffix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<fim_pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<code_to_intermediate>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "17": {
+      "content": "<intermediate_to_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "18": {
+      "content": "<pr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "19": {
+      "content": "<pr_status>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "20": {
+      "content": "<pr_is_merged>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "21": {
+      "content": "<pr_base>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "22": {
+      "content": "<pr_file>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "23": {
+      "content": "<pr_base_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "24": {
+      "content": "<pr_diff>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "25": {
+      "content": "<pr_diff_hunk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "26": {
+      "content": "<pr_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "27": {
+      "content": "<pr_event_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "28": {
+      "content": "<pr_review>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "29": {
+      "content": "<pr_review_state>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30": {
+      "content": "<pr_review_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "31": {
+      "content": "<pr_in_reply_to_review_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32": {
+      "content": "<pr_in_reply_to_comment_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "33": {
+      "content": "<pr_diff_hunk_comment_line>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "34": {
+      "content": "<NAME>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "35": {
+      "content": "<EMAIL>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "36": {
+      "content": "<KEY>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "37": {
+      "content": "<PASSWORD>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<fim_pad>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

trainer_state.json

@@ -0,0 +1,981 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.9870967741935484,
+  "eval_steps": 500,
+  "global_step": 308,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.03225806451612903,
+      "grad_norm": 11.747716903686523,
+      "learning_rate": 2.2086148233646147e-06,
+      "logits/chosen": -6.872015476226807,
+      "logits/rejected": -8.557083129882812,
+      "logps/chosen": -200.8333282470703,
+      "logps/rejected": -126.05131530761719,
+      "loss": 3.7785,
+      "rewards/accuracies": 0.5296875238418579,
+      "rewards/chosen": 0.6622785925865173,
+      "rewards/margins": 0.7673968076705933,
+      "rewards/rejected": -0.10511825233697891,
+      "step": 5
+    },
+    {
+      "epoch": 0.06451612903225806,
+      "grad_norm": 11.248347282409668,
+      "learning_rate": 4.9693833525703835e-06,
+      "logits/chosen": -6.8620195388793945,
+      "logits/rejected": -8.925396919250488,
+      "logps/chosen": -194.3622283935547,
+      "logps/rejected": -129.8681182861328,
+      "loss": 3.8044,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.45987948775291443,
+      "rewards/margins": 0.4599760174751282,
+      "rewards/rejected": -9.66489315032959e-05,
+      "step": 10
+    },
+    {
+      "epoch": 0.0967741935483871,
+      "grad_norm": 13.071877479553223,
+      "learning_rate": 7.730151881776152e-06,
+      "logits/chosen": -6.489249229431152,
+      "logits/rejected": -8.789779663085938,
+      "logps/chosen": -202.84713745117188,
+      "logps/rejected": -129.98233032226562,
+      "loss": 3.9664,
+      "rewards/accuracies": 0.526562511920929,
+      "rewards/chosen": 0.5739307999610901,
+      "rewards/margins": 0.5547459721565247,
+      "rewards/rejected": 0.019184818491339684,
+      "step": 15
+    },
+    {
+      "epoch": 0.12903225806451613,
+      "grad_norm": 8.609030723571777,
+      "learning_rate": 1.049092041098192e-05,
+      "logits/chosen": -6.255059719085693,
+      "logits/rejected": -9.302863121032715,
+      "logps/chosen": -176.4971466064453,
+      "logps/rejected": -134.4936981201172,
+      "loss": 2.6269,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": 1.0335595607757568,
+      "rewards/margins": 2.3927571773529053,
+      "rewards/rejected": -1.3591976165771484,
+      "step": 20
+    },
+    {
+      "epoch": 0.16129032258064516,
+      "grad_norm": 7.000743389129639,
+      "learning_rate": 1.325168894018769e-05,
+      "logits/chosen": -5.956324100494385,
+      "logits/rejected": -9.702302932739258,
+      "logps/chosen": -169.1029052734375,
+      "logps/rejected": -165.97474670410156,
+      "loss": 1.6851,
+      "rewards/accuracies": 0.8109375238418579,
+      "rewards/chosen": 2.0764987468719482,
+      "rewards/margins": 5.65482234954834,
+      "rewards/rejected": -3.5783238410949707,
+      "step": 25
+    },
+    {
+      "epoch": 0.1935483870967742,
+      "grad_norm": 2.4143693447113037,
+      "learning_rate": 1.601245746939346e-05,
+      "logits/chosen": -6.569132328033447,
+      "logits/rejected": -10.802966117858887,
+      "logps/chosen": -172.4666290283203,
+      "logps/rejected": -244.1662139892578,
+      "loss": 0.3164,
+      "rewards/accuracies": 0.932812511920929,
+      "rewards/chosen": 2.330841541290283,
+      "rewards/margins": 13.70948600769043,
+      "rewards/rejected": -11.378644943237305,
+      "step": 30
+    },
+    {
+      "epoch": 0.22580645161290322,
+      "grad_norm": 0.06904491782188416,
+      "learning_rate": 1.8773225998599227e-05,
+      "logits/chosen": -8.068090438842773,
+      "logits/rejected": -11.996877670288086,
+      "logps/chosen": -184.2109832763672,
+      "logps/rejected": -375.8692932128906,
+      "loss": 0.0338,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": 2.172847032546997,
+      "rewards/margins": 27.02886962890625,
+      "rewards/rejected": -24.85602569580078,
+      "step": 35
+    },
+    {
+      "epoch": 0.25806451612903225,
+      "grad_norm": 0.1487932652235031,
+      "learning_rate": 1.9317814719367415e-05,
+      "logits/chosen": -8.399168014526367,
+      "logits/rejected": -12.002328872680664,
+      "logps/chosen": -205.44052124023438,
+      "logps/rejected": -487.5409240722656,
+      "loss": 0.0144,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": -0.532281219959259,
+      "rewards/margins": 35.829750061035156,
+      "rewards/rejected": -36.362030029296875,
+      "step": 40
+    },
+    {
+      "epoch": 0.2903225806451613,
+      "grad_norm": 0.6289846301078796,
+      "learning_rate": 1.9287109033995323e-05,
+      "logits/chosen": -8.708343505859375,
+      "logits/rejected": -11.307577133178711,
+      "logps/chosen": -217.4675750732422,
+      "logps/rejected": -556.1575317382812,
+      "loss": 0.0232,
+      "rewards/accuracies": 0.995312511920929,
+      "rewards/chosen": -2.1770095825195312,
+      "rewards/margins": 40.92938995361328,
+      "rewards/rejected": -43.10639190673828,
+      "step": 45
+    },
+    {
+      "epoch": 0.3225806451612903,
+      "grad_norm": 1.6672333478927612,
+      "learning_rate": 1.923288991518115e-05,
+      "logits/chosen": -9.274807929992676,
+      "logits/rejected": -11.517184257507324,
+      "logps/chosen": -216.9584503173828,
+      "logps/rejected": -567.5177612304688,
+      "loss": 0.0117,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -2.5761454105377197,
+      "rewards/margins": 41.563804626464844,
+      "rewards/rejected": -44.13995361328125,
+      "step": 50
+    },
+    {
+      "epoch": 0.3548387096774194,
+      "grad_norm": 0.1163749024271965,
+      "learning_rate": 1.915533421442089e-05,
+      "logits/chosen": -8.658121109008789,
+      "logits/rejected": -10.576135635375977,
+      "logps/chosen": -204.00003051757812,
+      "logps/rejected": -574.2200317382812,
+      "loss": 0.0045,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": -1.2570573091506958,
+      "rewards/margins": 43.9434700012207,
+      "rewards/rejected": -45.200523376464844,
+      "step": 55
+    },
+    {
+      "epoch": 0.3870967741935484,
+      "grad_norm": 0.0025108063127845526,
+      "learning_rate": 1.9054694902290306e-05,
+      "logits/chosen": -7.479229927062988,
+      "logits/rejected": -8.57958698272705,
+      "logps/chosen": -195.77511596679688,
+      "logps/rejected": -588.6607666015625,
+      "loss": 0.0084,
+      "rewards/accuracies": 0.996874988079071,
+      "rewards/chosen": -0.20098623633384705,
+      "rewards/margins": 46.062320709228516,
+      "rewards/rejected": -46.263309478759766,
+      "step": 60
+    },
+    {
+      "epoch": 0.41935483870967744,
+      "grad_norm": 0.05390551686286926,
+      "learning_rate": 1.893130024330745e-05,
+      "logits/chosen": -5.855751991271973,
+      "logits/rejected": -6.480991363525391,
+      "logps/chosen": -181.93038940429688,
+      "logps/rejected": -579.6819458007812,
+      "loss": 0.0106,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 1.1638758182525635,
+      "rewards/margins": 46.36396408081055,
+      "rewards/rejected": -45.20009231567383,
+      "step": 65
+    },
+    {
+      "epoch": 0.45161290322580644,
+      "grad_norm": 0.008025813847780228,
+      "learning_rate": 1.8785552725202064e-05,
+      "logits/chosen": -4.496771335601807,
+      "logits/rejected": -4.330361843109131,
+      "logps/chosen": -177.8521728515625,
+      "logps/rejected": -569.95166015625,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.5159854888916016,
+      "rewards/margins": 46.066001892089844,
+      "rewards/rejected": -44.550018310546875,
+      "step": 70
+    },
+    {
+      "epoch": 0.4838709677419355,
+      "grad_norm": 0.2902831435203552,
+      "learning_rate": 1.8617927746084266e-05,
+      "logits/chosen": -3.704448699951172,
+      "logits/rejected": -3.2316722869873047,
+      "logps/chosen": -173.41717529296875,
+      "logps/rejected": -592.05908203125,
+      "loss": 0.0017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7170642614364624,
+      "rewards/margins": 47.78697967529297,
+      "rewards/rejected": -46.069915771484375,
+      "step": 75
+    },
+    {
+      "epoch": 0.5161290322580645,
+      "grad_norm": 1.5575469732284546,
+      "learning_rate": 1.8428972063794692e-05,
+      "logits/chosen": -3.1886916160583496,
+      "logits/rejected": -2.6457602977752686,
+      "logps/chosen": -166.82505798339844,
+      "logps/rejected": -598.5891723632812,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.995312511920929,
+      "rewards/chosen": 1.4597384929656982,
+      "rewards/margins": 48.52386474609375,
+      "rewards/rejected": -47.064125061035156,
+      "step": 80
+    },
+    {
+      "epoch": 0.5483870967741935,
+      "grad_norm": 6.104778731241822e-05,
+      "learning_rate": 1.8219302012494108e-05,
+      "logits/chosen": -3.119097948074341,
+      "logits/rejected": -2.784574508666992,
+      "logps/chosen": -176.2117462158203,
+      "logps/rejected": -601.7147827148438,
+      "loss": 0.0034,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 1.8994001150131226,
+      "rewards/margins": 49.24987030029297,
+      "rewards/rejected": -47.35047149658203,
+      "step": 85
+    },
+    {
+      "epoch": 0.5806451612903226,
+      "grad_norm": 0.021236000582575798,
+      "learning_rate": 1.7989601492309465e-05,
+      "logits/chosen": -2.805873394012451,
+      "logits/rejected": -2.618786334991455,
+      "logps/chosen": -170.4734649658203,
+      "logps/rejected": -609.2696533203125,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.078625202178955,
+      "rewards/margins": 50.50914764404297,
+      "rewards/rejected": -48.43052673339844,
+      "step": 90
+    },
+    {
+      "epoch": 0.6129032258064516,
+      "grad_norm": 0.006399027071893215,
+      "learning_rate": 1.774061973859382e-05,
+      "logits/chosen": -2.691558361053467,
+      "logits/rejected": -2.4888315200805664,
+      "logps/chosen": -177.30880737304688,
+      "logps/rejected": -640.2526245117188,
+      "loss": 0.0002,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.453687906265259,
+      "rewards/margins": 53.2391471862793,
+      "rewards/rejected": -50.78545379638672,
+      "step": 95
+    },
+    {
+      "epoch": 0.6451612903225806,
+      "grad_norm": 0.000686797546222806,
+      "learning_rate": 1.747316887807638e-05,
+      "logits/chosen": -2.461819887161255,
+      "logits/rejected": -2.338575601577759,
+      "logps/chosen": -170.4715118408203,
+      "logps/rejected": -635.7724609375,
+      "loss": 0.0015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.5472875833511353,
+      "rewards/margins": 52.81101608276367,
+      "rewards/rejected": -51.26373291015625,
+      "step": 100
+    },
+    {
+      "epoch": 0.6774193548387096,
+      "grad_norm": 0.0314163975417614,
+      "learning_rate": 1.718812127987388e-05,
+      "logits/chosen": -2.4023430347442627,
+      "logits/rejected": -2.294923782348633,
+      "logps/chosen": -169.93972778320312,
+      "logps/rejected": -621.2903442382812,
+      "loss": 0.0081,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 1.7308235168457031,
+      "rewards/margins": 51.537498474121094,
+      "rewards/rejected": -49.806678771972656,
+      "step": 105
+    },
+    {
+      "epoch": 0.7096774193548387,
+      "grad_norm": 1.8648837804794312,
+      "learning_rate": 1.688640671000386e-05,
+      "logits/chosen": -2.5111083984375,
+      "logits/rejected": -2.4942150115966797,
+      "logps/chosen": -186.0556182861328,
+      "logps/rejected": -643.71826171875,
+      "loss": 0.0384,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 1.8740406036376953,
+      "rewards/margins": 53.93976974487305,
+      "rewards/rejected": -52.06572341918945,
+      "step": 110
+    },
+    {
+      "epoch": 0.7419354838709677,
+      "grad_norm": 0.2866516709327698,
+      "learning_rate": 1.656900929868119e-05,
+      "logits/chosen": -2.851760149002075,
+      "logits/rejected": -2.680408477783203,
+      "logps/chosen": -180.36245727539062,
+      "logps/rejected": -650.8028564453125,
+      "loss": 0.0048,
+      "rewards/accuracies": 0.996874988079071,
+      "rewards/chosen": 1.7712488174438477,
+      "rewards/margins": 54.01469039916992,
+      "rewards/rejected": -52.243446350097656,
+      "step": 115
+    },
+    {
+      "epoch": 0.7741935483870968,
+      "grad_norm": 0.00485634570941329,
+      "learning_rate": 1.6236964330289824e-05,
+      "logits/chosen": -3.038351535797119,
+      "logits/rejected": -2.873201847076416,
+      "logps/chosen": -176.7779541015625,
+      "logps/rejected": -638.7794799804688,
+      "loss": 0.0015,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 2.315247058868408,
+      "rewards/margins": 53.3264045715332,
+      "rewards/rejected": -51.01116180419922,
+      "step": 120
+    },
+    {
+      "epoch": 0.8064516129032258,
+      "grad_norm": 0.00011157651897519827,
+      "learning_rate": 1.5891354866500355e-05,
+      "logits/chosen": -3.0750269889831543,
+      "logits/rejected": -2.8404810428619385,
+      "logps/chosen": -166.00732421875,
+      "logps/rejected": -626.678466796875,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.8882861137390137,
+      "rewards/margins": 53.35065841674805,
+      "rewards/rejected": -50.46237564086914,
+      "step": 125
+    },
+    {
+      "epoch": 0.8387096774193549,
+      "grad_norm": 0.037098996341228485,
+      "learning_rate": 1.5533308213547954e-05,
+      "logits/chosen": -2.93206787109375,
+      "logits/rejected": -2.821136474609375,
+      "logps/chosen": -163.5557098388672,
+      "logps/rejected": -600.4147338867188,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.8543925285339355,
+      "rewards/margins": 50.14447784423828,
+      "rewards/rejected": -47.29008865356445,
+      "step": 130
+    },
+    {
+      "epoch": 0.8709677419354839,
+      "grad_norm": 6.515044515253976e-05,
+      "learning_rate": 1.5163992245193735e-05,
+      "logits/chosen": -2.9613232612609863,
+      "logits/rejected": -2.806081771850586,
+      "logps/chosen": -167.47032165527344,
+      "logps/rejected": -627.2801513671875,
+      "loss": 0.0021,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 2.517486095428467,
+      "rewards/margins": 52.39359664916992,
+      "rewards/rejected": -49.87610626220703,
+      "step": 135
+    },
+    {
+      "epoch": 0.9032258064516129,
+      "grad_norm": 7.156098581617698e-05,
+      "learning_rate": 1.4784611593363314e-05,
+      "logits/chosen": -2.8768656253814697,
+      "logits/rejected": -2.8222408294677734,
+      "logps/chosen": -163.4990234375,
+      "logps/rejected": -613.9205932617188,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.8345298767089844,
+      "rewards/margins": 51.45341110229492,
+      "rewards/rejected": -48.6188850402832,
+      "step": 140
+    },
+    {
+      "epoch": 0.9354838709677419,
+      "grad_norm": 1.8226394331577467e-06,
+      "learning_rate": 1.439640371888783e-05,
+      "logits/chosen": -2.739189624786377,
+      "logits/rejected": -2.763517379760742,
+      "logps/chosen": -162.78518676757812,
+      "logps/rejected": -597.4986572265625,
+      "loss": 0.0027,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 3.1442718505859375,
+      "rewards/margins": 50.352046966552734,
+      "rewards/rejected": -47.2077751159668,
+      "step": 145
+    },
+    {
+      "epoch": 0.967741935483871,
+      "grad_norm": 6.970116373850033e-06,
+      "learning_rate": 1.4000634875163929e-05,
+      "logits/chosen": -2.6427102088928223,
+      "logits/rejected": -2.703989028930664,
+      "logps/chosen": -173.3591766357422,
+      "logps/rejected": -622.0789794921875,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.122755527496338,
+      "rewards/margins": 51.858497619628906,
+      "rewards/rejected": -49.73574447631836,
+      "step": 150
+    },
+    {
+      "epoch": 0.9935483870967742,
+      "eval_logits/chosen": -2.4111573696136475,
+      "eval_logits/rejected": -2.533914566040039,
+      "eval_logps/chosen": -121.9600601196289,
+      "eval_logps/rejected": -688.9767456054688,
+      "eval_loss": 0.0001295939728152007,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.3132941722869873,
+      "eval_rewards/margins": 61.79844284057617,
+      "eval_rewards/rejected": -62.11174011230469,
+      "eval_runtime": 13.3156,
+      "eval_samples_per_second": 15.02,
+      "eval_steps_per_second": 7.51,
+      "step": 154
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.1435479074716568,
+      "learning_rate": 1.3598595977898393e-05,
+      "logits/chosen": -2.470811128616333,
+      "logits/rejected": -2.687915086746216,
+      "logps/chosen": -176.60227966308594,
+      "logps/rejected": -602.3994750976562,
+      "loss": 0.003,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 1.7994766235351562,
+      "rewards/margins": 50.05354309082031,
+      "rewards/rejected": -48.254066467285156,
+      "step": 155
+    },
+    {
+      "epoch": 1.032258064516129,
+      "grad_norm": 4.26612663269043,
+      "learning_rate": 1.3191598394409396e-05,
+      "logits/chosen": -2.4601125717163086,
+      "logits/rejected": -2.6306862831115723,
+      "logps/chosen": -182.5096893310547,
+      "logps/rejected": -619.1905517578125,
+      "loss": 0.0084,
+      "rewards/accuracies": 0.996874988079071,
+      "rewards/chosen": 1.3070443868637085,
+      "rewards/margins": 50.604530334472656,
+      "rewards/rejected": -49.2974853515625,
+      "step": 160
+    },
+    {
+      "epoch": 1.064516129032258,
+      "grad_norm": 0.07364771515130997,
+      "learning_rate": 1.2780969666218915e-05,
+      "logits/chosen": -2.371699094772339,
+      "logits/rejected": -2.5179049968719482,
+      "logps/chosen": -171.65237426757812,
+      "logps/rejected": -626.7579345703125,
+      "loss": 0.0002,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8574281930923462,
+      "rewards/margins": 52.002235412597656,
+      "rewards/rejected": -50.144805908203125,
+      "step": 165
+    },
+    {
+      "epoch": 1.096774193548387,
+      "grad_norm": 2.0316293239593506,
+      "learning_rate": 1.2368049178888188e-05,
+      "logits/chosen": -2.3688201904296875,
+      "logits/rejected": -2.4845595359802246,
+      "logps/chosen": -167.29345703125,
+      "logps/rejected": -637.6082763671875,
+      "loss": 0.0059,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 2.473865032196045,
+      "rewards/margins": 53.60892868041992,
+      "rewards/rejected": -51.13506317138672,
+      "step": 170
+    },
+    {
+      "epoch": 1.129032258064516,
+      "grad_norm": 5.519531896425178e-06,
+      "learning_rate": 1.1954183793220426e-05,
+      "logits/chosen": -2.4739153385162354,
+      "logits/rejected": -2.5887179374694824,
+      "logps/chosen": -168.55252075195312,
+      "logps/rejected": -629.6751098632812,
+      "loss": 0.0003,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.837996006011963,
+      "rewards/margins": 54.0001106262207,
+      "rewards/rejected": -50.16211700439453,
+      "step": 175
+    },
+    {
+      "epoch": 1.1612903225806452,
+      "grad_norm": 4.96319489684538e-06,
+      "learning_rate": 1.154072345208082e-05,
+      "logits/chosen": -2.675598382949829,
+      "logits/rejected": -2.770794153213501,
+      "logps/chosen": -167.05870056152344,
+      "logps/rejected": -619.7980346679688,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.837059736251831,
+      "rewards/margins": 52.89777755737305,
+      "rewards/rejected": -49.06071853637695,
+      "step": 180
+    },
+    {
+      "epoch": 1.1935483870967742,
+      "grad_norm": 2.3110576876206324e-05,
+      "learning_rate": 1.1129016777163497e-05,
+      "logits/chosen": -2.7444872856140137,
+      "logits/rejected": -2.823488235473633,
+      "logps/chosen": -158.0244140625,
+      "logps/rejected": -612.7073974609375,
+      "loss": 0.0088,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 3.5461955070495605,
+      "rewards/margins": 52.555580139160156,
+      "rewards/rejected": -49.00938034057617,
+      "step": 185
+    },
+    {
+      "epoch": 1.2258064516129032,
+      "grad_norm": 0.00015435948444064707,
+      "learning_rate": 1.0720406670067866e-05,
+      "logits/chosen": -3.0422451496124268,
+      "logits/rejected": -3.078159809112549,
+      "logps/chosen": -162.6065216064453,
+      "logps/rejected": -648.2110595703125,
+      "loss": 0.009,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 2.963984966278076,
+      "rewards/margins": 54.6415901184082,
+      "rewards/rejected": -51.6776123046875,
+      "step": 190
+    },
+    {
+      "epoch": 1.2580645161290323,
+      "grad_norm": 0.23506908118724823,
+      "learning_rate": 1.0316225932032739e-05,
+      "logits/chosen": -3.1033904552459717,
+      "logits/rejected": -3.1072916984558105,
+      "logps/chosen": -162.05966186523438,
+      "logps/rejected": -627.5680541992188,
+      "loss": 0.0017,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 3.406363010406494,
+      "rewards/margins": 53.60102462768555,
+      "rewards/rejected": -50.194664001464844,
+      "step": 195
+    },
+    {
+      "epoch": 1.2903225806451613,
+      "grad_norm": 0.0005963341682218015,
+      "learning_rate": 9.917792916615701e-06,
+      "logits/chosen": -3.057863473892212,
+      "logits/rejected": -3.170236349105835,
+      "logps/chosen": -172.58238220214844,
+      "logps/rejected": -642.7207641601562,
+      "loss": 0.0015,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 3.198392629623413,
+      "rewards/margins": 54.85893630981445,
+      "rewards/rejected": -51.66054153442383,
+      "step": 200
+    },
+    {
+      "epoch": 1.3225806451612903,
+      "grad_norm": 3.264209590270184e-05,
+      "learning_rate": 9.526407229497827e-06,
+      "logits/chosen": -3.2984588146209717,
+      "logits/rejected": -3.360607624053955,
+      "logps/chosen": -163.1930694580078,
+      "logps/rejected": -648.0612182617188,
+      "loss": 0.0016,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.657193660736084,
+      "rewards/margins": 54.8690071105957,
+      "rewards/rejected": -52.211814880371094,
+      "step": 205
+    },
+    {
+      "epoch": 1.3548387096774195,
+      "grad_norm": 4.916101943308604e-07,
+      "learning_rate": 9.143345489440102e-06,
+      "logits/chosen": -3.411468982696533,
+      "logits/rejected": -3.4234490394592285,
+      "logps/chosen": -162.50698852539062,
+      "logps/rejected": -648.9544677734375,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.4228503704071045,
+      "rewards/margins": 56.02128982543945,
+      "rewards/rejected": -52.59843826293945,
+      "step": 210
+    },
+    {
+      "epoch": 1.3870967741935485,
+      "grad_norm": 3.84959868426904e-08,
+      "learning_rate": 8.769857164218336e-06,
+      "logits/chosen": -3.3674731254577637,
+      "logits/rejected": -3.465867519378662,
+      "logps/chosen": -156.56558227539062,
+      "logps/rejected": -662.9464721679688,
+      "loss": 0.014,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 3.516038179397583,
+      "rewards/margins": 56.7149772644043,
+      "rewards/rejected": -53.198936462402344,
+      "step": 215
+    },
+    {
+      "epoch": 1.4193548387096775,
+      "grad_norm": 1.2234114365128335e-05,
+      "learning_rate": 8.407160495119008e-06,
+      "logits/chosen": -3.2030277252197266,
+      "logits/rejected": -3.3925929069519043,
+      "logps/chosen": -170.909423828125,
+      "logps/rejected": -667.1768798828125,
+      "loss": 0.0004,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.376603364944458,
+      "rewards/margins": 57.6746826171875,
+      "rewards/rejected": -54.2980842590332,
+      "step": 220
+    },
+    {
+      "epoch": 1.4516129032258065,
+      "grad_norm": 0.19691216945648193,
+      "learning_rate": 8.056438523289393e-06,
+      "logits/chosen": -3.271845579147339,
+      "logits/rejected": -3.299424409866333,
+      "logps/chosen": -165.4629364013672,
+      "logps/rejected": -647.8621826171875,
+      "loss": 0.0047,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 2.843123435974121,
+      "rewards/margins": 54.9563102722168,
+      "rewards/rejected": -52.113182067871094,
+      "step": 225
+    },
+    {
+      "epoch": 1.4838709677419355,
+      "grad_norm": 0.0007676176610402763,
+      "learning_rate": 7.718835230903237e-06,
+      "logits/chosen": -3.305957794189453,
+      "logits/rejected": -3.3377411365509033,
+      "logps/chosen": -168.66317749023438,
+      "logps/rejected": -664.5413818359375,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.4339606761932373,
+      "rewards/margins": 57.25107955932617,
+      "rewards/rejected": -53.81711959838867,
+      "step": 230
+    },
+    {
+      "epoch": 1.5161290322580645,
+      "grad_norm": 1.1214526239200495e-05,
+      "learning_rate": 7.3954518097287005e-06,
+      "logits/chosen": -3.360619068145752,
+      "logits/rejected": -3.381943464279175,
+      "logps/chosen": -170.27052307128906,
+      "logps/rejected": -697.7355346679688,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.744898557662964,
+      "rewards/margins": 60.68017578125,
+      "rewards/rejected": -56.935279846191406,
+      "step": 235
+    },
+    {
+      "epoch": 1.5483870967741935,
+      "grad_norm": 0.001958629349246621,
+      "learning_rate": 7.08734306926967e-06,
+      "logits/chosen": -3.3265819549560547,
+      "logits/rejected": -3.33760404586792,
+      "logps/chosen": -158.65939331054688,
+      "logps/rejected": -662.5846557617188,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.8199775218963623,
+      "rewards/margins": 56.54740524291992,
+      "rewards/rejected": -53.7274284362793,
+      "step": 240
+    },
+    {
+      "epoch": 1.5806451612903225,
+      "grad_norm": 1.1762933354475535e-05,
+      "learning_rate": 6.7955139961964235e-06,
+      "logits/chosen": -3.417844295501709,
+      "logits/rejected": -3.355306625366211,
+      "logps/chosen": -163.48977661132812,
+      "logps/rejected": -671.9425659179688,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.6044082641601562,
+      "rewards/margins": 57.15119171142578,
+      "rewards/rejected": -54.546783447265625,
+      "step": 245
+    },
+    {
+      "epoch": 1.6129032258064515,
+      "grad_norm": 0.3113669157028198,
+      "learning_rate": 6.5209164762879876e-06,
+      "logits/chosen": -3.1726534366607666,
+      "logits/rejected": -3.2277159690856934,
+      "logps/chosen": -158.69281005859375,
+      "logps/rejected": -670.3931884765625,
+      "loss": 0.018,
+      "rewards/accuracies": 0.995312511920929,
+      "rewards/chosen": 2.9490036964416504,
+      "rewards/margins": 57.56342315673828,
+      "rewards/rejected": -54.61442184448242,
+      "step": 250
+    },
+    {
+      "epoch": 1.6451612903225805,
+      "grad_norm": 0.15534324944019318,
+      "learning_rate": 6.26444618957859e-06,
+      "logits/chosen": -2.9065983295440674,
+      "logits/rejected": -3.0085651874542236,
+      "logps/chosen": -162.54873657226562,
+      "logps/rejected": -677.47412109375,
+      "loss": 0.0006,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.7529945373535156,
+      "rewards/margins": 58.81446075439453,
+      "rewards/rejected": -55.06147003173828,
+      "step": 255
+    },
+    {
+      "epoch": 1.6774193548387095,
+      "grad_norm": 0.5288254022598267,
+      "learning_rate": 6.026939688835542e-06,
+      "logits/chosen": -2.7987606525421143,
+      "logits/rejected": -2.935157299041748,
+      "logps/chosen": -156.1405792236328,
+      "logps/rejected": -637.0443115234375,
+      "loss": 0.0059,
+      "rewards/accuracies": 0.996874988079071,
+      "rewards/chosen": 2.6880862712860107,
+      "rewards/margins": 53.83855056762695,
+      "rewards/rejected": -51.15046691894531,
+      "step": 260
+    },
+    {
+      "epoch": 1.7096774193548387,
+      "grad_norm": 9.21001483220607e-05,
+      "learning_rate": 5.809171670898002e-06,
+      "logits/chosen": -2.7774410247802734,
+      "logits/rejected": -2.9501843452453613,
+      "logps/chosen": -161.6907501220703,
+      "logps/rejected": -681.8929443359375,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.7917118072509766,
+      "rewards/margins": 58.00335693359375,
+      "rewards/rejected": -55.211647033691406,
+      "step": 265
+    },
+    {
+      "epoch": 1.7419354838709677,
+      "grad_norm": 0.0019305861787870526,
+      "learning_rate": 5.611852449776906e-06,
+      "logits/chosen": -2.70743989944458,
+      "logits/rejected": -2.9343791007995605,
+      "logps/chosen": -158.28378295898438,
+      "logps/rejected": -656.1748046875,
+      "loss": 0.0044,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 2.8768951892852783,
+      "rewards/margins": 56.165191650390625,
+      "rewards/rejected": -53.288291931152344,
+      "step": 270
+    },
+    {
+      "epoch": 1.7741935483870968,
+      "grad_norm": 0.05368852615356445,
+      "learning_rate": 5.435625639758302e-06,
+      "logits/chosen": -2.6554648876190186,
+      "logits/rejected": -2.9207005500793457,
+      "logps/chosen": -173.9909210205078,
+      "logps/rejected": -673.0507202148438,
+      "loss": 0.0003,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.8360095024108887,
+      "rewards/margins": 57.830604553222656,
+      "rewards/rejected": -54.994598388671875,
+      "step": 275
+    },
+    {
+      "epoch": 1.8064516129032258,
+      "grad_norm": 1.7408574422006495e-06,
+      "learning_rate": 5.281066056067326e-06,
+      "logits/chosen": -2.7118172645568848,
+      "logits/rejected": -2.919018268585205,
+      "logps/chosen": -169.70877075195312,
+      "logps/rejected": -689.2927856445312,
+      "loss": 0.008,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 2.9414725303649902,
+      "rewards/margins": 58.81359100341797,
+      "rewards/rejected": -55.87211990356445,
+      "step": 280
+    },
+    {
+      "epoch": 1.838709677419355,
+      "grad_norm": 0.02011033147573471,
+      "learning_rate": 5.148677839940424e-06,
+      "logits/chosen": -2.6920559406280518,
+      "logits/rejected": -2.856995105743408,
+      "logps/chosen": -170.58425903320312,
+      "logps/rejected": -718.7882690429688,
+      "loss": 0.0033,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 2.741574764251709,
+      "rewards/margins": 61.97066116333008,
+      "rewards/rejected": -59.22908401489258,
+      "step": 285
+    },
+    {
+      "epoch": 1.870967741935484,
+      "grad_norm": 0.07748307287693024,
+      "learning_rate": 5.038892814221403e-06,
+      "logits/chosen": -2.6701674461364746,
+      "logits/rejected": -2.828078031539917,
+      "logps/chosen": -166.5321807861328,
+      "logps/rejected": -721.7613525390625,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.030878782272339,
+      "rewards/margins": 61.87688446044922,
+      "rewards/rejected": -59.84600830078125,
+      "step": 290
+    },
+    {
+      "epoch": 1.903225806451613,
+      "grad_norm": 4.640487168217078e-05,
+      "learning_rate": 4.952069074845096e-06,
+      "logits/chosen": -2.712212324142456,
+      "logits/rejected": -2.777492046356201,
+      "logps/chosen": -163.62423706054688,
+      "logps/rejected": -726.83251953125,
+      "loss": 0.0021,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 2.233910083770752,
+      "rewards/margins": 62.41532516479492,
+      "rewards/rejected": -60.18141555786133,
+      "step": 295
+    },
+    {
+      "epoch": 1.935483870967742,
+      "grad_norm": 0.00010279910929966718,
+      "learning_rate": 4.888489822802835e-06,
+      "logits/chosen": -2.6591506004333496,
+      "logits/rejected": -2.798229694366455,
+      "logps/chosen": -177.29238891601562,
+      "logps/rejected": -769.4779663085938,
+      "loss": 0.0141,
+      "rewards/accuracies": 0.9984375238418579,
+      "rewards/chosen": 1.5061697959899902,
+      "rewards/margins": 65.76886749267578,
+      "rewards/rejected": -64.2626953125,
+      "step": 300
+    },
+    {
+      "epoch": 1.967741935483871,
+      "grad_norm": 1.4639048458775505e-05,
+      "learning_rate": 4.848362440399692e-06,
+      "logits/chosen": -2.73205828666687,
+      "logits/rejected": -2.761648178100586,
+      "logps/chosen": -175.31532287597656,
+      "logps/rejected": -765.2847900390625,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8591419458389282,
+      "rewards/margins": 65.3056869506836,
+      "rewards/rejected": -63.4465446472168,
+      "step": 305
+    },
+    {
+      "epoch": 1.9870967741935484,
+      "eval_logits/chosen": -2.4073026180267334,
+      "eval_logits/rejected": -2.258913516998291,
+      "eval_logps/chosen": -124.21800994873047,
+      "eval_logps/rejected": -902.7763061523438,
+      "eval_loss": 9.752377081895247e-06,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.5390886664390564,
+      "eval_rewards/margins": 82.95260620117188,
+      "eval_rewards/rejected": -83.49169921875,
+      "eval_runtime": 11.9364,
+      "eval_samples_per_second": 16.755,
+      "eval_steps_per_second": 8.378,
+      "step": 308
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 310,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9042f3417657c088979652fcee58d58ea78eae57afd750f0e3092ecdc42ca03e
+size 7121