Upload task output 63b2db3d-a057-429f-9319-84e8338dbfb9

README.md

@@ -0,0 +1,202 @@
+---
+base_model: None
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.15.1

adapter_config.json

@@ -0,0 +1,36 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": null,
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 256,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 128,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "qkv_proj",
+    "gate_up_proj",
+    "down_proj",
+    "o_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4ca99e8b2718ee22e8b79e961cc6b99c72ce364a268788e2c3335e891be5181
+size 805341552

added_tokens.json

@@ -0,0 +1,13 @@
+{
+  "<|assistant|>": 32001,
+  "<|endoftext|>": 32000,
+  "<|end|>": 32007,
+  "<|placeholder1|>": 32002,
+  "<|placeholder2|>": 32003,
+  "<|placeholder3|>": 32004,
+  "<|placeholder4|>": 32005,
+  "<|placeholder5|>": 32008,
+  "<|placeholder6|>": 32009,
+  "<|system|>": 32006,
+  "<|user|>": 32010
+}

loss.txt

@@ -0,0 +1 @@
+358,0.6298339366912842

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,132 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "32000": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "<|placeholder1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32003": {
+      "content": "<|placeholder2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32004": {
+      "content": "<|placeholder3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32005": {
+      "content": "<|placeholder4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32006": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32007": {
+      "content": "<|end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32008": {
+      "content": "<|placeholder5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32009": {
+      "content": "<|placeholder6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32010": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

trainer_state.json

@@ -0,0 +1,1115 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.99860529986053,
+  "eval_steps": 500,
+  "global_step": 358,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01394700139470014,
+      "grad_norm": 8.443673133850098,
+      "learning_rate": 3.1067090740782183e-06,
+      "logits/chosen": 13.965856552124023,
+      "logits/rejected": 15.354721069335938,
+      "logps/chosen": -263.2093200683594,
+      "logps/rejected": -286.745361328125,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.3083333373069763,
+      "rewards/chosen": 0.001035079127177596,
+      "rewards/margins": 0.004346395842730999,
+      "rewards/rejected": -0.003311316715553403,
+      "step": 5
+    },
+    {
+      "epoch": 0.02789400278940028,
+      "grad_norm": 9.05903434753418,
+      "learning_rate": 6.99009541667599e-06,
+      "logits/chosen": 13.079957962036133,
+      "logits/rejected": 15.2965726852417,
+      "logps/chosen": -246.61495971679688,
+      "logps/rejected": -292.19561767578125,
+      "loss": 0.6512,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.004167428705841303,
+      "rewards/margins": 0.10341107845306396,
+      "rewards/rejected": -0.10757851600646973,
+      "step": 10
+    },
+    {
+      "epoch": 0.04184100418410042,
+      "grad_norm": 7.4054036140441895,
+      "learning_rate": 1.0873481759273766e-05,
+      "logits/chosen": 13.588569641113281,
+      "logits/rejected": 14.664776802062988,
+      "logps/chosen": -250.9985809326172,
+      "logps/rejected": -271.63397216796875,
+      "loss": 0.6306,
+      "rewards/accuracies": 0.6750000715255737,
+      "rewards/chosen": -0.07446910440921783,
+      "rewards/margins": 0.2632213234901428,
+      "rewards/rejected": -0.33769041299819946,
+      "step": 15
+    },
+    {
+      "epoch": 0.05578800557880056,
+      "grad_norm": 7.973468780517578,
+      "learning_rate": 1.4756868101871536e-05,
+      "logits/chosen": 10.776374816894531,
+      "logits/rejected": 12.654977798461914,
+      "logps/chosen": -219.0859375,
+      "logps/rejected": -279.8912048339844,
+      "loss": 0.5803,
+      "rewards/accuracies": 0.7083333730697632,
+      "rewards/chosen": -0.15387776494026184,
+      "rewards/margins": 0.5580573678016663,
+      "rewards/rejected": -0.7119351625442505,
+      "step": 20
+    },
+    {
+      "epoch": 0.0697350069735007,
+      "grad_norm": 8.493500709533691,
+      "learning_rate": 1.864025444446931e-05,
+      "logits/chosen": 11.028432846069336,
+      "logits/rejected": 12.664867401123047,
+      "logps/chosen": -269.8415832519531,
+      "logps/rejected": -324.97613525390625,
+      "loss": 0.6957,
+      "rewards/accuracies": 0.6833333969116211,
+      "rewards/chosen": -0.31670600175857544,
+      "rewards/margins": 0.7376400232315063,
+      "rewards/rejected": -1.0543458461761475,
+      "step": 25
+    },
+    {
+      "epoch": 0.08368200836820083,
+      "grad_norm": 6.57462739944458,
+      "learning_rate": 2.2523640787067085e-05,
+      "logits/chosen": 12.154653549194336,
+      "logits/rejected": 13.542457580566406,
+      "logps/chosen": -250.53341674804688,
+      "logps/rejected": -290.4168395996094,
+      "loss": 0.6559,
+      "rewards/accuracies": 0.6750000715255737,
+      "rewards/chosen": -0.11686629056930542,
+      "rewards/margins": 0.5811147093772888,
+      "rewards/rejected": -0.6979809999465942,
+      "step": 30
+    },
+    {
+      "epoch": 0.09762900976290098,
+      "grad_norm": 7.757661819458008,
+      "learning_rate": 2.6407027129664858e-05,
+      "logits/chosen": 11.735052108764648,
+      "logits/rejected": 12.871235847473145,
+      "logps/chosen": -196.1747589111328,
+      "logps/rejected": -223.5957794189453,
+      "loss": 0.6182,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -0.00016996636986732483,
+      "rewards/margins": 0.3299906551837921,
+      "rewards/rejected": -0.33016061782836914,
+      "step": 35
+    },
+    {
+      "epoch": 0.11157601115760112,
+      "grad_norm": 9.74497127532959,
+      "learning_rate": 2.7182958819900885e-05,
+      "logits/chosen": 10.667373657226562,
+      "logits/rejected": 12.847865104675293,
+      "logps/chosen": -208.2180633544922,
+      "logps/rejected": -302.39971923828125,
+      "loss": 0.5985,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.03828881308436394,
+      "rewards/margins": 0.43971744179725647,
+      "rewards/rejected": -0.4780062735080719,
+      "step": 40
+    },
+    {
+      "epoch": 0.12552301255230125,
+      "grad_norm": 9.502607345581055,
+      "learning_rate": 2.7179930095042222e-05,
+      "logits/chosen": 11.666415214538574,
+      "logits/rejected": 13.399192810058594,
+      "logps/chosen": -252.95516967773438,
+      "logps/rejected": -297.4845275878906,
+      "loss": 0.6683,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.2012624740600586,
+      "rewards/margins": 0.4417993426322937,
+      "rewards/rejected": -0.6430618166923523,
+      "step": 45
+    },
+    {
+      "epoch": 0.1394700139470014,
+      "grad_norm": 6.734437942504883,
+      "learning_rate": 2.717457231667877e-05,
+      "logits/chosen": 11.400947570800781,
+      "logits/rejected": 12.905207633972168,
+      "logps/chosen": -199.80596923828125,
+      "logps/rejected": -254.4221954345703,
+      "loss": 0.5718,
+      "rewards/accuracies": 0.6416667103767395,
+      "rewards/chosen": -0.022540345788002014,
+      "rewards/margins": 0.5749102830886841,
+      "rewards/rejected": -0.5974506139755249,
+      "step": 50
+    },
+    {
+      "epoch": 0.15341701534170155,
+      "grad_norm": 6.349542617797852,
+      "learning_rate": 2.7166886709384802e-05,
+      "logits/chosen": 9.220715522766113,
+      "logits/rejected": 10.65298080444336,
+      "logps/chosen": -187.50601196289062,
+      "logps/rejected": -232.32241821289062,
+      "loss": 0.6379,
+      "rewards/accuracies": 0.6583333015441895,
+      "rewards/chosen": -0.3121258318424225,
+      "rewards/margins": 0.4689728617668152,
+      "rewards/rejected": -0.7810987234115601,
+      "step": 55
+    },
+    {
+      "epoch": 0.16736401673640167,
+      "grad_norm": 6.804388046264648,
+      "learning_rate": 2.715687502978336e-05,
+      "logits/chosen": 10.728836059570312,
+      "logits/rejected": 11.548182487487793,
+      "logps/chosen": -220.2592010498047,
+      "logps/rejected": -248.6510009765625,
+      "loss": 0.5624,
+      "rewards/accuracies": 0.6916667222976685,
+      "rewards/chosen": -0.24719354510307312,
+      "rewards/margins": 0.5707910656929016,
+      "rewards/rejected": -0.8179847002029419,
+      "step": 60
+    },
+    {
+      "epoch": 0.18131101813110181,
+      "grad_norm": 8.308326721191406,
+      "learning_rate": 2.714453956614478e-05,
+      "logits/chosen": 9.632620811462402,
+      "logits/rejected": 10.766716003417969,
+      "logps/chosen": -234.59494018554688,
+      "logps/rejected": -246.2835235595703,
+      "loss": 0.6495,
+      "rewards/accuracies": 0.6500000357627869,
+      "rewards/chosen": -0.4380224347114563,
+      "rewards/margins": 0.505369246006012,
+      "rewards/rejected": -0.9433916211128235,
+      "step": 65
+    },
+    {
+      "epoch": 0.19525801952580196,
+      "grad_norm": 5.469903945922852,
+      "learning_rate": 2.7129883137863668e-05,
+      "logits/chosen": 10.187846183776855,
+      "logits/rejected": 11.750052452087402,
+      "logps/chosen": -201.72952270507812,
+      "logps/rejected": -254.0623779296875,
+      "loss": 0.6407,
+      "rewards/accuracies": 0.6166666746139526,
+      "rewards/chosen": -0.28986990451812744,
+      "rewards/margins": 0.4815855026245117,
+      "rewards/rejected": -0.7714553475379944,
+      "step": 70
+    },
+    {
+      "epoch": 0.20920502092050208,
+      "grad_norm": 6.822120189666748,
+      "learning_rate": 2.7112909094814497e-05,
+      "logits/chosen": 11.570058822631836,
+      "logits/rejected": 12.092260360717773,
+      "logps/chosen": -201.8399658203125,
+      "logps/rejected": -255.96774291992188,
+      "loss": 0.5926,
+      "rewards/accuracies": 0.6500000357627869,
+      "rewards/chosen": -0.17586487531661987,
+      "rewards/margins": 0.4662502706050873,
+      "rewards/rejected": -0.6421152353286743,
+      "step": 75
+    },
+    {
+      "epoch": 0.22315202231520223,
+      "grad_norm": 6.789912223815918,
+      "learning_rate": 2.7093621316585976e-05,
+      "logits/chosen": 9.428289413452148,
+      "logits/rejected": 12.114514350891113,
+      "logps/chosen": -199.76634216308594,
+      "logps/rejected": -259.3163146972656,
+      "loss": 0.6011,
+      "rewards/accuracies": 0.7083333730697632,
+      "rewards/chosen": -0.12371706962585449,
+      "rewards/margins": 0.6005428433418274,
+      "rewards/rejected": -0.7242598533630371,
+      "step": 80
+    },
+    {
+      "epoch": 0.23709902370990238,
+      "grad_norm": 8.106010437011719,
+      "learning_rate": 2.7072024211594312e-05,
+      "logits/chosen": 10.688722610473633,
+      "logits/rejected": 11.625075340270996,
+      "logps/chosen": -221.6556854248047,
+      "logps/rejected": -274.34808349609375,
+      "loss": 0.6676,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.237966850399971,
+      "rewards/margins": 0.4688500761985779,
+      "rewards/rejected": -0.7068168520927429,
+      "step": 85
+    },
+    {
+      "epoch": 0.2510460251046025,
+      "grad_norm": 7.263269901275635,
+      "learning_rate": 2.7048122716075636e-05,
+      "logits/chosen": 12.357452392578125,
+      "logits/rejected": 14.314582824707031,
+      "logps/chosen": -224.59451293945312,
+      "logps/rejected": -291.54840087890625,
+      "loss": 0.6033,
+      "rewards/accuracies": 0.6666667461395264,
+      "rewards/chosen": -0.07308916002511978,
+      "rewards/margins": 0.504932165145874,
+      "rewards/rejected": -0.5780213475227356,
+      "step": 90
+    },
+    {
+      "epoch": 0.2649930264993027,
+      "grad_norm": 8.925468444824219,
+      "learning_rate": 2.7021922292957776e-05,
+      "logits/chosen": 12.812413215637207,
+      "logits/rejected": 13.796060562133789,
+      "logps/chosen": -230.6768798828125,
+      "logps/rejected": -276.98828125,
+      "loss": 0.6054,
+      "rewards/accuracies": 0.6916667222976685,
+      "rewards/chosen": -0.03974637761712074,
+      "rewards/margins": 0.5660965442657471,
+      "rewards/rejected": -0.6058429479598999,
+      "step": 95
+    },
+    {
+      "epoch": 0.2789400278940028,
+      "grad_norm": 5.422610282897949,
+      "learning_rate": 2.6993428930611634e-05,
+      "logits/chosen": 12.078475952148438,
+      "logits/rejected": 14.003347396850586,
+      "logps/chosen": -229.6229705810547,
+      "logps/rejected": -276.1091613769531,
+      "loss": 0.5103,
+      "rewards/accuracies": 0.7916666269302368,
+      "rewards/chosen": 0.10478214919567108,
+      "rewards/margins": 0.8226801156997681,
+      "rewards/rejected": -0.7178980112075806,
+      "step": 100
+    },
+    {
+      "epoch": 0.2928870292887029,
+      "grad_norm": 7.437990188598633,
+      "learning_rate": 2.69626491414825e-05,
+      "logits/chosen": 11.215384483337402,
+      "logits/rejected": 14.319795608520508,
+      "logps/chosen": -253.2958221435547,
+      "logps/rejected": -353.27728271484375,
+      "loss": 0.4647,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.20068514347076416,
+      "rewards/margins": 1.1105215549468994,
+      "rewards/rejected": -1.3112064599990845,
+      "step": 105
+    },
+    {
+      "epoch": 0.3068340306834031,
+      "grad_norm": 4.438156604766846,
+      "learning_rate": 2.6929589960601567e-05,
+      "logits/chosen": 10.04852294921875,
+      "logits/rejected": 11.834890365600586,
+      "logps/chosen": -177.2984619140625,
+      "logps/rejected": -235.3513946533203,
+      "loss": 0.5941,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.14255599677562714,
+      "rewards/margins": 1.0380289554595947,
+      "rewards/rejected": -1.1805849075317383,
+      "step": 110
+    },
+    {
+      "epoch": 0.3207810320781032,
+      "grad_norm": 5.81158971786499,
+      "learning_rate": 2.689425894397799e-05,
+      "logits/chosen": 11.626302719116211,
+      "logits/rejected": 13.641179084777832,
+      "logps/chosen": -258.5274353027344,
+      "logps/rejected": -290.1232604980469,
+      "loss": 0.7585,
+      "rewards/accuracies": 0.6916667222976685,
+      "rewards/chosen": -0.3435746729373932,
+      "rewards/margins": 0.7051381468772888,
+      "rewards/rejected": -1.0487128496170044,
+      "step": 115
+    },
+    {
+      "epoch": 0.33472803347280333,
+      "grad_norm": 6.80866003036499,
+      "learning_rate": 2.685666416687189e-05,
+      "logits/chosen": 12.343037605285645,
+      "logits/rejected": 14.431178092956543,
+      "logps/chosen": -240.32119750976562,
+      "logps/rejected": -306.0838623046875,
+      "loss": 0.595,
+      "rewards/accuracies": 0.7166666984558105,
+      "rewards/chosen": 0.007438424974679947,
+      "rewards/margins": 0.7371615171432495,
+      "rewards/rejected": -0.7297230362892151,
+      "step": 120
+    },
+    {
+      "epoch": 0.3486750348675035,
+      "grad_norm": 5.887704372406006,
+      "learning_rate": 2.6816814221948682e-05,
+      "logits/chosen": 12.360528945922852,
+      "logits/rejected": 13.354936599731445,
+      "logps/chosen": -229.65194702148438,
+      "logps/rejected": -286.4953918457031,
+      "loss": 0.5958,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.04780165106058121,
+      "rewards/margins": 0.5758650898933411,
+      "rewards/rejected": -0.5280634164810181,
+      "step": 125
+    },
+    {
+      "epoch": 0.36262203626220363,
+      "grad_norm": 6.528600215911865,
+      "learning_rate": 2.6774718217315124e-05,
+      "logits/chosen": 12.22896671295166,
+      "logits/rejected": 14.30839729309082,
+      "logps/chosen": -258.0196838378906,
+      "logps/rejected": -312.25018310546875,
+      "loss": 0.5731,
+      "rewards/accuracies": 0.7166666984558105,
+      "rewards/chosen": -0.1742248833179474,
+      "rewards/margins": 0.7956671118736267,
+      "rewards/rejected": -0.9698920249938965,
+      "step": 130
+    },
+    {
+      "epoch": 0.37656903765690375,
+      "grad_norm": 4.8263702392578125,
+      "learning_rate": 2.6730385774437575e-05,
+      "logits/chosen": 11.001718521118164,
+      "logits/rejected": 12.369864463806152,
+      "logps/chosen": -186.54306030273438,
+      "logps/rejected": -229.1658935546875,
+      "loss": 0.5747,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -0.10876262187957764,
+      "rewards/margins": 0.783820390701294,
+      "rewards/rejected": -0.8925830721855164,
+      "step": 135
+    },
+    {
+      "epoch": 0.3905160390516039,
+      "grad_norm": 6.88525915145874,
+      "learning_rate": 2.668382702594289e-05,
+      "logits/chosen": 12.428079605102539,
+      "logits/rejected": 13.376548767089844,
+      "logps/chosen": -244.35336303710938,
+      "logps/rejected": -314.2310485839844,
+      "loss": 0.5727,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.19110675156116486,
+      "rewards/margins": 0.8208184242248535,
+      "rewards/rejected": -1.011925220489502,
+      "step": 140
+    },
+    {
+      "epoch": 0.40446304044630405,
+      "grad_norm": 5.504042148590088,
+      "learning_rate": 2.663505261330254e-05,
+      "logits/chosen": 11.734379768371582,
+      "logits/rejected": 13.473363876342773,
+      "logps/chosen": -204.04251098632812,
+      "logps/rejected": -258.86737060546875,
+      "loss": 0.6128,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -0.17432288825511932,
+      "rewards/margins": 0.4802930951118469,
+      "rewards/rejected": -0.6546159982681274,
+      "step": 145
+    },
+    {
+      "epoch": 0.41841004184100417,
+      "grad_norm": 6.70412540435791,
+      "learning_rate": 2.6584073684400373e-05,
+      "logits/chosen": 13.045954704284668,
+      "logits/rejected": 13.6720552444458,
+      "logps/chosen": -220.8832550048828,
+      "logps/rejected": -269.233642578125,
+      "loss": 0.6032,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.04318929463624954,
+      "rewards/margins": 0.618157684803009,
+      "rewards/rejected": -0.6613470315933228,
+      "step": 150
+    },
+    {
+      "epoch": 0.43235704323570434,
+      "grad_norm": 12.566320419311523,
+      "learning_rate": 2.653090189098466e-05,
+      "logits/chosen": 11.808868408203125,
+      "logits/rejected": 13.49694538116455,
+      "logps/chosen": -248.18212890625,
+      "logps/rejected": -286.3559265136719,
+      "loss": 0.6096,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.15974543988704681,
+      "rewards/margins": 0.6279758214950562,
+      "rewards/rejected": -0.7877213954925537,
+      "step": 155
+    },
+    {
+      "epoch": 0.44630404463040446,
+      "grad_norm": 6.7717509269714355,
+      "learning_rate": 2.647554938600497e-05,
+      "logits/chosen": 11.906153678894043,
+      "logits/rejected": 13.252202033996582,
+      "logps/chosen": -230.92593383789062,
+      "logps/rejected": -248.78488159179688,
+      "loss": 0.6474,
+      "rewards/accuracies": 0.6166666746139526,
+      "rewards/chosen": -0.20924147963523865,
+      "rewards/margins": 0.49998435378074646,
+      "rewards/rejected": -0.7092257738113403,
+      "step": 160
+    },
+    {
+      "epoch": 0.4602510460251046,
+      "grad_norm": 7.113363742828369,
+      "learning_rate": 2.6418028820834483e-05,
+      "logits/chosen": 11.409849166870117,
+      "logits/rejected": 13.500089645385742,
+      "logps/chosen": -242.9384765625,
+      "logps/rejected": -325.8098449707031,
+      "loss": 0.5703,
+      "rewards/accuracies": 0.7000000476837158,
+      "rewards/chosen": -0.4405299127101898,
+      "rewards/margins": 0.8166629076004028,
+      "rewards/rejected": -1.257192850112915,
+      "step": 165
+    },
+    {
+      "epoch": 0.47419804741980476,
+      "grad_norm": 4.8473429679870605,
+      "learning_rate": 2.6358353342378405e-05,
+      "logits/chosen": 8.604534149169922,
+      "logits/rejected": 11.609006881713867,
+      "logps/chosen": -185.0061492919922,
+      "logps/rejected": -256.83465576171875,
+      "loss": 0.4518,
+      "rewards/accuracies": 0.8166667222976685,
+      "rewards/chosen": -0.5300418734550476,
+      "rewards/margins": 1.1527934074401855,
+      "rewards/rejected": -1.682835340499878,
+      "step": 170
+    },
+    {
+      "epoch": 0.4881450488145049,
+      "grad_norm": 6.59335994720459,
+      "learning_rate": 2.6296536590069104e-05,
+      "logits/chosen": 9.987462997436523,
+      "logits/rejected": 11.572959899902344,
+      "logps/chosen": -253.36050415039062,
+      "logps/rejected": -306.0450439453125,
+      "loss": 0.7439,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.0800180435180664,
+      "rewards/margins": 0.8873661160469055,
+      "rewards/rejected": -1.9673839807510376,
+      "step": 175
+    },
+    {
+      "epoch": 0.502092050209205,
+      "grad_norm": 7.2572808265686035,
+      "learning_rate": 2.6232592692748676e-05,
+      "logits/chosen": 9.321784019470215,
+      "logits/rejected": 10.399984359741211,
+      "logps/chosen": -234.3979949951172,
+      "logps/rejected": -294.28961181640625,
+      "loss": 0.489,
+      "rewards/accuracies": 0.7916667461395264,
+      "rewards/chosen": -0.8451215624809265,
+      "rewards/margins": 1.1555068492889404,
+      "rewards/rejected": -2.0006284713745117,
+      "step": 180
+    },
+    {
+      "epoch": 0.5160390516039052,
+      "grad_norm": 6.049355506896973,
+      "learning_rate": 2.6166536265439664e-05,
+      "logits/chosen": 11.809592247009277,
+      "logits/rejected": 13.855669021606445,
+      "logps/chosen": -235.9927215576172,
+      "logps/rejected": -315.3490295410156,
+      "loss": 0.5868,
+      "rewards/accuracies": 0.6583333015441895,
+      "rewards/chosen": -0.6016494631767273,
+      "rewards/margins": 0.7112706899642944,
+      "rewards/rejected": -1.3129202127456665,
+      "step": 185
+    },
+    {
+      "epoch": 0.5299860529986054,
+      "grad_norm": 11.038729667663574,
+      "learning_rate": 2.609838240600464e-05,
+      "logits/chosen": 11.31627082824707,
+      "logits/rejected": 12.770919799804688,
+      "logps/chosen": -221.79501342773438,
+      "logps/rejected": -262.8851013183594,
+      "loss": 0.5937,
+      "rewards/accuracies": 0.6916667222976685,
+      "rewards/chosen": -0.5558231472969055,
+      "rewards/margins": 0.6753066778182983,
+      "rewards/rejected": -1.2311298847198486,
+      "step": 190
+    },
+    {
+      "epoch": 0.5439330543933054,
+      "grad_norm": 4.267746925354004,
+      "learning_rate": 2.602814669169543e-05,
+      "logits/chosen": 11.312234878540039,
+      "logits/rejected": 13.296246528625488,
+      "logps/chosen": -221.83663940429688,
+      "logps/rejected": -302.32623291015625,
+      "loss": 0.5809,
+      "rewards/accuracies": 0.7166666984558105,
+      "rewards/chosen": -0.5700381994247437,
+      "rewards/margins": 0.8077206611633301,
+      "rewards/rejected": -1.3777590990066528,
+      "step": 195
+    },
+    {
+      "epoch": 0.5578800557880056,
+      "grad_norm": 5.395613193511963,
+      "learning_rate": 2.5955845175592813e-05,
+      "logits/chosen": 11.410247802734375,
+      "logits/rejected": 13.515039443969727,
+      "logps/chosen": -232.266845703125,
+      "logps/rejected": -293.22467041015625,
+      "loss": 0.5553,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5373277068138123,
+      "rewards/margins": 0.8851755857467651,
+      "rewards/rejected": -1.4225032329559326,
+      "step": 200
+    },
+    {
+      "epoch": 0.5718270571827058,
+      "grad_norm": 5.884355545043945,
+      "learning_rate": 2.5881494382937354e-05,
+      "logits/chosen": 11.658515930175781,
+      "logits/rejected": 13.375651359558105,
+      "logps/chosen": -212.52932739257812,
+      "logps/rejected": -286.71783447265625,
+      "loss": 0.5126,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.6171377897262573,
+      "rewards/margins": 1.1004387140274048,
+      "rewards/rejected": -1.7175763845443726,
+      "step": 205
+    },
+    {
+      "epoch": 0.5857740585774058,
+      "grad_norm": 8.411831855773926,
+      "learning_rate": 2.5805111307352483e-05,
+      "logits/chosen": 11.734652519226074,
+      "logits/rejected": 12.831028938293457,
+      "logps/chosen": -247.45736694335938,
+      "logps/rejected": -297.03485107421875,
+      "loss": 0.5907,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -0.6340610384941101,
+      "rewards/margins": 1.0966707468032837,
+      "rewards/rejected": -1.7307319641113281,
+      "step": 210
+    },
+    {
+      "epoch": 0.599721059972106,
+      "grad_norm": 6.429418087005615,
+      "learning_rate": 2.5726713406960365e-05,
+      "logits/chosen": 9.269506454467773,
+      "logits/rejected": 12.285856246948242,
+      "logps/chosen": -187.17591857910156,
+      "logps/rejected": -269.7003479003906,
+      "loss": 0.5349,
+      "rewards/accuracies": 0.7666667699813843,
+      "rewards/chosen": -0.5631422400474548,
+      "rewards/margins": 1.0805346965789795,
+      "rewards/rejected": -1.6436771154403687,
+      "step": 215
+    },
+    {
+      "epoch": 0.6136680613668062,
+      "grad_norm": 8.299524307250977,
+      "learning_rate": 2.5646318600391693e-05,
+      "logits/chosen": 10.429685592651367,
+      "logits/rejected": 11.87198543548584,
+      "logps/chosen": -234.75802612304688,
+      "logps/rejected": -284.33642578125,
+      "loss": 0.5775,
+      "rewards/accuracies": 0.6833333373069763,
+      "rewards/chosen": -0.5350970029830933,
+      "rewards/margins": 1.0643174648284912,
+      "rewards/rejected": -1.599414348602295,
+      "step": 220
+    },
+    {
+      "epoch": 0.6276150627615062,
+      "grad_norm": 3.9937915802001953,
+      "learning_rate": 2.556394526269021e-05,
+      "logits/chosen": 11.182074546813965,
+      "logits/rejected": 12.9561767578125,
+      "logps/chosen": -209.6283721923828,
+      "logps/rejected": -261.5816345214844,
+      "loss": 0.7124,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.651893675327301,
+      "rewards/margins": 0.631694495677948,
+      "rewards/rejected": -1.283588171005249,
+      "step": 225
+    },
+    {
+      "epoch": 0.6415620641562064,
+      "grad_norm": 8.43625259399414,
+      "learning_rate": 2.5479612221112888e-05,
+      "logits/chosen": 11.575922966003418,
+      "logits/rejected": 12.958845138549805,
+      "logps/chosen": -241.3083953857422,
+      "logps/rejected": -291.8955993652344,
+      "loss": 0.6766,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -0.6825530529022217,
+      "rewards/margins": 0.6574732661247253,
+      "rewards/rejected": -1.3400263786315918,
+      "step": 230
+    },
+    {
+      "epoch": 0.6555090655509066,
+      "grad_norm": 4.0093817710876465,
+      "learning_rate": 2.5393338750826796e-05,
+      "logits/chosen": 12.776809692382812,
+      "logits/rejected": 14.406654357910156,
+      "logps/chosen": -229.2129669189453,
+      "logps/rejected": -300.2899475097656,
+      "loss": 0.5466,
+      "rewards/accuracies": 0.7000000476837158,
+      "rewards/chosen": -0.5617952346801758,
+      "rewards/margins": 0.752461850643158,
+      "rewards/rejected": -1.314257264137268,
+      "step": 235
+    },
+    {
+      "epoch": 0.6694560669456067,
+      "grad_norm": 6.296603679656982,
+      "learning_rate": 2.5305144570503554e-05,
+      "logits/chosen": 12.071104049682617,
+      "logits/rejected": 14.155171394348145,
+      "logps/chosen": -226.9972381591797,
+      "logps/rejected": -299.86993408203125,
+      "loss": 0.5546,
+      "rewards/accuracies": 0.7166667580604553,
+      "rewards/chosen": -0.5309011936187744,
+      "rewards/margins": 0.8765324354171753,
+      "rewards/rejected": -1.4074336290359497,
+      "step": 240
+    },
+    {
+      "epoch": 0.6834030683403068,
+      "grad_norm": 6.713784694671631,
+      "learning_rate": 2.5215049837812413e-05,
+      "logits/chosen": 11.046672821044922,
+      "logits/rejected": 13.329177856445312,
+      "logps/chosen": -222.3082275390625,
+      "logps/rejected": -302.4698181152344,
+      "loss": 0.4944,
+      "rewards/accuracies": 0.7583333253860474,
+      "rewards/chosen": -0.6500851511955261,
+      "rewards/margins": 1.216412901878357,
+      "rewards/rejected": -1.8664979934692383,
+      "step": 245
+    },
+    {
+      "epoch": 0.697350069735007,
+      "grad_norm": 3.64483904838562,
+      "learning_rate": 2.5123075144813044e-05,
+      "logits/chosen": 9.95425796508789,
+      "logits/rejected": 12.83178997039795,
+      "logps/chosen": -245.83987426757812,
+      "logps/rejected": -347.21807861328125,
+      "loss": 0.4497,
+      "rewards/accuracies": 0.8083333969116211,
+      "rewards/chosen": -0.7453610897064209,
+      "rewards/margins": 1.5202277898788452,
+      "rewards/rejected": -2.2655892372131348,
+      "step": 250
+    },
+    {
+      "epoch": 0.7112970711297071,
+      "grad_norm": 6.323943138122559,
+      "learning_rate": 2.5029241513248992e-05,
+      "logits/chosen": 10.718851089477539,
+      "logits/rejected": 11.787945747375488,
+      "logps/chosen": -243.6724853515625,
+      "logps/rejected": -275.3341064453125,
+      "loss": 0.7227,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.8060539364814758,
+      "rewards/margins": 0.8095922470092773,
+      "rewards/rejected": -1.6156460046768188,
+      "step": 255
+    },
+    {
+      "epoch": 0.7252440725244073,
+      "grad_norm": 7.24354887008667,
+      "learning_rate": 2.4933570389742975e-05,
+      "logits/chosen": 10.517350196838379,
+      "logits/rejected": 12.290593147277832,
+      "logps/chosen": -217.3165740966797,
+      "logps/rejected": -277.9285888671875,
+      "loss": 0.5759,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -0.4403415620326996,
+      "rewards/margins": 0.91583251953125,
+      "rewards/rejected": -1.3561739921569824,
+      "step": 260
+    },
+    {
+      "epoch": 0.7391910739191074,
+      "grad_norm": 5.851767063140869,
+      "learning_rate": 2.4836083640895016e-05,
+      "logits/chosen": 11.360766410827637,
+      "logits/rejected": 13.0468111038208,
+      "logps/chosen": -253.7987518310547,
+      "logps/rejected": -321.97967529296875,
+      "loss": 0.552,
+      "rewards/accuracies": 0.73333340883255,
+      "rewards/chosen": -0.37671542167663574,
+      "rewards/margins": 1.0203909873962402,
+      "rewards/rejected": -1.3971065282821655,
+      "step": 265
+    },
+    {
+      "epoch": 0.7531380753138075,
+      "grad_norm": 5.513597011566162,
+      "learning_rate": 2.473680354828461e-05,
+      "logits/chosen": 10.999284744262695,
+      "logits/rejected": 12.660871505737305,
+      "logps/chosen": -218.94735717773438,
+      "logps/rejected": -259.43133544921875,
+      "loss": 0.5662,
+      "rewards/accuracies": 0.7083333730697632,
+      "rewards/chosen": -0.4751812517642975,
+      "rewards/margins": 0.7894707322120667,
+      "rewards/rejected": -1.264651894569397,
+      "step": 270
+    },
+    {
+      "epoch": 0.7670850767085077,
+      "grad_norm": 5.750617504119873,
+      "learning_rate": 2.4635752803378063e-05,
+      "logits/chosen": 11.961132049560547,
+      "logits/rejected": 14.314129829406738,
+      "logps/chosen": -221.6836700439453,
+      "logps/rejected": -307.97467041015625,
+      "loss": 0.5598,
+      "rewards/accuracies": 0.7166666984558105,
+      "rewards/chosen": -0.42271748185157776,
+      "rewards/margins": 0.7973464727401733,
+      "rewards/rejected": -1.2200638055801392,
+      "step": 275
+    },
+    {
+      "epoch": 0.7810320781032078,
+      "grad_norm": 6.243062496185303,
+      "learning_rate": 2.453295450234211e-05,
+      "logits/chosen": 11.414163589477539,
+      "logits/rejected": 13.276272773742676,
+      "logps/chosen": -223.52529907226562,
+      "logps/rejected": -324.9430236816406,
+      "loss": 0.516,
+      "rewards/accuracies": 0.7833333015441895,
+      "rewards/chosen": -0.5223814249038696,
+      "rewards/margins": 1.0250855684280396,
+      "rewards/rejected": -1.5474669933319092,
+      "step": 280
+    },
+    {
+      "epoch": 0.7949790794979079,
+      "grad_norm": 7.279259204864502,
+      "learning_rate": 2.442843214076507e-05,
+      "logits/chosen": 11.585288047790527,
+      "logits/rejected": 12.664071083068848,
+      "logps/chosen": -244.99154663085938,
+      "logps/rejected": -254.84158325195312,
+      "loss": 0.5448,
+      "rewards/accuracies": 0.6916667222976685,
+      "rewards/chosen": -0.7385099530220032,
+      "rewards/margins": 0.9241247177124023,
+      "rewards/rejected": -1.6626346111297607,
+      "step": 285
+    },
+    {
+      "epoch": 0.8089260808926081,
+      "grad_norm": 4.353116512298584,
+      "learning_rate": 2.4322209608286686e-05,
+      "logits/chosen": 8.986726760864258,
+      "logits/rejected": 12.175418853759766,
+      "logps/chosen": -206.93594360351562,
+      "logps/rejected": -292.4324645996094,
+      "loss": 0.5075,
+      "rewards/accuracies": 0.7166666984558105,
+      "rewards/chosen": -0.7221769094467163,
+      "rewards/margins": 1.2281521558761597,
+      "rewards/rejected": -1.9503291845321655,
+      "step": 290
+    },
+    {
+      "epoch": 0.8228730822873083,
+      "grad_norm": 8.33502197265625,
+      "learning_rate": 2.421431118313789e-05,
+      "logits/chosen": 10.475484848022461,
+      "logits/rejected": 11.844476699829102,
+      "logps/chosen": -240.02206420898438,
+      "logps/rejected": -279.66064453125,
+      "loss": 0.7693,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0663460493087769,
+      "rewards/margins": 0.940434455871582,
+      "rewards/rejected": -2.0067803859710693,
+      "step": 295
+    },
+    {
+      "epoch": 0.8368200836820083,
+      "grad_norm": 7.870750904083252,
+      "learning_rate": 2.41047615265918e-05,
+      "logits/chosen": 11.179079055786133,
+      "logits/rejected": 13.722526550292969,
+      "logps/chosen": -243.4619140625,
+      "logps/rejected": -338.6166687011719,
+      "loss": 0.5564,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -0.5845211744308472,
+      "rewards/margins": 1.252018690109253,
+      "rewards/rejected": -1.8365398645401,
+      "step": 300
+    },
+    {
+      "epoch": 0.8507670850767085,
+      "grad_norm": 5.815814018249512,
+      "learning_rate": 2.3993585677327107e-05,
+      "logits/chosen": 13.206197738647461,
+      "logits/rejected": 13.943652153015137,
+      "logps/chosen": -276.7415466308594,
+      "logps/rejected": -329.16949462890625,
+      "loss": 0.5341,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.4414283335208893,
+      "rewards/margins": 0.9674088358879089,
+      "rewards/rejected": -1.408837080001831,
+      "step": 305
+    },
+    {
+      "epoch": 0.8647140864714087,
+      "grad_norm": 8.953941345214844,
+      "learning_rate": 2.3880809045705262e-05,
+      "logits/chosen": 12.374226570129395,
+      "logits/rejected": 13.989924430847168,
+      "logps/chosen": -244.1103515625,
+      "logps/rejected": -294.4126281738281,
+      "loss": 0.5197,
+      "rewards/accuracies": 0.7416666746139526,
+      "rewards/chosen": -0.3111744523048401,
+      "rewards/margins": 1.076432228088379,
+      "rewards/rejected": -1.3876066207885742,
+      "step": 310
+    },
+    {
+      "epoch": 0.8786610878661087,
+      "grad_norm": 5.921610355377197,
+      "learning_rate": 2.3766457407962654e-05,
+      "logits/chosen": 11.312451362609863,
+      "logits/rejected": 13.527206420898438,
+      "logps/chosen": -234.7072296142578,
+      "logps/rejected": -290.8281555175781,
+      "loss": 0.6344,
+      "rewards/accuracies": 0.6583333611488342,
+      "rewards/chosen": -0.5945696234703064,
+      "rewards/margins": 0.7386828660964966,
+      "rewards/rejected": -1.3332524299621582,
+      "step": 315
+    },
+    {
+      "epoch": 0.8926080892608089,
+      "grad_norm": 7.662965774536133,
+      "learning_rate": 2.3650556900319204e-05,
+      "logits/chosen": 11.144214630126953,
+      "logits/rejected": 13.931811332702637,
+      "logps/chosen": -198.17422485351562,
+      "logps/rejected": -286.37152099609375,
+      "loss": 0.5295,
+      "rewards/accuracies": 0.7166666388511658,
+      "rewards/chosen": -0.3581700325012207,
+      "rewards/margins": 1.0556398630142212,
+      "rewards/rejected": -1.4138100147247314,
+      "step": 320
+    },
+    {
+      "epoch": 0.9065550906555091,
+      "grad_norm": 6.2693071365356445,
+      "learning_rate": 2.3533134013004666e-05,
+      "logits/chosen": 11.203069686889648,
+      "logits/rejected": 11.721672058105469,
+      "logps/chosen": -192.05380249023438,
+      "logps/rejected": -221.40042114257812,
+      "loss": 0.6338,
+      "rewards/accuracies": 0.7083333730697632,
+      "rewards/chosen": -0.56825852394104,
+      "rewards/margins": 0.6851181983947754,
+      "rewards/rejected": -1.253376841545105,
+      "step": 325
+    },
+    {
+      "epoch": 0.9205020920502092,
+      "grad_norm": 4.502702236175537,
+      "learning_rate": 2.341421558420403e-05,
+      "logits/chosen": 10.94641399383545,
+      "logits/rejected": 13.1921968460083,
+      "logps/chosen": -214.18911743164062,
+      "logps/rejected": -298.7375183105469,
+      "loss": 0.4812,
+      "rewards/accuracies": 0.7416666746139526,
+      "rewards/chosen": -0.6049180030822754,
+      "rewards/margins": 1.2558748722076416,
+      "rewards/rejected": -1.860793113708496,
+      "step": 330
+    },
+    {
+      "epoch": 0.9344490934449093,
+      "grad_norm": 6.939857482910156,
+      "learning_rate": 2.3293828793923365e-05,
+      "logits/chosen": 11.944158554077148,
+      "logits/rejected": 13.312121391296387,
+      "logps/chosen": -258.62774658203125,
+      "logps/rejected": -314.3907165527344,
+      "loss": 0.619,
+      "rewards/accuracies": 0.6583333015441895,
+      "rewards/chosen": -0.5666269063949585,
+      "rewards/margins": 1.0656505823135376,
+      "rewards/rejected": -1.632277488708496,
+      "step": 335
+    },
+    {
+      "epoch": 0.9483960948396095,
+      "grad_norm": 4.783708572387695,
+      "learning_rate": 2.3172001157777566e-05,
+      "logits/chosen": 11.259064674377441,
+      "logits/rejected": 12.700227737426758,
+      "logps/chosen": -222.77395629882812,
+      "logps/rejected": -291.24884033203125,
+      "loss": 0.6497,
+      "rewards/accuracies": 0.658333420753479,
+      "rewards/chosen": -0.5113255977630615,
+      "rewards/margins": 0.866672158241272,
+      "rewards/rejected": -1.377997636795044,
+      "step": 340
+    },
+    {
+      "epoch": 0.9623430962343096,
+      "grad_norm": 4.230240821838379,
+      "learning_rate": 2.3048760520701374e-05,
+      "logits/chosen": 11.741998672485352,
+      "logits/rejected": 13.120841979980469,
+      "logps/chosen": -246.1442108154297,
+      "logps/rejected": -311.65972900390625,
+      "loss": 0.485,
+      "rewards/accuracies": 0.7416666746139526,
+      "rewards/chosen": -0.4931299090385437,
+      "rewards/margins": 1.0040347576141357,
+      "rewards/rejected": -1.4971646070480347,
+      "step": 345
+    },
+    {
+      "epoch": 0.9762900976290098,
+      "grad_norm": 4.901912212371826,
+      "learning_rate": 2.2924135050585152e-05,
+      "logits/chosen": 11.310202598571777,
+      "logits/rejected": 13.040916442871094,
+      "logps/chosen": -223.5089111328125,
+      "logps/rejected": -253.26953125,
+      "loss": 0.5638,
+      "rewards/accuracies": 0.7416666746139526,
+      "rewards/chosen": -0.6479132771492004,
+      "rewards/margins": 0.7726107835769653,
+      "rewards/rejected": -1.4205242395401,
+      "step": 350
+    },
+    {
+      "epoch": 0.9902370990237099,
+      "grad_norm": 4.653822898864746,
+      "learning_rate": 2.2798153231836813e-05,
+      "logits/chosen": 12.201348304748535,
+      "logits/rejected": 13.601194381713867,
+      "logps/chosen": -268.86328125,
+      "logps/rejected": -299.11578369140625,
+      "loss": 0.5367,
+      "rewards/accuracies": 0.7416666746139526,
+      "rewards/chosen": -0.5867849588394165,
+      "rewards/margins": 1.1348934173583984,
+      "rewards/rejected": -1.721678376197815,
+      "step": 355
+    },
+    {
+      "epoch": 0.99860529986053,
+      "eval_logits/chosen": 11.91541862487793,
+      "eval_logits/rejected": 12.808130264282227,
+      "eval_logps/chosen": -229.4397430419922,
+      "eval_logps/rejected": -286.59234619140625,
+      "eval_loss": 0.6298339366912842,
+      "eval_rewards/accuracies": 0.7099999785423279,
+      "eval_rewards/chosen": -0.7923385500907898,
+      "eval_rewards/margins": 0.9493054151535034,
+      "eval_rewards/rejected": -1.7416436672210693,
+      "eval_runtime": 24.5973,
+      "eval_samples_per_second": 8.131,
+      "eval_steps_per_second": 8.131,
+      "step": 358
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1074,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 12,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ea11f63ac857a12c1f6cc99e62cd2c441d03650bbbe89deae1338518fdbfd2f
+size 6520