Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,202 @@
+---
+base_model: rica40325/10_14dpo
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.14.0

adapter_config.json

@@ -0,0 +1,37 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "rica40325/10_14dpo",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "gate_proj",
+    "k_proj",
+    "v_proj",
+    "down_proj",
+    "up_proj",
+    "o_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7160b2e9c92b8732b5556f6c0ea26dd8670ad48d0d10805038abe94fefd64f7d
+size 228140600

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41fc6469da62556a01f131a41f569b4a9ab18ac6cabf1d4d594e226890579265
+size 456603682

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1a97db8e41139aa1239ba7fb79ddeb0af5998c6305a440c1fe182e6ad02f2f5
+size 14244

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5fdef6dd9327359f5ee7231938d794ba5b9f33521375c374d6597f166da81dc
+size 1064

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0240ce510f08e6c2041724e9043e33be9d251d1e4a4d94eb68cd47b954b61d2
+size 17078292

trainer_state.json

@@ -0,0 +1,1361 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.518331226295828,
+  "eval_steps": 50,
+  "global_step": 400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0316055625790139,
+      "grad_norm": 10.339975357055664,
+      "learning_rate": 9.894514767932489e-07,
+      "logits/chosen": 0.7929186224937439,
+      "logits/rejected": -0.4776664972305298,
+      "logps/chosen": -153.9053192138672,
+      "logps/rejected": -99.09619903564453,
+      "loss": 0.692,
+      "rewards/accuracies": 0.4399999976158142,
+      "rewards/chosen": 0.006975799333304167,
+      "rewards/margins": 0.0027497103437781334,
+      "rewards/rejected": 0.004226089920848608,
+      "step": 5
+    },
+    {
+      "epoch": 0.0632111251580278,
+      "grad_norm": 9.067425727844238,
+      "learning_rate": 9.78902953586498e-07,
+      "logits/chosen": 0.8294331431388855,
+      "logits/rejected": -0.4984918236732483,
+      "logps/chosen": -156.86260986328125,
+      "logps/rejected": -110.65531921386719,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.5199999809265137,
+      "rewards/chosen": 0.0016689912881702185,
+      "rewards/margins": 0.006629813928157091,
+      "rewards/rejected": -0.004960823804140091,
+      "step": 10
+    },
+    {
+      "epoch": 0.09481668773704172,
+      "grad_norm": 10.66510009765625,
+      "learning_rate": 9.683544303797469e-07,
+      "logits/chosen": 0.8170953989028931,
+      "logits/rejected": -0.3735724687576294,
+      "logps/chosen": -166.33270263671875,
+      "logps/rejected": -108.52173614501953,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.5799999833106995,
+      "rewards/chosen": 0.00982923898845911,
+      "rewards/margins": 0.01228736899793148,
+      "rewards/rejected": -0.002458130242303014,
+      "step": 15
+    },
+    {
+      "epoch": 0.1264222503160556,
+      "grad_norm": 11.072169303894043,
+      "learning_rate": 9.578059071729958e-07,
+      "logits/chosen": 0.8109498620033264,
+      "logits/rejected": -0.5970823764801025,
+      "logps/chosen": -169.12843322753906,
+      "logps/rejected": -107.89179992675781,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.46000000834465027,
+      "rewards/chosen": 0.012243811972439289,
+      "rewards/margins": 0.002129364525899291,
+      "rewards/rejected": 0.010114450007677078,
+      "step": 20
+    },
+    {
+      "epoch": 0.15802781289506954,
+      "grad_norm": 10.236513137817383,
+      "learning_rate": 9.472573839662447e-07,
+      "logits/chosen": 0.7613813281059265,
+      "logits/rejected": -0.4820778965950012,
+      "logps/chosen": -166.7922821044922,
+      "logps/rejected": -111.9711685180664,
+      "loss": 0.6788,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.041573330760002136,
+      "rewards/margins": 0.02995917573571205,
+      "rewards/rejected": 0.01161415595561266,
+      "step": 25
+    },
+    {
+      "epoch": 0.18963337547408343,
+      "grad_norm": 11.859560012817383,
+      "learning_rate": 9.367088607594936e-07,
+      "logits/chosen": 0.7980347275733948,
+      "logits/rejected": -0.30686867237091064,
+      "logps/chosen": -159.4391632080078,
+      "logps/rejected": -102.94271850585938,
+      "loss": 0.6714,
+      "rewards/accuracies": 0.7400000095367432,
+      "rewards/chosen": 0.034902799874544144,
+      "rewards/margins": 0.04492241516709328,
+      "rewards/rejected": -0.010019614361226559,
+      "step": 30
+    },
+    {
+      "epoch": 0.22123893805309736,
+      "grad_norm": 11.4252347946167,
+      "learning_rate": 9.261603375527426e-07,
+      "logits/chosen": 0.8221642971038818,
+      "logits/rejected": -0.6532940864562988,
+      "logps/chosen": -157.35171508789062,
+      "logps/rejected": -103.97557830810547,
+      "loss": 0.6624,
+      "rewards/accuracies": 0.8199999928474426,
+      "rewards/chosen": 0.06025644764304161,
+      "rewards/margins": 0.06370121240615845,
+      "rewards/rejected": -0.003444763831794262,
+      "step": 35
+    },
+    {
+      "epoch": 0.2528445006321112,
+      "grad_norm": 11.550130844116211,
+      "learning_rate": 9.156118143459915e-07,
+      "logits/chosen": 0.8232347965240479,
+      "logits/rejected": -0.5209521055221558,
+      "logps/chosen": -161.43267822265625,
+      "logps/rejected": -110.23632049560547,
+      "loss": 0.6603,
+      "rewards/accuracies": 0.9200000166893005,
+      "rewards/chosen": 0.07129205018281937,
+      "rewards/margins": 0.06766117364168167,
+      "rewards/rejected": 0.00363088957965374,
+      "step": 40
+    },
+    {
+      "epoch": 0.28445006321112515,
+      "grad_norm": 9.030801773071289,
+      "learning_rate": 9.050632911392405e-07,
+      "logits/chosen": 0.7879684567451477,
+      "logits/rejected": -0.5709326267242432,
+      "logps/chosen": -155.87847900390625,
+      "logps/rejected": -103.713134765625,
+      "loss": 0.6603,
+      "rewards/accuracies": 0.8399999737739563,
+      "rewards/chosen": 0.06003337725996971,
+      "rewards/margins": 0.06810689717531204,
+      "rewards/rejected": -0.008073524571955204,
+      "step": 45
+    },
+    {
+      "epoch": 0.31605562579013907,
+      "grad_norm": 9.76406192779541,
+      "learning_rate": 8.945147679324893e-07,
+      "logits/chosen": 0.7950836420059204,
+      "logits/rejected": -0.42198893427848816,
+      "logps/chosen": -148.63394165039062,
+      "logps/rejected": -101.21693420410156,
+      "loss": 0.6437,
+      "rewards/accuracies": 0.9399999976158142,
+      "rewards/chosen": 0.07873085141181946,
+      "rewards/margins": 0.10321204364299774,
+      "rewards/rejected": -0.024481192231178284,
+      "step": 50
+    },
+    {
+      "epoch": 0.31605562579013907,
+      "eval_logits/chosen": 0.7988272309303284,
+      "eval_logits/rejected": -0.5165692567825317,
+      "eval_logps/chosen": -157.75689697265625,
+      "eval_logps/rejected": -109.79812622070312,
+      "eval_loss": 0.6324340105056763,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.10482321679592133,
+      "eval_rewards/margins": 0.1264423131942749,
+      "eval_rewards/rejected": -0.021619105711579323,
+      "eval_runtime": 5.7029,
+      "eval_samples_per_second": 2.806,
+      "eval_steps_per_second": 2.806,
+      "step": 50
+    },
+    {
+      "epoch": 0.347661188369153,
+      "grad_norm": 8.975409507751465,
+      "learning_rate": 8.839662447257383e-07,
+      "logits/chosen": 0.8465293645858765,
+      "logits/rejected": -0.3923209011554718,
+      "logps/chosen": -161.0301513671875,
+      "logps/rejected": -110.12336730957031,
+      "loss": 0.6253,
+      "rewards/accuracies": 0.9599999785423279,
+      "rewards/chosen": 0.11860670894384384,
+      "rewards/margins": 0.14395156502723694,
+      "rewards/rejected": -0.025344856083393097,
+      "step": 55
+    },
+    {
+      "epoch": 0.37926675094816686,
+      "grad_norm": 11.271946907043457,
+      "learning_rate": 8.734177215189873e-07,
+      "logits/chosen": 0.8341310620307922,
+      "logits/rejected": -0.45204219222068787,
+      "logps/chosen": -166.23269653320312,
+      "logps/rejected": -109.92120361328125,
+      "loss": 0.6151,
+      "rewards/accuracies": 0.9599999785423279,
+      "rewards/chosen": 0.14352449774742126,
+      "rewards/margins": 0.16602401435375214,
+      "rewards/rejected": -0.022499512881040573,
+      "step": 60
+    },
+    {
+      "epoch": 0.4108723135271808,
+      "grad_norm": 9.486306190490723,
+      "learning_rate": 8.628691983122362e-07,
+      "logits/chosen": 0.7777873873710632,
+      "logits/rejected": -0.48896849155426025,
+      "logps/chosen": -161.54290771484375,
+      "logps/rejected": -109.1745834350586,
+      "loss": 0.6136,
+      "rewards/accuracies": 0.9800000190734863,
+      "rewards/chosen": 0.14903154969215393,
+      "rewards/margins": 0.1686798632144928,
+      "rewards/rejected": -0.019648319110274315,
+      "step": 65
+    },
+    {
+      "epoch": 0.4424778761061947,
+      "grad_norm": 8.906453132629395,
+      "learning_rate": 8.523206751054853e-07,
+      "logits/chosen": 0.763322114944458,
+      "logits/rejected": -0.6100043654441833,
+      "logps/chosen": -157.4031219482422,
+      "logps/rejected": -102.33517456054688,
+      "loss": 0.6055,
+      "rewards/accuracies": 0.9800000190734863,
+      "rewards/chosen": 0.15665321052074432,
+      "rewards/margins": 0.18765796720981598,
+      "rewards/rejected": -0.03100474365055561,
+      "step": 70
+    },
+    {
+      "epoch": 0.4740834386852086,
+      "grad_norm": 8.623327255249023,
+      "learning_rate": 8.417721518987342e-07,
+      "logits/chosen": 0.7523571848869324,
+      "logits/rejected": -0.4898630976676941,
+      "logps/chosen": -156.9187469482422,
+      "logps/rejected": -106.943115234375,
+      "loss": 0.5987,
+      "rewards/accuracies": 0.9800000190734863,
+      "rewards/chosen": 0.16381792724132538,
+      "rewards/margins": 0.2020380049943924,
+      "rewards/rejected": -0.038220059126615524,
+      "step": 75
+    },
+    {
+      "epoch": 0.5056890012642224,
+      "grad_norm": 23.04412269592285,
+      "learning_rate": 8.312236286919831e-07,
+      "logits/chosen": 0.7943715453147888,
+      "logits/rejected": -0.5809293389320374,
+      "logps/chosen": -157.24765014648438,
+      "logps/rejected": -108.35530090332031,
+      "loss": 0.5752,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.204318568110466,
+      "rewards/margins": 0.25807440280914307,
+      "rewards/rejected": -0.053755830973386765,
+      "step": 80
+    },
+    {
+      "epoch": 0.5372945638432364,
+      "grad_norm": 8.798491477966309,
+      "learning_rate": 8.20675105485232e-07,
+      "logits/chosen": 0.84209144115448,
+      "logits/rejected": -0.4210968315601349,
+      "logps/chosen": -155.2298126220703,
+      "logps/rejected": -101.53964233398438,
+      "loss": 0.5571,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.245680034160614,
+      "rewards/margins": 0.30093926191329956,
+      "rewards/rejected": -0.05525921657681465,
+      "step": 85
+    },
+    {
+      "epoch": 0.5689001264222503,
+      "grad_norm": 10.881470680236816,
+      "learning_rate": 8.10126582278481e-07,
+      "logits/chosen": 0.8002663254737854,
+      "logits/rejected": -0.41633889079093933,
+      "logps/chosen": -145.74266052246094,
+      "logps/rejected": -101.3503189086914,
+      "loss": 0.5688,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.21307054162025452,
+      "rewards/margins": 0.2720237374305725,
+      "rewards/rejected": -0.05895319581031799,
+      "step": 90
+    },
+    {
+      "epoch": 0.6005056890012642,
+      "grad_norm": 8.707910537719727,
+      "learning_rate": 7.995780590717299e-07,
+      "logits/chosen": 0.7864500284194946,
+      "logits/rejected": -0.7528146505355835,
+      "logps/chosen": -158.0437469482422,
+      "logps/rejected": -109.78366088867188,
+      "loss": 0.5043,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3314324915409088,
+      "rewards/margins": 0.4454871416091919,
+      "rewards/rejected": -0.11405465006828308,
+      "step": 95
+    },
+    {
+      "epoch": 0.6321112515802781,
+      "grad_norm": 9.02295970916748,
+      "learning_rate": 7.890295358649789e-07,
+      "logits/chosen": 0.8295723795890808,
+      "logits/rejected": -0.597999632358551,
+      "logps/chosen": -158.6008758544922,
+      "logps/rejected": -112.83537292480469,
+      "loss": 0.5112,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.31477904319763184,
+      "rewards/margins": 0.4191209673881531,
+      "rewards/rejected": -0.10434195399284363,
+      "step": 100
+    },
+    {
+      "epoch": 0.6321112515802781,
+      "eval_logits/chosen": 0.8237414360046387,
+      "eval_logits/rejected": -0.5530629754066467,
+      "eval_logps/chosen": -155.5559844970703,
+      "eval_logps/rejected": -111.12935638427734,
+      "eval_loss": 0.4854588806629181,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.3249143064022064,
+      "eval_rewards/margins": 0.47965648770332336,
+      "eval_rewards/rejected": -0.15474216639995575,
+      "eval_runtime": 5.7006,
+      "eval_samples_per_second": 2.807,
+      "eval_steps_per_second": 2.807,
+      "step": 100
+    },
+    {
+      "epoch": 0.6637168141592921,
+      "grad_norm": 9.607686996459961,
+      "learning_rate": 7.784810126582278e-07,
+      "logits/chosen": 0.8056095838546753,
+      "logits/rejected": -0.5340779423713684,
+      "logps/chosen": -158.3437042236328,
+      "logps/rejected": -107.14030456542969,
+      "loss": 0.5056,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.348066508769989,
+      "rewards/margins": 0.43299809098243713,
+      "rewards/rejected": -0.08493158221244812,
+      "step": 105
+    },
+    {
+      "epoch": 0.695322376738306,
+      "grad_norm": 7.763835430145264,
+      "learning_rate": 7.679324894514767e-07,
+      "logits/chosen": 0.7881090044975281,
+      "logits/rejected": -0.5530937314033508,
+      "logps/chosen": -156.03094482421875,
+      "logps/rejected": -108.07464599609375,
+      "loss": 0.4694,
+      "rewards/accuracies": 0.9800000190734863,
+      "rewards/chosen": 0.3952154815196991,
+      "rewards/margins": 0.533521831035614,
+      "rewards/rejected": -0.13830631971359253,
+      "step": 110
+    },
+    {
+      "epoch": 0.7269279393173198,
+      "grad_norm": 7.892477035522461,
+      "learning_rate": 7.573839662447257e-07,
+      "logits/chosen": 0.7714009284973145,
+      "logits/rejected": -0.5417795777320862,
+      "logps/chosen": -161.7781219482422,
+      "logps/rejected": -109.03459167480469,
+      "loss": 0.4588,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4125593900680542,
+      "rewards/margins": 0.567659318447113,
+      "rewards/rejected": -0.15509992837905884,
+      "step": 115
+    },
+    {
+      "epoch": 0.7585335018963337,
+      "grad_norm": 6.997534275054932,
+      "learning_rate": 7.468354430379746e-07,
+      "logits/chosen": 0.9488077759742737,
+      "logits/rejected": -0.46034353971481323,
+      "logps/chosen": -163.64418029785156,
+      "logps/rejected": -107.16482543945312,
+      "loss": 0.439,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4228503406047821,
+      "rewards/margins": 0.6315315365791321,
+      "rewards/rejected": -0.2086811512708664,
+      "step": 120
+    },
+    {
+      "epoch": 0.7901390644753477,
+      "grad_norm": 7.597814559936523,
+      "learning_rate": 7.362869198312236e-07,
+      "logits/chosen": 0.885678768157959,
+      "logits/rejected": -0.5294114947319031,
+      "logps/chosen": -154.77304077148438,
+      "logps/rejected": -106.53233337402344,
+      "loss": 0.43,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4804726839065552,
+      "rewards/margins": 0.6722790002822876,
+      "rewards/rejected": -0.1918063759803772,
+      "step": 125
+    },
+    {
+      "epoch": 0.8217446270543616,
+      "grad_norm": 7.842750549316406,
+      "learning_rate": 7.257383966244725e-07,
+      "logits/chosen": 0.8075446486473083,
+      "logits/rejected": -0.5433291792869568,
+      "logps/chosen": -158.94253540039062,
+      "logps/rejected": -108.86349487304688,
+      "loss": 0.4276,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.46029525995254517,
+      "rewards/margins": 0.6777289509773254,
+      "rewards/rejected": -0.21743372082710266,
+      "step": 130
+    },
+    {
+      "epoch": 0.8533501896333755,
+      "grad_norm": 6.6370062828063965,
+      "learning_rate": 7.151898734177216e-07,
+      "logits/chosen": 0.9542413353919983,
+      "logits/rejected": -0.47683554887771606,
+      "logps/chosen": -155.0622100830078,
+      "logps/rejected": -108.30109405517578,
+      "loss": 0.4059,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5105652809143066,
+      "rewards/margins": 0.7540386915206909,
+      "rewards/rejected": -0.2434733808040619,
+      "step": 135
+    },
+    {
+      "epoch": 0.8849557522123894,
+      "grad_norm": 7.874971866607666,
+      "learning_rate": 7.046413502109705e-07,
+      "logits/chosen": 0.8691486120223999,
+      "logits/rejected": -0.688520073890686,
+      "logps/chosen": -162.5826416015625,
+      "logps/rejected": -112.48107147216797,
+      "loss": 0.4022,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5319894552230835,
+      "rewards/margins": 0.7736017107963562,
+      "rewards/rejected": -0.2416123002767563,
+      "step": 140
+    },
+    {
+      "epoch": 0.9165613147914032,
+      "grad_norm": 6.782280445098877,
+      "learning_rate": 6.940928270042194e-07,
+      "logits/chosen": 0.9024925231933594,
+      "logits/rejected": -0.48865240812301636,
+      "logps/chosen": -157.64137268066406,
+      "logps/rejected": -109.9500503540039,
+      "loss": 0.3857,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5103211402893066,
+      "rewards/margins": 0.7939779162406921,
+      "rewards/rejected": -0.2836567759513855,
+      "step": 145
+    },
+    {
+      "epoch": 0.9481668773704172,
+      "grad_norm": 6.642027378082275,
+      "learning_rate": 6.835443037974683e-07,
+      "logits/chosen": 0.8508256673812866,
+      "logits/rejected": -0.6600081920623779,
+      "logps/chosen": -158.06771850585938,
+      "logps/rejected": -115.92665100097656,
+      "loss": 0.3391,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6296125650405884,
+      "rewards/margins": 0.9785265922546387,
+      "rewards/rejected": -0.34891390800476074,
+      "step": 150
+    },
+    {
+      "epoch": 0.9481668773704172,
+      "eval_logits/chosen": 0.8691748976707458,
+      "eval_logits/rejected": -0.6211389303207397,
+      "eval_logps/chosen": -152.68296813964844,
+      "eval_logps/rejected": -114.10419464111328,
+      "eval_loss": 0.3081452548503876,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.6122143864631653,
+      "eval_rewards/margins": 1.064440131187439,
+      "eval_rewards/rejected": -0.45222577452659607,
+      "eval_runtime": 5.7326,
+      "eval_samples_per_second": 2.791,
+      "eval_steps_per_second": 2.791,
+      "step": 150
+    },
+    {
+      "epoch": 0.9797724399494311,
+      "grad_norm": 5.967626094818115,
+      "learning_rate": 6.729957805907173e-07,
+      "logits/chosen": 0.9301344156265259,
+      "logits/rejected": -0.5499186515808105,
+      "logps/chosen": -150.24380493164062,
+      "logps/rejected": -104.84210205078125,
+      "loss": 0.349,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5856583118438721,
+      "rewards/margins": 0.9493656158447266,
+      "rewards/rejected": -0.3637073040008545,
+      "step": 155
+    },
+    {
+      "epoch": 1.0063211125158027,
+      "grad_norm": 10.495283126831055,
+      "learning_rate": 6.624472573839662e-07,
+      "logits/chosen": 0.9606431126594543,
+      "logits/rejected": -0.5025166273117065,
+      "logps/chosen": -163.14552307128906,
+      "logps/rejected": -116.9787368774414,
+      "loss": 0.285,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6027621030807495,
+      "rewards/margins": 0.9733738899230957,
+      "rewards/rejected": -0.3706117868423462,
+      "step": 160
+    },
+    {
+      "epoch": 1.0379266750948166,
+      "grad_norm": 6.406510353088379,
+      "learning_rate": 6.518987341772152e-07,
+      "logits/chosen": 0.9654910564422607,
+      "logits/rejected": -0.742189347743988,
+      "logps/chosen": -146.29498291015625,
+      "logps/rejected": -105.1583480834961,
+      "loss": 0.3248,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5812354683876038,
+      "rewards/margins": 1.044042944908142,
+      "rewards/rejected": -0.46280738711357117,
+      "step": 165
+    },
+    {
+      "epoch": 1.0695322376738305,
+      "grad_norm": 5.3434929847717285,
+      "learning_rate": 6.413502109704642e-07,
+      "logits/chosen": 0.9043405652046204,
+      "logits/rejected": -0.5751953721046448,
+      "logps/chosen": -153.95059204101562,
+      "logps/rejected": -112.80074310302734,
+      "loss": 0.3061,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5781198143959045,
+      "rewards/margins": 1.0888361930847168,
+      "rewards/rejected": -0.5107164978981018,
+      "step": 170
+    },
+    {
+      "epoch": 1.1011378002528445,
+      "grad_norm": 5.284570693969727,
+      "learning_rate": 6.30801687763713e-07,
+      "logits/chosen": 0.7719966769218445,
+      "logits/rejected": -0.6168527603149414,
+      "logps/chosen": -145.92726135253906,
+      "logps/rejected": -107.75357055664062,
+      "loss": 0.3065,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.595247745513916,
+      "rewards/margins": 1.1324158906936646,
+      "rewards/rejected": -0.5371681451797485,
+      "step": 175
+    },
+    {
+      "epoch": 1.1327433628318584,
+      "grad_norm": 5.735872268676758,
+      "learning_rate": 6.20253164556962e-07,
+      "logits/chosen": 0.834024965763092,
+      "logits/rejected": -0.7654290199279785,
+      "logps/chosen": -153.1019287109375,
+      "logps/rejected": -114.1486587524414,
+      "loss": 0.2771,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6445808410644531,
+      "rewards/margins": 1.2771058082580566,
+      "rewards/rejected": -0.6325249671936035,
+      "step": 180
+    },
+    {
+      "epoch": 1.1643489254108723,
+      "grad_norm": 4.696133613586426,
+      "learning_rate": 6.097046413502109e-07,
+      "logits/chosen": 0.9559400081634521,
+      "logits/rejected": -0.5612914562225342,
+      "logps/chosen": -155.5025634765625,
+      "logps/rejected": -114.08667755126953,
+      "loss": 0.2495,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6708691120147705,
+      "rewards/margins": 1.368281602859497,
+      "rewards/rejected": -0.6974124312400818,
+      "step": 185
+    },
+    {
+      "epoch": 1.1959544879898862,
+      "grad_norm": 6.697731971740723,
+      "learning_rate": 5.991561181434599e-07,
+      "logits/chosen": 0.8690047264099121,
+      "logits/rejected": -0.7378618121147156,
+      "logps/chosen": -158.60435485839844,
+      "logps/rejected": -120.35429382324219,
+      "loss": 0.2348,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6325040459632874,
+      "rewards/margins": 1.4803410768508911,
+      "rewards/rejected": -0.847836971282959,
+      "step": 190
+    },
+    {
+      "epoch": 1.2275600505689002,
+      "grad_norm": 6.123443603515625,
+      "learning_rate": 5.886075949367088e-07,
+      "logits/chosen": 0.8898682594299316,
+      "logits/rejected": -0.7053573727607727,
+      "logps/chosen": -156.8765106201172,
+      "logps/rejected": -114.8968276977539,
+      "loss": 0.2338,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6635453104972839,
+      "rewards/margins": 1.5678775310516357,
+      "rewards/rejected": -0.9043322205543518,
+      "step": 195
+    },
+    {
+      "epoch": 1.259165613147914,
+      "grad_norm": 4.868192195892334,
+      "learning_rate": 5.780590717299578e-07,
+      "logits/chosen": 0.8993723392486572,
+      "logits/rejected": -0.7870872616767883,
+      "logps/chosen": -153.87742614746094,
+      "logps/rejected": -117.11913299560547,
+      "loss": 0.2397,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5956121683120728,
+      "rewards/margins": 1.497450590133667,
+      "rewards/rejected": -0.9018382430076599,
+      "step": 200
+    },
+    {
+      "epoch": 1.259165613147914,
+      "eval_logits/chosen": 0.9185884594917297,
+      "eval_logits/rejected": -0.7732617855072021,
+      "eval_logps/chosen": -152.10044860839844,
+      "eval_logps/rejected": -122.02890014648438,
+      "eval_loss": 0.1629265695810318,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.6704683303833008,
+      "eval_rewards/margins": 1.9151642322540283,
+      "eval_rewards/rejected": -1.2446959018707275,
+      "eval_runtime": 5.9347,
+      "eval_samples_per_second": 2.696,
+      "eval_steps_per_second": 2.696,
+      "step": 200
+    },
+    {
+      "epoch": 1.290771175726928,
+      "grad_norm": 4.5561747550964355,
+      "learning_rate": 5.675105485232066e-07,
+      "logits/chosen": 0.9104704856872559,
+      "logits/rejected": -0.6324939131736755,
+      "logps/chosen": -165.21856689453125,
+      "logps/rejected": -120.56816101074219,
+      "loss": 0.2374,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.615541398525238,
+      "rewards/margins": 1.4608503580093384,
+      "rewards/rejected": -0.8453088998794556,
+      "step": 205
+    },
+    {
+      "epoch": 1.322376738305942,
+      "grad_norm": 5.0898895263671875,
+      "learning_rate": 5.569620253164557e-07,
+      "logits/chosen": 0.8807806968688965,
+      "logits/rejected": -0.8468418717384338,
+      "logps/chosen": -150.10260009765625,
+      "logps/rejected": -116.68962097167969,
+      "loss": 0.2067,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6738828420639038,
+      "rewards/margins": 1.7353729009628296,
+      "rewards/rejected": -1.0614900588989258,
+      "step": 210
+    },
+    {
+      "epoch": 1.3539823008849559,
+      "grad_norm": 3.205390214920044,
+      "learning_rate": 5.464135021097046e-07,
+      "logits/chosen": 0.8711918592453003,
+      "logits/rejected": -0.7964326739311218,
+      "logps/chosen": -147.24038696289062,
+      "logps/rejected": -110.6238784790039,
+      "loss": 0.2135,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6835477352142334,
+      "rewards/margins": 1.7037204504013062,
+      "rewards/rejected": -1.0201725959777832,
+      "step": 215
+    },
+    {
+      "epoch": 1.3855878634639698,
+      "grad_norm": 4.114279270172119,
+      "learning_rate": 5.358649789029536e-07,
+      "logits/chosen": 0.9505433440208435,
+      "logits/rejected": -0.8043610453605652,
+      "logps/chosen": -157.4142303466797,
+      "logps/rejected": -120.68592071533203,
+      "loss": 0.1414,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7591128349304199,
+      "rewards/margins": 2.2252755165100098,
+      "rewards/rejected": -1.4661625623703003,
+      "step": 220
+    },
+    {
+      "epoch": 1.4171934260429835,
+      "grad_norm": 4.277466297149658,
+      "learning_rate": 5.253164556962026e-07,
+      "logits/chosen": 1.0081183910369873,
+      "logits/rejected": -0.6150592565536499,
+      "logps/chosen": -160.62301635742188,
+      "logps/rejected": -117.5049057006836,
+      "loss": 0.1618,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.679344117641449,
+      "rewards/margins": 2.0494327545166016,
+      "rewards/rejected": -1.3700886964797974,
+      "step": 225
+    },
+    {
+      "epoch": 1.4487989886219974,
+      "grad_norm": 3.798518180847168,
+      "learning_rate": 5.147679324894515e-07,
+      "logits/chosen": 0.9845036864280701,
+      "logits/rejected": -0.7662844657897949,
+      "logps/chosen": -152.2828369140625,
+      "logps/rejected": -121.86825561523438,
+      "loss": 0.1678,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6634464263916016,
+      "rewards/margins": 2.1404311656951904,
+      "rewards/rejected": -1.4769845008850098,
+      "step": 230
+    },
+    {
+      "epoch": 1.4804045512010113,
+      "grad_norm": 3.4062438011169434,
+      "learning_rate": 5.042194092827004e-07,
+      "logits/chosen": 0.8833762407302856,
+      "logits/rejected": -0.7040042877197266,
+      "logps/chosen": -156.48309326171875,
+      "logps/rejected": -123.51959228515625,
+      "loss": 0.1493,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.7038917541503906,
+      "rewards/margins": 2.063302516937256,
+      "rewards/rejected": -1.3594110012054443,
+      "step": 235
+    },
+    {
+      "epoch": 1.5120101137800253,
+      "grad_norm": 3.338238477706909,
+      "learning_rate": 4.936708860759494e-07,
+      "logits/chosen": 1.024337649345398,
+      "logits/rejected": -0.9010278582572937,
+      "logps/chosen": -157.03439331054688,
+      "logps/rejected": -124.58073425292969,
+      "loss": 0.1426,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6442054510116577,
+      "rewards/margins": 2.2925338745117188,
+      "rewards/rejected": -1.6483283042907715,
+      "step": 240
+    },
+    {
+      "epoch": 1.5436156763590392,
+      "grad_norm": 4.070380210876465,
+      "learning_rate": 4.831223628691983e-07,
+      "logits/chosen": 0.9164025783538818,
+      "logits/rejected": -0.7858352661132812,
+      "logps/chosen": -151.3623504638672,
+      "logps/rejected": -122.046630859375,
+      "loss": 0.1562,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6087484955787659,
+      "rewards/margins": 2.3244431018829346,
+      "rewards/rejected": -1.715694785118103,
+      "step": 245
+    },
+    {
+      "epoch": 1.575221238938053,
+      "grad_norm": 3.3322041034698486,
+      "learning_rate": 4.725738396624472e-07,
+      "logits/chosen": 0.9566361308097839,
+      "logits/rejected": -0.6725453734397888,
+      "logps/chosen": -159.6315155029297,
+      "logps/rejected": -128.08526611328125,
+      "loss": 0.133,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6768271923065186,
+      "rewards/margins": 2.3868112564086914,
+      "rewards/rejected": -1.7099841833114624,
+      "step": 250
+    },
+    {
+      "epoch": 1.575221238938053,
+      "eval_logits/chosen": 0.9665680527687073,
+      "eval_logits/rejected": -0.9290347099304199,
+      "eval_logps/chosen": -152.50790405273438,
+      "eval_logps/rejected": -133.14015197753906,
+      "eval_loss": 0.07605654001235962,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.6297206282615662,
+      "eval_rewards/margins": 2.9855422973632812,
+      "eval_rewards/rejected": -2.355821371078491,
+      "eval_runtime": 5.734,
+      "eval_samples_per_second": 2.79,
+      "eval_steps_per_second": 2.79,
+      "step": 250
+    },
+    {
+      "epoch": 1.606826801517067,
+      "grad_norm": 1.9533931016921997,
+      "learning_rate": 4.620253164556962e-07,
+      "logits/chosen": 1.0786921977996826,
+      "logits/rejected": -0.7568433880805969,
+      "logps/chosen": -158.71527099609375,
+      "logps/rejected": -122.67300415039062,
+      "loss": 0.1302,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5455527305603027,
+      "rewards/margins": 2.3328702449798584,
+      "rewards/rejected": -1.7873177528381348,
+      "step": 255
+    },
+    {
+      "epoch": 1.638432364096081,
+      "grad_norm": 2.984175682067871,
+      "learning_rate": 4.5147679324894513e-07,
+      "logits/chosen": 0.9486113786697388,
+      "logits/rejected": -0.9164045453071594,
+      "logps/chosen": -153.20921325683594,
+      "logps/rejected": -123.53470611572266,
+      "loss": 0.1223,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6579499840736389,
+      "rewards/margins": 2.4546985626220703,
+      "rewards/rejected": -1.796748399734497,
+      "step": 260
+    },
+    {
+      "epoch": 1.6700379266750947,
+      "grad_norm": 3.6064512729644775,
+      "learning_rate": 4.409282700421941e-07,
+      "logits/chosen": 0.9844273328781128,
+      "logits/rejected": -0.8227636218070984,
+      "logps/chosen": -151.73770141601562,
+      "logps/rejected": -128.2213897705078,
+      "loss": 0.1197,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.6511099934577942,
+      "rewards/margins": 2.535490036010742,
+      "rewards/rejected": -1.8843797445297241,
+      "step": 265
+    },
+    {
+      "epoch": 1.7016434892541086,
+      "grad_norm": 3.0833349227905273,
+      "learning_rate": 4.3037974683544305e-07,
+      "logits/chosen": 1.0600930452346802,
+      "logits/rejected": -0.8472379446029663,
+      "logps/chosen": -155.49119567871094,
+      "logps/rejected": -125.29644775390625,
+      "loss": 0.1114,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5597690343856812,
+      "rewards/margins": 2.7540347576141357,
+      "rewards/rejected": -2.194265842437744,
+      "step": 270
+    },
+    {
+      "epoch": 1.7332490518331225,
+      "grad_norm": 2.314880132675171,
+      "learning_rate": 4.1983122362869195e-07,
+      "logits/chosen": 0.990541398525238,
+      "logits/rejected": -0.8019971251487732,
+      "logps/chosen": -155.95314025878906,
+      "logps/rejected": -126.25853729248047,
+      "loss": 0.1061,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5866345167160034,
+      "rewards/margins": 2.7441272735595703,
+      "rewards/rejected": -2.1574926376342773,
+      "step": 275
+    },
+    {
+      "epoch": 1.7648546144121364,
+      "grad_norm": 3.4949941635131836,
+      "learning_rate": 4.092827004219409e-07,
+      "logits/chosen": 1.0804252624511719,
+      "logits/rejected": -1.0062919855117798,
+      "logps/chosen": -154.00534057617188,
+      "logps/rejected": -132.72201538085938,
+      "loss": 0.1079,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5344264507293701,
+      "rewards/margins": 2.9083995819091797,
+      "rewards/rejected": -2.3739731311798096,
+      "step": 280
+    },
+    {
+      "epoch": 1.7964601769911503,
+      "grad_norm": 3.3386101722717285,
+      "learning_rate": 3.9873417721518987e-07,
+      "logits/chosen": 0.9868491888046265,
+      "logits/rejected": -0.9694197177886963,
+      "logps/chosen": -152.30015563964844,
+      "logps/rejected": -127.70690155029297,
+      "loss": 0.1056,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5393376350402832,
+      "rewards/margins": 2.850891351699829,
+      "rewards/rejected": -2.311553716659546,
+      "step": 285
+    },
+    {
+      "epoch": 1.8280657395701643,
+      "grad_norm": 2.1566314697265625,
+      "learning_rate": 3.8818565400843877e-07,
+      "logits/chosen": 0.960561990737915,
+      "logits/rejected": -0.8585584163665771,
+      "logps/chosen": -157.81039428710938,
+      "logps/rejected": -128.45408630371094,
+      "loss": 0.1201,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5334513187408447,
+      "rewards/margins": 2.7658531665802,
+      "rewards/rejected": -2.2324020862579346,
+      "step": 290
+    },
+    {
+      "epoch": 1.8596713021491782,
+      "grad_norm": 2.643030881881714,
+      "learning_rate": 3.7763713080168773e-07,
+      "logits/chosen": 0.9850192070007324,
+      "logits/rejected": -0.9574072360992432,
+      "logps/chosen": -160.3083038330078,
+      "logps/rejected": -131.57269287109375,
+      "loss": 0.0927,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5686212778091431,
+      "rewards/margins": 2.922128915786743,
+      "rewards/rejected": -2.3535072803497314,
+      "step": 295
+    },
+    {
+      "epoch": 1.8912768647281921,
+      "grad_norm": 2.3175697326660156,
+      "learning_rate": 3.670886075949367e-07,
+      "logits/chosen": 0.9753783345222473,
+      "logits/rejected": -0.7726050615310669,
+      "logps/chosen": -157.6334991455078,
+      "logps/rejected": -132.01675415039062,
+      "loss": 0.1053,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5510329604148865,
+      "rewards/margins": 3.0005292892456055,
+      "rewards/rejected": -2.449496030807495,
+      "step": 300
+    },
+    {
+      "epoch": 1.8912768647281921,
+      "eval_logits/chosen": 1.009190320968628,
+      "eval_logits/rejected": -1.052826166152954,
+      "eval_logps/chosen": -154.10565185546875,
+      "eval_logps/rejected": -146.7685089111328,
+      "eval_loss": 0.035761214792728424,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.4699467420578003,
+      "eval_rewards/margins": 4.18860387802124,
+      "eval_rewards/rejected": -3.7186567783355713,
+      "eval_runtime": 5.7354,
+      "eval_samples_per_second": 2.79,
+      "eval_steps_per_second": 2.79,
+      "step": 300
+    },
+    {
+      "epoch": 1.922882427307206,
+      "grad_norm": 3.111605405807495,
+      "learning_rate": 3.565400843881856e-07,
+      "logits/chosen": 0.9777363538742065,
+      "logits/rejected": -0.9437763690948486,
+      "logps/chosen": -158.83944702148438,
+      "logps/rejected": -135.89527893066406,
+      "loss": 0.0901,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.48101624846458435,
+      "rewards/margins": 3.0204601287841797,
+      "rewards/rejected": -2.5394439697265625,
+      "step": 305
+    },
+    {
+      "epoch": 1.95448798988622,
+      "grad_norm": 2.0300815105438232,
+      "learning_rate": 3.4599156118143455e-07,
+      "logits/chosen": 1.0862417221069336,
+      "logits/rejected": -0.9694156050682068,
+      "logps/chosen": -158.91456604003906,
+      "logps/rejected": -137.14682006835938,
+      "loss": 0.077,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.42779281735420227,
+      "rewards/margins": 3.3323452472686768,
+      "rewards/rejected": -2.9045522212982178,
+      "step": 310
+    },
+    {
+      "epoch": 1.986093552465234,
+      "grad_norm": 2.512455701828003,
+      "learning_rate": 3.3544303797468356e-07,
+      "logits/chosen": 0.9629431366920471,
+      "logits/rejected": -0.8894481062889099,
+      "logps/chosen": -148.05079650878906,
+      "logps/rejected": -129.8061981201172,
+      "loss": 0.0913,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3557136654853821,
+      "rewards/margins": 3.1383042335510254,
+      "rewards/rejected": -2.782590389251709,
+      "step": 315
+    },
+    {
+      "epoch": 2.0126422250316054,
+      "grad_norm": 2.8620548248291016,
+      "learning_rate": 3.2489451476793247e-07,
+      "logits/chosen": 0.8875166773796082,
+      "logits/rejected": -0.9138175249099731,
+      "logps/chosen": -163.05441284179688,
+      "logps/rejected": -144.6278533935547,
+      "loss": 0.0514,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5428631901741028,
+      "rewards/margins": 3.6013400554656982,
+      "rewards/rejected": -3.058476686477661,
+      "step": 320
+    },
+    {
+      "epoch": 2.0442477876106193,
+      "grad_norm": 2.0968728065490723,
+      "learning_rate": 3.143459915611814e-07,
+      "logits/chosen": 1.0299267768859863,
+      "logits/rejected": -0.8118210434913635,
+      "logps/chosen": -158.1188507080078,
+      "logps/rejected": -133.52955627441406,
+      "loss": 0.0603,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5612393617630005,
+      "rewards/margins": 3.5689892768859863,
+      "rewards/rejected": -3.0077502727508545,
+      "step": 325
+    },
+    {
+      "epoch": 2.0758533501896332,
+      "grad_norm": 3.4302992820739746,
+      "learning_rate": 3.037974683544304e-07,
+      "logits/chosen": 1.0188157558441162,
+      "logits/rejected": -1.0047935247421265,
+      "logps/chosen": -168.2453155517578,
+      "logps/rejected": -142.3535919189453,
+      "loss": 0.055,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4401624798774719,
+      "rewards/margins": 3.662224054336548,
+      "rewards/rejected": -3.2220618724823,
+      "step": 330
+    },
+    {
+      "epoch": 2.107458912768647,
+      "grad_norm": 1.689516305923462,
+      "learning_rate": 2.9324894514767934e-07,
+      "logits/chosen": 0.9318011403083801,
+      "logits/rejected": -0.9510775804519653,
+      "logps/chosen": -158.6299285888672,
+      "logps/rejected": -141.5879364013672,
+      "loss": 0.075,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.31305503845214844,
+      "rewards/margins": 3.578648328781128,
+      "rewards/rejected": -3.2655932903289795,
+      "step": 335
+    },
+    {
+      "epoch": 2.139064475347661,
+      "grad_norm": 2.7359039783477783,
+      "learning_rate": 2.8270042194092825e-07,
+      "logits/chosen": 1.0490792989730835,
+      "logits/rejected": -1.0458027124404907,
+      "logps/chosen": -156.5770721435547,
+      "logps/rejected": -139.2061004638672,
+      "loss": 0.0707,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3262902796268463,
+      "rewards/margins": 3.555952548980713,
+      "rewards/rejected": -3.2296624183654785,
+      "step": 340
+    },
+    {
+      "epoch": 2.170670037926675,
+      "grad_norm": 1.3942673206329346,
+      "learning_rate": 2.721518987341772e-07,
+      "logits/chosen": 1.0303372144699097,
+      "logits/rejected": -0.9317108392715454,
+      "logps/chosen": -161.4883270263672,
+      "logps/rejected": -139.76275634765625,
+      "loss": 0.057,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.44392240047454834,
+      "rewards/margins": 4.022792816162109,
+      "rewards/rejected": -3.5788707733154297,
+      "step": 345
+    },
+    {
+      "epoch": 2.202275600505689,
+      "grad_norm": 2.6043851375579834,
+      "learning_rate": 2.6160337552742616e-07,
+      "logits/chosen": 1.0524630546569824,
+      "logits/rejected": -0.8903059959411621,
+      "logps/chosen": -161.5210418701172,
+      "logps/rejected": -142.73471069335938,
+      "loss": 0.0512,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.25128641724586487,
+      "rewards/margins": 3.7466254234313965,
+      "rewards/rejected": -3.4953391551971436,
+      "step": 350
+    },
+    {
+      "epoch": 2.202275600505689,
+      "eval_logits/chosen": 1.0315576791763306,
+      "eval_logits/rejected": -1.146601915359497,
+      "eval_logps/chosen": -157.10855102539062,
+      "eval_logps/rejected": -160.81405639648438,
+      "eval_loss": 0.019293567165732384,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.16965742409229279,
+      "eval_rewards/margins": 5.292869567871094,
+      "eval_rewards/rejected": -5.123211860656738,
+      "eval_runtime": 5.702,
+      "eval_samples_per_second": 2.806,
+      "eval_steps_per_second": 2.806,
+      "step": 350
+    },
+    {
+      "epoch": 2.233881163084703,
+      "grad_norm": 1.663423776626587,
+      "learning_rate": 2.5105485232067507e-07,
+      "logits/chosen": 1.0018548965454102,
+      "logits/rejected": -1.0198403596878052,
+      "logps/chosen": -161.3214111328125,
+      "logps/rejected": -148.89273071289062,
+      "loss": 0.0552,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3175838887691498,
+      "rewards/margins": 4.002824783325195,
+      "rewards/rejected": -3.6852409839630127,
+      "step": 355
+    },
+    {
+      "epoch": 2.265486725663717,
+      "grad_norm": 2.691444158554077,
+      "learning_rate": 2.40506329113924e-07,
+      "logits/chosen": 1.0729763507843018,
+      "logits/rejected": -0.9446109533309937,
+      "logps/chosen": -151.8990478515625,
+      "logps/rejected": -143.46170043945312,
+      "loss": 0.0637,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2433631718158722,
+      "rewards/margins": 3.872161865234375,
+      "rewards/rejected": -3.628798484802246,
+      "step": 360
+    },
+    {
+      "epoch": 2.2970922882427307,
+      "grad_norm": 1.7708544731140137,
+      "learning_rate": 2.2995780590717298e-07,
+      "logits/chosen": 1.1254976987838745,
+      "logits/rejected": -1.0846221446990967,
+      "logps/chosen": -151.911865234375,
+      "logps/rejected": -141.7873992919922,
+      "loss": 0.0381,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3414432108402252,
+      "rewards/margins": 4.305066108703613,
+      "rewards/rejected": -3.963623046875,
+      "step": 365
+    },
+    {
+      "epoch": 2.3286978508217446,
+      "grad_norm": 2.7277867794036865,
+      "learning_rate": 2.1940928270042194e-07,
+      "logits/chosen": 1.0135225057601929,
+      "logits/rejected": -1.0739619731903076,
+      "logps/chosen": -164.70875549316406,
+      "logps/rejected": -147.31813049316406,
+      "loss": 0.0567,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.10428227484226227,
+      "rewards/margins": 4.259421348571777,
+      "rewards/rejected": -4.155139446258545,
+      "step": 370
+    },
+    {
+      "epoch": 2.3603034134007586,
+      "grad_norm": 2.4887077808380127,
+      "learning_rate": 2.0886075949367087e-07,
+      "logits/chosen": 1.0406650304794312,
+      "logits/rejected": -1.0156916379928589,
+      "logps/chosen": -151.97161865234375,
+      "logps/rejected": -142.9573211669922,
+      "loss": 0.0458,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.20924396812915802,
+      "rewards/margins": 4.204397201538086,
+      "rewards/rejected": -3.9951531887054443,
+      "step": 375
+    },
+    {
+      "epoch": 2.3919089759797725,
+      "grad_norm": 1.972943663597107,
+      "learning_rate": 1.983122362869198e-07,
+      "logits/chosen": 1.0882539749145508,
+      "logits/rejected": -1.0857642889022827,
+      "logps/chosen": -157.22930908203125,
+      "logps/rejected": -144.57322692871094,
+      "loss": 0.0777,
+      "rewards/accuracies": 0.9800000190734863,
+      "rewards/chosen": 0.07933851331472397,
+      "rewards/margins": 4.109725475311279,
+      "rewards/rejected": -4.030386447906494,
+      "step": 380
+    },
+    {
+      "epoch": 2.4235145385587864,
+      "grad_norm": 0.6846963763237,
+      "learning_rate": 1.877637130801688e-07,
+      "logits/chosen": 0.9714919924736023,
+      "logits/rejected": -1.133774995803833,
+      "logps/chosen": -156.46595764160156,
+      "logps/rejected": -150.5989990234375,
+      "loss": 0.0577,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.045257944613695145,
+      "rewards/margins": 4.0961012840271,
+      "rewards/rejected": -4.141359329223633,
+      "step": 385
+    },
+    {
+      "epoch": 2.4551201011378003,
+      "grad_norm": 3.072885751724243,
+      "learning_rate": 1.7721518987341772e-07,
+      "logits/chosen": 1.1129729747772217,
+      "logits/rejected": -1.008325457572937,
+      "logps/chosen": -156.78793334960938,
+      "logps/rejected": -150.4031524658203,
+      "loss": 0.0434,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.13875991106033325,
+      "rewards/margins": 4.480461120605469,
+      "rewards/rejected": -4.619221210479736,
+      "step": 390
+    },
+    {
+      "epoch": 2.4867256637168142,
+      "grad_norm": 3.30064058303833,
+      "learning_rate": 1.6666666666666665e-07,
+      "logits/chosen": 1.0585930347442627,
+      "logits/rejected": -1.1667245626449585,
+      "logps/chosen": -155.1703643798828,
+      "logps/rejected": -147.69415283203125,
+      "loss": 0.0607,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.009059485048055649,
+      "rewards/margins": 4.336320877075195,
+      "rewards/rejected": -4.345380783081055,
+      "step": 395
+    },
+    {
+      "epoch": 2.518331226295828,
+      "grad_norm": 1.6333876848220825,
+      "learning_rate": 1.561181434599156e-07,
+      "logits/chosen": 1.0707764625549316,
+      "logits/rejected": -1.114976406097412,
+      "logps/chosen": -158.34593200683594,
+      "logps/rejected": -150.01637268066406,
+      "loss": 0.055,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.030936947092413902,
+      "rewards/margins": 4.401369571685791,
+      "rewards/rejected": -4.3704328536987305,
+      "step": 400
+    },
+    {
+      "epoch": 2.518331226295828,
+      "eval_logits/chosen": 1.0462472438812256,
+      "eval_logits/rejected": -1.2050833702087402,
+      "eval_logps/chosen": -159.44815063476562,
+      "eval_logps/rejected": -170.6011962890625,
+      "eval_loss": 0.013500434346497059,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.06430274248123169,
+      "eval_rewards/margins": 6.037621974945068,
+      "eval_rewards/rejected": -6.101924419403076,
+      "eval_runtime": 5.6929,
+      "eval_samples_per_second": 2.811,
+      "eval_steps_per_second": 2.811,
+      "step": 400
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 474,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 400,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08c349a698643ae807c0727adae15ef25964abd41d2aad65335fa0c8b4b256ef
+size 6200