Model save

README.md

@@ -0,0 +1,75 @@
+---
+license: apache-2.0
+library_name: peft
+tags:
+- trl
+- dpo
+- generated_from_trainer
+base_model: alignment-handbook/zephyr-7b-sft-full
+model-index:
+- name: vanilla_doff_iter_2
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# vanilla_doff_iter_2
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6681
+- Rewards/chosen: -0.1963
+- Rewards/rejected: -0.2541
+- Rewards/accuracies: 0.5560
+- Rewards/margins: 0.0578
+- Logps/rejected: -295.9663
+- Logps/chosen: -311.6507
+- Logits/rejected: -2.5643
+- Logits/chosen: -2.6639
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 2
+- eval_batch_size: 2
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 128
+- total_eval_batch_size: 8
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.66          | 0.65  | 100  | 0.6681          | -0.1963        | -0.2541          | 0.5560             | 0.0578          | -295.9663      | -311.6507    | -2.5643         | -2.6639       |
+
+
+### Framework versions
+
+- PEFT 0.7.1
+- Transformers 4.36.2
+- Pytorch 2.3.0+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.6699569872002197,
+    "train_runtime": 6332.363,
+    "train_samples": 19600,
+    "train_samples_per_second": 3.095,
+    "train_steps_per_second": 0.024
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.6699569872002197,
+    "train_runtime": 6332.363,
+    "train_samples": 19600,
+    "train_samples_per_second": 3.095,
+    "train_steps_per_second": 0.024
+}

trainer_state.json

@@ -0,0 +1,270 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9991836734693877,
+  "eval_steps": 100,
+  "global_step": 153,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.125e-07,
+      "logits/chosen": -2.641124725341797,
+      "logits/rejected": -2.6616296768188477,
+      "logps/chosen": -264.0085754394531,
+      "logps/rejected": -342.70855712890625,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.125e-06,
+      "logits/chosen": -2.6937146186828613,
+      "logits/rejected": -2.6783552169799805,
+      "logps/chosen": -251.7361602783203,
+      "logps/rejected": -279.3586120605469,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4722222089767456,
+      "rewards/chosen": 0.00643091881647706,
+      "rewards/margins": 0.0015065649058669806,
+      "rewards/rejected": 0.004924353677779436,
+      "step": 10
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.989490450759331e-06,
+      "logits/chosen": -2.7149226665496826,
+      "logits/rejected": -2.664808511734009,
+      "logps/chosen": -246.8320770263672,
+      "logps/rejected": -270.1576232910156,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.012097175233066082,
+      "rewards/margins": 0.003004220314323902,
+      "rewards/rejected": 0.00909295491874218,
+      "step": 20
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.872270441827174e-06,
+      "logits/chosen": -2.734833002090454,
+      "logits/rejected": -2.667550563812256,
+      "logps/chosen": -276.6678161621094,
+      "logps/rejected": -292.2188415527344,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.03442594036459923,
+      "rewards/margins": 0.021055880934000015,
+      "rewards/rejected": -0.05548182129859924,
+      "step": 30
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.630851211353007e-06,
+      "logits/chosen": -2.623716115951538,
+      "logits/rejected": -2.599712371826172,
+      "logps/chosen": -253.99520874023438,
+      "logps/rejected": -301.3703308105469,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.05057894438505173,
+      "rewards/margins": 0.03473568707704544,
+      "rewards/rejected": -0.08531462401151657,
+      "step": 40
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.277872161641682e-06,
+      "logits/chosen": -2.6934971809387207,
+      "logits/rejected": -2.6622395515441895,
+      "logps/chosen": -248.6873016357422,
+      "logps/rejected": -270.53009033203125,
+      "loss": 0.6776,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": 0.012412898242473602,
+      "rewards/margins": 0.027583047747612,
+      "rewards/rejected": -0.015170147642493248,
+      "step": 50
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.831813362428005e-06,
+      "logits/chosen": -2.6209137439727783,
+      "logits/rejected": -2.5777764320373535,
+      "logps/chosen": -261.93841552734375,
+      "logps/rejected": -297.962890625,
+      "loss": 0.667,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.15015549957752228,
+      "rewards/margins": 0.06109144166111946,
+      "rewards/rejected": -0.21124692261219025,
+      "step": 60
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.3160280345958614e-06,
+      "logits/chosen": -2.623276710510254,
+      "logits/rejected": -2.576505184173584,
+      "logps/chosen": -280.05535888671875,
+      "logps/rejected": -315.9161682128906,
+      "loss": 0.6632,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.14175915718078613,
+      "rewards/margins": 0.05709190294146538,
+      "rewards/rejected": -0.198851078748703,
+      "step": 70
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.757519902117886e-06,
+      "logits/chosen": -2.6120524406433105,
+      "logits/rejected": -2.598693370819092,
+      "logps/chosen": -259.7000732421875,
+      "logps/rejected": -271.61346435546875,
+      "loss": 0.6655,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.15165603160858154,
+      "rewards/margins": 0.07243518531322479,
+      "rewards/rejected": -0.22409121692180634,
+      "step": 80
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.185529423440807e-06,
+      "logits/chosen": -2.615450382232666,
+      "logits/rejected": -2.60538911819458,
+      "logps/chosen": -273.26263427734375,
+      "logps/rejected": -324.07928466796875,
+      "loss": 0.6591,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.2031743824481964,
+      "rewards/margins": 0.08501346409320831,
+      "rewards/rejected": -0.28818783164024353,
+      "step": 90
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6300029195778454e-06,
+      "logits/chosen": -2.6159863471984863,
+      "logits/rejected": -2.5994787216186523,
+      "logps/chosen": -279.80194091796875,
+      "logps/rejected": -321.82025146484375,
+      "loss": 0.66,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.2041420042514801,
+      "rewards/margins": 0.09220623970031738,
+      "rewards/rejected": -0.2963482439517975,
+      "step": 100
+    },
+    {
+      "epoch": 0.65,
+      "eval_logits/chosen": -2.663874626159668,
+      "eval_logits/rejected": -2.564279794692993,
+      "eval_logps/chosen": -311.6506652832031,
+      "eval_logps/rejected": -295.9662780761719,
+      "eval_loss": 0.6680780053138733,
+      "eval_rewards/accuracies": 0.5559999942779541,
+      "eval_rewards/chosen": -0.19626890122890472,
+      "eval_rewards/margins": 0.05780958756804466,
+      "eval_rewards/rejected": -0.2540785074234009,
+      "eval_runtime": 384.0259,
+      "eval_samples_per_second": 5.208,
+      "eval_steps_per_second": 0.651,
+      "step": 100
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1200247470632394e-06,
+      "logits/chosen": -2.6203360557556152,
+      "logits/rejected": -2.6212260723114014,
+      "logps/chosen": -263.1394958496094,
+      "logps/rejected": -310.59674072265625,
+      "loss": 0.6647,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.2304689884185791,
+      "rewards/margins": 0.06938486546278,
+      "rewards/rejected": -0.2998538315296173,
+      "step": 110
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.822945986946386e-07,
+      "logits/chosen": -2.6103806495666504,
+      "logits/rejected": -2.583449602127075,
+      "logps/chosen": -245.1764678955078,
+      "logps/rejected": -280.20208740234375,
+      "loss": 0.6618,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.1879294216632843,
+      "rewards/margins": 0.08095093071460724,
+      "rewards/rejected": -0.26888033747673035,
+      "step": 120
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.397296523427807e-07,
+      "logits/chosen": -2.570099353790283,
+      "logits/rejected": -2.5483436584472656,
+      "logps/chosen": -261.6362609863281,
+      "logps/rejected": -306.3728942871094,
+      "loss": 0.6604,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.19585388898849487,
+      "rewards/margins": 0.09675396978855133,
+      "rewards/rejected": -0.292607843875885,
+      "step": 130
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.102647517397798e-07,
+      "logits/chosen": -2.606142520904541,
+      "logits/rejected": -2.5906078815460205,
+      "logps/chosen": -271.9281921386719,
+      "logps/rejected": -317.7606201171875,
+      "loss": 0.6633,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.20845098793506622,
+      "rewards/margins": 0.101742222905159,
+      "rewards/rejected": -0.31019318103790283,
+      "step": 140
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 5.9134352763748345e-09,
+      "logits/chosen": -2.620689868927002,
+      "logits/rejected": -2.607283115386963,
+      "logps/chosen": -296.2304382324219,
+      "logps/rejected": -331.72760009765625,
+      "loss": 0.6602,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.22927439212799072,
+      "rewards/margins": 0.0815407782793045,
+      "rewards/rejected": -0.31081515550613403,
+      "step": 150
+    },
+    {
+      "epoch": 1.0,
+      "step": 153,
+      "total_flos": 0.0,
+      "train_loss": 0.6699569872002197,
+      "train_runtime": 6332.363,
+      "train_samples_per_second": 3.095,
+      "train_steps_per_second": 0.024
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 153,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}