Model save

README.md

@@ -2,14 +2,9 @@
 license: apache-2.0
 base_model: YYYYYYibo/full_vanilla_dpo_iter_1
 tags:
-- alignment-handbook
-- generated_from_trainer
 - trl
 - dpo
 - generated_from_trainer
-datasets:
-- updated
-- original
 model-index:
 - name: full_simple_online_iter_2
   results: []
@@ -20,7 +15,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # full_simple_online_iter_2
 
-This model is a fine-tuned version of [YYYYYYibo/full_vanilla_dpo_iter_1](https://huggingface.co/YYYYYYibo/full_vanilla_dpo_iter_1) on the updated and the original datasets.
+This model is a fine-tuned version of [YYYYYYibo/full_vanilla_dpo_iter_1](https://huggingface.co/YYYYYYibo/full_vanilla_dpo_iter_1) on the None dataset.
 
 ## Model description
 

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6894009854342487,
-    "train_runtime": 9025.6123,
-    "train_samples": 19000,
-    "train_samples_per_second": 2.105,
+    "train_loss": 0.5776262069359804,
+    "train_runtime": 9474.0276,
+    "train_samples": 20000,
+    "train_samples_per_second": 2.111,
     "train_steps_per_second": 0.016
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fe86e2c25ad9ae983c498811e5a480ac3727f642766fef0236a11db3e13dcab0
+oid sha256:48cf9bd654703f1ed9fef98526910ec9c025c0d695da0b63c75a56bcd66db8fb
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:307c5e0242d47c8559d5c54f0a41b72bd4bc5610338b0aa2e8a209cfe8d24638
+oid sha256:dcb19293e1e1d6426e90b5a675ba75b3277ca27d3bf76b59acb6072c131a4432
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:00c242505930094b5941241990bc3657e0a553a43da834b995015f08c0718f9d
+oid sha256:2366e046327d3da5e5c7b3a83c25c1d3e1f07246246f7c73ac37b26d9f758300
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6894009854342487,
-    "train_runtime": 9025.6123,
-    "train_samples": 19000,
-    "train_samples_per_second": 2.105,
+    "train_loss": 0.5776262069359804,
+    "train_runtime": 9474.0276,
+    "train_samples": 20000,
+    "train_samples_per_second": 2.111,
     "train_steps_per_second": 0.016
 }

trainer_state.json

@@ -1,20 +1,20 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9966329966329966,
+  "epoch": 0.9984,
   "eval_steps": 500,
-  "global_step": 148,
+  "global_step": 156,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.01,
-      "learning_rate": 3.3333333333333334e-08,
-      "logits/chosen": -2.023646593093872,
-      "logits/rejected": -1.861999750137329,
-      "logps/chosen": -160.15196228027344,
-      "logps/rejected": -164.30947875976562,
+      "learning_rate": 3.125e-08,
+      "logits/chosen": -1.4567933082580566,
+      "logits/rejected": -0.871229887008667,
+      "logps/chosen": -244.365234375,
+      "logps/rejected": -212.26486206054688,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -23,213 +23,227 @@
       "step": 1
     },
     {
-      "epoch": 0.07,
-      "learning_rate": 3.333333333333333e-07,
-      "logits/chosen": -1.8246960639953613,
-      "logits/rejected": -1.874166488647461,
-      "logps/chosen": -186.62855529785156,
-      "logps/rejected": -191.06869506835938,
-      "loss": 0.693,
-      "rewards/accuracies": 0.4513888955116272,
-      "rewards/chosen": -0.0028373675886541605,
-      "rewards/margins": -9.477811545366421e-06,
-      "rewards/rejected": -0.002827889285981655,
+      "epoch": 0.06,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -1.83387291431427,
+      "logits/rejected": -1.0804697275161743,
+      "logps/chosen": -206.00912475585938,
+      "logps/rejected": -202.784912109375,
+      "loss": 0.6817,
+      "rewards/accuracies": 0.5486111044883728,
+      "rewards/chosen": -0.039022047072649,
+      "rewards/margins": 0.04178649187088013,
+      "rewards/rejected": -0.08080853521823883,
       "step": 10
     },
     {
       "epoch": 0.13,
-      "learning_rate": 4.98258427321406e-07,
-      "logits/chosen": -1.5834639072418213,
-      "logits/rejected": -1.6468555927276611,
-      "logps/chosen": -191.36276245117188,
-      "logps/rejected": -201.68360900878906,
-      "loss": 0.6936,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.09236270189285278,
-      "rewards/margins": 0.01027429848909378,
-      "rewards/rejected": -0.10263700783252716,
+      "learning_rate": 4.989935734988097e-07,
+      "logits/chosen": -1.0675297975540161,
+      "logits/rejected": -0.5359733700752258,
+      "logps/chosen": -237.27444458007812,
+      "logps/rejected": -251.00753784179688,
+      "loss": 0.6561,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.7787758111953735,
+      "rewards/margins": 0.11565746366977692,
+      "rewards/rejected": -0.8944332003593445,
       "step": 20
     },
     {
-      "epoch": 0.2,
-      "learning_rate": 4.844710954430464e-07,
-      "logits/chosen": -1.572912335395813,
-      "logits/rejected": -1.6341121196746826,
-      "logps/chosen": -189.92250061035156,
-      "logps/rejected": -188.98448181152344,
-      "loss": 0.6947,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.10429297387599945,
-      "rewards/margins": -0.007742973975837231,
-      "rewards/rejected": -0.09655000269412994,
+      "epoch": 0.19,
+      "learning_rate": 4.877641290737883e-07,
+      "logits/chosen": -1.095473289489746,
+      "logits/rejected": -0.37094515562057495,
+      "logps/chosen": -244.32162475585938,
+      "logps/rejected": -296.1733703613281,
+      "loss": 0.5953,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.6706979870796204,
+      "rewards/margins": 0.5164287090301514,
+      "rewards/rejected": -1.187126636505127,
       "step": 30
     },
     {
-      "epoch": 0.27,
-      "learning_rate": 4.576621278295557e-07,
-      "logits/chosen": -1.6369476318359375,
-      "logits/rejected": -1.5702846050262451,
-      "logps/chosen": -203.3102264404297,
-      "logps/rejected": -203.18093872070312,
-      "loss": 0.6897,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.03835677355527878,
-      "rewards/margins": 0.012528707273304462,
-      "rewards/rejected": -0.05088547617197037,
+      "epoch": 0.26,
+      "learning_rate": 4.646121984004665e-07,
+      "logits/chosen": -0.8634458780288696,
+      "logits/rejected": 0.12595783174037933,
+      "logps/chosen": -242.0459442138672,
+      "logps/rejected": -296.41595458984375,
+      "loss": 0.5648,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.5677449703216553,
+      "rewards/margins": 0.5976042747497559,
+      "rewards/rejected": -1.1653492450714111,
       "step": 40
     },
     {
-      "epoch": 0.34,
-      "learning_rate": 4.193203929064353e-07,
-      "logits/chosen": -1.3983080387115479,
-      "logits/rejected": -1.405611276626587,
-      "logps/chosen": -200.58035278320312,
-      "logps/rejected": -205.0247802734375,
-      "loss": 0.6879,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.2730618119239807,
-      "rewards/margins": 0.03439151123166084,
-      "rewards/rejected": -0.30745333433151245,
+      "epoch": 0.32,
+      "learning_rate": 4.3069871595684787e-07,
+      "logits/chosen": -0.6954927444458008,
+      "logits/rejected": 0.03154268115758896,
+      "logps/chosen": -246.68258666992188,
+      "logps/rejected": -295.62884521484375,
+      "loss": 0.5913,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.8166979551315308,
+      "rewards/margins": 0.5098680257797241,
+      "rewards/rejected": -1.3265659809112549,
       "step": 50
     },
     {
-      "epoch": 0.4,
-      "learning_rate": 3.715752452735703e-07,
-      "logits/chosen": -1.2504160404205322,
-      "logits/rejected": -1.3270930051803589,
-      "logps/chosen": -236.2527313232422,
-      "logps/rejected": -239.9370574951172,
-      "loss": 0.6856,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.5804754495620728,
-      "rewards/margins": 0.015670539811253548,
-      "rewards/rejected": -0.5961459279060364,
+      "epoch": 0.38,
+      "learning_rate": 3.877242453630256e-07,
+      "logits/chosen": -0.768271267414093,
+      "logits/rejected": 0.022685179486870766,
+      "logps/chosen": -245.92782592773438,
+      "logps/rejected": -300.2510681152344,
+      "loss": 0.5887,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.7170382738113403,
+      "rewards/margins": 0.5133967399597168,
+      "rewards/rejected": -1.2304350137710571,
       "step": 60
     },
     {
-      "epoch": 0.47,
-      "learning_rate": 3.170782694233712e-07,
-      "logits/chosen": -1.1783835887908936,
-      "logits/rejected": -1.141601800918579,
-      "logps/chosen": -263.32073974609375,
-      "logps/rejected": -272.93450927734375,
-      "loss": 0.6956,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.9808699488639832,
-      "rewards/margins": 0.05901496857404709,
-      "rewards/rejected": -1.0398849248886108,
+      "epoch": 0.45,
+      "learning_rate": 3.378437060203357e-07,
+      "logits/chosen": -0.5168389081954956,
+      "logits/rejected": 0.45852264761924744,
+      "logps/chosen": -256.852294921875,
+      "logps/rejected": -309.4953308105469,
+      "loss": 0.5836,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.9044780731201172,
+      "rewards/margins": 0.5655065178871155,
+      "rewards/rejected": -1.4699846506118774,
       "step": 70
     },
     {
-      "epoch": 0.54,
-      "learning_rate": 2.588560207905135e-07,
-      "logits/chosen": -1.2210887670516968,
-      "logits/rejected": -1.3275476694107056,
-      "logps/chosen": -277.7283020019531,
-      "logps/rejected": -284.2450256347656,
-      "loss": 0.6953,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.8450358510017395,
-      "rewards/margins": -0.01733619160950184,
-      "rewards/rejected": -0.8276995420455933,
+      "epoch": 0.51,
+      "learning_rate": 2.8355831645441387e-07,
+      "logits/chosen": -0.3654092848300934,
+      "logits/rejected": 0.10795004665851593,
+      "logps/chosen": -251.9696502685547,
+      "logps/rejected": -292.9334716796875,
+      "loss": 0.5522,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.7978931069374084,
+      "rewards/margins": 0.38232654333114624,
+      "rewards/rejected": -1.1802196502685547,
       "step": 80
     },
     {
-      "epoch": 0.61,
-      "learning_rate": 2.001419423371019e-07,
-      "logits/chosen": -1.2033240795135498,
-      "logits/rejected": -1.2934813499450684,
-      "logps/chosen": -233.0209197998047,
-      "logps/rejected": -232.87161254882812,
-      "loss": 0.6984,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -0.7631824612617493,
-      "rewards/margins": -0.014957061037421227,
-      "rewards/rejected": -0.7482253313064575,
+      "epoch": 0.58,
+      "learning_rate": 2.2759017277414164e-07,
+      "logits/chosen": -0.8108726739883423,
+      "logits/rejected": 0.14660978317260742,
+      "logps/chosen": -273.36419677734375,
+      "logps/rejected": -320.58209228515625,
+      "loss": 0.5671,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.6586915254592896,
+      "rewards/margins": 0.6224299669265747,
+      "rewards/rejected": -1.2811213731765747,
       "step": 90
     },
     {
-      "epoch": 0.67,
-      "learning_rate": 1.4419679138889375e-07,
-      "logits/chosen": -1.5676350593566895,
-      "logits/rejected": -1.5352352857589722,
-      "logps/chosen": -243.58340454101562,
-      "logps/rejected": -249.83670043945312,
-      "loss": 0.6839,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.431363046169281,
-      "rewards/margins": 0.026667874306440353,
-      "rewards/rejected": -0.45803093910217285,
+      "epoch": 0.64,
+      "learning_rate": 1.7274575140626315e-07,
+      "logits/chosen": -0.46979203820228577,
+      "logits/rejected": 0.5494852066040039,
+      "logps/chosen": -272.69427490234375,
+      "logps/rejected": -317.7990417480469,
+      "loss": 0.5547,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.9002830386161804,
+      "rewards/margins": 0.5319327116012573,
+      "rewards/rejected": -1.432215690612793,
       "step": 100
     },
     {
-      "epoch": 0.74,
-      "learning_rate": 9.412754953531663e-08,
-      "logits/chosen": -1.4186036586761475,
-      "logits/rejected": -1.4321409463882446,
-      "logps/chosen": -207.4648895263672,
-      "logps/rejected": -220.63540649414062,
-      "loss": 0.6783,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.3499959409236908,
-      "rewards/margins": 0.08647538721561432,
-      "rewards/rejected": -0.4364713132381439,
+      "epoch": 0.7,
+      "learning_rate": 1.2177518064852348e-07,
+      "logits/chosen": -0.3219306170940399,
+      "logits/rejected": 0.26910799741744995,
+      "logps/chosen": -251.5453338623047,
+      "logps/rejected": -299.8834533691406,
+      "loss": 0.56,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.7972058057785034,
+      "rewards/margins": 0.43246564269065857,
+      "rewards/rejected": -1.2296714782714844,
       "step": 110
     },
     {
-      "epoch": 0.81,
-      "learning_rate": 5.271487265090163e-08,
-      "logits/chosen": -1.3899421691894531,
-      "logits/rejected": -1.4946931600570679,
-      "logps/chosen": -199.7452392578125,
-      "logps/rejected": -206.2049102783203,
-      "loss": 0.6857,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.31818634271621704,
-      "rewards/margins": 0.03093295730650425,
-      "rewards/rejected": -0.34911927580833435,
+      "epoch": 0.77,
+      "learning_rate": 7.723433775328384e-08,
+      "logits/chosen": -0.37325382232666016,
+      "logits/rejected": 0.5774334669113159,
+      "logps/chosen": -233.79562377929688,
+      "logps/rejected": -328.5582580566406,
+      "loss": 0.5585,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.6402639150619507,
+      "rewards/margins": 0.7515830397605896,
+      "rewards/rejected": -1.3918468952178955,
       "step": 120
     },
     {
-      "epoch": 0.88,
-      "learning_rate": 2.2258663809784888e-08,
-      "logits/chosen": -1.5326082706451416,
-      "logits/rejected": -1.4976835250854492,
-      "logps/chosen": -203.8050079345703,
-      "logps/rejected": -208.89431762695312,
-      "loss": 0.6865,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.27058011293411255,
-      "rewards/margins": 0.03228816017508507,
-      "rewards/rejected": -0.3028682768344879,
+      "epoch": 0.83,
+      "learning_rate": 4.1356686569674335e-08,
+      "logits/chosen": -0.3119003176689148,
+      "logits/rejected": 0.8427650332450867,
+      "logps/chosen": -233.98971557617188,
+      "logps/rejected": -324.93316650390625,
+      "loss": 0.5265,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.7347938418388367,
+      "rewards/margins": 0.7224765419960022,
+      "rewards/rejected": -1.4572702646255493,
       "step": 130
     },
     {
-      "epoch": 0.94,
-      "learning_rate": 4.45034538815614e-09,
-      "logits/chosen": -1.4072165489196777,
-      "logits/rejected": -1.4480218887329102,
-      "logps/chosen": -210.232666015625,
-      "logps/rejected": -218.79763793945312,
-      "loss": 0.6827,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.29495373368263245,
-      "rewards/margins": 0.03974684700369835,
-      "rewards/rejected": -0.3347005844116211,
+      "epoch": 0.9,
+      "learning_rate": 1.5941282340065697e-08,
+      "logits/chosen": -0.20903070271015167,
+      "logits/rejected": 0.7928945422172546,
+      "logps/chosen": -274.28704833984375,
+      "logps/rejected": -331.6188049316406,
+      "loss": 0.5484,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.9145911931991577,
+      "rewards/margins": 0.5992218255996704,
+      "rewards/rejected": -1.5138130187988281,
       "step": 140
     },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2625595580163247e-09,
+      "logits/chosen": 0.10685434192419052,
+      "logits/rejected": 0.766906201839447,
+      "logps/chosen": -257.482666015625,
+      "logps/rejected": -326.8499450683594,
+      "loss": 0.5539,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.9127100706100464,
+      "rewards/margins": 0.6432833075523376,
+      "rewards/rejected": -1.5559935569763184,
+      "step": 150
+    },
     {
       "epoch": 1.0,
-      "step": 148,
+      "step": 156,
       "total_flos": 0.0,
-      "train_loss": 0.6894009854342487,
-      "train_runtime": 9025.6123,
-      "train_samples_per_second": 2.105,
+      "train_loss": 0.5776262069359804,
+      "train_runtime": 9474.0276,
+      "train_samples_per_second": 2.111,
       "train_steps_per_second": 0.016
     }
   ],
   "logging_steps": 10,
-  "max_steps": 148,
+  "max_steps": 156,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,