End of training

README.md

@@ -3,9 +3,10 @@ library_name: peft
 license: llama3.1
 base_model: meta-llama/Meta-Llama-3.1-8B-Instruct
 tags:
+- llama-factory
+- lora
 - trl
 - kto
-- llama-factory
 - generated_from_trainer
 model-index:
 - name: Llama-3.1-8B-Instruct-KTO-100
@@ -17,7 +18,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # Llama-3.1-8B-Instruct-KTO-100
 
-This model is a fine-tuned version of [meta-llama/Meta-Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct) on an unknown dataset.
+This model is a fine-tuned version of [meta-llama/Meta-Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct) on the bct_non_cot_kto_100 dataset.
 It achieves the following results on the evaluation set:
 - Loss: 0.4997
 - Rewards/chosen: 0.0050

all_results.json

@@ -0,0 +1,20 @@
+{
+    "epoch": 9.777777777777779,
+    "eval_logits/chosen": -5053702.857142857,
+    "eval_logits/rejected": -7957526.666666667,
+    "eval_logps/chosen": -17.07437242780413,
+    "eval_logps/rejected": -23.829910278320312,
+    "eval_loss": 0.4997052252292633,
+    "eval_rewards/chosen": 0.005012544138090951,
+    "eval_rewards/margins": -0.0027697735599109107,
+    "eval_rewards/rejected": 0.007782317698001862,
+    "eval_runtime": 3.8441,
+    "eval_samples_per_second": 5.203,
+    "eval_steps_per_second": 2.601,
+    "kl": 0.0,
+    "total_flos": 1.0124196800495616e+16,
+    "train_loss": 0.4910273031754927,
+    "train_runtime": 636.4533,
+    "train_samples_per_second": 2.828,
+    "train_steps_per_second": 0.173
+}

eval_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 9.777777777777779,
+    "eval_logits/chosen": -5053702.857142857,
+    "eval_logits/rejected": -7957526.666666667,
+    "eval_logps/chosen": -17.07437242780413,
+    "eval_logps/rejected": -23.829910278320312,
+    "eval_loss": 0.4997052252292633,
+    "eval_rewards/chosen": 0.005012544138090951,
+    "eval_rewards/margins": -0.0027697735599109107,
+    "eval_rewards/rejected": 0.007782317698001862,
+    "eval_runtime": 3.8441,
+    "eval_samples_per_second": 5.203,
+    "eval_steps_per_second": 2.601,
+    "kl": 0.0
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 9.777777777777779,
+    "total_flos": 1.0124196800495616e+16,
+    "train_loss": 0.4910273031754927,
+    "train_runtime": 636.4533,
+    "train_samples_per_second": 2.828,
+    "train_steps_per_second": 0.173
+}

trainer_state.json

@@ -0,0 +1,239 @@
+{
+  "best_metric": 0.4997052252292633,
+  "best_model_checkpoint": "saves/sycophancy/Llama-8B-3.1-Instruct/kto-100/checkpoint-100",
+  "epoch": 9.777777777777779,
+  "eval_steps": 50,
+  "global_step": 110,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.8888888888888888,
+      "grad_norm": 0.6491909027099609,
+      "kl": 3.4562063217163086,
+      "learning_rate": 4.5454545454545455e-06,
+      "logits/chosen": -6489723.7894736845,
+      "logits/rejected": -7186845.714285715,
+      "logps/chosen": -19.671953703227796,
+      "logps/rejected": -20.928447905040922,
+      "loss": 0.5005,
+      "rewards/chosen": -0.001196424623853282,
+      "rewards/margins": -0.004711177834965531,
+      "rewards/rejected": 0.0035147532111122494,
+      "step": 10
+    },
+    {
+      "epoch": 1.7777777777777777,
+      "grad_norm": 0.8529065251350403,
+      "kl": 4.441736221313477,
+      "learning_rate": 4.8987324340362445e-06,
+      "logits/chosen": -6480347.428571428,
+      "logits/rejected": -7280319.2289156625,
+      "logps/chosen": -20.52882127637987,
+      "logps/rejected": -20.018041403896838,
+      "loss": 0.4998,
+      "rewards/chosen": -0.0007010871036486192,
+      "rewards/margins": 0.0022902732504876525,
+      "rewards/rejected": -0.0029913603541362717,
+      "step": 20
+    },
+    {
+      "epoch": 2.6666666666666665,
+      "grad_norm": 0.6697812676429749,
+      "kl": 2.5247769355773926,
+      "learning_rate": 4.559191453574582e-06,
+      "logits/chosen": -6102143.1466666665,
+      "logits/rejected": -7669873.694117647,
+      "logps/chosen": -19.55682291666667,
+      "logps/rejected": -20.801958869485293,
+      "loss": 0.499,
+      "rewards/chosen": 0.0057321461041768395,
+      "rewards/margins": 0.00854761147031597,
+      "rewards/rejected": -0.0028154653661391312,
+      "step": 30
+    },
+    {
+      "epoch": 3.5555555555555554,
+      "grad_norm": 0.9257539510726929,
+      "kl": 2.3885769844055176,
+      "learning_rate": 4.014024217844167e-06,
+      "logits/chosen": -6635654.320987654,
+      "logits/rejected": -7498068.253164557,
+      "logps/chosen": -17.21456344039352,
+      "logps/rejected": -20.309491507614716,
+      "loss": 0.4977,
+      "rewards/chosen": 0.014323502411077053,
+      "rewards/margins": 0.01919812530833383,
+      "rewards/rejected": -0.004874622897256779,
+      "step": 40
+    },
+    {
+      "epoch": 4.444444444444445,
+      "grad_norm": 0.8241211771965027,
+      "kl": 1.9659223556518555,
+      "learning_rate": 3.3176699082935546e-06,
+      "logits/chosen": -6149508.155844156,
+      "logits/rejected": -7053594.21686747,
+      "logps/chosen": -20.374757444703732,
+      "logps/rejected": -20.175387095256024,
+      "loss": 0.4944,
+      "rewards/chosen": 0.031391688755580356,
+      "rewards/margins": 0.04555150604904621,
+      "rewards/rejected": -0.014159817293465856,
+      "step": 50
+    },
+    {
+      "epoch": 4.444444444444445,
+      "eval_logits/chosen": -5154306.285714285,
+      "eval_logits/rejected": -7920785.333333333,
+      "eval_logps/chosen": -17.13888440813337,
+      "eval_logps/rejected": -23.718454996744793,
+      "eval_loss": 0.501809298992157,
+      "eval_rewards/chosen": -0.0014386344701051712,
+      "eval_rewards/margins": -0.02036647933224837,
+      "eval_rewards/rejected": 0.0189278448621432,
+      "eval_runtime": 3.9297,
+      "eval_samples_per_second": 5.089,
+      "eval_steps_per_second": 2.545,
+      "kl": 0.07577133178710938,
+      "step": 50
+    },
+    {
+      "epoch": 5.333333333333333,
+      "grad_norm": 0.7947250008583069,
+      "kl": 2.237136125564575,
+      "learning_rate": 2.53966490958702e-06,
+      "logits/chosen": -6028127.05882353,
+      "logits/rejected": -7365730.782608695,
+      "logps/chosen": -20.794239717371322,
+      "logps/rejected": -20.906398607336957,
+      "loss": 0.4925,
+      "rewards/chosen": 0.03433014364803538,
+      "rewards/margins": 0.06258551818330574,
+      "rewards/rejected": -0.02825537453527036,
+      "step": 60
+    },
+    {
+      "epoch": 6.222222222222222,
+      "grad_norm": 1.0035609006881714,
+      "kl": 0.19663143157958984,
+      "learning_rate": 1.7576990616793139e-06,
+      "logits/chosen": -5932590.8292682925,
+      "logits/rejected": -7592413.538461538,
+      "logps/chosen": -19.798485732660062,
+      "logps/rejected": -20.841768704927883,
+      "loss": 0.4885,
+      "rewards/chosen": 0.041047639963103504,
+      "rewards/margins": 0.0920561089822842,
+      "rewards/rejected": -0.05100846901918069,
+      "step": 70
+    },
+    {
+      "epoch": 7.111111111111111,
+      "grad_norm": 1.3625916242599487,
+      "kl": 1.2758498191833496,
+      "learning_rate": 1.049857726072005e-06,
+      "logits/chosen": -6768110.08,
+      "logits/rejected": -6676467.9529411765,
+      "logps/chosen": -17.8654541015625,
+      "logps/rejected": -21.48829704733456,
+      "loss": 0.4841,
+      "rewards/chosen": 0.053166917165120446,
+      "rewards/margins": 0.1267875899520575,
+      "rewards/rejected": -0.07362067278693704,
+      "step": 80
+    },
+    {
+      "epoch": 8.0,
+      "grad_norm": 1.2151498794555664,
+      "kl": 1.8016748428344727,
+      "learning_rate": 4.868243561723535e-07,
+      "logits/chosen": -5652549.818181818,
+      "logits/rejected": -7915004.915662651,
+      "logps/chosen": -19.880209390219157,
+      "logps/rejected": -21.150170016001507,
+      "loss": 0.4834,
+      "rewards/chosen": 0.05748867059682871,
+      "rewards/margins": 0.1355446326193573,
+      "rewards/rejected": -0.0780559620225286,
+      "step": 90
+    },
+    {
+      "epoch": 8.88888888888889,
+      "grad_norm": 1.0270472764968872,
+      "kl": 1.7680234909057617,
+      "learning_rate": 1.2482220564763669e-07,
+      "logits/chosen": -6533376.432432433,
+      "logits/rejected": -6922219.162790698,
+      "logps/chosen": -19.65522065033784,
+      "logps/rejected": -21.008502339207848,
+      "loss": 0.4809,
+      "rewards/chosen": 0.0715047024391793,
+      "rewards/margins": 0.15263205215962408,
+      "rewards/rejected": -0.08112734972044479,
+      "step": 100
+    },
+    {
+      "epoch": 8.88888888888889,
+      "eval_logits/chosen": -5053702.857142857,
+      "eval_logits/rejected": -7957526.666666667,
+      "eval_logps/chosen": -17.07437242780413,
+      "eval_logps/rejected": -23.829910278320312,
+      "eval_loss": 0.4997052252292633,
+      "eval_rewards/chosen": 0.005012544138090951,
+      "eval_rewards/margins": -0.0027697735599109107,
+      "eval_rewards/rejected": 0.007782317698001862,
+      "eval_runtime": 3.937,
+      "eval_samples_per_second": 5.08,
+      "eval_steps_per_second": 2.54,
+      "kl": 0.0,
+      "step": 100
+    },
+    {
+      "epoch": 9.777777777777779,
+      "grad_norm": 0.8866751194000244,
+      "kl": 1.2843537330627441,
+      "learning_rate": 0.0,
+      "logits/chosen": -6480412.307692308,
+      "logits/rejected": -8124753.1707317075,
+      "logps/chosen": -19.448489458133015,
+      "logps/rejected": -22.091482767244663,
+      "loss": 0.4805,
+      "rewards/chosen": 0.06529526221446502,
+      "rewards/margins": 0.1558387591139535,
+      "rewards/rejected": -0.0905434968994885,
+      "step": 110
+    },
+    {
+      "epoch": 9.777777777777779,
+      "step": 110,
+      "total_flos": 1.0124196800495616e+16,
+      "train_loss": 0.4910273031754927,
+      "train_runtime": 636.4533,
+      "train_samples_per_second": 2.828,
+      "train_steps_per_second": 0.173
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 110,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.0124196800495616e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}