End of training

README.md

@@ -3,9 +3,10 @@ library_name: peft
 license: llama3.1
 base_model: meta-llama/Meta-Llama-3.1-8B-Instruct
 tags:
+- llama-factory
+- lora
 - trl
 - kto
-- llama-factory
 - generated_from_trainer
 model-index:
 - name: Llama-3.1-8B-Instruct-KTO-300
@@ -17,7 +18,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # Llama-3.1-8B-Instruct-KTO-300
 
-This model is a fine-tuned version of [meta-llama/Meta-Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct) on an unknown dataset.
+This model is a fine-tuned version of [meta-llama/Meta-Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct) on the bct_non_cot_kto_300 dataset.
 It achieves the following results on the evaluation set:
 - Loss: 0.2807
 - Rewards/chosen: 0.7524

all_results.json

@@ -0,0 +1,20 @@
+{
+    "epoch": 9.844444444444445,
+    "eval_logits/chosen": -3652706.0,
+    "eval_logits/rejected": -7997593.777777778,
+    "eval_logps/chosen": -12.900739034016928,
+    "eval_logps/rejected": -37.902974446614586,
+    "eval_loss": 0.2807026207447052,
+    "eval_rewards/chosen": 0.7524267832438151,
+    "eval_rewards/margins": 2.2908316718207464,
+    "eval_rewards/rejected": -1.5384048885769315,
+    "eval_runtime": 13.115,
+    "eval_samples_per_second": 4.575,
+    "eval_steps_per_second": 2.287,
+    "kl": 0.663330078125,
+    "total_flos": 2.9519334394822656e+16,
+    "train_loss": 0.3735494534174601,
+    "train_runtime": 1879.5093,
+    "train_samples_per_second": 2.873,
+    "train_steps_per_second": 0.176
+}

eval_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 9.844444444444445,
+    "eval_logits/chosen": -3652706.0,
+    "eval_logits/rejected": -7997593.777777778,
+    "eval_logps/chosen": -12.900739034016928,
+    "eval_logps/rejected": -37.902974446614586,
+    "eval_loss": 0.2807026207447052,
+    "eval_rewards/chosen": 0.7524267832438151,
+    "eval_rewards/margins": 2.2908316718207464,
+    "eval_rewards/rejected": -1.5384048885769315,
+    "eval_runtime": 13.115,
+    "eval_samples_per_second": 4.575,
+    "eval_steps_per_second": 2.287,
+    "kl": 0.663330078125
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 9.844444444444445,
+    "total_flos": 2.9519334394822656e+16,
+    "train_loss": 0.3735494534174601,
+    "train_runtime": 1879.5093,
+    "train_samples_per_second": 2.873,
+    "train_steps_per_second": 0.176
+}

trainer_state.json

@@ -0,0 +1,633 @@
+{
+  "best_metric": 0.2807026207447052,
+  "best_model_checkpoint": "saves/sycophancy/Llama-8B-3.1-Instruct/kto-300/checkpoint-300",
+  "epoch": 9.844444444444445,
+  "eval_steps": 50,
+  "global_step": 330,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.2962962962962963,
+      "grad_norm": 0.6046058535575867,
+      "kl": 4.538087844848633,
+      "learning_rate": 1.5151515151515152e-06,
+      "logits/chosen": -5854173.364705882,
+      "logits/rejected": -6722548.053333334,
+      "logps/chosen": -16.62459357766544,
+      "logps/rejected": -19.6134912109375,
+      "loss": 0.5008,
+      "rewards/chosen": -0.001938680690877578,
+      "rewards/margins": -0.006729768795125625,
+      "rewards/rejected": 0.004791088104248047,
+      "step": 10
+    },
+    {
+      "epoch": 0.5925925925925926,
+      "grad_norm": 0.5337051153182983,
+      "kl": 3.5176525115966797,
+      "learning_rate": 3.0303030303030305e-06,
+      "logits/chosen": -5356078.12987013,
+      "logits/rejected": -8177576.096385542,
+      "logps/chosen": -16.870456447849026,
+      "logps/rejected": -19.49406708866717,
+      "loss": 0.4995,
+      "rewards/chosen": 0.0029797325660656025,
+      "rewards/margins": 0.004846458838143048,
+      "rewards/rejected": -0.0018667262720774456,
+      "step": 20
+    },
+    {
+      "epoch": 0.8888888888888888,
+      "grad_norm": 1.3099695444107056,
+      "kl": 5.179463863372803,
+      "learning_rate": 4.5454545454545455e-06,
+      "logits/chosen": -6825626.482758621,
+      "logits/rejected": -7491633.095890411,
+      "logps/chosen": -17.327173693426722,
+      "logps/rejected": -19.826569857662673,
+      "loss": 0.4998,
+      "rewards/chosen": 0.005853150767841558,
+      "rewards/margins": 0.0036841889799645406,
+      "rewards/rejected": 0.0021689617878770176,
+      "step": 30
+    },
+    {
+      "epoch": 1.1925925925925926,
+      "grad_norm": 0.740423321723938,
+      "kl": 6.535464763641357,
+      "learning_rate": 4.993149937871306e-06,
+      "logits/chosen": -6410937.173333333,
+      "logits/rejected": -6861577.035294117,
+      "logps/chosen": -17.7991552734375,
+      "logps/rejected": -19.352366727941178,
+      "loss": 0.4997,
+      "rewards/chosen": 0.014202831586201985,
+      "rewards/margins": 0.004197974579007018,
+      "rewards/rejected": 0.010004857007194968,
+      "step": 40
+    },
+    {
+      "epoch": 1.488888888888889,
+      "grad_norm": 0.7895795702934265,
+      "kl": 6.706003189086914,
+      "learning_rate": 4.959688949822748e-06,
+      "logits/chosen": -5263670.325581395,
+      "logits/rejected": -7425049.945945946,
+      "logps/chosen": -17.420151821402616,
+      "logps/rejected": -20.32705276076858,
+      "loss": 0.4973,
+      "rewards/chosen": 0.03406415983688,
+      "rewards/margins": 0.018683696752220938,
+      "rewards/rejected": 0.015380463084659061,
+      "step": 50
+    },
+    {
+      "epoch": 1.488888888888889,
+      "eval_logits/chosen": -5131056.333333333,
+      "eval_logits/rejected": -8269521.777777778,
+      "eval_logps/chosen": -20.01836903889974,
+      "eval_logps/rejected": -22.1998291015625,
+      "eval_loss": 0.49953240156173706,
+      "eval_rewards/chosen": 0.04066378871599833,
+      "eval_rewards/margins": 0.008754322926203408,
+      "eval_rewards/rejected": 0.03190946578979492,
+      "eval_runtime": 13.3665,
+      "eval_samples_per_second": 4.489,
+      "eval_steps_per_second": 2.244,
+      "kl": 4.248124122619629,
+      "step": 50
+    },
+    {
+      "epoch": 1.7851851851851852,
+      "grad_norm": 1.0223308801651,
+      "kl": 10.039009094238281,
+      "learning_rate": 4.8987324340362445e-06,
+      "logits/chosen": -6110996.8,
+      "logits/rejected": -6743119.2,
+      "logps/chosen": -15.2028076171875,
+      "logps/rejected": -17.942332458496093,
+      "loss": 0.4959,
+      "rewards/chosen": 0.060177081823349,
+      "rewards/margins": 0.03066384792327881,
+      "rewards/rejected": 0.029513233900070192,
+      "step": 60
+    },
+    {
+      "epoch": 2.088888888888889,
+      "grad_norm": 0.8759526014328003,
+      "kl": 11.317947387695312,
+      "learning_rate": 4.810961790316731e-06,
+      "logits/chosen": -6347121.116279069,
+      "logits/rejected": -7174790.918918919,
+      "logps/chosen": -17.81520257994186,
+      "logps/rejected": -20.214766218855573,
+      "loss": 0.4929,
+      "rewards/chosen": 0.10434515531672987,
+      "rewards/margins": 0.05541059575239717,
+      "rewards/rejected": 0.0489345595643327,
+      "step": 70
+    },
+    {
+      "epoch": 2.3851851851851853,
+      "grad_norm": 0.9635448455810547,
+      "kl": 14.32188606262207,
+      "learning_rate": 4.697358159051549e-06,
+      "logits/chosen": -6680833.215189873,
+      "logits/rejected": -6814345.481481481,
+      "logps/chosen": -14.122315998318829,
+      "logps/rejected": -18.411060474537038,
+      "loss": 0.4833,
+      "rewards/chosen": 0.18152446988262708,
+      "rewards/margins": 0.1343007253136257,
+      "rewards/rejected": 0.04722374456900137,
+      "step": 80
+    },
+    {
+      "epoch": 2.6814814814814816,
+      "grad_norm": 1.1595202684402466,
+      "kl": 22.486248016357422,
+      "learning_rate": 4.559191453574582e-06,
+      "logits/chosen": -5439367.191011236,
+      "logits/rejected": -7333265.126760564,
+      "logps/chosen": -15.327505047401685,
+      "logps/rejected": -19.205719423965668,
+      "loss": 0.4725,
+      "rewards/chosen": 0.27227440308988765,
+      "rewards/margins": 0.2085925856993383,
+      "rewards/rejected": 0.06368181739054934,
+      "step": 90
+    },
+    {
+      "epoch": 2.977777777777778,
+      "grad_norm": 1.3290456533432007,
+      "kl": 32.22382354736328,
+      "learning_rate": 4.398006164494358e-06,
+      "logits/chosen": -6466890.520547945,
+      "logits/rejected": -7224656.91954023,
+      "logps/chosen": -12.733187740796232,
+      "logps/rejected": -18.703230233028016,
+      "loss": 0.4683,
+      "rewards/chosen": 0.3788376377053457,
+      "rewards/margins": 0.31561340427533824,
+      "rewards/rejected": 0.06322423343000741,
+      "step": 100
+    },
+    {
+      "epoch": 2.977777777777778,
+      "eval_logits/chosen": -4839914.0,
+      "eval_logits/rejected": -8194471.111111111,
+      "eval_logps/chosen": -16.739178975423176,
+      "eval_logps/rejected": -22.005423651801216,
+      "eval_loss": 0.4692111313343048,
+      "eval_rewards/chosen": 0.36858288447062176,
+      "eval_rewards/margins": 0.3172328074773153,
+      "eval_rewards/rejected": 0.05135007699330648,
+      "eval_runtime": 13.3695,
+      "eval_samples_per_second": 4.488,
+      "eval_steps_per_second": 2.244,
+      "kl": 12.38624095916748,
+      "step": 100
+    },
+    {
+      "epoch": 3.2814814814814817,
+      "grad_norm": 2.0513880252838135,
+      "kl": 9.37839412689209,
+      "learning_rate": 4.215604094671835e-06,
+      "logits/chosen": -4592428.631578947,
+      "logits/rejected": -7237347.047619048,
+      "logps/chosen": -12.766847309313322,
+      "logps/rejected": -21.028982979910715,
+      "loss": 0.4456,
+      "rewards/chosen": 0.38844979436774,
+      "rewards/margins": 0.4612913705352554,
+      "rewards/rejected": -0.07284157616751534,
+      "step": 110
+    },
+    {
+      "epoch": 3.5777777777777775,
+      "grad_norm": 1.8086739778518677,
+      "kl": 4.136274337768555,
+      "learning_rate": 4.014024217844167e-06,
+      "logits/chosen": -5455928.847058823,
+      "logits/rejected": -6883678.72,
+      "logps/chosen": -12.671711282169118,
+      "logps/rejected": -21.248759765625,
+      "loss": 0.4288,
+      "rewards/chosen": 0.4551482256721048,
+      "rewards/margins": 0.5749030042162129,
+      "rewards/rejected": -0.11975477854410807,
+      "step": 120
+    },
+    {
+      "epoch": 3.8740740740740742,
+      "grad_norm": 1.9404373168945312,
+      "kl": 6.438965797424316,
+      "learning_rate": 3.7955198860439892e-06,
+      "logits/chosen": -6478493.268292683,
+      "logits/rejected": -6934217.025641026,
+      "logps/chosen": -13.174736209032012,
+      "logps/rejected": -20.87556653145032,
+      "loss": 0.4103,
+      "rewards/chosen": 0.4654149311344798,
+      "rewards/margins": 0.7423959914559942,
+      "rewards/rejected": -0.2769810603215144,
+      "step": 130
+    },
+    {
+      "epoch": 4.177777777777778,
+      "grad_norm": 1.794080376625061,
+      "kl": 3.989973306655884,
+      "learning_rate": 3.5625336406000752e-06,
+      "logits/chosen": -5239293.155555556,
+      "logits/rejected": -6854407.314285714,
+      "logps/chosen": -12.975401475694444,
+      "logps/rejected": -24.620901925223215,
+      "loss": 0.3852,
+      "rewards/chosen": 0.4861636691623264,
+      "rewards/margins": 0.9952043926905072,
+      "rewards/rejected": -0.5090407235281807,
+      "step": 140
+    },
+    {
+      "epoch": 4.474074074074074,
+      "grad_norm": 1.1039807796478271,
+      "kl": 6.615535736083984,
+      "learning_rate": 3.3176699082935546e-06,
+      "logits/chosen": -5551139.764705882,
+      "logits/rejected": -6892623.36,
+      "logps/chosen": -12.373483455882353,
+      "logps/rejected": -26.899697265625,
+      "loss": 0.3709,
+      "rewards/chosen": 0.5205115823184743,
+      "rewards/margins": 1.1443341124291515,
+      "rewards/rejected": -0.6238225301106771,
+      "step": 150
+    },
+    {
+      "epoch": 4.474074074074074,
+      "eval_logits/chosen": -4248186.666666667,
+      "eval_logits/rejected": -8021952.0,
+      "eval_logps/chosen": -14.744425455729166,
+      "eval_logps/rejected": -28.396257188585068,
+      "eval_loss": 0.3691816031932831,
+      "eval_rewards/chosen": 0.568058172861735,
+      "eval_rewards/margins": 1.155791653527154,
+      "eval_rewards/rejected": -0.5877334806654189,
+      "eval_runtime": 13.365,
+      "eval_samples_per_second": 4.489,
+      "eval_steps_per_second": 2.245,
+      "kl": 2.0649871826171875,
+      "step": 150
+    },
+    {
+      "epoch": 4.770370370370371,
+      "grad_norm": 1.5495309829711914,
+      "kl": 5.70380163192749,
+      "learning_rate": 3.0636658878845116e-06,
+      "logits/chosen": -3638242.086956522,
+      "logits/rejected": -7199811.516483516,
+      "logps/chosen": -13.276876698369565,
+      "logps/rejected": -26.58016665951236,
+      "loss": 0.3675,
+      "rewards/chosen": 0.40610935377038043,
+      "rewards/margins": 1.2144003593346404,
+      "rewards/rejected": -0.8082910055642599,
+      "step": 160
+    },
+    {
+      "epoch": 5.074074074074074,
+      "grad_norm": 3.531503200531006,
+      "kl": 3.2827072143554688,
+      "learning_rate": 2.803360952452705e-06,
+      "logits/chosen": -5859259.2,
+      "logits/rejected": -5897748.8,
+      "logps/chosen": -10.869898986816406,
+      "logps/rejected": -31.441567993164064,
+      "loss": 0.3254,
+      "rewards/chosen": 0.5347123622894288,
+      "rewards/margins": 1.6758206844329835,
+      "rewards/rejected": -1.1411083221435547,
+      "step": 170
+    },
+    {
+      "epoch": 5.37037037037037,
+      "grad_norm": 1.9355641603469849,
+      "kl": 6.933535575866699,
+      "learning_rate": 2.53966490958702e-06,
+      "logits/chosen": -5240330.541176471,
+      "logits/rejected": -6676705.28,
+      "logps/chosen": -14.63643008961397,
+      "logps/rejected": -31.603450520833334,
+      "loss": 0.3377,
+      "rewards/chosen": 0.4823792401482077,
+      "rewards/margins": 1.6440772381950826,
+      "rewards/rejected": -1.161697998046875,
+      "step": 180
+    },
+    {
+      "epoch": 5.666666666666667,
+      "grad_norm": 2.1604580879211426,
+      "kl": 0.7298488616943359,
+      "learning_rate": 2.275525474225771e-06,
+      "logits/chosen": -3473293.7721518986,
+      "logits/rejected": -6992762.469135802,
+      "logps/chosen": -12.097724238528482,
+      "logps/rejected": -32.88131148726852,
+      "loss": 0.3295,
+      "rewards/chosen": 0.3602371698693384,
+      "rewards/margins": 1.7327642099505982,
+      "rewards/rejected": -1.3725270400812597,
+      "step": 190
+    },
+    {
+      "epoch": 5.962962962962963,
+      "grad_norm": 1.2311595678329468,
+      "kl": 0.0,
+      "learning_rate": 2.013895317751323e-06,
+      "logits/chosen": -5824722.75862069,
+      "logits/rejected": -6419050.95890411,
+      "logps/chosen": -12.355808301903735,
+      "logps/rejected": -34.44381421232877,
+      "loss": 0.3219,
+      "rewards/chosen": 0.4446083156541846,
+      "rewards/margins": 1.962539692072207,
+      "rewards/rejected": -1.5179313764180222,
+      "step": 200
+    },
+    {
+      "epoch": 5.962962962962963,
+      "eval_logits/chosen": -3768132.0,
+      "eval_logits/rejected": -7958945.777777778,
+      "eval_logps/chosen": -14.012957255045572,
+      "eval_logps/rejected": -34.88158840603299,
+      "eval_loss": 0.30443304777145386,
+      "eval_rewards/chosen": 0.6412049929300944,
+      "eval_rewards/margins": 1.8774711290995278,
+      "eval_rewards/rejected": -1.2362661361694336,
+      "eval_runtime": 13.3919,
+      "eval_samples_per_second": 4.48,
+      "eval_steps_per_second": 2.24,
+      "kl": 1.4612131118774414,
+      "step": 200
+    },
+    {
+      "epoch": 6.266666666666667,
+      "grad_norm": 0.925562858581543,
+      "kl": 0.0,
+      "learning_rate": 1.7576990616793139e-06,
+      "logits/chosen": -4411608.094117647,
+      "logits/rejected": -6808450.986666666,
+      "logps/chosen": -11.909748391544118,
+      "logps/rejected": -36.503896484375,
+      "loss": 0.3074,
+      "rewards/chosen": 0.45500959508559286,
+      "rewards/margins": 2.139190096387676,
+      "rewards/rejected": -1.6841805013020834,
+      "step": 210
+    },
+    {
+      "epoch": 6.562962962962963,
+      "grad_norm": 1.7327585220336914,
+      "kl": 0.9251070022583008,
+      "learning_rate": 1.509800584902108e-06,
+      "logits/chosen": -4351900.049382716,
+      "logits/rejected": -7359590.075949367,
+      "logps/chosen": -11.922060366030093,
+      "logps/rejected": -35.884592563291136,
+      "loss": 0.3064,
+      "rewards/chosen": 0.49753245600947626,
+      "rewards/margins": 2.070165450842348,
+      "rewards/rejected": -1.5726329948328719,
+      "step": 220
+    },
+    {
+      "epoch": 6.859259259259259,
+      "grad_norm": 4.674618721008301,
+      "kl": 4.636810302734375,
+      "learning_rate": 1.2729710099410802e-06,
+      "logits/chosen": -5216737.129411764,
+      "logits/rejected": -5893650.773333333,
+      "logps/chosen": -11.36463551240809,
+      "logps/rejected": -36.05122395833333,
+      "loss": 0.2968,
+      "rewards/chosen": 0.5570346607881433,
+      "rewards/margins": 2.3463927743949142,
+      "rewards/rejected": -1.7893581136067709,
+      "step": 230
+    },
+    {
+      "epoch": 7.162962962962963,
+      "grad_norm": 1.30046808719635,
+      "kl": 7.6185832023620605,
+      "learning_rate": 1.049857726072005e-06,
+      "logits/chosen": -5359469.894736842,
+      "logits/rejected": -6327644.952380952,
+      "logps/chosen": -12.688995361328125,
+      "logps/rejected": -39.67555454799107,
+      "loss": 0.2733,
+      "rewards/chosen": 0.538983194451583,
+      "rewards/margins": 2.463327938452699,
+      "rewards/rejected": -1.924344744001116,
+      "step": 240
+    },
+    {
+      "epoch": 7.459259259259259,
+      "grad_norm": 1.7867010831832886,
+      "kl": 0.22487592697143555,
+      "learning_rate": 8.4295479559726e-07,
+      "logits/chosen": -5227298.285714285,
+      "logits/rejected": -6265175.578947368,
+      "logps/chosen": -12.42079089936756,
+      "logps/rejected": -38.650663677014805,
+      "loss": 0.2882,
+      "rewards/chosen": 0.5529203414916992,
+      "rewards/margins": 2.4198797627499227,
+      "rewards/rejected": -1.8669594212582237,
+      "step": 250
+    },
+    {
+      "epoch": 7.459259259259259,
+      "eval_logits/chosen": -3685730.3333333335,
+      "eval_logits/rejected": -7991116.444444444,
+      "eval_logps/chosen": -13.086176554361979,
+      "eval_logps/rejected": -37.038543701171875,
+      "eval_loss": 0.2863730192184448,
+      "eval_rewards/chosen": 0.7338830629984537,
+      "eval_rewards/margins": 2.1858451101515026,
+      "eval_rewards/rejected": -1.451962047153049,
+      "eval_runtime": 13.3948,
+      "eval_samples_per_second": 4.479,
+      "eval_steps_per_second": 2.24,
+      "kl": 0.8725299835205078,
+      "step": 250
+    },
+    {
+      "epoch": 7.7555555555555555,
+      "grad_norm": 9.230957984924316,
+      "kl": 9.669639587402344,
+      "learning_rate": 6.545750740770338e-07,
+      "logits/chosen": -4307272.481927711,
+      "logits/rejected": -6674962.285714285,
+      "logps/chosen": -9.58886277532003,
+      "logps/rejected": -38.9914741020698,
+      "loss": 0.2772,
+      "rewards/chosen": 0.610903268837067,
+      "rewards/margins": 2.649724798869535,
+      "rewards/rejected": -2.0388215300324677,
+      "step": 260
+    },
+    {
+      "epoch": 8.059259259259258,
+      "grad_norm": 1.193448781967163,
+      "kl": 3.7342185974121094,
+      "learning_rate": 4.868243561723535e-07,
+      "logits/chosen": -5346473.643835616,
+      "logits/rejected": -6204017.287356322,
+      "logps/chosen": -14.12191647046233,
+      "logps/rejected": -39.27419517780172,
+      "loss": 0.2837,
+      "rewards/chosen": 0.40948598678797893,
+      "rewards/margins": 2.4313709488479645,
+      "rewards/rejected": -2.0218849620599855,
+      "step": 270
+    },
+    {
+      "epoch": 8.355555555555556,
+      "grad_norm": 1.557085633277893,
+      "kl": 0.0,
+      "learning_rate": 3.4157783610952263e-07,
+      "logits/chosen": -5714791.696202531,
+      "logits/rejected": -6967618.37037037,
+      "logps/chosen": -12.57650254647943,
+      "logps/rejected": -41.757679880401234,
+      "loss": 0.2867,
+      "rewards/chosen": 0.3982359004926078,
+      "rewards/margins": 2.571001561661589,
+      "rewards/rejected": -2.1727656611689814,
+      "step": 280
+    },
+    {
+      "epoch": 8.651851851851852,
+      "grad_norm": 1.3718146085739136,
+      "kl": 2.2029647827148438,
+      "learning_rate": 2.2045914590165252e-07,
+      "logits/chosen": -4378275.720930233,
+      "logits/rejected": -6295708.972972973,
+      "logps/chosen": -12.955159032067588,
+      "logps/rejected": -36.683788402660475,
+      "loss": 0.2996,
+      "rewards/chosen": 0.5802536010742188,
+      "rewards/margins": 2.36948992754962,
+      "rewards/rejected": -1.7892363264754012,
+      "step": 290
+    },
+    {
+      "epoch": 8.948148148148148,
+      "grad_norm": 2.0397913455963135,
+      "kl": 0.0,
+      "learning_rate": 1.2482220564763669e-07,
+      "logits/chosen": -5213185.488372093,
+      "logits/rejected": -6215361.72972973,
+      "logps/chosen": -9.879340593204942,
+      "logps/rejected": -41.936137431376686,
+      "loss": 0.2574,
+      "rewards/chosen": 0.6615725228952807,
+      "rewards/margins": 2.8659850347723923,
+      "rewards/rejected": -2.2044125118771114,
+      "step": 300
+    },
+    {
+      "epoch": 8.948148148148148,
+      "eval_logits/chosen": -3652706.0,
+      "eval_logits/rejected": -7997593.777777778,
+      "eval_logps/chosen": -12.900739034016928,
+      "eval_logps/rejected": -37.902974446614586,
+      "eval_loss": 0.2807026207447052,
+      "eval_rewards/chosen": 0.7524267832438151,
+      "eval_rewards/margins": 2.2908316718207464,
+      "eval_rewards/rejected": -1.5384048885769315,
+      "eval_runtime": 13.382,
+      "eval_samples_per_second": 4.484,
+      "eval_steps_per_second": 2.242,
+      "kl": 0.663330078125,
+      "step": 300
+    },
+    {
+      "epoch": 9.251851851851852,
+      "grad_norm": 1.2545690536499023,
+      "kl": 6.414087772369385,
+      "learning_rate": 5.573608879422876e-08,
+      "logits/chosen": -4187727.36,
+      "logits/rejected": -7287492.517647059,
+      "logps/chosen": -12.690235188802083,
+      "logps/rejected": -43.10070657169118,
+      "loss": 0.2534,
+      "rewards/chosen": 0.5371046956380209,
+      "rewards/margins": 2.892158586090686,
+      "rewards/rejected": -2.355053890452665,
+      "step": 310
+    },
+    {
+      "epoch": 9.548148148148147,
+      "grad_norm": 2.7359890937805176,
+      "kl": 0.0,
+      "learning_rate": 1.3973071544233219e-08,
+      "logits/chosen": -4982654.379746836,
+      "logits/rejected": -7114389.333333333,
+      "logps/chosen": -9.849223849139635,
+      "logps/rejected": -36.88050672743056,
+      "loss": 0.2753,
+      "rewards/chosen": 0.7713173492045342,
+      "rewards/margins": 2.553770721359241,
+      "rewards/rejected": -1.782453372154707,
+      "step": 320
+    },
+    {
+      "epoch": 9.844444444444445,
+      "grad_norm": 1.730713129043579,
+      "kl": 0.3280200958251953,
+      "learning_rate": 0.0,
+      "logits/chosen": -4839422.528735632,
+      "logits/rejected": -6777751.671232876,
+      "logps/chosen": -11.5740931719199,
+      "logps/rejected": -41.290196248929796,
+      "loss": 0.2887,
+      "rewards/chosen": 0.46058505705033226,
+      "rewards/margins": 2.5596534766243906,
+      "rewards/rejected": -2.099068419574058,
+      "step": 330
+    },
+    {
+      "epoch": 9.844444444444445,
+      "step": 330,
+      "total_flos": 2.9519334394822656e+16,
+      "train_loss": 0.3735494534174601,
+      "train_runtime": 1879.5093,
+      "train_samples_per_second": 2.873,
+      "train_steps_per_second": 0.176
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 330,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.9519334394822656e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}