{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.0,
  "eval_steps": 500,
  "global_step": 68,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.03018867924528302,
      "grad_norm": 138.31220610930498,
      "learning_rate": 0.0,
      "logits/chosen": -2.109619140625,
      "logits/rejected": -2.166259765625,
      "logps/chosen": -515.4921875,
      "logps/rejected": -403.890625,
      "loss": 0.69140625,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.06037735849056604,
      "grad_norm": 103.0810019302269,
      "learning_rate": 7.142857142857142e-08,
      "logits/chosen": -2.19482421875,
      "logits/rejected": -2.22802734375,
      "logps/chosen": -248.8828125,
      "logps/rejected": -282.2421875,
      "loss": 0.69140625,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2
    },
    {
      "epoch": 0.09056603773584905,
      "grad_norm": 110.9946117237372,
      "learning_rate": 1.4285714285714285e-07,
      "logits/chosen": -2.11962890625,
      "logits/rejected": -2.154052734375,
      "logps/chosen": -385.7265625,
      "logps/rejected": -336.3046875,
      "loss": 0.66015625,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": 0.044574737548828125,
      "rewards/margins": 0.07429122924804688,
      "rewards/rejected": -0.02971649169921875,
      "step": 3
    },
    {
      "epoch": 0.12075471698113208,
      "grad_norm": 134.1044963548161,
      "learning_rate": 2.1428571428571426e-07,
      "logits/chosen": -2.161865234375,
      "logits/rejected": -2.15185546875,
      "logps/chosen": -416.0,
      "logps/rejected": -410.796875,
      "loss": 0.7122802734375,
      "rewards/accuracies": 0.28125,
      "rewards/chosen": -0.02658843994140625,
      "rewards/margins": -0.01718902587890625,
      "rewards/rejected": -0.0093841552734375,
      "step": 4
    },
    {
      "epoch": 0.1509433962264151,
      "grad_norm": 181.29875886515373,
      "learning_rate": 2.857142857142857e-07,
      "logits/chosen": -2.147216796875,
      "logits/rejected": -2.2158203125,
      "logps/chosen": -337.8984375,
      "logps/rejected": -268.15625,
      "loss": 0.694091796875,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01094818115234375,
      "rewards/margins": -0.00392913818359375,
      "rewards/rejected": -0.007038116455078125,
      "step": 5
    },
    {
      "epoch": 0.1811320754716981,
      "grad_norm": 127.48792819761363,
      "learning_rate": 3.5714285714285716e-07,
      "logits/chosen": -2.21826171875,
      "logits/rejected": -2.229248046875,
      "logps/chosen": -425.4140625,
      "logps/rejected": -476.859375,
      "loss": 0.672119140625,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": 0.019550323486328125,
      "rewards/margins": 0.055568695068359375,
      "rewards/rejected": -0.03597259521484375,
      "step": 6
    },
    {
      "epoch": 0.21132075471698114,
      "grad_norm": 104.85395093542431,
      "learning_rate": 4.285714285714285e-07,
      "logits/chosen": -2.1513671875,
      "logits/rejected": -2.2177734375,
      "logps/chosen": -324.6328125,
      "logps/rejected": -291.40625,
      "loss": 0.690185546875,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.00390625,
      "rewards/margins": 0.00390625,
      "rewards/rejected": -0.00782012939453125,
      "step": 7
    },
    {
      "epoch": 0.24150943396226415,
      "grad_norm": 110.67144826119416,
      "learning_rate": 5e-07,
      "logits/chosen": -2.196533203125,
      "logits/rejected": -2.22265625,
      "logps/chosen": -309.921875,
      "logps/rejected": -344.0,
      "loss": 0.6737060546875,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.043010711669921875,
      "rewards/margins": 0.04144287109375,
      "rewards/rejected": 0.00156402587890625,
      "step": 8
    },
    {
      "epoch": 0.27169811320754716,
      "grad_norm": 113.76857363403872,
      "learning_rate": 4.996685224712076e-07,
      "logits/chosen": -2.184326171875,
      "logits/rejected": -2.20166015625,
      "logps/chosen": -269.8671875,
      "logps/rejected": -311.5625,
      "loss": 0.6883544921875,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0031280517578125,
      "rewards/margins": 0.01567840576171875,
      "rewards/rejected": -0.018768310546875,
      "step": 9
    },
    {
      "epoch": 0.3018867924528302,
      "grad_norm": 116.528380644011,
      "learning_rate": 4.986749689036473e-07,
      "logits/chosen": -2.19580078125,
      "logits/rejected": -2.22314453125,
      "logps/chosen": -328.390625,
      "logps/rejected": -301.921875,
      "loss": 0.666748046875,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0203399658203125,
      "rewards/margins": 0.05944061279296875,
      "rewards/rejected": -0.03910064697265625,
      "step": 10
    },
    {
      "epoch": 0.3320754716981132,
      "grad_norm": 114.41976508331743,
      "learning_rate": 4.970219740227693e-07,
      "logits/chosen": -2.25244140625,
      "logits/rejected": -2.267578125,
      "logps/chosen": -230.1171875,
      "logps/rejected": -271.609375,
      "loss": 0.6549072265625,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.014858245849609375,
      "rewards/margins": 0.08209609985351562,
      "rewards/rejected": -0.06724929809570312,
      "step": 11
    },
    {
      "epoch": 0.3622641509433962,
      "grad_norm": 127.78939520989171,
      "learning_rate": 4.947139212738394e-07,
      "logits/chosen": -2.19775390625,
      "logits/rejected": -2.24169921875,
      "logps/chosen": -368.9609375,
      "logps/rejected": -386.296875,
      "loss": 0.6553955078125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.028934478759765625,
      "rewards/margins": 0.08996200561523438,
      "rewards/rejected": -0.06099700927734375,
      "step": 12
    },
    {
      "epoch": 0.39245283018867927,
      "grad_norm": 160.43709922042387,
      "learning_rate": 4.917569311978301e-07,
      "logits/chosen": -2.16552734375,
      "logits/rejected": -2.205322265625,
      "logps/chosen": -325.640625,
      "logps/rejected": -357.0859375,
      "loss": 0.640380859375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03597259521484375,
      "rewards/margins": 0.13688278198242188,
      "rewards/rejected": -0.10087966918945312,
      "step": 13
    },
    {
      "epoch": 0.4226415094339623,
      "grad_norm": 128.59812939912248,
      "learning_rate": 4.881588452008456e-07,
      "logits/chosen": -2.119140625,
      "logits/rejected": -2.1953125,
      "logps/chosen": -354.609375,
      "logps/rejected": -295.546875,
      "loss": 0.63433837890625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.016429901123046875,
      "rewards/margins": 0.13383102416992188,
      "rewards/rejected": -0.11734771728515625,
      "step": 14
    },
    {
      "epoch": 0.4528301886792453,
      "grad_norm": 120.56983987783293,
      "learning_rate": 4.839292047601234e-07,
      "logits/chosen": -2.1904296875,
      "logits/rejected": -2.263916015625,
      "logps/chosen": -415.4140625,
      "logps/rejected": -337.859375,
      "loss": 0.629638671875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0328369140625,
      "rewards/margins": 0.1298675537109375,
      "rewards/rejected": -0.16265869140625,
      "step": 15
    },
    {
      "epoch": 0.4830188679245283,
      "grad_norm": 120.33867564906456,
      "learning_rate": 4.790792261217512e-07,
      "logits/chosen": -2.14404296875,
      "logits/rejected": -2.19482421875,
      "logps/chosen": -369.359375,
      "logps/rejected": -369.453125,
      "loss": 0.635009765625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03910064697265625,
      "rewards/margins": 0.14395904541015625,
      "rewards/rejected": -0.183013916015625,
      "step": 16
    },
    {
      "epoch": 0.5132075471698113,
      "grad_norm": 102.63014934519077,
      "learning_rate": 4.7362177055719886e-07,
      "logits/chosen": -2.176513671875,
      "logits/rejected": -2.178466796875,
      "logps/chosen": -244.3671875,
      "logps/rejected": -294.7890625,
      "loss": 0.60015869140625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.0203399658203125,
      "rewards/margins": 0.21128082275390625,
      "rewards/rejected": -0.19091033935546875,
      "step": 17
    },
    {
      "epoch": 0.5433962264150943,
      "grad_norm": 106.93923521151912,
      "learning_rate": 4.675713102575388e-07,
      "logits/chosen": -2.198974609375,
      "logits/rejected": -2.231689453125,
      "logps/chosen": -320.1171875,
      "logps/rejected": -339.6953125,
      "loss": 0.5753173828125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.01172637939453125,
      "rewards/margins": 0.2574005126953125,
      "rewards/rejected": -0.24556732177734375,
      "step": 18
    },
    {
      "epoch": 0.5735849056603773,
      "grad_norm": 96.40829818733613,
      "learning_rate": 4.6094388995579634e-07,
      "logits/chosen": -2.16357421875,
      "logits/rejected": -2.264404296875,
      "logps/chosen": -367.75,
      "logps/rejected": -296.609375,
      "loss": 0.61151123046875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.09149932861328125,
      "rewards/margins": 0.1924896240234375,
      "rewards/rejected": -0.2839202880859375,
      "step": 19
    },
    {
      "epoch": 0.6037735849056604,
      "grad_norm": 104.01103241458425,
      "learning_rate": 4.537570843792028e-07,
      "logits/chosen": -2.16064453125,
      "logits/rejected": -2.170654296875,
      "logps/chosen": -340.203125,
      "logps/rejected": -415.078125,
      "loss": 0.55609130859375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.032871246337890625,
      "rewards/margins": 0.328399658203125,
      "rewards/rejected": -0.36133575439453125,
      "step": 20
    },
    {
      "epoch": 0.6339622641509434,
      "grad_norm": 102.48543340920472,
      "learning_rate": 4.460299516441776e-07,
      "logits/chosen": -2.183837890625,
      "logits/rejected": -2.28662109375,
      "logps/chosen": -474.2421875,
      "logps/rejected": -356.5234375,
      "loss": 0.54766845703125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0742950439453125,
      "rewards/margins": 0.3504180908203125,
      "rewards/rejected": -0.42476654052734375,
      "step": 21
    },
    {
      "epoch": 0.6641509433962264,
      "grad_norm": 101.64915161645655,
      "learning_rate": 4.3778298271762993e-07,
      "logits/chosen": -2.17626953125,
      "logits/rejected": -2.209716796875,
      "logps/chosen": -303.9921875,
      "logps/rejected": -357.921875,
      "loss": 0.52899169921875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.03277587890625,
      "rewards/margins": 0.3879852294921875,
      "rewards/rejected": -0.3552093505859375,
      "step": 22
    },
    {
      "epoch": 0.6943396226415094,
      "grad_norm": 99.4155872940957,
      "learning_rate": 4.290380470785983e-07,
      "logits/chosen": -2.16943359375,
      "logits/rejected": -2.249267578125,
      "logps/chosen": -401.28125,
      "logps/rejected": -417.7734375,
      "loss": 0.48834228515625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.004669189453125,
      "rewards/margins": 0.5399398803710938,
      "rewards/rejected": -0.5444259643554688,
      "step": 23
    },
    {
      "epoch": 0.7245283018867924,
      "grad_norm": 90.52873672236639,
      "learning_rate": 4.198183347243233e-07,
      "logits/chosen": -2.23193359375,
      "logits/rejected": -2.27392578125,
      "logps/chosen": -373.828125,
      "logps/rejected": -369.609375,
      "loss": 0.47064208984375,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.044586181640625,
      "rewards/margins": 0.5651931762695312,
      "rewards/rejected": -0.6098709106445312,
      "step": 24
    },
    {
      "epoch": 0.7547169811320755,
      "grad_norm": 89.70939620633418,
      "learning_rate": 4.101482946745438e-07,
      "logits/chosen": -2.24365234375,
      "logits/rejected": -2.263916015625,
      "logps/chosen": -296.6640625,
      "logps/rejected": -297.609375,
      "loss": 0.52313232421875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08766937255859375,
      "rewards/margins": 0.43157958984375,
      "rewards/rejected": -0.51934814453125,
      "step": 25
    },
    {
      "epoch": 0.7849056603773585,
      "grad_norm": 106.35938326021778,
      "learning_rate": 4.000535701370921e-07,
      "logits/chosen": -2.2880859375,
      "logits/rejected": -2.29931640625,
      "logps/chosen": -342.4609375,
      "logps/rejected": -469.5625,
      "loss": 0.4821319580078125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21507644653320312,
      "rewards/margins": 0.6137771606445312,
      "rewards/rejected": -0.8289947509765625,
      "step": 26
    },
    {
      "epoch": 0.8150943396226416,
      "grad_norm": 89.47071292174518,
      "learning_rate": 3.895609305067161e-07,
      "logits/chosen": -2.19677734375,
      "logits/rejected": -2.234375,
      "logps/chosen": -310.375,
      "logps/rejected": -317.515625,
      "loss": 0.460784912109375,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.15332794189453125,
      "rewards/margins": 0.607879638671875,
      "rewards/rejected": -0.76116943359375,
      "step": 27
    },
    {
      "epoch": 0.8452830188679246,
      "grad_norm": 82.71880874204284,
      "learning_rate": 3.786982003774577e-07,
      "logits/chosen": -2.19189453125,
      "logits/rejected": -2.216064453125,
      "logps/chosen": -238.6875,
      "logps/rejected": -300.8359375,
      "loss": 0.4767303466796875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2033233642578125,
      "rewards/margins": 0.59539794921875,
      "rewards/rejected": -0.7988433837890625,
      "step": 28
    },
    {
      "epoch": 0.8754716981132076,
      "grad_norm": 95.70798094262803,
      "learning_rate": 3.6749418575683e-07,
      "logits/chosen": -2.20166015625,
      "logits/rejected": -2.242431640625,
      "logps/chosen": -298.8203125,
      "logps/rejected": -324.796875,
      "loss": 0.4431915283203125,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.1024017333984375,
      "rewards/margins": 0.679779052734375,
      "rewards/rejected": -0.7819290161132812,
      "step": 29
    },
    {
      "epoch": 0.9056603773584906,
      "grad_norm": 99.73199372695603,
      "learning_rate": 3.559785976774652e-07,
      "logits/chosen": -2.2548828125,
      "logits/rejected": -2.306640625,
      "logps/chosen": -329.125,
      "logps/rejected": -343.3515625,
      "loss": 0.4751110076904297,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.26758575439453125,
      "rewards/margins": 0.6523895263671875,
      "rewards/rejected": -0.9197998046875,
      "step": 30
    },
    {
      "epoch": 0.9358490566037736,
      "grad_norm": 84.8861552760989,
      "learning_rate": 3.4418197340879627e-07,
      "logits/chosen": -2.24267578125,
      "logits/rejected": -2.27783203125,
      "logps/chosen": -388.859375,
      "logps/rejected": -397.546875,
      "loss": 0.3660297393798828,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17905426025390625,
      "rewards/margins": 1.132171630859375,
      "rewards/rejected": -1.311431884765625,
      "step": 31
    },
    {
      "epoch": 0.9660377358490566,
      "grad_norm": 84.17818925735007,
      "learning_rate": 3.321355954777087e-07,
      "logits/chosen": -2.186279296875,
      "logits/rejected": -2.217041015625,
      "logps/chosen": -303.9296875,
      "logps/rejected": -323.9375,
      "loss": 0.410308837890625,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.390289306640625,
      "rewards/margins": 0.807708740234375,
      "rewards/rejected": -1.1977081298828125,
      "step": 32
    },
    {
      "epoch": 0.9962264150943396,
      "grad_norm": 89.71705544962349,
      "learning_rate": 3.1987140871290236e-07,
      "logits/chosen": -2.248779296875,
      "logits/rejected": -2.27880859375,
      "logps/chosen": -322.6640625,
      "logps/rejected": -355.671875,
      "loss": 0.4383659362792969,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.3761749267578125,
      "rewards/margins": 0.867095947265625,
      "rewards/rejected": -1.2427978515625,
      "step": 33
    },
    {
      "epoch": 1.0,
      "grad_norm": 183.46893679118816,
      "learning_rate": 3.074219355329489e-07,
      "logits/chosen": -2.00390625,
      "logits/rejected": -2.03515625,
      "logps/chosen": -213.0,
      "logps/rejected": -232.5,
      "loss": 0.41015625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2001953125,
      "rewards/margins": 0.72607421875,
      "rewards/rejected": -0.92578125,
      "step": 34
    },
    {
      "epoch": 1.030188679245283,
      "grad_norm": 53.96764181203498,
      "learning_rate": 2.948201897026839e-07,
      "logits/chosen": -2.30712890625,
      "logits/rejected": -2.344970703125,
      "logps/chosen": -225.59375,
      "logps/rejected": -293.5,
      "loss": 0.3011283874511719,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03855133056640625,
      "rewards/margins": 1.488525390625,
      "rewards/rejected": -1.526702880859375,
      "step": 35
    },
    {
      "epoch": 1.060377358490566,
      "grad_norm": 62.43795666242199,
      "learning_rate": 2.8209958878663776e-07,
      "logits/chosen": -2.279296875,
      "logits/rejected": -2.30224609375,
      "logps/chosen": -333.53125,
      "logps/rejected": -399.6875,
      "loss": 0.2984161376953125,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.2995033264160156,
      "rewards/margins": 1.37225341796875,
      "rewards/rejected": -1.67138671875,
      "step": 36
    },
    {
      "epoch": 1.090566037735849,
      "grad_norm": 51.16866512137806,
      "learning_rate": 2.692938655316616e-07,
      "logits/chosen": -2.2197265625,
      "logits/rejected": -2.24462890625,
      "logps/chosen": -274.34375,
      "logps/rejected": -341.8125,
      "loss": 0.28313589096069336,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.3532257080078125,
      "rewards/margins": 1.56646728515625,
      "rewards/rejected": -1.920501708984375,
      "step": 37
    },
    {
      "epoch": 1.120754716981132,
      "grad_norm": 66.72467012226434,
      "learning_rate": 2.5643697841374715e-07,
      "logits/chosen": -2.1904296875,
      "logits/rejected": -2.17822265625,
      "logps/chosen": -216.140625,
      "logps/rejected": -349.375,
      "loss": 0.2605581283569336,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.30327606201171875,
      "rewards/margins": 1.6783447265625,
      "rewards/rejected": -1.98095703125,
      "step": 38
    },
    {
      "epoch": 1.150943396226415,
      "grad_norm": 62.738746790894204,
      "learning_rate": 2.435630215862529e-07,
      "logits/chosen": -2.218017578125,
      "logits/rejected": -2.300048828125,
      "logps/chosen": -327.2109375,
      "logps/rejected": -246.84375,
      "loss": 0.3275489807128906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.29560089111328125,
      "rewards/margins": 1.1746826171875,
      "rewards/rejected": -1.4700088500976562,
      "step": 39
    },
    {
      "epoch": 1.181132075471698,
      "grad_norm": 64.97314574211953,
      "learning_rate": 2.307061344683384e-07,
      "logits/chosen": -2.24853515625,
      "logits/rejected": -2.289794921875,
      "logps/chosen": -378.265625,
      "logps/rejected": -409.421875,
      "loss": 0.26163482666015625,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.325836181640625,
      "rewards/margins": 1.73931884765625,
      "rewards/rejected": -2.06549072265625,
      "step": 40
    },
    {
      "epoch": 1.211320754716981,
      "grad_norm": 70.926273227677,
      "learning_rate": 2.1790041121336222e-07,
      "logits/chosen": -2.247314453125,
      "logits/rejected": -2.322265625,
      "logps/chosen": -495.953125,
      "logps/rejected": -395.453125,
      "loss": 0.3062589168548584,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.44409942626953125,
      "rewards/margins": 1.86004638671875,
      "rewards/rejected": -2.304534912109375,
      "step": 41
    },
    {
      "epoch": 1.241509433962264,
      "grad_norm": 48.82537565235591,
      "learning_rate": 2.0517981029731612e-07,
      "logits/chosen": -2.271240234375,
      "logits/rejected": -2.32568359375,
      "logps/chosen": -385.1875,
      "logps/rejected": -355.1875,
      "loss": 0.22526264190673828,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.21823883056640625,
      "rewards/margins": 1.83306884765625,
      "rewards/rejected": -2.051666259765625,
      "step": 42
    },
    {
      "epoch": 1.271698113207547,
      "grad_norm": 78.04521274122948,
      "learning_rate": 1.9257806446705112e-07,
      "logits/chosen": -2.318359375,
      "logits/rejected": -2.340087890625,
      "logps/chosen": -283.8671875,
      "logps/rejected": -317.15625,
      "loss": 0.27616119384765625,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.51422119140625,
      "rewards/margins": 1.6151123046875,
      "rewards/rejected": -2.1287841796875,
      "step": 43
    },
    {
      "epoch": 1.3018867924528301,
      "grad_norm": 51.25580086600066,
      "learning_rate": 1.8012859128709764e-07,
      "logits/chosen": -2.290771484375,
      "logits/rejected": -2.2978515625,
      "logps/chosen": -425.6796875,
      "logps/rejected": -484.390625,
      "loss": 0.13787996768951416,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5703582763671875,
      "rewards/margins": 2.62646484375,
      "rewards/rejected": -3.19580078125,
      "step": 44
    },
    {
      "epoch": 1.3320754716981131,
      "grad_norm": 58.73013327845229,
      "learning_rate": 1.678644045222913e-07,
      "logits/chosen": -2.2548828125,
      "logits/rejected": -2.3232421875,
      "logps/chosen": -425.2578125,
      "logps/rejected": -417.0,
      "loss": 0.1794424057006836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3956565856933594,
      "rewards/margins": 2.46533203125,
      "rewards/rejected": -2.862060546875,
      "step": 45
    },
    {
      "epoch": 1.3622641509433961,
      "grad_norm": 62.34773299199525,
      "learning_rate": 1.5581802659120368e-07,
      "logits/chosen": -2.2158203125,
      "logits/rejected": -2.242431640625,
      "logps/chosen": -410.671875,
      "logps/rejected": -398.625,
      "loss": 0.1958155632019043,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.283416748046875,
      "rewards/margins": 2.351318359375,
      "rewards/rejected": -2.6341323852539062,
      "step": 46
    },
    {
      "epoch": 1.3924528301886792,
      "grad_norm": 55.39949009295557,
      "learning_rate": 1.4402140232253484e-07,
      "logits/chosen": -2.2275390625,
      "logits/rejected": -2.26171875,
      "logps/chosen": -332.5,
      "logps/rejected": -322.046875,
      "loss": 0.23157119750976562,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.635040283203125,
      "rewards/margins": 1.67864990234375,
      "rewards/rejected": -2.314208984375,
      "step": 47
    },
    {
      "epoch": 1.4226415094339622,
      "grad_norm": 49.0523957870456,
      "learning_rate": 1.3250581424317008e-07,
      "logits/chosen": -2.234375,
      "logits/rejected": -2.29443359375,
      "logps/chosen": -404.1796875,
      "logps/rejected": -373.6875,
      "loss": 0.1951136589050293,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.49471282958984375,
      "rewards/margins": 2.03558349609375,
      "rewards/rejected": -2.53033447265625,
      "step": 48
    },
    {
      "epoch": 1.4528301886792452,
      "grad_norm": 43.85948277562909,
      "learning_rate": 1.213017996225424e-07,
      "logits/chosen": -2.27734375,
      "logits/rejected": -2.277099609375,
      "logps/chosen": -396.1328125,
      "logps/rejected": -428.21875,
      "loss": 0.15965144336223602,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4569244384765625,
      "rewards/margins": 2.53271484375,
      "rewards/rejected": -2.9884033203125,
      "step": 49
    },
    {
      "epoch": 1.4830188679245282,
      "grad_norm": 67.30462143329488,
      "learning_rate": 1.1043906949328385e-07,
      "logits/chosen": -2.2587890625,
      "logits/rejected": -2.29833984375,
      "logps/chosen": -310.765625,
      "logps/rejected": -351.59375,
      "loss": 0.31104373931884766,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.7111129760742188,
      "rewards/margins": 1.73919677734375,
      "rewards/rejected": -2.45050048828125,
      "step": 50
    },
    {
      "epoch": 1.5132075471698112,
      "grad_norm": 54.97036033515127,
      "learning_rate": 9.994642986290797e-08,
      "logits/chosen": -2.265625,
      "logits/rejected": -2.28955078125,
      "logps/chosen": -449.4609375,
      "logps/rejected": -496.03125,
      "loss": 0.1978926658630371,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.75494384765625,
      "rewards/margins": 2.4698486328125,
      "rewards/rejected": -3.2242431640625,
      "step": 51
    },
    {
      "epoch": 1.5433962264150942,
      "grad_norm": 58.7410932105306,
      "learning_rate": 8.985170532545622e-08,
      "logits/chosen": -2.202880859375,
      "logits/rejected": -2.2685546875,
      "logps/chosen": -392.7734375,
      "logps/rejected": -294.8671875,
      "loss": 0.21066856384277344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3479804992675781,
      "rewards/margins": 1.939697265625,
      "rewards/rejected": -2.2872314453125,
      "step": 52
    },
    {
      "epoch": 1.5735849056603772,
      "grad_norm": 70.52365689566419,
      "learning_rate": 8.018166527567672e-08,
      "logits/chosen": -2.240966796875,
      "logits/rejected": -2.307373046875,
      "logps/chosen": -342.15625,
      "logps/rejected": -294.140625,
      "loss": 0.29566192626953125,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.7627105712890625,
      "rewards/margins": 1.51177978515625,
      "rewards/rejected": -2.274658203125,
      "step": 53
    },
    {
      "epoch": 1.6037735849056602,
      "grad_norm": 91.59109368101038,
      "learning_rate": 7.096195292140172e-08,
      "logits/chosen": -2.281494140625,
      "logits/rejected": -2.3701171875,
      "logps/chosen": -473.859375,
      "logps/rejected": -352.1953125,
      "loss": 0.30013465881347656,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.8317108154296875,
      "rewards/margins": 1.802398681640625,
      "rewards/rejected": -2.634796142578125,
      "step": 54
    },
    {
      "epoch": 1.6339622641509433,
      "grad_norm": 57.582350815239245,
      "learning_rate": 6.221701728237008e-08,
      "logits/chosen": -2.204833984375,
      "logits/rejected": -2.232666015625,
      "logps/chosen": -314.7265625,
      "logps/rejected": -399.859375,
      "loss": 0.2029261589050293,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.4932861328125,
      "rewards/margins": 2.4166259765625,
      "rewards/rejected": -2.9102783203125,
      "step": 55
    },
    {
      "epoch": 1.6641509433962263,
      "grad_norm": 45.97477082609319,
      "learning_rate": 5.3970048355822416e-08,
      "logits/chosen": -2.27880859375,
      "logits/rejected": -2.33447265625,
      "logps/chosen": -253.046875,
      "logps/rejected": -313.453125,
      "loss": 0.2117919921875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5623092651367188,
      "rewards/margins": 1.8668212890625,
      "rewards/rejected": -2.42919921875,
      "step": 56
    },
    {
      "epoch": 1.6943396226415093,
      "grad_norm": 44.62211760281082,
      "learning_rate": 4.6242915620797186e-08,
      "logits/chosen": -2.330078125,
      "logits/rejected": -2.33447265625,
      "logps/chosen": -279.0859375,
      "logps/rejected": -318.984375,
      "loss": 0.21802997589111328,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.40540313720703125,
      "rewards/margins": 2.044219970703125,
      "rewards/rejected": -2.450164794921875,
      "step": 57
    },
    {
      "epoch": 1.7245283018867923,
      "grad_norm": 51.2670325277809,
      "learning_rate": 3.905611004420359e-08,
      "logits/chosen": -2.32958984375,
      "logits/rejected": -2.3330078125,
      "logps/chosen": -352.4296875,
      "logps/rejected": -507.390625,
      "loss": 0.20676660537719727,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.807769775390625,
      "rewards/margins": 2.173065185546875,
      "rewards/rejected": -2.979888916015625,
      "step": 58
    },
    {
      "epoch": 1.7547169811320755,
      "grad_norm": 44.04885960141364,
      "learning_rate": 3.2428689742461187e-08,
      "logits/chosen": -2.22802734375,
      "logits/rejected": -2.23876953125,
      "logps/chosen": -285.890625,
      "logps/rejected": -427.265625,
      "loss": 0.16101567447185516,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5487899780273438,
      "rewards/margins": 2.62451171875,
      "rewards/rejected": -3.1728515625,
      "step": 59
    },
    {
      "epoch": 1.7849056603773585,
      "grad_norm": 43.187388393833025,
      "learning_rate": 2.6378229442801157e-08,
      "logits/chosen": -2.24658203125,
      "logits/rejected": -2.252685546875,
      "logps/chosen": -336.703125,
      "logps/rejected": -402.09375,
      "loss": 0.16387367248535156,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5001144409179688,
      "rewards/margins": 2.2327880859375,
      "rewards/rejected": -2.732666015625,
      "step": 60
    },
    {
      "epoch": 1.8150943396226416,
      "grad_norm": 44.49559340775631,
      "learning_rate": 2.0920773878248837e-08,
      "logits/chosen": -2.23388671875,
      "logits/rejected": -2.287353515625,
      "logps/chosen": -433.109375,
      "logps/rejected": -437.796875,
      "loss": 0.17702722549438477,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5224227905273438,
      "rewards/margins": 2.38623046875,
      "rewards/rejected": -2.90966796875,
      "step": 61
    },
    {
      "epoch": 1.8452830188679246,
      "grad_norm": 38.58001237378031,
      "learning_rate": 1.6070795239876617e-08,
      "logits/chosen": -2.1708984375,
      "logits/rejected": -2.24072265625,
      "logps/chosen": -330.2734375,
      "logps/rejected": -324.390625,
      "loss": 0.17849206924438477,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.43164825439453125,
      "rewards/margins": 2.142822265625,
      "rewards/rejected": -2.5733642578125,
      "step": 62
    },
    {
      "epoch": 1.8754716981132076,
      "grad_norm": 47.091837159623296,
      "learning_rate": 1.1841154799154373e-08,
      "logits/chosen": -2.34326171875,
      "logits/rejected": -2.378173828125,
      "logps/chosen": -282.71875,
      "logps/rejected": -280.9140625,
      "loss": 0.20744895935058594,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.43671417236328125,
      "rewards/margins": 1.98101806640625,
      "rewards/rejected": -2.41754150390625,
      "step": 63
    },
    {
      "epoch": 1.9056603773584906,
      "grad_norm": 51.17603737519097,
      "learning_rate": 8.243068802169905e-09,
      "logits/chosen": -2.2236328125,
      "logits/rejected": -2.307861328125,
      "logps/chosen": -394.0390625,
      "logps/rejected": -326.078125,
      "loss": 0.2276473045349121,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.34835052490234375,
      "rewards/margins": 1.89508056640625,
      "rewards/rejected": -2.2437744140625,
      "step": 64
    },
    {
      "epoch": 1.9358490566037736,
      "grad_norm": 55.15415072058887,
      "learning_rate": 5.286078726160548e-09,
      "logits/chosen": -2.25244140625,
      "logits/rejected": -2.32275390625,
      "logps/chosen": -358.90625,
      "logps/rejected": -276.796875,
      "loss": 0.23507118225097656,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5408477783203125,
      "rewards/margins": 1.8223876953125,
      "rewards/rejected": -2.36279296875,
      "step": 65
    },
    {
      "epoch": 1.9660377358490566,
      "grad_norm": 33.48144189144124,
      "learning_rate": 2.978025977230736e-09,
      "logits/chosen": -2.30712890625,
      "logits/rejected": -2.3076171875,
      "logps/chosen": -230.609375,
      "logps/rejected": -380.734375,
      "loss": 0.1398305892944336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.37613677978515625,
      "rewards/margins": 2.4521484375,
      "rewards/rejected": -2.828125,
      "step": 66
    },
    {
      "epoch": 1.9962264150943396,
      "grad_norm": 64.2728983817751,
      "learning_rate": 1.3250310963527355e-09,
      "logits/chosen": -2.22412109375,
      "logits/rejected": -2.228515625,
      "logps/chosen": -281.109375,
      "logps/rejected": -359.625,
      "loss": 0.2590675354003906,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.548980712890625,
      "rewards/margins": 1.7811279296875,
      "rewards/rejected": -2.331298828125,
      "step": 67
    },
    {
      "epoch": 2.0,
      "grad_norm": 209.37960293088204,
      "learning_rate": 3.314775287923677e-10,
      "logits/chosen": -2.42578125,
      "logits/rejected": -2.3203125,
      "logps/chosen": -309.0,
      "logps/rejected": -929.625,
      "loss": 0.41407907009124756,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4259033203125,
      "rewards/margins": 2.6611328125,
      "rewards/rejected": -3.081298828125,
      "step": 68
    },
    {
      "epoch": 2.0,
      "step": 68,
      "total_flos": 119628768952320.0,
      "train_loss": 0.4046005795107168,
      "train_runtime": 1834.6485,
      "train_samples_per_second": 1.154,
      "train_steps_per_second": 0.037
    }
  ],
  "logging_steps": 1,
  "max_steps": 68,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 10,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 119628768952320.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}