{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 939,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.016,
      "grad_norm": 3.5558087825775146,
      "learning_rate": 6.896551724137931e-07,
      "logits/chosen": -1.0070860385894775,
      "logits/rejected": -0.978116512298584,
      "logps/chosen": -211.4395294189453,
      "logps/rejected": -259.907958984375,
      "loss": 0.6918,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.003007554914802313,
      "rewards/margins": 0.00275672459974885,
      "rewards/rejected": 0.0002508305769879371,
      "step": 5
    },
    {
      "epoch": 0.032,
      "grad_norm": 4.531008243560791,
      "learning_rate": 1.5517241379310346e-06,
      "logits/chosen": -1.0405652523040771,
      "logits/rejected": -1.108757734298706,
      "logps/chosen": -225.45095825195312,
      "logps/rejected": -228.9770965576172,
      "loss": 0.6757,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.029836362227797508,
      "rewards/margins": 0.03624327853322029,
      "rewards/rejected": -0.00640691677108407,
      "step": 10
    },
    {
      "epoch": 0.048,
      "grad_norm": 3.2132256031036377,
      "learning_rate": 2.4137931034482762e-06,
      "logits/chosen": -0.9458276033401489,
      "logits/rejected": -0.8570151329040527,
      "logps/chosen": -229.1313934326172,
      "logps/rejected": -243.45675659179688,
      "loss": 0.6407,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.10421393811702728,
      "rewards/margins": 0.11481194198131561,
      "rewards/rejected": -0.010597997345030308,
      "step": 15
    },
    {
      "epoch": 0.064,
      "grad_norm": 2.8418328762054443,
      "learning_rate": 3.2758620689655175e-06,
      "logits/chosen": -1.0654828548431396,
      "logits/rejected": -1.0304937362670898,
      "logps/chosen": -220.5907440185547,
      "logps/rejected": -223.35400390625,
      "loss": 0.6019,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.20341631770133972,
      "rewards/margins": 0.2232368290424347,
      "rewards/rejected": -0.01982051134109497,
      "step": 20
    },
    {
      "epoch": 0.08,
      "grad_norm": 3.0401999950408936,
      "learning_rate": 4.137931034482759e-06,
      "logits/chosen": -1.1209378242492676,
      "logits/rejected": -1.0013506412506104,
      "logps/chosen": -222.6129150390625,
      "logps/rejected": -236.6962127685547,
      "loss": 0.5516,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.34540629386901855,
      "rewards/margins": 0.3849923014640808,
      "rewards/rejected": -0.039586011320352554,
      "step": 25
    },
    {
      "epoch": 0.096,
      "grad_norm": 2.5417866706848145,
      "learning_rate": 5e-06,
      "logits/chosen": -1.1623287200927734,
      "logits/rejected": -1.1034960746765137,
      "logps/chosen": -210.07327270507812,
      "logps/rejected": -253.70626831054688,
      "loss": 0.4866,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.51075279712677,
      "rewards/margins": 0.6025584936141968,
      "rewards/rejected": -0.09180571138858795,
      "step": 30
    },
    {
      "epoch": 0.112,
      "grad_norm": 2.4233880043029785,
      "learning_rate": 4.999627560102124e-06,
      "logits/chosen": -1.158803105354309,
      "logits/rejected": -1.154788851737976,
      "logps/chosen": -210.58798217773438,
      "logps/rejected": -244.96206665039062,
      "loss": 0.4686,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.6626904606819153,
      "rewards/margins": 0.7746375203132629,
      "rewards/rejected": -0.11194711923599243,
      "step": 35
    },
    {
      "epoch": 0.128,
      "grad_norm": 2.218043088912964,
      "learning_rate": 4.998510351377676e-06,
      "logits/chosen": -1.1345102787017822,
      "logits/rejected": -1.017688512802124,
      "logps/chosen": -211.61611938476562,
      "logps/rejected": -234.57315063476562,
      "loss": 0.4477,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.7445486187934875,
      "rewards/margins": 0.899127185344696,
      "rewards/rejected": -0.15457859635353088,
      "step": 40
    },
    {
      "epoch": 0.144,
      "grad_norm": 2.5353105068206787,
      "learning_rate": 4.996648706701141e-06,
      "logits/chosen": -1.1606733798980713,
      "logits/rejected": -1.1573020219802856,
      "logps/chosen": -192.83447265625,
      "logps/rejected": -236.6874542236328,
      "loss": 0.4351,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.8413721323013306,
      "rewards/margins": 1.077791452407837,
      "rewards/rejected": -0.23641936480998993,
      "step": 45
    },
    {
      "epoch": 0.16,
      "grad_norm": 3.485431671142578,
      "learning_rate": 4.99404318075312e-06,
      "logits/chosen": -1.1803537607192993,
      "logits/rejected": -1.0980823040008545,
      "logps/chosen": -221.9828643798828,
      "logps/rejected": -223.74551391601562,
      "loss": 0.4058,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 1.0138968229293823,
      "rewards/margins": 1.2538959980010986,
      "rewards/rejected": -0.23999913036823273,
      "step": 50
    },
    {
      "epoch": 0.176,
      "grad_norm": 1.676897406578064,
      "learning_rate": 4.990694549855066e-06,
      "logits/chosen": -1.1700044870376587,
      "logits/rejected": -1.1240018606185913,
      "logps/chosen": -201.88095092773438,
      "logps/rejected": -241.2851104736328,
      "loss": 0.3752,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 1.2212125062942505,
      "rewards/margins": 1.552638292312622,
      "rewards/rejected": -0.33142566680908203,
      "step": 55
    },
    {
      "epoch": 0.192,
      "grad_norm": 2.7544007301330566,
      "learning_rate": 4.986603811737982e-06,
      "logits/chosen": -1.1804865598678589,
      "logits/rejected": -1.1661874055862427,
      "logps/chosen": -226.8423309326172,
      "logps/rejected": -266.3681945800781,
      "loss": 0.3889,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 0.9815112352371216,
      "rewards/margins": 1.3240571022033691,
      "rewards/rejected": -0.34254592657089233,
      "step": 60
    },
    {
      "epoch": 0.208,
      "grad_norm": 2.14679217338562,
      "learning_rate": 4.981772185245135e-06,
      "logits/chosen": -1.1490398645401,
      "logits/rejected": -1.1126385927200317,
      "logps/chosen": -208.37521362304688,
      "logps/rejected": -251.33090209960938,
      "loss": 0.3573,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 1.256662130355835,
      "rewards/margins": 1.747520089149475,
      "rewards/rejected": -0.4908579885959625,
      "step": 65
    },
    {
      "epoch": 0.224,
      "grad_norm": 2.0803537368774414,
      "learning_rate": 4.976201109968909e-06,
      "logits/chosen": -1.2029155492782593,
      "logits/rejected": -1.1176910400390625,
      "logps/chosen": -207.2776336669922,
      "logps/rejected": -223.60400390625,
      "loss": 0.3721,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.2059996128082275,
      "rewards/margins": 1.7384388446807861,
      "rewards/rejected": -0.5324392318725586,
      "step": 70
    },
    {
      "epoch": 0.24,
      "grad_norm": 2.2244112491607666,
      "learning_rate": 4.969892245821867e-06,
      "logits/chosen": -1.2101242542266846,
      "logits/rejected": -1.2214491367340088,
      "logps/chosen": -191.7788543701172,
      "logps/rejected": -240.3572235107422,
      "loss": 0.3254,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.2775731086730957,
      "rewards/margins": 1.9345057010650635,
      "rewards/rejected": -0.656932532787323,
      "step": 75
    },
    {
      "epoch": 0.256,
      "grad_norm": 2.4931509494781494,
      "learning_rate": 4.9628474725421845e-06,
      "logits/chosen": -1.2070999145507812,
      "logits/rejected": -1.125981092453003,
      "logps/chosen": -214.82461547851562,
      "logps/rejected": -259.69122314453125,
      "loss": 0.3627,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.1662296056747437,
      "rewards/margins": 1.7516357898712158,
      "rewards/rejected": -0.5854062438011169,
      "step": 80
    },
    {
      "epoch": 0.272,
      "grad_norm": 2.0334951877593994,
      "learning_rate": 4.955068889133577e-06,
      "logits/chosen": -1.257777214050293,
      "logits/rejected": -1.2115817070007324,
      "logps/chosen": -201.50735473632812,
      "logps/rejected": -231.436767578125,
      "loss": 0.3581,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 1.2795923948287964,
      "rewards/margins": 1.8879226446151733,
      "rewards/rejected": -0.6083300709724426,
      "step": 85
    },
    {
      "epoch": 0.288,
      "grad_norm": NaN,
      "learning_rate": 4.946558813239889e-06,
      "logits/chosen": -1.26598060131073,
      "logits/rejected": -1.211882472038269,
      "logps/chosen": -204.62889099121094,
      "logps/rejected": -248.060546875,
      "loss": 0.3342,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.2854232788085938,
      "rewards/margins": 2.1888251304626465,
      "rewards/rejected": -0.9034019708633423,
      "step": 90
    },
    {
      "epoch": 0.304,
      "grad_norm": 2.3634698390960693,
      "learning_rate": 4.937319780454559e-06,
      "logits/chosen": -1.350507378578186,
      "logits/rejected": -1.2982906103134155,
      "logps/chosen": -194.1533966064453,
      "logps/rejected": -239.66055297851562,
      "loss": 0.3292,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.2391879558563232,
      "rewards/margins": 2.0913984775543213,
      "rewards/rejected": -0.8522104024887085,
      "step": 95
    },
    {
      "epoch": 0.32,
      "grad_norm": 3.3314573764801025,
      "learning_rate": 4.927354543565131e-06,
      "logits/chosen": -1.2342275381088257,
      "logits/rejected": -1.1977652311325073,
      "logps/chosen": -217.9714813232422,
      "logps/rejected": -260.415283203125,
      "loss": 0.3733,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 1.1353275775909424,
      "rewards/margins": 1.9522864818572998,
      "rewards/rejected": -0.816959023475647,
      "step": 100
    },
    {
      "epoch": 0.336,
      "grad_norm": 2.4203505516052246,
      "learning_rate": 4.9166660717330496e-06,
      "logits/chosen": -1.3102691173553467,
      "logits/rejected": -1.2377029657363892,
      "logps/chosen": -187.3673858642578,
      "logps/rejected": -238.6230926513672,
      "loss": 0.3051,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 1.4179824590682983,
      "rewards/margins": 2.268038272857666,
      "rewards/rejected": -0.8500560522079468,
      "step": 105
    },
    {
      "epoch": 0.352,
      "grad_norm": 2.4510674476623535,
      "learning_rate": 4.905257549609002e-06,
      "logits/chosen": -1.305759072303772,
      "logits/rejected": -1.2166777849197388,
      "logps/chosen": -207.846435546875,
      "logps/rejected": -244.6707000732422,
      "loss": 0.281,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 1.5557820796966553,
      "rewards/margins": 2.547685146331787,
      "rewards/rejected": -0.9919027090072632,
      "step": 110
    },
    {
      "epoch": 0.368,
      "grad_norm": 2.3315584659576416,
      "learning_rate": 4.89313237638404e-06,
      "logits/chosen": -1.2491403818130493,
      "logits/rejected": -1.2307875156402588,
      "logps/chosen": -213.06405639648438,
      "logps/rejected": -255.96499633789062,
      "loss": 0.3359,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.2078200578689575,
      "rewards/margins": 2.0568363666534424,
      "rewards/rejected": -0.8490163087844849,
      "step": 115
    },
    {
      "epoch": 0.384,
      "grad_norm": 2.4910717010498047,
      "learning_rate": 4.880294164776785e-06,
      "logits/chosen": -1.3219393491744995,
      "logits/rejected": -1.3105016946792603,
      "logps/chosen": -225.34326171875,
      "logps/rejected": -267.8374938964844,
      "loss": 0.2741,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 1.3729044198989868,
      "rewards/margins": 2.491551637649536,
      "rewards/rejected": -1.1186472177505493,
      "step": 120
    },
    {
      "epoch": 0.4,
      "grad_norm": 2.8208534717559814,
      "learning_rate": 4.866746739957016e-06,
      "logits/chosen": -1.2614940404891968,
      "logits/rejected": -1.3334258794784546,
      "logps/chosen": -209.5437774658203,
      "logps/rejected": -268.48028564453125,
      "loss": 0.3283,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.9299019575119019,
      "rewards/margins": 2.1982016563415527,
      "rewards/rejected": -1.2682996988296509,
      "step": 125
    },
    {
      "epoch": 0.416,
      "grad_norm": 2.4839680194854736,
      "learning_rate": 4.852494138405942e-06,
      "logits/chosen": -1.3196954727172852,
      "logits/rejected": -1.242071509361267,
      "logps/chosen": -222.31631469726562,
      "logps/rejected": -253.88211059570312,
      "loss": 0.2927,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.3626937866210938,
      "rewards/margins": 2.622807025909424,
      "rewards/rejected": -1.2601131200790405,
      "step": 130
    },
    {
      "epoch": 0.432,
      "grad_norm": 2.6777167320251465,
      "learning_rate": 4.837540606713538e-06,
      "logits/chosen": -1.3383070230484009,
      "logits/rejected": -1.2611116170883179,
      "logps/chosen": -215.638916015625,
      "logps/rejected": -233.745361328125,
      "loss": 0.3246,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 1.1914758682250977,
      "rewards/margins": 2.3704941272735596,
      "rewards/rejected": -1.179018259048462,
      "step": 135
    },
    {
      "epoch": 0.448,
      "grad_norm": 2.0181572437286377,
      "learning_rate": 4.821890600313256e-06,
      "logits/chosen": -1.2718770503997803,
      "logits/rejected": -1.2419809103012085,
      "logps/chosen": -207.59140014648438,
      "logps/rejected": -246.1212158203125,
      "loss": 0.294,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 1.152263879776001,
      "rewards/margins": 2.4711754322052,
      "rewards/rejected": -1.3189116716384888,
      "step": 140
    },
    {
      "epoch": 0.464,
      "grad_norm": 2.557889461517334,
      "learning_rate": 4.805548782154527e-06,
      "logits/chosen": -1.3852113485336304,
      "logits/rejected": -1.317633867263794,
      "logps/chosen": -207.5449676513672,
      "logps/rejected": -258.5287780761719,
      "loss": 0.2983,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.0099232196807861,
      "rewards/margins": 2.554460048675537,
      "rewards/rejected": -1.544536828994751,
      "step": 145
    },
    {
      "epoch": 0.48,
      "grad_norm": 2.6911659240722656,
      "learning_rate": 4.788520021313416e-06,
      "logits/chosen": -1.3613696098327637,
      "logits/rejected": -1.3260200023651123,
      "logps/chosen": -203.2208251953125,
      "logps/rejected": -252.5656280517578,
      "loss": 0.2815,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.1206295490264893,
      "rewards/margins": 2.8727269172668457,
      "rewards/rejected": -1.7520973682403564,
      "step": 150
    },
    {
      "epoch": 0.496,
      "grad_norm": 2.3627328872680664,
      "learning_rate": 4.770809391541885e-06,
      "logits/chosen": -1.4218701124191284,
      "logits/rejected": -1.4015001058578491,
      "logps/chosen": -208.8138427734375,
      "logps/rejected": -256.91021728515625,
      "loss": 0.3085,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.9383912086486816,
      "rewards/margins": 2.533010721206665,
      "rewards/rejected": -1.5946193933486938,
      "step": 155
    },
    {
      "epoch": 0.512,
      "grad_norm": 3.0470287799835205,
      "learning_rate": 4.752422169756048e-06,
      "logits/chosen": -1.367304801940918,
      "logits/rejected": -1.3290746212005615,
      "logps/chosen": -195.11813354492188,
      "logps/rejected": -259.2236022949219,
      "loss": 0.3068,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.8705726861953735,
      "rewards/margins": 2.558838367462158,
      "rewards/rejected": -1.6882654428482056,
      "step": 160
    },
    {
      "epoch": 0.528,
      "grad_norm": 2.524092435836792,
      "learning_rate": 4.733363834463908e-06,
      "logits/chosen": -1.3467628955841064,
      "logits/rejected": -1.3406982421875,
      "logps/chosen": -209.3506622314453,
      "logps/rejected": -261.6465759277344,
      "loss": 0.2852,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 0.982197105884552,
      "rewards/margins": 3.0284159183502197,
      "rewards/rejected": -2.0462186336517334,
      "step": 165
    },
    {
      "epoch": 0.544,
      "grad_norm": 2.598055601119995,
      "learning_rate": 4.7136400641330245e-06,
      "logits/chosen": -1.3876451253890991,
      "logits/rejected": -1.3981963396072388,
      "logps/chosen": -198.97781372070312,
      "logps/rejected": -261.360595703125,
      "loss": 0.3937,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.6369429230690002,
      "rewards/margins": 2.3431198596954346,
      "rewards/rejected": -1.706176996231079,
      "step": 170
    },
    {
      "epoch": 0.56,
      "grad_norm": 2.1487221717834473,
      "learning_rate": 4.693256735498605e-06,
      "logits/chosen": -1.27901291847229,
      "logits/rejected": -1.2287042140960693,
      "logps/chosen": -207.0474395751953,
      "logps/rejected": -225.71542358398438,
      "loss": 0.3105,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.0815143585205078,
      "rewards/margins": 2.551909923553467,
      "rewards/rejected": -1.470395565032959,
      "step": 175
    },
    {
      "epoch": 0.576,
      "grad_norm": 2.4707322120666504,
      "learning_rate": 4.672219921812517e-06,
      "logits/chosen": -1.3839061260223389,
      "logits/rejected": -1.394359827041626,
      "logps/chosen": -202.1638946533203,
      "logps/rejected": -260.03961181640625,
      "loss": 0.3076,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 0.9943957328796387,
      "rewards/margins": 2.6735429763793945,
      "rewards/rejected": -1.6791471242904663,
      "step": 180
    },
    {
      "epoch": 0.592,
      "grad_norm": 2.918832540512085,
      "learning_rate": 4.650535891033752e-06,
      "logits/chosen": -1.425442099571228,
      "logits/rejected": -1.400042176246643,
      "logps/chosen": -212.57003784179688,
      "logps/rejected": -266.183349609375,
      "loss": 0.277,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.072869896888733,
      "rewards/margins": 2.737441301345825,
      "rewards/rejected": -1.6645710468292236,
      "step": 185
    },
    {
      "epoch": 0.608,
      "grad_norm": 2.2655458450317383,
      "learning_rate": 4.6282111039608786e-06,
      "logits/chosen": -1.4577895402908325,
      "logits/rejected": -1.412097692489624,
      "logps/chosen": -211.5107421875,
      "logps/rejected": -260.93280029296875,
      "loss": 0.2882,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.102726697921753,
      "rewards/margins": 2.8348031044006348,
      "rewards/rejected": -1.7320764064788818,
      "step": 190
    },
    {
      "epoch": 0.624,
      "grad_norm": 2.698307991027832,
      "learning_rate": 4.60525221230703e-06,
      "logits/chosen": -1.428170919418335,
      "logits/rejected": -1.317826509475708,
      "logps/chosen": -202.50094604492188,
      "logps/rejected": -244.787841796875,
      "loss": 0.2431,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.3533462285995483,
      "rewards/margins": 3.1192307472229004,
      "rewards/rejected": -1.7658843994140625,
      "step": 195
    },
    {
      "epoch": 0.64,
      "grad_norm": 4.034365177154541,
      "learning_rate": 4.581666056718016e-06,
      "logits/chosen": -1.3452891111373901,
      "logits/rejected": -1.3130595684051514,
      "logps/chosen": -216.2296905517578,
      "logps/rejected": -259.84454345703125,
      "loss": 0.29,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.1018333435058594,
      "rewards/margins": 2.8792738914489746,
      "rewards/rejected": -1.7774404287338257,
      "step": 200
    },
    {
      "epoch": 0.656,
      "grad_norm": 3.709115982055664,
      "learning_rate": 4.5574596647341414e-06,
      "logits/chosen": -1.4108755588531494,
      "logits/rejected": -1.3489899635314941,
      "logps/chosen": -215.44210815429688,
      "logps/rejected": -245.09335327148438,
      "loss": 0.323,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 0.8439299464225769,
      "rewards/margins": 2.641413688659668,
      "rewards/rejected": -1.7974838018417358,
      "step": 205
    },
    {
      "epoch": 0.672,
      "grad_norm": 2.070817470550537,
      "learning_rate": 4.532640248696331e-06,
      "logits/chosen": -1.4538488388061523,
      "logits/rejected": -1.37591552734375,
      "logps/chosen": -207.18453979492188,
      "logps/rejected": -269.21502685546875,
      "loss": 0.2839,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 1.192795991897583,
      "rewards/margins": 3.0717904567718506,
      "rewards/rejected": -1.878994345664978,
      "step": 210
    },
    {
      "epoch": 0.688,
      "grad_norm": 2.577613592147827,
      "learning_rate": 4.5072152035972065e-06,
      "logits/chosen": -1.371187448501587,
      "logits/rejected": -1.313145399093628,
      "logps/chosen": -216.05545043945312,
      "logps/rejected": -257.0478820800781,
      "loss": 0.3076,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 1.1174415349960327,
      "rewards/margins": 2.8477675914764404,
      "rewards/rejected": -1.7303262948989868,
      "step": 215
    },
    {
      "epoch": 0.704,
      "grad_norm": 2.084120750427246,
      "learning_rate": 4.481192104877727e-06,
      "logits/chosen": -1.4179775714874268,
      "logits/rejected": -1.4000885486602783,
      "logps/chosen": -199.14834594726562,
      "logps/rejected": -259.9361267089844,
      "loss": 0.2761,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.2094013690948486,
      "rewards/margins": 2.9881272315979004,
      "rewards/rejected": -1.7787256240844727,
      "step": 220
    },
    {
      "epoch": 0.72,
      "grad_norm": 2.672084331512451,
      "learning_rate": 4.454578706170075e-06,
      "logits/chosen": -1.4160033464431763,
      "logits/rejected": -1.381691336631775,
      "logps/chosen": -212.1450653076172,
      "logps/rejected": -264.46978759765625,
      "loss": 0.2776,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.1448103189468384,
      "rewards/margins": 3.152707099914551,
      "rewards/rejected": -2.007896661758423,
      "step": 225
    },
    {
      "epoch": 0.736,
      "grad_norm": 2.2846269607543945,
      "learning_rate": 4.427382936987449e-06,
      "logits/chosen": -1.4172203540802002,
      "logits/rejected": -1.3477474451065063,
      "logps/chosen": -201.10195922851562,
      "logps/rejected": -253.28067016601562,
      "loss": 0.2495,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.2827614545822144,
      "rewards/margins": 3.226905345916748,
      "rewards/rejected": -1.9441440105438232,
      "step": 230
    },
    {
      "epoch": 0.752,
      "grad_norm": 2.592393398284912,
      "learning_rate": 4.3996129003614476e-06,
      "logits/chosen": -1.3589997291564941,
      "logits/rejected": -1.329886794090271,
      "logps/chosen": -269.73968505859375,
      "logps/rejected": -273.5852355957031,
      "loss": 0.234,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.4174926280975342,
      "rewards/margins": 3.680424213409424,
      "rewards/rejected": -2.2629313468933105,
      "step": 235
    },
    {
      "epoch": 0.768,
      "grad_norm": 3.361455202102661,
      "learning_rate": 4.3712768704277535e-06,
      "logits/chosen": -1.3936150074005127,
      "logits/rejected": -1.396785855293274,
      "logps/chosen": -199.318115234375,
      "logps/rejected": -256.87396240234375,
      "loss": 0.245,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 1.1769810914993286,
      "rewards/margins": 3.183269739151001,
      "rewards/rejected": -2.006288766860962,
      "step": 240
    },
    {
      "epoch": 0.784,
      "grad_norm": 2.744699239730835,
      "learning_rate": 4.342383289960842e-06,
      "logits/chosen": -1.4649932384490967,
      "logits/rejected": -1.3714121580123901,
      "logps/chosen": -190.34593200683594,
      "logps/rejected": -262.5522766113281,
      "loss": 0.2459,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 1.1242296695709229,
      "rewards/margins": 3.3215816020965576,
      "rewards/rejected": -2.1973519325256348,
      "step": 245
    },
    {
      "epoch": 0.8,
      "grad_norm": 2.5331287384033203,
      "learning_rate": 4.312940767858442e-06,
      "logits/chosen": -1.5104731321334839,
      "logits/rejected": -1.4501935243606567,
      "logps/chosen": -218.8745880126953,
      "logps/rejected": -274.6693115234375,
      "loss": 0.2521,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.0992300510406494,
      "rewards/margins": 3.235896348953247,
      "rewards/rejected": -2.1366658210754395,
      "step": 250
    },
    {
      "epoch": 0.816,
      "grad_norm": 2.387657880783081,
      "learning_rate": 4.282958076576491e-06,
      "logits/chosen": -1.495585560798645,
      "logits/rejected": -1.4296038150787354,
      "logps/chosen": -249.4803466796875,
      "logps/rejected": -273.93994140625,
      "loss": 0.25,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 1.0271832942962646,
      "rewards/margins": 3.5675711631774902,
      "rewards/rejected": -2.5403876304626465,
      "step": 255
    },
    {
      "epoch": 0.832,
      "grad_norm": 2.605201244354248,
      "learning_rate": 4.252444149515374e-06,
      "logits/chosen": -1.4731948375701904,
      "logits/rejected": -1.4050095081329346,
      "logps/chosen": -223.21548461914062,
      "logps/rejected": -277.0063171386719,
      "loss": 0.2489,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.0619561672210693,
      "rewards/margins": 3.671313524246216,
      "rewards/rejected": -2.6093573570251465,
      "step": 260
    },
    {
      "epoch": 0.848,
      "grad_norm": 2.5914604663848877,
      "learning_rate": 4.221408078358193e-06,
      "logits/chosen": -1.582438349723816,
      "logits/rejected": -1.4810370206832886,
      "logps/chosen": -196.49365234375,
      "logps/rejected": -268.8774108886719,
      "loss": 0.229,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.138505220413208,
      "rewards/margins": 3.519007921218872,
      "rewards/rejected": -2.380502462387085,
      "step": 265
    },
    {
      "epoch": 0.864,
      "grad_norm": 2.9997308254241943,
      "learning_rate": 4.189859110361886e-06,
      "logits/chosen": -1.4258604049682617,
      "logits/rejected": -1.2593319416046143,
      "logps/chosen": -233.5980224609375,
      "logps/rejected": -268.10211181640625,
      "loss": 0.2899,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.0289585590362549,
      "rewards/margins": 3.284339189529419,
      "rewards/rejected": -2.255380868911743,
      "step": 270
    },
    {
      "epoch": 0.88,
      "grad_norm": 3.4195799827575684,
      "learning_rate": 4.1578066456019885e-06,
      "logits/chosen": -1.4348300695419312,
      "logits/rejected": -1.3643486499786377,
      "logps/chosen": -201.9141082763672,
      "logps/rejected": -248.4167022705078,
      "loss": 0.2646,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7493404150009155,
      "rewards/margins": 3.0913987159729004,
      "rewards/rejected": -2.3420581817626953,
      "step": 275
    },
    {
      "epoch": 0.896,
      "grad_norm": 2.695443868637085,
      "learning_rate": 4.125260234171861e-06,
      "logits/chosen": -1.4055596590042114,
      "logits/rejected": -1.4040924310684204,
      "logps/chosen": -191.24880981445312,
      "logps/rejected": -268.87591552734375,
      "loss": 0.2082,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 1.337165355682373,
      "rewards/margins": 4.025852203369141,
      "rewards/rejected": -2.6886870861053467,
      "step": 280
    },
    {
      "epoch": 0.912,
      "grad_norm": 2.7110860347747803,
      "learning_rate": 4.092229573337223e-06,
      "logits/chosen": -1.5467432737350464,
      "logits/rejected": -1.4956064224243164,
      "logps/chosen": -217.4794158935547,
      "logps/rejected": -271.25726318359375,
      "loss": 0.2636,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.9303010106086731,
      "rewards/margins": 3.6637215614318848,
      "rewards/rejected": -2.7334208488464355,
      "step": 285
    },
    {
      "epoch": 0.928,
      "grad_norm": 3.4274234771728516,
      "learning_rate": 4.058724504646834e-06,
      "logits/chosen": -1.4762035608291626,
      "logits/rejected": -1.4678940773010254,
      "logps/chosen": -215.96139526367188,
      "logps/rejected": -281.8223876953125,
      "loss": 0.2417,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.948248565196991,
      "rewards/margins": 3.616924285888672,
      "rewards/rejected": -2.668675661087036,
      "step": 290
    },
    {
      "epoch": 0.944,
      "grad_norm": 2.92130708694458,
      "learning_rate": 4.0247550110001825e-06,
      "logits/chosen": -1.5313862562179565,
      "logits/rejected": -1.5171687602996826,
      "logps/chosen": -214.7728271484375,
      "logps/rejected": -268.3896179199219,
      "loss": 0.2797,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.846409797668457,
      "rewards/margins": 3.2781710624694824,
      "rewards/rejected": -2.4317612648010254,
      "step": 295
    },
    {
      "epoch": 0.96,
      "grad_norm": 3.7328531742095947,
      "learning_rate": 3.990331213673064e-06,
      "logits/chosen": -1.4543683528900146,
      "logits/rejected": -1.4041635990142822,
      "logps/chosen": -221.00619506835938,
      "logps/rejected": -261.167236328125,
      "loss": 0.328,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 0.697174072265625,
      "rewards/margins": 3.078673839569092,
      "rewards/rejected": -2.381499767303467,
      "step": 300
    },
    {
      "epoch": 0.976,
      "grad_norm": 3.6233444213867188,
      "learning_rate": 3.9554633693019275e-06,
      "logits/chosen": -1.4626250267028809,
      "logits/rejected": -1.4502376317977905,
      "logps/chosen": -224.6941375732422,
      "logps/rejected": -265.57562255859375,
      "loss": 0.2784,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9059398770332336,
      "rewards/margins": 3.199237823486328,
      "rewards/rejected": -2.29329776763916,
      "step": 305
    },
    {
      "epoch": 0.992,
      "grad_norm": 3.0609471797943115,
      "learning_rate": 3.92016186682789e-06,
      "logits/chosen": -1.4352532625198364,
      "logits/rejected": -1.3892724514007568,
      "logps/chosen": -232.20510864257812,
      "logps/rejected": -249.70175170898438,
      "loss": 0.3045,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.0156091451644897,
      "rewards/margins": 2.6612770557403564,
      "rewards/rejected": -1.6456680297851562,
      "step": 310
    },
    {
      "epoch": 1.0064,
      "grad_norm": 2.452713966369629,
      "learning_rate": 3.884437224401333e-06,
      "logits/chosen": -1.4115742444992065,
      "logits/rejected": -1.425600528717041,
      "logps/chosen": -197.7564697265625,
      "logps/rejected": -261.188232421875,
      "loss": 0.2462,
      "rewards/accuracies": 0.9097222089767456,
      "rewards/chosen": 0.8398535251617432,
      "rewards/margins": 3.0637478828430176,
      "rewards/rejected": -2.223893880844116,
      "step": 315
    },
    {
      "epoch": 1.0224,
      "grad_norm": 2.764373779296875,
      "learning_rate": 3.848300086247998e-06,
      "logits/chosen": -1.480515718460083,
      "logits/rejected": -1.4369785785675049,
      "logps/chosen": -227.99496459960938,
      "logps/rejected": -269.6391296386719,
      "loss": 0.2132,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.1980942487716675,
      "rewards/margins": 3.635913133621216,
      "rewards/rejected": -2.4378185272216797,
      "step": 320
    },
    {
      "epoch": 1.0384,
      "grad_norm": 2.349464178085327,
      "learning_rate": 3.81176121949752e-06,
      "logits/chosen": -1.439501166343689,
      "logits/rejected": -1.4518448114395142,
      "logps/chosen": -221.046630859375,
      "logps/rejected": -268.6219482421875,
      "loss": 0.1815,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 1.5582630634307861,
      "rewards/margins": 3.5910651683807373,
      "rewards/rejected": -2.0328023433685303,
      "step": 325
    },
    {
      "epoch": 1.0544,
      "grad_norm": 2.8748230934143066,
      "learning_rate": 3.7748315109753402e-06,
      "logits/chosen": -1.473581075668335,
      "logits/rejected": -1.3876584768295288,
      "logps/chosen": -233.5598907470703,
      "logps/rejected": -265.6020812988281,
      "loss": 0.1887,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.539259433746338,
      "rewards/margins": 3.6530539989471436,
      "rewards/rejected": -2.1137945652008057,
      "step": 330
    },
    {
      "epoch": 1.0704,
      "grad_norm": 2.4874422550201416,
      "learning_rate": 3.737521963958954e-06,
      "logits/chosen": -1.4513788223266602,
      "logits/rejected": -1.4336411952972412,
      "logps/chosen": -223.05093383789062,
      "logps/rejected": -278.9050598144531,
      "loss": 0.1933,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 1.3904491662979126,
      "rewards/margins": 4.047357082366943,
      "rewards/rejected": -2.6569082736968994,
      "step": 335
    },
    {
      "epoch": 1.0864,
      "grad_norm": 1.6010459661483765,
      "learning_rate": 3.6998436948994664e-06,
      "logits/chosen": -1.4964760541915894,
      "logits/rejected": -1.4747968912124634,
      "logps/chosen": -206.62960815429688,
      "logps/rejected": -263.10955810546875,
      "loss": 0.1859,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.4117496013641357,
      "rewards/margins": 4.006656169891357,
      "rewards/rejected": -2.5949065685272217,
      "step": 340
    },
    {
      "epoch": 1.1024,
      "grad_norm": 1.9789769649505615,
      "learning_rate": 3.661807930109422e-06,
      "logits/chosen": -1.485519528388977,
      "logits/rejected": -1.4273617267608643,
      "logps/chosen": -211.47769165039062,
      "logps/rejected": -276.42559814453125,
      "loss": 0.206,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.1727440357208252,
      "rewards/margins": 3.8037447929382324,
      "rewards/rejected": -2.6310009956359863,
      "step": 345
    },
    {
      "epoch": 1.1184,
      "grad_norm": 2.5497982501983643,
      "learning_rate": 3.6234260024179036e-06,
      "logits/chosen": -1.4089610576629639,
      "logits/rejected": -1.302335500717163,
      "logps/chosen": -213.62350463867188,
      "logps/rejected": -266.78814697265625,
      "loss": 0.2387,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.048295021057129,
      "rewards/margins": 3.6779956817626953,
      "rewards/rejected": -2.6297006607055664,
      "step": 350
    },
    {
      "epoch": 1.1344,
      "grad_norm": 1.9107615947723389,
      "learning_rate": 3.5847093477938955e-06,
      "logits/chosen": -1.5314279794692993,
      "logits/rejected": -1.4192278385162354,
      "logps/chosen": -193.95803833007812,
      "logps/rejected": -266.6246643066406,
      "loss": 0.1881,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.1405754089355469,
      "rewards/margins": 3.900803804397583,
      "rewards/rejected": -2.7602286338806152,
      "step": 355
    },
    {
      "epoch": 1.1504,
      "grad_norm": 2.248903274536133,
      "learning_rate": 3.545669501938913e-06,
      "logits/chosen": -1.3808566331863403,
      "logits/rejected": -1.3538700342178345,
      "logps/chosen": -229.12655639648438,
      "logps/rejected": -274.583251953125,
      "loss": 0.1907,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.035677433013916,
      "rewards/margins": 3.7907638549804688,
      "rewards/rejected": -2.7550864219665527,
      "step": 360
    },
    {
      "epoch": 1.1663999999999999,
      "grad_norm": 2.1139893531799316,
      "learning_rate": 3.506318096849919e-06,
      "logits/chosen": -1.5006558895111084,
      "logits/rejected": -1.4343509674072266,
      "logps/chosen": -210.866943359375,
      "logps/rejected": -255.4267120361328,
      "loss": 0.2048,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.8776493072509766,
      "rewards/margins": 3.7163491249084473,
      "rewards/rejected": -2.8386993408203125,
      "step": 365
    },
    {
      "epoch": 1.1824,
      "grad_norm": 2.827563524246216,
      "learning_rate": 3.466666857353547e-06,
      "logits/chosen": -1.4484174251556396,
      "logits/rejected": -1.3797426223754883,
      "logps/chosen": -228.63720703125,
      "logps/rejected": -262.3194274902344,
      "loss": 0.24,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 1.26779043674469,
      "rewards/margins": 3.7388503551483154,
      "rewards/rejected": -2.471060276031494,
      "step": 370
    },
    {
      "epoch": 1.1984,
      "grad_norm": 5.360053539276123,
      "learning_rate": 3.4267275976126697e-06,
      "logits/chosen": -1.5434255599975586,
      "logits/rejected": -1.4696922302246094,
      "logps/chosen": -205.205322265625,
      "logps/rejected": -279.88714599609375,
      "loss": 0.1568,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 1.2206922769546509,
      "rewards/margins": 4.399322032928467,
      "rewards/rejected": -3.1786296367645264,
      "step": 375
    },
    {
      "epoch": 1.2144,
      "grad_norm": 2.5186727046966553,
      "learning_rate": 3.386512217606339e-06,
      "logits/chosen": -1.481487512588501,
      "logits/rejected": -1.4067240953445435,
      "logps/chosen": -199.1824951171875,
      "logps/rejected": -262.4269714355469,
      "loss": 0.1983,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 1.1041431427001953,
      "rewards/margins": 3.9722588062286377,
      "rewards/rejected": -2.8681156635284424,
      "step": 380
    },
    {
      "epoch": 1.2304,
      "grad_norm": 2.019970417022705,
      "learning_rate": 3.346032699584176e-06,
      "logits/chosen": -1.5157972574234009,
      "logits/rejected": -1.4852614402770996,
      "logps/chosen": -197.70455932617188,
      "logps/rejected": -257.8968505859375,
      "loss": 0.2178,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 1.0223619937896729,
      "rewards/margins": 3.5193912982940674,
      "rewards/rejected": -2.4970290660858154,
      "step": 385
    },
    {
      "epoch": 1.2464,
      "grad_norm": 2.2139105796813965,
      "learning_rate": 3.3053011044962268e-06,
      "logits/chosen": -1.564191222190857,
      "logits/rejected": -1.5597954988479614,
      "logps/chosen": -203.78013610839844,
      "logps/rejected": -282.21136474609375,
      "loss": 0.1759,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.0344898700714111,
      "rewards/margins": 4.304172992706299,
      "rewards/rejected": -3.2696831226348877,
      "step": 390
    },
    {
      "epoch": 1.2624,
      "grad_norm": 3.2327651977539062,
      "learning_rate": 3.2643295683993827e-06,
      "logits/chosen": -1.504197120666504,
      "logits/rejected": -1.4315654039382935,
      "logps/chosen": -217.15869140625,
      "logps/rejected": -265.8768310546875,
      "loss": 0.2149,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.929955780506134,
      "rewards/margins": 4.235381126403809,
      "rewards/rejected": -3.3054251670837402,
      "step": 395
    },
    {
      "epoch": 1.2784,
      "grad_norm": 3.1434683799743652,
      "learning_rate": 3.2231302988414198e-06,
      "logits/chosen": -1.4585120677947998,
      "logits/rejected": -1.4169162511825562,
      "logps/chosen": -210.6068115234375,
      "logps/rejected": -273.0541687011719,
      "loss": 0.2233,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.8234063982963562,
      "rewards/margins": 4.173564910888672,
      "rewards/rejected": -3.35015869140625,
      "step": 400
    },
    {
      "epoch": 1.2944,
      "grad_norm": 1.7649414539337158,
      "learning_rate": 3.1817155712237347e-06,
      "logits/chosen": -1.4741700887680054,
      "logits/rejected": -1.3722097873687744,
      "logps/chosen": -226.2657928466797,
      "logps/rejected": -277.5989990234375,
      "loss": 0.221,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.9392329454421997,
      "rewards/margins": 3.8948147296905518,
      "rewards/rejected": -2.9555816650390625,
      "step": 405
    },
    {
      "epoch": 1.3104,
      "grad_norm": 3.173729658126831,
      "learning_rate": 3.140097725143868e-06,
      "logits/chosen": -1.4934290647506714,
      "logits/rejected": -1.412099838256836,
      "logps/chosen": -197.9171905517578,
      "logps/rejected": -267.22479248046875,
      "loss": 0.1667,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.9158923029899597,
      "rewards/margins": 4.115560531616211,
      "rewards/rejected": -3.1996684074401855,
      "step": 410
    },
    {
      "epoch": 1.3264,
      "grad_norm": 3.198397159576416,
      "learning_rate": 3.0982891607188948e-06,
      "logits/chosen": -1.490950345993042,
      "logits/rejected": -1.4556682109832764,
      "logps/chosen": -238.0470428466797,
      "logps/rejected": -283.5078430175781,
      "loss": 0.2348,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.9115694165229797,
      "rewards/margins": 3.748671054840088,
      "rewards/rejected": -2.837101697921753,
      "step": 415
    },
    {
      "epoch": 1.3424,
      "grad_norm": 2.510286331176758,
      "learning_rate": 3.056302334890786e-06,
      "logits/chosen": -1.4701006412506104,
      "logits/rejected": -1.441331386566162,
      "logps/chosen": -260.3006591796875,
      "logps/rejected": -261.59100341796875,
      "loss": 0.202,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.0139939785003662,
      "rewards/margins": 3.927891254425049,
      "rewards/rejected": -2.913897752761841,
      "step": 420
    },
    {
      "epoch": 1.3584,
      "grad_norm": 3.634739637374878,
      "learning_rate": 3.0141497577148416e-06,
      "logits/chosen": -1.5961452722549438,
      "logits/rejected": -1.57537841796875,
      "logps/chosen": -205.42941284179688,
      "logps/rejected": -282.03997802734375,
      "loss": 0.2036,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.551750659942627,
      "rewards/margins": 4.165204048156738,
      "rewards/rejected": -3.6134536266326904,
      "step": 425
    },
    {
      "epoch": 1.3744,
      "grad_norm": 3.698240041732788,
      "learning_rate": 2.971843988632292e-06,
      "logits/chosen": -1.4715049266815186,
      "logits/rejected": -1.3983992338180542,
      "logps/chosen": -231.0693359375,
      "logps/rejected": -257.81280517578125,
      "loss": 0.2249,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.7772096991539001,
      "rewards/margins": 3.659933567047119,
      "rewards/rejected": -2.882723808288574,
      "step": 430
    },
    {
      "epoch": 1.3904,
      "grad_norm": 3.1096959114074707,
      "learning_rate": 2.929397632728191e-06,
      "logits/chosen": -1.5016343593597412,
      "logits/rejected": -1.4842969179153442,
      "logps/chosen": -207.78262329101562,
      "logps/rejected": -275.2723388671875,
      "loss": 0.1894,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 1.0016006231307983,
      "rewards/margins": 4.241255760192871,
      "rewards/rejected": -3.2396552562713623,
      "step": 435
    },
    {
      "epoch": 1.4064,
      "grad_norm": 3.0866386890411377,
      "learning_rate": 2.886823336975703e-06,
      "logits/chosen": -1.5346386432647705,
      "logits/rejected": -1.437782645225525,
      "logps/chosen": -202.8625030517578,
      "logps/rejected": -287.2112731933594,
      "loss": 0.2577,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 0.8549686670303345,
      "rewards/margins": 3.8706657886505127,
      "rewards/rejected": -3.0156967639923096,
      "step": 440
    },
    {
      "epoch": 1.4224,
      "grad_norm": 4.283688545227051,
      "learning_rate": 2.8441337864679213e-06,
      "logits/chosen": -1.4911736249923706,
      "logits/rejected": -1.426809310913086,
      "logps/chosen": -198.9265899658203,
      "logps/rejected": -265.6153564453125,
      "loss": 0.1915,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 1.185701608657837,
      "rewards/margins": 4.455019950866699,
      "rewards/rejected": -3.2693183422088623,
      "step": 445
    },
    {
      "epoch": 1.4384000000000001,
      "grad_norm": 2.7477753162384033,
      "learning_rate": 2.8013417006383078e-06,
      "logits/chosen": -1.4503509998321533,
      "logits/rejected": -1.4662247896194458,
      "logps/chosen": -197.8849334716797,
      "logps/rejected": -275.83831787109375,
      "loss": 0.1803,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.975446879863739,
      "rewards/margins": 4.213322162628174,
      "rewards/rejected": -3.237874984741211,
      "step": 450
    },
    {
      "epoch": 1.4544000000000001,
      "grad_norm": 2.467759132385254,
      "learning_rate": 2.758459829470925e-06,
      "logits/chosen": -1.5110101699829102,
      "logits/rejected": -1.5128719806671143,
      "logps/chosen": -222.1479949951172,
      "logps/rejected": -272.8663330078125,
      "loss": 0.1873,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.9241386651992798,
      "rewards/margins": 4.218895435333252,
      "rewards/rejected": -3.2947566509246826,
      "step": 455
    },
    {
      "epoch": 1.4704,
      "grad_norm": 2.5419845581054688,
      "learning_rate": 2.7155009497015487e-06,
      "logits/chosen": -1.549300193786621,
      "logits/rejected": -1.4836896657943726,
      "logps/chosen": -195.63414001464844,
      "logps/rejected": -276.05230712890625,
      "loss": 0.1573,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 1.0924745798110962,
      "rewards/margins": 4.838970184326172,
      "rewards/rejected": -3.7464957237243652,
      "step": 460
    },
    {
      "epoch": 1.4864,
      "grad_norm": 2.3716790676116943,
      "learning_rate": 2.6724778610108123e-06,
      "logits/chosen": -1.5963730812072754,
      "logits/rejected": -1.545782446861267,
      "logps/chosen": -204.3538818359375,
      "logps/rejected": -261.65625,
      "loss": 0.1937,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.0204435586929321,
      "rewards/margins": 4.1371049880981445,
      "rewards/rejected": -3.116661787033081,
      "step": 465
    },
    {
      "epoch": 1.5024,
      "grad_norm": 3.689908981323242,
      "learning_rate": 2.629403382210524e-06,
      "logits/chosen": -1.5731112957000732,
      "logits/rejected": -1.458322525024414,
      "logps/chosen": -189.004638671875,
      "logps/rejected": -253.52145385742188,
      "loss": 0.2311,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.8893483281135559,
      "rewards/margins": 3.971355438232422,
      "rewards/rejected": -3.08200740814209,
      "step": 470
    },
    {
      "epoch": 1.5184,
      "grad_norm": 2.376858949661255,
      "learning_rate": 2.5862903474242707e-06,
      "logits/chosen": -1.4432743787765503,
      "logits/rejected": -1.369016408920288,
      "logps/chosen": -203.9170684814453,
      "logps/rejected": -268.00201416015625,
      "loss": 0.1701,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 1.1208674907684326,
      "rewards/margins": 4.389878273010254,
      "rewards/rejected": -3.269010543823242,
      "step": 475
    },
    {
      "epoch": 1.5344,
      "grad_norm": 3.032092332839966,
      "learning_rate": 2.5431516022634718e-06,
      "logits/chosen": -1.5671919584274292,
      "logits/rejected": -1.4809482097625732,
      "logps/chosen": -200.5327606201172,
      "logps/rejected": -268.4273986816406,
      "loss": 0.1361,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 1.168104887008667,
      "rewards/margins": 4.882475852966309,
      "rewards/rejected": -3.7143707275390625,
      "step": 480
    },
    {
      "epoch": 1.5504,
      "grad_norm": 3.788256883621216,
      "learning_rate": 2.5e-06,
      "logits/chosen": -1.5944483280181885,
      "logits/rejected": -1.4670864343643188,
      "logps/chosen": -216.2355194091797,
      "logps/rejected": -270.3284606933594,
      "loss": 0.1597,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.7749621272087097,
      "rewards/margins": 4.562745094299316,
      "rewards/rejected": -3.787782669067383,
      "step": 485
    },
    {
      "epoch": 1.5664,
      "grad_norm": 2.4552183151245117,
      "learning_rate": 2.456848397736529e-06,
      "logits/chosen": -1.586390733718872,
      "logits/rejected": -1.5117604732513428,
      "logps/chosen": -226.70364379882812,
      "logps/rejected": -263.7825012207031,
      "loss": 0.221,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.6633533239364624,
      "rewards/margins": 4.1091156005859375,
      "rewards/rejected": -3.4457621574401855,
      "step": 490
    },
    {
      "epoch": 1.5824,
      "grad_norm": 2.8182895183563232,
      "learning_rate": 2.4137096525757306e-06,
      "logits/chosen": -1.445420503616333,
      "logits/rejected": -1.5098474025726318,
      "logps/chosen": -211.97216796875,
      "logps/rejected": -244.12893676757812,
      "loss": 0.1896,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.7570455074310303,
      "rewards/margins": 4.359705924987793,
      "rewards/rejected": -3.6026604175567627,
      "step": 495
    },
    {
      "epoch": 1.5984,
      "grad_norm": 2.062126636505127,
      "learning_rate": 2.3705966177894763e-06,
      "logits/chosen": -1.5797353982925415,
      "logits/rejected": -1.5158863067626953,
      "logps/chosen": -191.18060302734375,
      "logps/rejected": -275.715576171875,
      "loss": 0.1436,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.751899778842926,
      "rewards/margins": 4.745448112487793,
      "rewards/rejected": -3.9935481548309326,
      "step": 500
    },
    {
      "epoch": 1.6143999999999998,
      "grad_norm": 2.1525497436523438,
      "learning_rate": 2.3275221389891877e-06,
      "logits/chosen": -1.4490973949432373,
      "logits/rejected": -1.4142483472824097,
      "logps/chosen": -223.93295288085938,
      "logps/rejected": -267.38995361328125,
      "loss": 0.1898,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.8560250401496887,
      "rewards/margins": 4.311391353607178,
      "rewards/rejected": -3.4553661346435547,
      "step": 505
    },
    {
      "epoch": 1.6303999999999998,
      "grad_norm": 3.303349018096924,
      "learning_rate": 2.2844990502984517e-06,
      "logits/chosen": -1.5563932657241821,
      "logits/rejected": -1.5370930433273315,
      "logps/chosen": -207.7513885498047,
      "logps/rejected": -284.32928466796875,
      "loss": 0.2041,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.6124199628829956,
      "rewards/margins": 4.572120666503906,
      "rewards/rejected": -3.9597008228302,
      "step": 510
    },
    {
      "epoch": 1.6463999999999999,
      "grad_norm": 3.3144888877868652,
      "learning_rate": 2.2415401705290753e-06,
      "logits/chosen": -1.5163837671279907,
      "logits/rejected": -1.477477788925171,
      "logps/chosen": -215.9023895263672,
      "logps/rejected": -288.0118713378906,
      "loss": 0.2097,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.6554079055786133,
      "rewards/margins": 4.482608795166016,
      "rewards/rejected": -3.8272006511688232,
      "step": 515
    },
    {
      "epoch": 1.6623999999999999,
      "grad_norm": 3.2930357456207275,
      "learning_rate": 2.1986582993616926e-06,
      "logits/chosen": -1.4894039630889893,
      "logits/rejected": -1.4551079273223877,
      "logps/chosen": -201.579833984375,
      "logps/rejected": -288.08392333984375,
      "loss": 0.179,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.6922101974487305,
      "rewards/margins": 4.913764953613281,
      "rewards/rejected": -4.221554756164551,
      "step": 520
    },
    {
      "epoch": 1.6784,
      "grad_norm": 3.814143657684326,
      "learning_rate": 2.15586621353208e-06,
      "logits/chosen": -1.499711513519287,
      "logits/rejected": -1.4773409366607666,
      "logps/chosen": -211.90798950195312,
      "logps/rejected": -274.95892333984375,
      "loss": 0.1965,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.6713994741439819,
      "rewards/margins": 4.400740623474121,
      "rewards/rejected": -3.7293410301208496,
      "step": 525
    },
    {
      "epoch": 1.6944,
      "grad_norm": 2.245388984680176,
      "learning_rate": 2.113176663024297e-06,
      "logits/chosen": -1.5954089164733887,
      "logits/rejected": -1.5338319540023804,
      "logps/chosen": -255.9158172607422,
      "logps/rejected": -277.632080078125,
      "loss": 0.2137,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.6438032388687134,
      "rewards/margins": 4.144532203674316,
      "rewards/rejected": -3.5007293224334717,
      "step": 530
    },
    {
      "epoch": 1.7104,
      "grad_norm": 3.450652837753296,
      "learning_rate": 2.07060236727181e-06,
      "logits/chosen": -1.555140733718872,
      "logits/rejected": -1.5839719772338867,
      "logps/chosen": -215.4705810546875,
      "logps/rejected": -282.3954162597656,
      "loss": 0.1855,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.5197333097457886,
      "rewards/margins": 4.1995954513549805,
      "rewards/rejected": -3.6798622608184814,
      "step": 535
    },
    {
      "epoch": 1.7264,
      "grad_norm": 3.3166344165802,
      "learning_rate": 2.0281560113677085e-06,
      "logits/chosen": -1.4750263690948486,
      "logits/rejected": -1.4056106805801392,
      "logps/chosen": -227.0777587890625,
      "logps/rejected": -278.08203125,
      "loss": 0.1951,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.8925558924674988,
      "rewards/margins": 4.614056587219238,
      "rewards/rejected": -3.7215003967285156,
      "step": 540
    },
    {
      "epoch": 1.7424,
      "grad_norm": 2.740474224090576,
      "learning_rate": 1.9858502422851588e-06,
      "logits/chosen": -1.4764506816864014,
      "logits/rejected": -1.5012338161468506,
      "logps/chosen": -214.189453125,
      "logps/rejected": -290.9407653808594,
      "loss": 0.1765,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.7208338379859924,
      "rewards/margins": 4.320445537567139,
      "rewards/rejected": -3.599611759185791,
      "step": 545
    },
    {
      "epoch": 1.7584,
      "grad_norm": 1.636268138885498,
      "learning_rate": 1.9436976651092143e-06,
      "logits/chosen": -1.4227992296218872,
      "logits/rejected": -1.5083410739898682,
      "logps/chosen": -190.32284545898438,
      "logps/rejected": -269.3763427734375,
      "loss": 0.1326,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.1259499788284302,
      "rewards/margins": 4.734012603759766,
      "rewards/rejected": -3.608062267303467,
      "step": 550
    },
    {
      "epoch": 1.7744,
      "grad_norm": 3.211097240447998,
      "learning_rate": 1.9017108392811065e-06,
      "logits/chosen": -1.6095863580703735,
      "logits/rejected": -1.6652412414550781,
      "logps/chosen": -198.27940368652344,
      "logps/rejected": -273.74053955078125,
      "loss": 0.1919,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.6028407216072083,
      "rewards/margins": 4.351764678955078,
      "rewards/rejected": -3.7489237785339355,
      "step": 555
    },
    {
      "epoch": 1.7904,
      "grad_norm": 3.039146900177002,
      "learning_rate": 1.8599022748561324e-06,
      "logits/chosen": -1.6256183385849,
      "logits/rejected": -1.5564262866973877,
      "logps/chosen": -207.5953826904297,
      "logps/rejected": -292.1823425292969,
      "loss": 0.192,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.6595524549484253,
      "rewards/margins": 4.907757759094238,
      "rewards/rejected": -4.248204708099365,
      "step": 560
    },
    {
      "epoch": 1.8064,
      "grad_norm": 2.752476215362549,
      "learning_rate": 1.8182844287762657e-06,
      "logits/chosen": -1.540544867515564,
      "logits/rejected": -1.529164433479309,
      "logps/chosen": -222.51266479492188,
      "logps/rejected": -307.46258544921875,
      "loss": 0.1604,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.8695205450057983,
      "rewards/margins": 5.046850204467773,
      "rewards/rejected": -4.177329063415527,
      "step": 565
    },
    {
      "epoch": 1.8224,
      "grad_norm": 2.99002742767334,
      "learning_rate": 1.776869701158581e-06,
      "logits/chosen": -1.5254290103912354,
      "logits/rejected": -1.4722168445587158,
      "logps/chosen": -221.71719360351562,
      "logps/rejected": -269.9870910644531,
      "loss": 0.1846,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.9478768110275269,
      "rewards/margins": 4.497414588928223,
      "rewards/rejected": -3.5495376586914062,
      "step": 570
    },
    {
      "epoch": 1.8384,
      "grad_norm": 4.398141384124756,
      "learning_rate": 1.7356704316006185e-06,
      "logits/chosen": -1.501982569694519,
      "logits/rejected": -1.4923641681671143,
      "logps/chosen": -239.463623046875,
      "logps/rejected": -276.9585876464844,
      "loss": 0.2674,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.11826063692569733,
      "rewards/margins": 3.5606513023376465,
      "rewards/rejected": -3.4423904418945312,
      "step": 575
    },
    {
      "epoch": 1.8544,
      "grad_norm": 3.2502195835113525,
      "learning_rate": 1.694698895503774e-06,
      "logits/chosen": -1.521781325340271,
      "logits/rejected": -1.445136308670044,
      "logps/chosen": -222.3773651123047,
      "logps/rejected": -268.7160339355469,
      "loss": 0.1946,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.7389503717422485,
      "rewards/margins": 4.303567886352539,
      "rewards/rejected": -3.56461763381958,
      "step": 580
    },
    {
      "epoch": 1.8704,
      "grad_norm": 3.165996551513672,
      "learning_rate": 1.653967300415824e-06,
      "logits/chosen": -1.4852544069290161,
      "logits/rejected": -1.4845850467681885,
      "logps/chosen": -214.6300811767578,
      "logps/rejected": -295.7364501953125,
      "loss": 0.1623,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.7326163053512573,
      "rewards/margins": 5.156301975250244,
      "rewards/rejected": -4.4236860275268555,
      "step": 585
    },
    {
      "epoch": 1.8864,
      "grad_norm": 4.204633712768555,
      "learning_rate": 1.613487782393661e-06,
      "logits/chosen": -1.598530650138855,
      "logits/rejected": -1.5577715635299683,
      "logps/chosen": -200.7525634765625,
      "logps/rejected": -280.6080017089844,
      "loss": 0.2124,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.6972262263298035,
      "rewards/margins": 4.553775787353516,
      "rewards/rejected": -3.8565497398376465,
      "step": 590
    },
    {
      "epoch": 1.9024,
      "grad_norm": 2.8563499450683594,
      "learning_rate": 1.5732724023873313e-06,
      "logits/chosen": -1.4955016374588013,
      "logits/rejected": -1.526990294456482,
      "logps/chosen": -195.61587524414062,
      "logps/rejected": -277.1825256347656,
      "loss": 0.1821,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.7116910219192505,
      "rewards/margins": 4.586084842681885,
      "rewards/rejected": -3.874393939971924,
      "step": 595
    },
    {
      "epoch": 1.9184,
      "grad_norm": 2.4262192249298096,
      "learning_rate": 1.5333331426464532e-06,
      "logits/chosen": -1.5559725761413574,
      "logits/rejected": -1.4919229745864868,
      "logps/chosen": -226.164306640625,
      "logps/rejected": -290.63018798828125,
      "loss": 0.1933,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.4328826069831848,
      "rewards/margins": 4.367493629455566,
      "rewards/rejected": -3.9346108436584473,
      "step": 600
    },
    {
      "epoch": 1.9344000000000001,
      "grad_norm": 4.227684497833252,
      "learning_rate": 1.4936819031500816e-06,
      "logits/chosen": -1.45127272605896,
      "logits/rejected": -1.4866652488708496,
      "logps/chosen": -222.46701049804688,
      "logps/rejected": -293.39520263671875,
      "loss": 0.1848,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.8863641619682312,
      "rewards/margins": 4.865546226501465,
      "rewards/rejected": -3.979182004928589,
      "step": 605
    },
    {
      "epoch": 1.9504000000000001,
      "grad_norm": 3.343823194503784,
      "learning_rate": 1.4543304980610879e-06,
      "logits/chosen": -1.5442382097244263,
      "logits/rejected": -1.5331324338912964,
      "logps/chosen": -207.97799682617188,
      "logps/rejected": -280.4158935546875,
      "loss": 0.2311,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.5620196461677551,
      "rewards/margins": 4.68914270401001,
      "rewards/rejected": -4.1271233558654785,
      "step": 610
    },
    {
      "epoch": 1.9664000000000001,
      "grad_norm": 2.6413931846618652,
      "learning_rate": 1.415290652206105e-06,
      "logits/chosen": -1.5758134126663208,
      "logits/rejected": -1.5755757093429565,
      "logps/chosen": -194.215576171875,
      "logps/rejected": -279.52703857421875,
      "loss": 0.191,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.5947729349136353,
      "rewards/margins": 4.675285339355469,
      "rewards/rejected": -4.080512046813965,
      "step": 615
    },
    {
      "epoch": 1.9824000000000002,
      "grad_norm": 4.83159065246582,
      "learning_rate": 1.3765739975820964e-06,
      "logits/chosen": -1.5410311222076416,
      "logits/rejected": -1.46727454662323,
      "logps/chosen": -205.7766876220703,
      "logps/rejected": -287.05706787109375,
      "loss": 0.1981,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.6785244345664978,
      "rewards/margins": 4.577788829803467,
      "rewards/rejected": -3.8992645740509033,
      "step": 620
    },
    {
      "epoch": 1.9984,
      "grad_norm": 3.0916264057159424,
      "learning_rate": 1.3381920698905788e-06,
      "logits/chosen": -1.5568665266036987,
      "logits/rejected": -1.487683892250061,
      "logps/chosen": -197.45993041992188,
      "logps/rejected": -270.3889465332031,
      "loss": 0.1795,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.41252049803733826,
      "rewards/margins": 4.434720039367676,
      "rewards/rejected": -4.0221991539001465,
      "step": 625
    },
    {
      "epoch": 2.0128,
      "grad_norm": 3.156958818435669,
      "learning_rate": 1.3001563051005348e-06,
      "logits/chosen": -1.5213212966918945,
      "logits/rejected": -1.4491430521011353,
      "logps/chosen": -214.64614868164062,
      "logps/rejected": -271.3830871582031,
      "loss": 0.1604,
      "rewards/accuracies": 0.9444444179534912,
      "rewards/chosen": 0.5151737928390503,
      "rewards/margins": 4.692739963531494,
      "rewards/rejected": -4.177565574645996,
      "step": 630
    },
    {
      "epoch": 2.0288,
      "grad_norm": 3.899384021759033,
      "learning_rate": 1.2624780360410465e-06,
      "logits/chosen": -1.5627011060714722,
      "logits/rejected": -1.5318704843521118,
      "logps/chosen": -258.4732971191406,
      "logps/rejected": -268.80194091796875,
      "loss": 0.2228,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.5114110112190247,
      "rewards/margins": 4.497435569763184,
      "rewards/rejected": -3.9860243797302246,
      "step": 635
    },
    {
      "epoch": 2.0448,
      "grad_norm": 1.9564834833145142,
      "learning_rate": 1.225168489024661e-06,
      "logits/chosen": -1.570343255996704,
      "logits/rejected": -1.539205551147461,
      "logps/chosen": -221.6617889404297,
      "logps/rejected": -303.56744384765625,
      "loss": 0.1121,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.5438711047172546,
      "rewards/margins": 5.046863079071045,
      "rewards/rejected": -4.502991676330566,
      "step": 640
    },
    {
      "epoch": 2.0608,
      "grad_norm": 3.1204640865325928,
      "learning_rate": 1.18823878050248e-06,
      "logits/chosen": -1.4727885723114014,
      "logits/rejected": -1.416222333908081,
      "logps/chosen": -217.26416015625,
      "logps/rejected": -290.2035827636719,
      "loss": 0.1569,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.9948328733444214,
      "rewards/margins": 5.736484527587891,
      "rewards/rejected": -4.741652011871338,
      "step": 645
    },
    {
      "epoch": 2.0768,
      "grad_norm": 3.6491470336914062,
      "learning_rate": 1.1516999137520023e-06,
      "logits/chosen": -1.4982631206512451,
      "logits/rejected": -1.4064592123031616,
      "logps/chosen": -214.3899383544922,
      "logps/rejected": -274.42340087890625,
      "loss": 0.164,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.9111789464950562,
      "rewards/margins": 4.822937965393066,
      "rewards/rejected": -3.9117591381073,
      "step": 650
    },
    {
      "epoch": 2.0928,
      "grad_norm": 2.44218373298645,
      "learning_rate": 1.1155627755986673e-06,
      "logits/chosen": -1.4995273351669312,
      "logits/rejected": -1.399436354637146,
      "logps/chosen": -200.63796997070312,
      "logps/rejected": -254.8256072998047,
      "loss": 0.1737,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.758064866065979,
      "rewards/margins": 4.460127353668213,
      "rewards/rejected": -3.7020626068115234,
      "step": 655
    },
    {
      "epoch": 2.1088,
      "grad_norm": 2.5277323722839355,
      "learning_rate": 1.079838133172111e-06,
      "logits/chosen": -1.5891015529632568,
      "logits/rejected": -1.5126923322677612,
      "logps/chosen": -214.89492797851562,
      "logps/rejected": -271.93463134765625,
      "loss": 0.1561,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.9002867937088013,
      "rewards/margins": 4.9788923263549805,
      "rewards/rejected": -4.0786051750183105,
      "step": 660
    },
    {
      "epoch": 2.1248,
      "grad_norm": 3.2694408893585205,
      "learning_rate": 1.044536630698074e-06,
      "logits/chosen": -1.4245771169662476,
      "logits/rejected": -1.4857099056243896,
      "logps/chosen": -227.65475463867188,
      "logps/rejected": -280.5589904785156,
      "loss": 0.1571,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.8741976618766785,
      "rewards/margins": 4.782471656799316,
      "rewards/rejected": -3.9082744121551514,
      "step": 665
    },
    {
      "epoch": 2.1408,
      "grad_norm": 3.1822643280029297,
      "learning_rate": 1.0096687863269368e-06,
      "logits/chosen": -1.6040668487548828,
      "logits/rejected": -1.5315070152282715,
      "logps/chosen": -216.56002807617188,
      "logps/rejected": -272.2256774902344,
      "loss": 0.1276,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.028387188911438,
      "rewards/margins": 5.059295654296875,
      "rewards/rejected": -4.030908584594727,
      "step": 670
    },
    {
      "epoch": 2.1568,
      "grad_norm": 1.985778570175171,
      "learning_rate": 9.752449889998177e-07,
      "logits/chosen": -1.5215928554534912,
      "logits/rejected": -1.4317477941513062,
      "logps/chosen": -220.37283325195312,
      "logps/rejected": -282.06463623046875,
      "loss": 0.1527,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.8296896815299988,
      "rewards/margins": 4.47981595993042,
      "rewards/rejected": -3.6501266956329346,
      "step": 675
    },
    {
      "epoch": 2.1728,
      "grad_norm": 2.4375979900360107,
      "learning_rate": 9.412754953531664e-07,
      "logits/chosen": -1.5252634286880493,
      "logits/rejected": -1.471828818321228,
      "logps/chosen": -214.5895538330078,
      "logps/rejected": -291.8826904296875,
      "loss": 0.126,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.6852453947067261,
      "rewards/margins": 5.221624851226807,
      "rewards/rejected": -4.536379814147949,
      "step": 680
    },
    {
      "epoch": 2.1888,
      "grad_norm": 2.8848726749420166,
      "learning_rate": 9.077704266627777e-07,
      "logits/chosen": -1.5290806293487549,
      "logits/rejected": -1.5034786462783813,
      "logps/chosen": -207.0309600830078,
      "logps/rejected": -277.1182556152344,
      "loss": 0.1651,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.6543574333190918,
      "rewards/margins": 4.433718681335449,
      "rewards/rejected": -3.7793612480163574,
      "step": 685
    },
    {
      "epoch": 2.2048,
      "grad_norm": 2.8718600273132324,
      "learning_rate": 8.747397658281396e-07,
      "logits/chosen": -1.5480024814605713,
      "logits/rejected": -1.5038466453552246,
      "logps/chosen": -229.7807159423828,
      "logps/rejected": -297.35284423828125,
      "loss": 0.1557,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.940424919128418,
      "rewards/margins": 5.296424388885498,
      "rewards/rejected": -4.355999946594238,
      "step": 690
    },
    {
      "epoch": 2.2208,
      "grad_norm": 1.3786566257476807,
      "learning_rate": 8.421933543980126e-07,
      "logits/chosen": -1.4576784372329712,
      "logits/rejected": -1.5554091930389404,
      "logps/chosen": -215.29415893554688,
      "logps/rejected": -277.7773132324219,
      "loss": 0.1193,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.7863073348999023,
      "rewards/margins": 5.157085418701172,
      "rewards/rejected": -4.3707780838012695,
      "step": 695
    },
    {
      "epoch": 2.2368,
      "grad_norm": 4.430466651916504,
      "learning_rate": 8.101408896381141e-07,
      "logits/chosen": -1.5009676218032837,
      "logits/rejected": -1.4523426294326782,
      "logps/chosen": -229.9296875,
      "logps/rejected": -281.8008728027344,
      "loss": 0.1953,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.5093691349029541,
      "rewards/margins": 5.013900279998779,
      "rewards/rejected": -4.504530906677246,
      "step": 700
    },
    {
      "epoch": 2.2528,
      "grad_norm": 3.049445152282715,
      "learning_rate": 7.78591921641807e-07,
      "logits/chosen": -1.5671838521957397,
      "logits/rejected": -1.508034348487854,
      "logps/chosen": -223.31234741210938,
      "logps/rejected": -278.34185791015625,
      "loss": 0.1751,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.33358827233314514,
      "rewards/margins": 4.723074436187744,
      "rewards/rejected": -4.389485836029053,
      "step": 705
    },
    {
      "epoch": 2.2688,
      "grad_norm": 2.8345718383789062,
      "learning_rate": 7.475558504846264e-07,
      "logits/chosen": -1.5353517532348633,
      "logits/rejected": -1.5015804767608643,
      "logps/chosen": -213.0556182861328,
      "logps/rejected": -278.02197265625,
      "loss": 0.1505,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.681841254234314,
      "rewards/margins": 5.173250675201416,
      "rewards/rejected": -4.4914093017578125,
      "step": 710
    },
    {
      "epoch": 2.2848,
      "grad_norm": 3.8184285163879395,
      "learning_rate": 7.170419234235099e-07,
      "logits/chosen": -1.5723527669906616,
      "logits/rejected": -1.5454447269439697,
      "logps/chosen": -220.0974578857422,
      "logps/rejected": -285.02471923828125,
      "loss": 0.1645,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.32882851362228394,
      "rewards/margins": 4.826219081878662,
      "rewards/rejected": -4.4973907470703125,
      "step": 715
    },
    {
      "epoch": 2.3008,
      "grad_norm": 1.8742855787277222,
      "learning_rate": 6.870592321415595e-07,
      "logits/chosen": -1.5036770105361938,
      "logits/rejected": -1.5060780048370361,
      "logps/chosen": -258.08514404296875,
      "logps/rejected": -288.4046630859375,
      "loss": 0.1302,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.4841408133506775,
      "rewards/margins": 4.661401271820068,
      "rewards/rejected": -4.177260398864746,
      "step": 720
    },
    {
      "epoch": 2.3168,
      "grad_norm": 2.858125686645508,
      "learning_rate": 6.576167100391583e-07,
      "logits/chosen": -1.5316962003707886,
      "logits/rejected": -1.5005123615264893,
      "logps/chosen": -200.00645446777344,
      "logps/rejected": -274.73797607421875,
      "loss": 0.1702,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.2455310821533203,
      "rewards/margins": 4.6962361335754395,
      "rewards/rejected": -4.450705051422119,
      "step": 725
    },
    {
      "epoch": 2.3327999999999998,
      "grad_norm": 2.4755117893218994,
      "learning_rate": 6.28723129572247e-07,
      "logits/chosen": -1.4955435991287231,
      "logits/rejected": -1.450901985168457,
      "logps/chosen": -215.6560821533203,
      "logps/rejected": -282.6337890625,
      "loss": 0.1465,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.7359136343002319,
      "rewards/margins": 5.175809860229492,
      "rewards/rejected": -4.4398956298828125,
      "step": 730
    },
    {
      "epoch": 2.3487999999999998,
      "grad_norm": 2.082693338394165,
      "learning_rate": 6.003870996385533e-07,
      "logits/chosen": -1.5163285732269287,
      "logits/rejected": -1.4916880130767822,
      "logps/chosen": -223.2830810546875,
      "logps/rejected": -304.0013122558594,
      "loss": 0.1236,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.4678799510002136,
      "rewards/margins": 5.262962341308594,
      "rewards/rejected": -4.7950825691223145,
      "step": 735
    },
    {
      "epoch": 2.3648,
      "grad_norm": 3.079440116882324,
      "learning_rate": 5.72617063012551e-07,
      "logits/chosen": -1.4600715637207031,
      "logits/rejected": -1.4372762441635132,
      "logps/chosen": -200.4602813720703,
      "logps/rejected": -298.87261962890625,
      "loss": 0.1165,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.6318634748458862,
      "rewards/margins": 5.3328537940979,
      "rewards/rejected": -4.700989723205566,
      "step": 740
    },
    {
      "epoch": 2.3808,
      "grad_norm": 2.735065221786499,
      "learning_rate": 5.454212938299256e-07,
      "logits/chosen": -1.5163583755493164,
      "logits/rejected": -1.4097801446914673,
      "logps/chosen": -222.529052734375,
      "logps/rejected": -293.6036682128906,
      "loss": 0.1375,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.6007111072540283,
      "rewards/margins": 5.02976655960083,
      "rewards/rejected": -4.429055213928223,
      "step": 745
    },
    {
      "epoch": 2.3968,
      "grad_norm": 3.09622859954834,
      "learning_rate": 5.188078951222745e-07,
      "logits/chosen": -1.5613389015197754,
      "logits/rejected": -1.421422004699707,
      "logps/chosen": -225.4494171142578,
      "logps/rejected": -287.4031982421875,
      "loss": 0.1347,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.6756893992424011,
      "rewards/margins": 5.113866329193115,
      "rewards/rejected": -4.438177585601807,
      "step": 750
    },
    {
      "epoch": 2.4128,
      "grad_norm": 4.569268703460693,
      "learning_rate": 4.92784796402794e-07,
      "logits/chosen": -1.5446864366531372,
      "logits/rejected": -1.5669465065002441,
      "logps/chosen": -213.2777862548828,
      "logps/rejected": -303.1000671386719,
      "loss": 0.1699,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.12444484233856201,
      "rewards/margins": 4.8673295974731445,
      "rewards/rejected": -4.742884635925293,
      "step": 755
    },
    {
      "epoch": 2.4288,
      "grad_norm": 1.9897822141647339,
      "learning_rate": 4.673597513036684e-07,
      "logits/chosen": -1.5755380392074585,
      "logits/rejected": -1.5424786806106567,
      "logps/chosen": -217.6551971435547,
      "logps/rejected": -267.38604736328125,
      "loss": 0.1802,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.19554877281188965,
      "rewards/margins": 4.060165882110596,
      "rewards/rejected": -3.864617109298706,
      "step": 760
    },
    {
      "epoch": 2.4448,
      "grad_norm": 1.4262510538101196,
      "learning_rate": 4.4254033526585917e-07,
      "logits/chosen": -1.4375520944595337,
      "logits/rejected": -1.4742629528045654,
      "logps/chosen": -217.225341796875,
      "logps/rejected": -281.43182373046875,
      "loss": 0.1281,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.7436832189559937,
      "rewards/margins": 5.335335731506348,
      "rewards/rejected": -4.591652870178223,
      "step": 765
    },
    {
      "epoch": 2.4608,
      "grad_norm": 3.0517566204071045,
      "learning_rate": 4.183339432819844e-07,
      "logits/chosen": -1.5090440511703491,
      "logits/rejected": -1.4726752042770386,
      "logps/chosen": -222.68173217773438,
      "logps/rejected": -276.00872802734375,
      "loss": 0.1476,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.494690477848053,
      "rewards/margins": 4.942327976226807,
      "rewards/rejected": -4.447638034820557,
      "step": 770
    },
    {
      "epoch": 2.4768,
      "grad_norm": 2.9095730781555176,
      "learning_rate": 3.947477876929706e-07,
      "logits/chosen": -1.5281938314437866,
      "logits/rejected": -1.4848145246505737,
      "logps/chosen": -214.83279418945312,
      "logps/rejected": -294.42193603515625,
      "loss": 0.1613,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.24510130286216736,
      "rewards/margins": 5.043450355529785,
      "rewards/rejected": -4.798348903656006,
      "step": 775
    },
    {
      "epoch": 2.4928,
      "grad_norm": 1.4994947910308838,
      "learning_rate": 3.717888960391222e-07,
      "logits/chosen": -1.5372960567474365,
      "logits/rejected": -1.5170810222625732,
      "logps/chosen": -230.2967987060547,
      "logps/rejected": -311.89129638671875,
      "loss": 0.1262,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.27188095450401306,
      "rewards/margins": 5.6029887199401855,
      "rewards/rejected": -5.331108570098877,
      "step": 780
    },
    {
      "epoch": 2.5088,
      "grad_norm": 3.1163201332092285,
      "learning_rate": 3.4946410896624816e-07,
      "logits/chosen": -1.5684281587600708,
      "logits/rejected": -1.5009348392486572,
      "logps/chosen": -204.79660034179688,
      "logps/rejected": -296.203857421875,
      "loss": 0.1305,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.44786787033081055,
      "rewards/margins": 5.553648948669434,
      "rewards/rejected": -5.105780601501465,
      "step": 785
    },
    {
      "epoch": 2.5248,
      "grad_norm": 4.7359795570373535,
      "learning_rate": 3.2778007818748376e-07,
      "logits/chosen": -1.495949149131775,
      "logits/rejected": -1.5144822597503662,
      "logps/chosen": -235.7005615234375,
      "logps/rejected": -299.01824951171875,
      "loss": 0.2294,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.26778358221054077,
      "rewards/margins": 4.562989234924316,
      "rewards/rejected": -4.295205593109131,
      "step": 790
    },
    {
      "epoch": 2.5408,
      "grad_norm": 3.072680711746216,
      "learning_rate": 3.0674326450139516e-07,
      "logits/chosen": -1.386596918106079,
      "logits/rejected": -1.4416801929473877,
      "logps/chosen": -222.2554168701172,
      "logps/rejected": -300.201171875,
      "loss": 0.1592,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.29924696683883667,
      "rewards/margins": 5.083845615386963,
      "rewards/rejected": -4.784598350524902,
      "step": 795
    },
    {
      "epoch": 2.5568,
      "grad_norm": 2.27390193939209,
      "learning_rate": 2.8635993586697555e-07,
      "logits/chosen": -1.5572984218597412,
      "logits/rejected": -1.4875266551971436,
      "logps/chosen": -209.93283081054688,
      "logps/rejected": -285.22491455078125,
      "loss": 0.1291,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.31827136874198914,
      "rewards/margins": 5.4527387619018555,
      "rewards/rejected": -5.134467124938965,
      "step": 800
    },
    {
      "epoch": 2.5728,
      "grad_norm": 3.9169278144836426,
      "learning_rate": 2.6663616553609273e-07,
      "logits/chosen": -1.5840399265289307,
      "logits/rejected": -1.533908486366272,
      "logps/chosen": -218.97805786132812,
      "logps/rejected": -276.19989013671875,
      "loss": 0.169,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1213858500123024,
      "rewards/margins": 4.739739418029785,
      "rewards/rejected": -4.618352890014648,
      "step": 805
    },
    {
      "epoch": 2.5888,
      "grad_norm": 2.456167697906494,
      "learning_rate": 2.4757783024395244e-07,
      "logits/chosen": -1.463147521018982,
      "logits/rejected": -1.3830325603485107,
      "logps/chosen": -219.8505401611328,
      "logps/rejected": -282.1579284667969,
      "loss": 0.1611,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.33140283823013306,
      "rewards/margins": 4.928020477294922,
      "rewards/rejected": -4.596617698669434,
      "step": 810
    },
    {
      "epoch": 2.6048,
      "grad_norm": 1.365941047668457,
      "learning_rate": 2.2919060845811498e-07,
      "logits/chosen": -1.585924744606018,
      "logits/rejected": -1.5624620914459229,
      "logps/chosen": -207.739013671875,
      "logps/rejected": -302.1373291015625,
      "loss": 0.1271,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.21847054362297058,
      "rewards/margins": 5.681158542633057,
      "rewards/rejected": -5.4626874923706055,
      "step": 815
    },
    {
      "epoch": 2.6208,
      "grad_norm": 3.1357429027557373,
      "learning_rate": 2.1147997868658427e-07,
      "logits/chosen": -1.5514700412750244,
      "logits/rejected": -1.6299651861190796,
      "logps/chosen": -210.6826934814453,
      "logps/rejected": -283.8253173828125,
      "loss": 0.174,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.06875920295715332,
      "rewards/margins": 5.225507736206055,
      "rewards/rejected": -5.1567487716674805,
      "step": 820
    },
    {
      "epoch": 2.6368,
      "grad_norm": 2.4194045066833496,
      "learning_rate": 1.9445121784547437e-07,
      "logits/chosen": -1.5748323202133179,
      "logits/rejected": -1.5742114782333374,
      "logps/chosen": -209.5052947998047,
      "logps/rejected": -294.8690185546875,
      "loss": 0.1372,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.09001219272613525,
      "rewards/margins": 4.978179454803467,
      "rewards/rejected": -4.888167858123779,
      "step": 825
    },
    {
      "epoch": 2.6528,
      "grad_norm": 1.028726577758789,
      "learning_rate": 1.7810939968674418e-07,
      "logits/chosen": -1.6239761114120483,
      "logits/rejected": -1.5699875354766846,
      "logps/chosen": -211.88369750976562,
      "logps/rejected": -284.1302795410156,
      "loss": 0.0973,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": 0.3777560591697693,
      "rewards/margins": 5.902177333831787,
      "rewards/rejected": -5.524421691894531,
      "step": 830
    },
    {
      "epoch": 2.6688,
      "grad_norm": 2.883683681488037,
      "learning_rate": 1.6245939328646322e-07,
      "logits/chosen": -1.4824936389923096,
      "logits/rejected": -1.535343050956726,
      "logps/chosen": -221.0633087158203,
      "logps/rejected": -297.4151611328125,
      "loss": 0.1735,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.1900520920753479,
      "rewards/margins": 5.30252742767334,
      "rewards/rejected": -5.112476348876953,
      "step": 835
    },
    {
      "epoch": 2.6848,
      "grad_norm": 3.589721918106079,
      "learning_rate": 1.4750586159405917e-07,
      "logits/chosen": -1.5421957969665527,
      "logits/rejected": -1.5106103420257568,
      "logps/chosen": -210.3826904296875,
      "logps/rejected": -274.41046142578125,
      "loss": 0.2073,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.28717076778411865,
      "rewards/margins": 4.679255485534668,
      "rewards/rejected": -4.392085075378418,
      "step": 840
    },
    {
      "epoch": 2.7008,
      "grad_norm": 3.69374418258667,
      "learning_rate": 1.3325326004298533e-07,
      "logits/chosen": -1.558068037033081,
      "logits/rejected": -1.5273410081863403,
      "logps/chosen": -231.698974609375,
      "logps/rejected": -311.16204833984375,
      "loss": 0.2213,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.017372488975524902,
      "rewards/margins": 4.810871124267578,
      "rewards/rejected": -4.828243732452393,
      "step": 845
    },
    {
      "epoch": 2.7168,
      "grad_norm": 3.5620641708374023,
      "learning_rate": 1.197058352232147e-07,
      "logits/chosen": -1.561061143875122,
      "logits/rejected": -1.5442357063293457,
      "logps/chosen": -220.67630004882812,
      "logps/rejected": -293.6253356933594,
      "loss": 0.1862,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.19432000815868378,
      "rewards/margins": 5.36147928237915,
      "rewards/rejected": -5.167159080505371,
      "step": 850
    },
    {
      "epoch": 2.7328,
      "grad_norm": 1.8132295608520508,
      "learning_rate": 1.0686762361596075e-07,
      "logits/chosen": -1.6041780710220337,
      "logits/rejected": -1.501832365989685,
      "logps/chosen": -213.18447875976562,
      "logps/rejected": -277.6732177734375,
      "loss": 0.136,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.015193851664662361,
      "rewards/margins": 4.558278560638428,
      "rewards/rejected": -4.54308557510376,
      "step": 855
    },
    {
      "epoch": 2.7488,
      "grad_norm": 4.747691631317139,
      "learning_rate": 9.474245039099883e-08,
      "logits/chosen": -1.4521749019622803,
      "logits/rejected": -1.3812751770019531,
      "logps/chosen": -219.65438842773438,
      "logps/rejected": -263.45977783203125,
      "loss": 0.1762,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.5863351821899414,
      "rewards/margins": 4.733705997467041,
      "rewards/rejected": -4.147371292114258,
      "step": 860
    },
    {
      "epoch": 2.7648,
      "grad_norm": 2.4241857528686523,
      "learning_rate": 8.333392826695125e-08,
      "logits/chosen": -1.6109880208969116,
      "logits/rejected": -1.5573093891143799,
      "logps/chosen": -207.94308471679688,
      "logps/rejected": -267.8423767089844,
      "loss": 0.1597,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.2518244683742523,
      "rewards/margins": 4.8867506980896,
      "rewards/rejected": -4.6349263191223145,
      "step": 865
    },
    {
      "epoch": 2.7808,
      "grad_norm": 3.291642189025879,
      "learning_rate": 7.264545643486997e-08,
      "logits/chosen": -1.5958778858184814,
      "logits/rejected": -1.5443265438079834,
      "logps/chosen": -229.17300415039062,
      "logps/rejected": -307.0359802246094,
      "loss": 0.1593,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.4986957013607025,
      "rewards/margins": 5.3819475173950195,
      "rewards/rejected": -4.883251667022705,
      "step": 870
    },
    {
      "epoch": 2.7968,
      "grad_norm": 3.9328432083129883,
      "learning_rate": 6.268021954544095e-08,
      "logits/chosen": -1.5294109582901,
      "logits/rejected": -1.5232346057891846,
      "logps/chosen": -215.351318359375,
      "logps/rejected": -281.74481201171875,
      "loss": 0.1916,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.16991707682609558,
      "rewards/margins": 4.9698686599731445,
      "rewards/rejected": -4.799951076507568,
      "step": 875
    },
    {
      "epoch": 2.8128,
      "grad_norm": 1.6685928106307983,
      "learning_rate": 5.344118676011173e-08,
      "logits/chosen": -1.517317295074463,
      "logits/rejected": -1.5692014694213867,
      "logps/chosen": -202.7231903076172,
      "logps/rejected": -289.62152099609375,
      "loss": 0.1346,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.6345641016960144,
      "rewards/margins": 5.580275058746338,
      "rewards/rejected": -4.945711135864258,
      "step": 880
    },
    {
      "epoch": 2.8288,
      "grad_norm": 2.229893207550049,
      "learning_rate": 4.493111086642438e-08,
      "logits/chosen": -1.568162202835083,
      "logits/rejected": -1.5379860401153564,
      "logps/chosen": -197.50967407226562,
      "logps/rejected": -292.4415283203125,
      "loss": 0.1472,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.33922216296195984,
      "rewards/margins": 5.462419509887695,
      "rewards/rejected": -5.123197555541992,
      "step": 885
    },
    {
      "epoch": 2.8448,
      "grad_norm": 2.519378900527954,
      "learning_rate": 3.7152527457815504e-08,
      "logits/chosen": -1.5041329860687256,
      "logits/rejected": -1.5115489959716797,
      "logps/chosen": -207.9475555419922,
      "logps/rejected": -274.80438232421875,
      "loss": 0.1982,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.23130455613136292,
      "rewards/margins": 4.7877349853515625,
      "rewards/rejected": -4.556429862976074,
      "step": 890
    },
    {
      "epoch": 2.8608000000000002,
      "grad_norm": 3.1473171710968018,
      "learning_rate": 3.010775417813388e-08,
      "logits/chosen": -1.522871971130371,
      "logits/rejected": -1.5445659160614014,
      "logps/chosen": -196.432861328125,
      "logps/rejected": -287.74359130859375,
      "loss": 0.2051,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.02917574718594551,
      "rewards/margins": 4.636673927307129,
      "rewards/rejected": -4.665849685668945,
      "step": 895
    },
    {
      "epoch": 2.8768000000000002,
      "grad_norm": 3.1271698474884033,
      "learning_rate": 2.3798890031092037e-08,
      "logits/chosen": -1.5359059572219849,
      "logits/rejected": -1.4754010438919067,
      "logps/chosen": -223.46484375,
      "logps/rejected": -294.39752197265625,
      "loss": 0.1698,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.7118966579437256,
      "rewards/margins": 5.0428667068481445,
      "rewards/rejected": -4.33096981048584,
      "step": 900
    },
    {
      "epoch": 2.8928000000000003,
      "grad_norm": 1.7171223163604736,
      "learning_rate": 1.822781475486507e-08,
      "logits/chosen": -1.5836378335952759,
      "logits/rejected": -1.5234249830245972,
      "logps/chosen": -209.51748657226562,
      "logps/rejected": -297.83343505859375,
      "loss": 0.1182,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.35548344254493713,
      "rewards/margins": 5.583395481109619,
      "rewards/rejected": -5.227911949157715,
      "step": 905
    },
    {
      "epoch": 2.9088000000000003,
      "grad_norm": 2.023606538772583,
      "learning_rate": 1.3396188262018439e-08,
      "logits/chosen": -1.486228346824646,
      "logits/rejected": -1.506572961807251,
      "logps/chosen": -211.44338989257812,
      "logps/rejected": -291.0889892578125,
      "loss": 0.1161,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": 0.8655111193656921,
      "rewards/margins": 5.486937522888184,
      "rewards/rejected": -4.621427059173584,
      "step": 910
    },
    {
      "epoch": 2.9248,
      "grad_norm": 2.3930633068084717,
      "learning_rate": 9.305450144933836e-09,
      "logits/chosen": -1.4977796077728271,
      "logits/rejected": -1.5438463687896729,
      "logps/chosen": -212.6026611328125,
      "logps/rejected": -279.10528564453125,
      "loss": 0.1812,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.12152069807052612,
      "rewards/margins": 4.457780838012695,
      "rewards/rejected": -4.336259841918945,
      "step": 915
    },
    {
      "epoch": 2.9408,
      "grad_norm": 2.2170960903167725,
      "learning_rate": 5.9568192468811844e-09,
      "logits/chosen": -1.6552884578704834,
      "logits/rejected": -1.611583948135376,
      "logps/chosen": -212.38027954101562,
      "logps/rejected": -305.21142578125,
      "loss": 0.1603,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.3693106472492218,
      "rewards/margins": 5.326704978942871,
      "rewards/rejected": -4.957394599914551,
      "step": 920
    },
    {
      "epoch": 2.9568,
      "grad_norm": 3.3993589878082275,
      "learning_rate": 3.3512932988596924e-09,
      "logits/chosen": -1.5539171695709229,
      "logits/rejected": -1.5044705867767334,
      "logps/chosen": -212.821044921875,
      "logps/rejected": -275.41632080078125,
      "loss": 0.1584,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.17114581167697906,
      "rewards/margins": 4.8249664306640625,
      "rewards/rejected": -4.653820991516113,
      "step": 925
    },
    {
      "epoch": 2.9728,
      "grad_norm": 2.7821054458618164,
      "learning_rate": 1.4896486223239802e-09,
      "logits/chosen": -1.6075645685195923,
      "logits/rejected": -1.5205880403518677,
      "logps/chosen": -211.8487091064453,
      "logps/rejected": -275.73980712890625,
      "loss": 0.1672,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3071829676628113,
      "rewards/margins": 4.825101375579834,
      "rewards/rejected": -4.517918586730957,
      "step": 930
    },
    {
      "epoch": 2.9888,
      "grad_norm": 3.1003592014312744,
      "learning_rate": 3.7243989787633106e-10,
      "logits/chosen": -1.535521149635315,
      "logits/rejected": -1.5245225429534912,
      "logps/chosen": -230.11026000976562,
      "logps/rejected": -291.1710510253906,
      "loss": 0.1983,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.3992445170879364,
      "rewards/margins": 4.833321571350098,
      "rewards/rejected": -4.43407678604126,
      "step": 935
    }
  ],
  "logging_steps": 5,
  "max_steps": 939,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}