{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 1407,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010666666666666666,
      "grad_norm": 3.7003118991851807,
      "learning_rate": 4.651162790697675e-07,
      "logits/chosen": -1.455744981765747,
      "logits/rejected": -1.3772783279418945,
      "logps/chosen": -222.02456665039062,
      "logps/rejected": -294.1808776855469,
      "loss": 0.6926,
      "rewards/accuracies": 0.3687500059604645,
      "rewards/chosen": -0.0002770518767647445,
      "rewards/margins": 0.0010710380738601089,
      "rewards/rejected": -0.001348090241663158,
      "step": 5
    },
    {
      "epoch": 0.021333333333333333,
      "grad_norm": 3.347348928451538,
      "learning_rate": 1.0465116279069768e-06,
      "logits/chosen": -1.360552191734314,
      "logits/rejected": -1.2121436595916748,
      "logps/chosen": -243.914306640625,
      "logps/rejected": -300.3173828125,
      "loss": 0.6895,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.0027127263601869345,
      "rewards/margins": 0.007307104766368866,
      "rewards/rejected": -0.010019831359386444,
      "step": 10
    },
    {
      "epoch": 0.032,
      "grad_norm": 6.566436290740967,
      "learning_rate": 1.6279069767441862e-06,
      "logits/chosen": -1.3644325733184814,
      "logits/rejected": -1.2686518430709839,
      "logps/chosen": -259.2509460449219,
      "logps/rejected": -282.755615234375,
      "loss": 0.6851,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.00618976354598999,
      "rewards/margins": 0.01635187305510044,
      "rewards/rejected": -0.02254163660109043,
      "step": 15
    },
    {
      "epoch": 0.042666666666666665,
      "grad_norm": 3.0041937828063965,
      "learning_rate": 2.2093023255813954e-06,
      "logits/chosen": -1.3592755794525146,
      "logits/rejected": -1.2850176095962524,
      "logps/chosen": -255.9427947998047,
      "logps/rejected": -290.93572998046875,
      "loss": 0.6765,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.008222861215472221,
      "rewards/margins": 0.034090589731931686,
      "rewards/rejected": -0.04231344908475876,
      "step": 20
    },
    {
      "epoch": 0.05333333333333334,
      "grad_norm": 3.1462841033935547,
      "learning_rate": 2.790697674418605e-06,
      "logits/chosen": -1.4638659954071045,
      "logits/rejected": -1.2584081888198853,
      "logps/chosen": -242.3787384033203,
      "logps/rejected": -286.44012451171875,
      "loss": 0.6654,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.015348866581916809,
      "rewards/margins": 0.057769518345594406,
      "rewards/rejected": -0.07311838120222092,
      "step": 25
    },
    {
      "epoch": 0.064,
      "grad_norm": 3.3971853256225586,
      "learning_rate": 3.372093023255814e-06,
      "logits/chosen": -1.4179127216339111,
      "logits/rejected": -1.2499792575836182,
      "logps/chosen": -258.70672607421875,
      "logps/rejected": -291.22808837890625,
      "loss": 0.6444,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02679530717432499,
      "rewards/margins": 0.10471153259277344,
      "rewards/rejected": -0.13150686025619507,
      "step": 30
    },
    {
      "epoch": 0.07466666666666667,
      "grad_norm": 3.0295073986053467,
      "learning_rate": 3.953488372093024e-06,
      "logits/chosen": -1.508451223373413,
      "logits/rejected": -1.3404954671859741,
      "logps/chosen": -245.9871368408203,
      "logps/rejected": -296.29498291015625,
      "loss": 0.6181,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.03217829018831253,
      "rewards/margins": 0.16509227454662323,
      "rewards/rejected": -0.19727057218551636,
      "step": 35
    },
    {
      "epoch": 0.08533333333333333,
      "grad_norm": 2.999821901321411,
      "learning_rate": 4.5348837209302326e-06,
      "logits/chosen": -1.4148328304290771,
      "logits/rejected": -1.3169008493423462,
      "logps/chosen": -231.54904174804688,
      "logps/rejected": -299.49896240234375,
      "loss": 0.5936,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.0469132661819458,
      "rewards/margins": 0.22682125866413116,
      "rewards/rejected": -0.27373450994491577,
      "step": 40
    },
    {
      "epoch": 0.096,
      "grad_norm": 3.179332733154297,
      "learning_rate": 4.99999336897255e-06,
      "logits/chosen": -1.4154040813446045,
      "logits/rejected": -1.2963824272155762,
      "logps/chosen": -250.27114868164062,
      "logps/rejected": -313.90899658203125,
      "loss": 0.5755,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.12327556312084198,
      "rewards/margins": 0.2791060507297516,
      "rewards/rejected": -0.4023815989494324,
      "step": 45
    },
    {
      "epoch": 0.10666666666666667,
      "grad_norm": 2.8430349826812744,
      "learning_rate": 4.999761286705285e-06,
      "logits/chosen": -1.5061628818511963,
      "logits/rejected": -1.407204270362854,
      "logps/chosen": -268.7781677246094,
      "logps/rejected": -301.4381408691406,
      "loss": 0.5144,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.1702246367931366,
      "rewards/margins": 0.46079692244529724,
      "rewards/rejected": -0.6310215592384338,
      "step": 50
    },
    {
      "epoch": 0.11733333333333333,
      "grad_norm": 2.7913994789123535,
      "learning_rate": 4.999197688241076e-06,
      "logits/chosen": -1.4658788442611694,
      "logits/rejected": -1.3267757892608643,
      "logps/chosen": -233.3777313232422,
      "logps/rejected": -285.8099670410156,
      "loss": 0.5025,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.16926373541355133,
      "rewards/margins": 0.5302549600601196,
      "rewards/rejected": -0.6995187401771545,
      "step": 55
    },
    {
      "epoch": 0.128,
      "grad_norm": 2.3396801948547363,
      "learning_rate": 4.9983026483238685e-06,
      "logits/chosen": -1.5207116603851318,
      "logits/rejected": -1.4026588201522827,
      "logps/chosen": -243.4673309326172,
      "logps/rejected": -300.88372802734375,
      "loss": 0.4772,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.31207606196403503,
      "rewards/margins": 0.6151299476623535,
      "rewards/rejected": -0.9272059202194214,
      "step": 60
    },
    {
      "epoch": 0.13866666666666666,
      "grad_norm": 2.555133819580078,
      "learning_rate": 4.997076285653087e-06,
      "logits/chosen": -1.5522791147232056,
      "logits/rejected": -1.3812358379364014,
      "logps/chosen": -273.4674987792969,
      "logps/rejected": -318.3159484863281,
      "loss": 0.423,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.3257208466529846,
      "rewards/margins": 0.8692599534988403,
      "rewards/rejected": -1.1949808597564697,
      "step": 65
    },
    {
      "epoch": 0.14933333333333335,
      "grad_norm": 3.6287903785705566,
      "learning_rate": 4.995518762867897e-06,
      "logits/chosen": -1.6529529094696045,
      "logits/rejected": -1.3509896993637085,
      "logps/chosen": -231.0438232421875,
      "logps/rejected": -295.55517578125,
      "loss": 0.4277,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.40942874550819397,
      "rewards/margins": 0.8625381588935852,
      "rewards/rejected": -1.2719669342041016,
      "step": 70
    },
    {
      "epoch": 0.16,
      "grad_norm": 3.1857149600982666,
      "learning_rate": 4.993630286525634e-06,
      "logits/chosen": -1.5783509016036987,
      "logits/rejected": -1.4265633821487427,
      "logps/chosen": -243.6620635986328,
      "logps/rejected": -316.5206604003906,
      "loss": 0.3824,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.5047813653945923,
      "rewards/margins": 1.0900554656982422,
      "rewards/rejected": -1.594836950302124,
      "step": 75
    },
    {
      "epoch": 0.17066666666666666,
      "grad_norm": 3.0437979698181152,
      "learning_rate": 4.99141110707441e-06,
      "logits/chosen": -1.6324541568756104,
      "logits/rejected": -1.4887969493865967,
      "logps/chosen": -243.0408172607422,
      "logps/rejected": -290.43109130859375,
      "loss": 0.4195,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.4802798628807068,
      "rewards/margins": 0.9819287061691284,
      "rewards/rejected": -1.4622085094451904,
      "step": 80
    },
    {
      "epoch": 0.18133333333333335,
      "grad_norm": 2.822033166885376,
      "learning_rate": 4.9888615188199e-06,
      "logits/chosen": -1.6993844509124756,
      "logits/rejected": -1.4935144186019897,
      "logps/chosen": -262.30218505859375,
      "logps/rejected": -311.645263671875,
      "loss": 0.3555,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.5132590532302856,
      "rewards/margins": 1.3637869358062744,
      "rewards/rejected": -1.8770462274551392,
      "step": 85
    },
    {
      "epoch": 0.192,
      "grad_norm": 2.7532804012298584,
      "learning_rate": 4.9859818598863104e-06,
      "logits/chosen": -1.6364755630493164,
      "logits/rejected": -1.4478092193603516,
      "logps/chosen": -241.4347686767578,
      "logps/rejected": -297.4327087402344,
      "loss": 0.4077,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.6576055288314819,
      "rewards/margins": 1.298345685005188,
      "rewards/rejected": -1.9559510946273804,
      "step": 90
    },
    {
      "epoch": 0.20266666666666666,
      "grad_norm": 2.444952964782715,
      "learning_rate": 4.982772512171538e-06,
      "logits/chosen": -1.601289987564087,
      "logits/rejected": -1.5096111297607422,
      "logps/chosen": -233.38088989257812,
      "logps/rejected": -311.9754333496094,
      "loss": 0.3462,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.654367208480835,
      "rewards/margins": 1.4337469339370728,
      "rewards/rejected": -2.0881142616271973,
      "step": 95
    },
    {
      "epoch": 0.21333333333333335,
      "grad_norm": 3.0096845626831055,
      "learning_rate": 4.979233901296523e-06,
      "logits/chosen": -1.6930478811264038,
      "logits/rejected": -1.5883949995040894,
      "logps/chosen": -241.365966796875,
      "logps/rejected": -299.60540771484375,
      "loss": 0.2976,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5582109689712524,
      "rewards/margins": 1.7530122995376587,
      "rewards/rejected": -2.311223030090332,
      "step": 100
    },
    {
      "epoch": 0.224,
      "grad_norm": 2.8001527786254883,
      "learning_rate": 4.975366496548804e-06,
      "logits/chosen": -1.6890506744384766,
      "logits/rejected": -1.476367712020874,
      "logps/chosen": -253.2794647216797,
      "logps/rejected": -304.4421691894531,
      "loss": 0.3065,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6882883906364441,
      "rewards/margins": 1.7531687021255493,
      "rewards/rejected": -2.4414572715759277,
      "step": 105
    },
    {
      "epoch": 0.23466666666666666,
      "grad_norm": 2.144157648086548,
      "learning_rate": 4.971170810820279e-06,
      "logits/chosen": -1.6170241832733154,
      "logits/rejected": -1.5461595058441162,
      "logps/chosen": -248.46035766601562,
      "logps/rejected": -327.1305236816406,
      "loss": 0.2433,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.5820830464363098,
      "rewards/margins": 2.0067741870880127,
      "rewards/rejected": -2.5888569355010986,
      "step": 110
    },
    {
      "epoch": 0.24533333333333332,
      "grad_norm": 3.336944103240967,
      "learning_rate": 4.96664740053919e-06,
      "logits/chosen": -1.6426379680633545,
      "logits/rejected": -1.5436229705810547,
      "logps/chosen": -260.5669860839844,
      "logps/rejected": -313.0763854980469,
      "loss": 0.3159,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.7669323682785034,
      "rewards/margins": 1.7966525554656982,
      "rewards/rejected": -2.563584804534912,
      "step": 115
    },
    {
      "epoch": 0.256,
      "grad_norm": 2.8522286415100098,
      "learning_rate": 4.961796865596328e-06,
      "logits/chosen": -1.6780723333358765,
      "logits/rejected": -1.5482378005981445,
      "logps/chosen": -230.629150390625,
      "logps/rejected": -312.26068115234375,
      "loss": 0.329,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.5399895310401917,
      "rewards/margins": 1.8903239965438843,
      "rewards/rejected": -2.4303135871887207,
      "step": 120
    },
    {
      "epoch": 0.26666666666666666,
      "grad_norm": 2.774472236633301,
      "learning_rate": 4.956619849265473e-06,
      "logits/chosen": -1.7566455602645874,
      "logits/rejected": -1.535705327987671,
      "logps/chosen": -238.42520141601562,
      "logps/rejected": -296.03424072265625,
      "loss": 0.2868,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4130798280239105,
      "rewards/margins": 2.085447072982788,
      "rewards/rejected": -2.4985270500183105,
      "step": 125
    },
    {
      "epoch": 0.2773333333333333,
      "grad_norm": 2.236811876296997,
      "learning_rate": 4.951117038118091e-06,
      "logits/chosen": -1.7088298797607422,
      "logits/rejected": -1.5699352025985718,
      "logps/chosen": -240.57534790039062,
      "logps/rejected": -314.7195739746094,
      "loss": 0.2552,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.4095377027988434,
      "rewards/margins": 2.295804977416992,
      "rewards/rejected": -2.7053427696228027,
      "step": 130
    },
    {
      "epoch": 0.288,
      "grad_norm": 2.135939121246338,
      "learning_rate": 4.945289161932273e-06,
      "logits/chosen": -1.747797966003418,
      "logits/rejected": -1.6278316974639893,
      "logps/chosen": -245.04721069335938,
      "logps/rejected": -334.56060791015625,
      "loss": 0.1957,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.33136236667633057,
      "rewards/margins": 2.5914673805236816,
      "rewards/rejected": -2.9228296279907227,
      "step": 135
    },
    {
      "epoch": 0.2986666666666667,
      "grad_norm": 4.112620830535889,
      "learning_rate": 4.93913699359596e-06,
      "logits/chosen": -1.7526079416275024,
      "logits/rejected": -1.58306086063385,
      "logps/chosen": -238.26718139648438,
      "logps/rejected": -328.44317626953125,
      "loss": 0.2467,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.6260155439376831,
      "rewards/margins": 2.4649341106414795,
      "rewards/rejected": -3.090949535369873,
      "step": 140
    },
    {
      "epoch": 0.30933333333333335,
      "grad_norm": 3.598618507385254,
      "learning_rate": 4.9326613490044364e-06,
      "logits/chosen": -1.7870292663574219,
      "logits/rejected": -1.6332266330718994,
      "logps/chosen": -234.255126953125,
      "logps/rejected": -304.899658203125,
      "loss": 0.3035,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.6351826786994934,
      "rewards/margins": 2.2837603092193604,
      "rewards/rejected": -2.918942928314209,
      "step": 145
    },
    {
      "epoch": 0.32,
      "grad_norm": 2.3492329120635986,
      "learning_rate": 4.925863086952132e-06,
      "logits/chosen": -1.805809736251831,
      "logits/rejected": -1.6427209377288818,
      "logps/chosen": -250.2255401611328,
      "logps/rejected": -336.9315490722656,
      "loss": 0.2122,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.4687698483467102,
      "rewards/margins": 2.872528076171875,
      "rewards/rejected": -3.3412978649139404,
      "step": 150
    },
    {
      "epoch": 0.33066666666666666,
      "grad_norm": 3.2796857357025146,
      "learning_rate": 4.918743109018727e-06,
      "logits/chosen": -1.870831847190857,
      "logits/rejected": -1.696987509727478,
      "logps/chosen": -249.03280639648438,
      "logps/rejected": -338.06231689453125,
      "loss": 0.2102,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.7773758769035339,
      "rewards/margins": 2.9179739952087402,
      "rewards/rejected": -3.695349931716919,
      "step": 155
    },
    {
      "epoch": 0.3413333333333333,
      "grad_norm": 3.265937089920044,
      "learning_rate": 4.911302359449585e-06,
      "logits/chosen": -1.862051010131836,
      "logits/rejected": -1.6175568103790283,
      "logps/chosen": -245.9069061279297,
      "logps/rejected": -347.184326171875,
      "loss": 0.1894,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.8534000515937805,
      "rewards/margins": 3.094893455505371,
      "rewards/rejected": -3.948293685913086,
      "step": 160
    },
    {
      "epoch": 0.352,
      "grad_norm": 4.633449077606201,
      "learning_rate": 4.9035418250305314e-06,
      "logits/chosen": -1.7832374572753906,
      "logits/rejected": -1.5434472560882568,
      "logps/chosen": -257.94879150390625,
      "logps/rejected": -334.8266296386719,
      "loss": 0.2446,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.8249349594116211,
      "rewards/margins": 2.9752113819122314,
      "rewards/rejected": -3.8001461029052734,
      "step": 165
    },
    {
      "epoch": 0.3626666666666667,
      "grad_norm": 2.3466925621032715,
      "learning_rate": 4.895462534956981e-06,
      "logits/chosen": -1.7460002899169922,
      "logits/rejected": -1.6494872570037842,
      "logps/chosen": -239.1138153076172,
      "logps/rejected": -327.669921875,
      "loss": 0.2073,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.773691713809967,
      "rewards/margins": 3.2550785541534424,
      "rewards/rejected": -4.028769493103027,
      "step": 170
    },
    {
      "epoch": 0.37333333333333335,
      "grad_norm": 3.3231382369995117,
      "learning_rate": 4.887065560697449e-06,
      "logits/chosen": -1.7860323190689087,
      "logits/rejected": -1.664729356765747,
      "logps/chosen": -262.41534423828125,
      "logps/rejected": -338.5781555175781,
      "loss": 0.2815,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.7041072249412537,
      "rewards/margins": 2.8621432781219482,
      "rewards/rejected": -3.5662505626678467,
      "step": 175
    },
    {
      "epoch": 0.384,
      "grad_norm": 1.6720951795578003,
      "learning_rate": 4.878352015851459e-06,
      "logits/chosen": -1.7511796951293945,
      "logits/rejected": -1.5772809982299805,
      "logps/chosen": -259.29962158203125,
      "logps/rejected": -330.5465087890625,
      "loss": 0.163,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.5976866483688354,
      "rewards/margins": 3.216883420944214,
      "rewards/rejected": -3.814570188522339,
      "step": 180
    },
    {
      "epoch": 0.39466666666666667,
      "grad_norm": 4.071167469024658,
      "learning_rate": 4.869323056001852e-06,
      "logits/chosen": -1.7100389003753662,
      "logits/rejected": -1.5902552604675293,
      "logps/chosen": -238.5122833251953,
      "logps/rejected": -319.55145263671875,
      "loss": 0.2501,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.7488561868667603,
      "rewards/margins": 2.900449514389038,
      "rewards/rejected": -3.649306058883667,
      "step": 185
    },
    {
      "epoch": 0.4053333333333333,
      "grad_norm": 2.2532527446746826,
      "learning_rate": 4.859979878561536e-06,
      "logits/chosen": -1.741315245628357,
      "logits/rejected": -1.579534888267517,
      "logps/chosen": -246.63308715820312,
      "logps/rejected": -334.5790710449219,
      "loss": 0.1882,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.4792085289955139,
      "rewards/margins": 3.239213466644287,
      "rewards/rejected": -3.7184219360351562,
      "step": 190
    },
    {
      "epoch": 0.416,
      "grad_norm": 5.985015869140625,
      "learning_rate": 4.850323722614685e-06,
      "logits/chosen": -1.751989722251892,
      "logits/rejected": -1.5852595567703247,
      "logps/chosen": -254.85635375976562,
      "logps/rejected": -345.94757080078125,
      "loss": 0.2467,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.5594536066055298,
      "rewards/margins": 3.1026852130889893,
      "rewards/rejected": -3.6621384620666504,
      "step": 195
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 4.524343967437744,
      "learning_rate": 4.840355868752416e-06,
      "logits/chosen": -1.688010811805725,
      "logits/rejected": -1.5757395029067993,
      "logps/chosen": -293.441162109375,
      "logps/rejected": -335.3061828613281,
      "loss": 0.285,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.47422847151756287,
      "rewards/margins": 2.8011474609375,
      "rewards/rejected": -3.2753758430480957,
      "step": 200
    },
    {
      "epoch": 0.43733333333333335,
      "grad_norm": 1.6691768169403076,
      "learning_rate": 4.830077638902955e-06,
      "logits/chosen": -1.8881086111068726,
      "logits/rejected": -1.6681621074676514,
      "logps/chosen": -265.43914794921875,
      "logps/rejected": -306.125,
      "loss": 0.2008,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.5452998876571655,
      "rewards/margins": 3.2147326469421387,
      "rewards/rejected": -3.7600326538085938,
      "step": 205
    },
    {
      "epoch": 0.448,
      "grad_norm": 3.2968130111694336,
      "learning_rate": 4.8194903961563255e-06,
      "logits/chosen": -1.8698129653930664,
      "logits/rejected": -1.733386754989624,
      "logps/chosen": -236.70834350585938,
      "logps/rejected": -325.7504577636719,
      "loss": 0.2263,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.6137224435806274,
      "rewards/margins": 3.213418960571289,
      "rewards/rejected": -3.827141284942627,
      "step": 210
    },
    {
      "epoch": 0.45866666666666667,
      "grad_norm": 1.8006588220596313,
      "learning_rate": 4.808595544583578e-06,
      "logits/chosen": -1.7925153970718384,
      "logits/rejected": -1.62285578250885,
      "logps/chosen": -244.73388671875,
      "logps/rejected": -338.7201232910156,
      "loss": 0.1987,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.5871314406394958,
      "rewards/margins": 3.5609264373779297,
      "rewards/rejected": -4.14805793762207,
      "step": 215
    },
    {
      "epoch": 0.4693333333333333,
      "grad_norm": 2.2668755054473877,
      "learning_rate": 4.797394529050577e-06,
      "logits/chosen": -1.8083196878433228,
      "logits/rejected": -1.6195793151855469,
      "logps/chosen": -254.16452026367188,
      "logps/rejected": -340.5189514160156,
      "loss": 0.1648,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.307282954454422,
      "rewards/margins": 3.5338077545166016,
      "rewards/rejected": -3.841090440750122,
      "step": 220
    },
    {
      "epoch": 0.48,
      "grad_norm": 2.5740833282470703,
      "learning_rate": 4.785888835026393e-06,
      "logits/chosen": -1.737634301185608,
      "logits/rejected": -1.5834619998931885,
      "logps/chosen": -234.80917358398438,
      "logps/rejected": -308.63641357421875,
      "loss": 0.2302,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.40825945138931274,
      "rewards/margins": 3.2277324199676514,
      "rewards/rejected": -3.6359925270080566,
      "step": 225
    },
    {
      "epoch": 0.49066666666666664,
      "grad_norm": 2.8291969299316406,
      "learning_rate": 4.7740799883862966e-06,
      "logits/chosen": -1.7801179885864258,
      "logits/rejected": -1.6636883020401,
      "logps/chosen": -237.48291015625,
      "logps/rejected": -349.30694580078125,
      "loss": 0.1692,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.503801167011261,
      "rewards/margins": 3.6341793537139893,
      "rewards/rejected": -4.1379804611206055,
      "step": 230
    },
    {
      "epoch": 0.5013333333333333,
      "grad_norm": 1.6978325843811035,
      "learning_rate": 4.761969555209394e-06,
      "logits/chosen": -1.79836905002594,
      "logits/rejected": -1.7076466083526611,
      "logps/chosen": -229.66592407226562,
      "logps/rejected": -336.6689758300781,
      "loss": 0.1825,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.48729997873306274,
      "rewards/margins": 3.8662896156311035,
      "rewards/rejected": -4.353589057922363,
      "step": 235
    },
    {
      "epoch": 0.512,
      "grad_norm": 2.3878140449523926,
      "learning_rate": 4.7495591415709435e-06,
      "logits/chosen": -1.8353424072265625,
      "logits/rejected": -1.6892439126968384,
      "logps/chosen": -265.48736572265625,
      "logps/rejected": -356.034912109375,
      "loss": 0.1745,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.8327113389968872,
      "rewards/margins": 3.900333881378174,
      "rewards/rejected": -4.73304557800293,
      "step": 240
    },
    {
      "epoch": 0.5226666666666666,
      "grad_norm": 2.2849137783050537,
      "learning_rate": 4.736850393329353e-06,
      "logits/chosen": -1.8372564315795898,
      "logits/rejected": -1.6298954486846924,
      "logps/chosen": -255.60238647460938,
      "logps/rejected": -361.90948486328125,
      "loss": 0.1689,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.9175726771354675,
      "rewards/margins": 4.157658576965332,
      "rewards/rejected": -5.075230598449707,
      "step": 245
    },
    {
      "epoch": 0.5333333333333333,
      "grad_norm": 3.430100917816162,
      "learning_rate": 4.7238449959079095e-06,
      "logits/chosen": -1.8621412515640259,
      "logits/rejected": -1.753686547279358,
      "logps/chosen": -237.799072265625,
      "logps/rejected": -339.875732421875,
      "loss": 0.1646,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.8456540107727051,
      "rewards/margins": 4.005865573883057,
      "rewards/rejected": -4.851519584655762,
      "step": 250
    },
    {
      "epoch": 0.544,
      "grad_norm": 3.102550506591797,
      "learning_rate": 4.710544674071259e-06,
      "logits/chosen": -1.8992693424224854,
      "logits/rejected": -1.7382304668426514,
      "logps/chosen": -244.0569305419922,
      "logps/rejected": -320.05047607421875,
      "loss": 0.181,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.7346665263175964,
      "rewards/margins": 4.076887607574463,
      "rewards/rejected": -4.811553955078125,
      "step": 255
    },
    {
      "epoch": 0.5546666666666666,
      "grad_norm": 4.898008346557617,
      "learning_rate": 4.696951191696673e-06,
      "logits/chosen": -1.8656402826309204,
      "logits/rejected": -1.6837927103042603,
      "logps/chosen": -246.1433868408203,
      "logps/rejected": -354.2840881347656,
      "loss": 0.2213,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.977743923664093,
      "rewards/margins": 4.141406536102295,
      "rewards/rejected": -5.119150638580322,
      "step": 260
    },
    {
      "epoch": 0.5653333333333334,
      "grad_norm": 2.6836533546447754,
      "learning_rate": 4.6830663515401196e-06,
      "logits/chosen": -1.8147302865982056,
      "logits/rejected": -1.6920936107635498,
      "logps/chosen": -246.29776000976562,
      "logps/rejected": -344.7865295410156,
      "loss": 0.2436,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.7966432571411133,
      "rewards/margins": 3.7020270824432373,
      "rewards/rejected": -4.4986701011657715,
      "step": 265
    },
    {
      "epoch": 0.576,
      "grad_norm": 4.104752063751221,
      "learning_rate": 4.6688919949971855e-06,
      "logits/chosen": -1.8584072589874268,
      "logits/rejected": -1.6018040180206299,
      "logps/chosen": -246.09945678710938,
      "logps/rejected": -320.53314208984375,
      "loss": 0.2292,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.820715606212616,
      "rewards/margins": 3.767307996749878,
      "rewards/rejected": -4.588023662567139,
      "step": 270
    },
    {
      "epoch": 0.5866666666666667,
      "grad_norm": 2.240436553955078,
      "learning_rate": 4.654430001858874e-06,
      "logits/chosen": -1.672431230545044,
      "logits/rejected": -1.5979852676391602,
      "logps/chosen": -249.657470703125,
      "logps/rejected": -345.25433349609375,
      "loss": 0.199,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.5963641405105591,
      "rewards/margins": 3.945819139480591,
      "rewards/rejected": -4.5421833992004395,
      "step": 275
    },
    {
      "epoch": 0.5973333333333334,
      "grad_norm": 3.5626108646392822,
      "learning_rate": 4.6396822900623075e-06,
      "logits/chosen": -1.8363211154937744,
      "logits/rejected": -1.6771224737167358,
      "logps/chosen": -258.85400390625,
      "logps/rejected": -342.59814453125,
      "loss": 0.1781,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.793243408203125,
      "rewards/margins": 4.015771865844727,
      "rewards/rejected": -4.809015274047852,
      "step": 280
    },
    {
      "epoch": 0.608,
      "grad_norm": 2.846989870071411,
      "learning_rate": 4.624650815436368e-06,
      "logits/chosen": -1.8874149322509766,
      "logits/rejected": -1.7275638580322266,
      "logps/chosen": -266.51983642578125,
      "logps/rejected": -327.6214599609375,
      "loss": 0.2161,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.5533876419067383,
      "rewards/margins": 3.9066085815429688,
      "rewards/rejected": -4.459996223449707,
      "step": 285
    },
    {
      "epoch": 0.6186666666666667,
      "grad_norm": 2.3967525959014893,
      "learning_rate": 4.609337571442324e-06,
      "logits/chosen": -1.8907324075698853,
      "logits/rejected": -1.7417322397232056,
      "logps/chosen": -232.366943359375,
      "logps/rejected": -336.410400390625,
      "loss": 0.1725,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.7537919878959656,
      "rewards/margins": 3.960203170776367,
      "rewards/rejected": -4.713995456695557,
      "step": 290
    },
    {
      "epoch": 0.6293333333333333,
      "grad_norm": 2.485433578491211,
      "learning_rate": 4.5937445889094535e-06,
      "logits/chosen": -1.7409560680389404,
      "logits/rejected": -1.6088342666625977,
      "logps/chosen": -268.9187927246094,
      "logps/rejected": -331.84161376953125,
      "loss": 0.1863,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.6697766780853271,
      "rewards/margins": 4.0562920570373535,
      "rewards/rejected": -4.72606897354126,
      "step": 295
    },
    {
      "epoch": 0.64,
      "grad_norm": 3.9823858737945557,
      "learning_rate": 4.577873935765722e-06,
      "logits/chosen": -1.893109917640686,
      "logits/rejected": -1.7313368320465088,
      "logps/chosen": -240.6698455810547,
      "logps/rejected": -339.091552734375,
      "loss": 0.1527,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.630599856376648,
      "rewards/margins": 4.312032222747803,
      "rewards/rejected": -4.942632675170898,
      "step": 300
    },
    {
      "epoch": 0.6506666666666666,
      "grad_norm": 3.2793948650360107,
      "learning_rate": 4.561727716763532e-06,
      "logits/chosen": -1.7743825912475586,
      "logits/rejected": -1.70479416847229,
      "logps/chosen": -263.1169738769531,
      "logps/rejected": -332.9526062011719,
      "loss": 0.1769,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.5394365787506104,
      "rewards/margins": 4.297283172607422,
      "rewards/rejected": -4.836719036102295,
      "step": 305
    },
    {
      "epoch": 0.6613333333333333,
      "grad_norm": 5.001033306121826,
      "learning_rate": 4.545308073200591e-06,
      "logits/chosen": -1.7639503479003906,
      "logits/rejected": -1.6067688465118408,
      "logps/chosen": -255.49417114257812,
      "logps/rejected": -350.12554931640625,
      "loss": 0.1838,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.6203233003616333,
      "rewards/margins": 4.100062370300293,
      "rewards/rejected": -4.720385551452637,
      "step": 310
    },
    {
      "epoch": 0.672,
      "grad_norm": 2.9403929710388184,
      "learning_rate": 4.528617182635943e-06,
      "logits/chosen": -1.8818168640136719,
      "logits/rejected": -1.7447856664657593,
      "logps/chosen": -251.169921875,
      "logps/rejected": -358.80645751953125,
      "loss": 0.1575,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.9520936012268066,
      "rewards/margins": 4.413204193115234,
      "rewards/rejected": -5.365297317504883,
      "step": 315
    },
    {
      "epoch": 0.6826666666666666,
      "grad_norm": 3.748673439025879,
      "learning_rate": 4.511657258601171e-06,
      "logits/chosen": -1.8479007482528687,
      "logits/rejected": -1.6725494861602783,
      "logps/chosen": -257.38983154296875,
      "logps/rejected": -335.9569396972656,
      "loss": 0.1584,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.9364475011825562,
      "rewards/margins": 4.432442665100098,
      "rewards/rejected": -5.368889808654785,
      "step": 320
    },
    {
      "epoch": 0.6933333333333334,
      "grad_norm": 3.777651309967041,
      "learning_rate": 4.494430550306848e-06,
      "logits/chosen": -1.757094383239746,
      "logits/rejected": -1.6897532939910889,
      "logps/chosen": -263.840087890625,
      "logps/rejected": -342.526611328125,
      "loss": 0.1966,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.5220140218734741,
      "rewards/margins": 4.274970054626465,
      "rewards/rejected": -4.796984672546387,
      "step": 325
    },
    {
      "epoch": 0.704,
      "grad_norm": 2.858718156814575,
      "learning_rate": 4.476939342344246e-06,
      "logits/chosen": -1.8669666051864624,
      "logits/rejected": -1.7618488073349,
      "logps/chosen": -231.40560913085938,
      "logps/rejected": -309.1484375,
      "loss": 0.191,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.6370632648468018,
      "rewards/margins": 3.9954731464385986,
      "rewards/rejected": -4.6325364112854,
      "step": 330
    },
    {
      "epoch": 0.7146666666666667,
      "grad_norm": 1.4209452867507935,
      "learning_rate": 4.45918595438236e-06,
      "logits/chosen": -1.7774779796600342,
      "logits/rejected": -1.7063461542129517,
      "logps/chosen": -253.4152069091797,
      "logps/rejected": -339.8118896484375,
      "loss": 0.2015,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.38608410954475403,
      "rewards/margins": 3.99464750289917,
      "rewards/rejected": -4.380731105804443,
      "step": 335
    },
    {
      "epoch": 0.7253333333333334,
      "grad_norm": 1.5503495931625366,
      "learning_rate": 4.441172740860267e-06,
      "logits/chosen": -1.767425298690796,
      "logits/rejected": -1.6601545810699463,
      "logps/chosen": -263.5538024902344,
      "logps/rejected": -362.27203369140625,
      "loss": 0.1889,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.3973921835422516,
      "rewards/margins": 4.149497032165527,
      "rewards/rejected": -4.546889781951904,
      "step": 340
    },
    {
      "epoch": 0.736,
      "grad_norm": 2.47979998588562,
      "learning_rate": 4.422902090674886e-06,
      "logits/chosen": -1.7320607900619507,
      "logits/rejected": -1.5309160947799683,
      "logps/chosen": -260.5031433105469,
      "logps/rejected": -330.44012451171875,
      "loss": 0.1594,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.32683077454566956,
      "rewards/margins": 3.9723198413848877,
      "rewards/rejected": -4.299150466918945,
      "step": 345
    },
    {
      "epoch": 0.7466666666666667,
      "grad_norm": 2.5703437328338623,
      "learning_rate": 4.4043764268641705e-06,
      "logits/chosen": -1.7610089778900146,
      "logits/rejected": -1.6328073740005493,
      "logps/chosen": -271.28472900390625,
      "logps/rejected": -344.73504638671875,
      "loss": 0.2003,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.8157901763916016,
      "rewards/margins": 4.135244846343994,
      "rewards/rejected": -4.9510345458984375,
      "step": 350
    },
    {
      "epoch": 0.7573333333333333,
      "grad_norm": 3.8566060066223145,
      "learning_rate": 4.385598206285758e-06,
      "logits/chosen": -1.927005410194397,
      "logits/rejected": -1.8474489450454712,
      "logps/chosen": -237.0289306640625,
      "logps/rejected": -356.33978271484375,
      "loss": 0.1774,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.3049725592136383,
      "rewards/margins": 4.615744590759277,
      "rewards/rejected": -4.920717716217041,
      "step": 355
    },
    {
      "epoch": 0.768,
      "grad_norm": 3.5054588317871094,
      "learning_rate": 4.3665699192911495e-06,
      "logits/chosen": -1.807989478111267,
      "logits/rejected": -1.707664132118225,
      "logps/chosen": -243.15560913085938,
      "logps/rejected": -323.3631896972656,
      "loss": 0.1703,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5694507956504822,
      "rewards/margins": 3.8626322746276855,
      "rewards/rejected": -4.4320831298828125,
      "step": 360
    },
    {
      "epoch": 0.7786666666666666,
      "grad_norm": 2.2553398609161377,
      "learning_rate": 4.347294089395438e-06,
      "logits/chosen": -1.8030427694320679,
      "logits/rejected": -1.7392162084579468,
      "logps/chosen": -276.47113037109375,
      "logps/rejected": -363.3248291015625,
      "loss": 0.1399,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.549434244632721,
      "rewards/margins": 4.955690383911133,
      "rewards/rejected": -5.505124092102051,
      "step": 365
    },
    {
      "epoch": 0.7893333333333333,
      "grad_norm": 3.9264116287231445,
      "learning_rate": 4.327773272942647e-06,
      "logits/chosen": -1.831888198852539,
      "logits/rejected": -1.679851770401001,
      "logps/chosen": -241.8992156982422,
      "logps/rejected": -336.3548278808594,
      "loss": 0.2017,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.47854313254356384,
      "rewards/margins": 4.543624401092529,
      "rewards/rejected": -5.022168159484863,
      "step": 370
    },
    {
      "epoch": 0.8,
      "grad_norm": 3.0306379795074463,
      "learning_rate": 4.308010058766703e-06,
      "logits/chosen": -1.8648799657821655,
      "logits/rejected": -1.7884979248046875,
      "logps/chosen": -247.47683715820312,
      "logps/rejected": -332.400146484375,
      "loss": 0.1547,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.6630781292915344,
      "rewards/margins": 4.304313659667969,
      "rewards/rejected": -4.967392444610596,
      "step": 375
    },
    {
      "epoch": 0.8106666666666666,
      "grad_norm": 4.7305145263671875,
      "learning_rate": 4.2880070678481135e-06,
      "logits/chosen": -1.8011143207550049,
      "logits/rejected": -1.725319266319275,
      "logps/chosen": -253.3076171875,
      "logps/rejected": -360.08831787109375,
      "loss": 0.1978,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.0798097848892212,
      "rewards/margins": 4.478891849517822,
      "rewards/rejected": -5.558701515197754,
      "step": 380
    },
    {
      "epoch": 0.8213333333333334,
      "grad_norm": 3.582191228866577,
      "learning_rate": 4.267766952966369e-06,
      "logits/chosen": -1.6858046054840088,
      "logits/rejected": -1.5931075811386108,
      "logps/chosen": -261.5023498535156,
      "logps/rejected": -340.2249450683594,
      "loss": 0.1993,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.9488990902900696,
      "rewards/margins": 4.209158897399902,
      "rewards/rejected": -5.158058166503906,
      "step": 385
    },
    {
      "epoch": 0.832,
      "grad_norm": 2.397202730178833,
      "learning_rate": 4.24729239834814e-06,
      "logits/chosen": -1.8469274044036865,
      "logits/rejected": -1.6774814128875732,
      "logps/chosen": -242.9910888671875,
      "logps/rejected": -323.50433349609375,
      "loss": 0.19,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.6384455561637878,
      "rewards/margins": 4.31686544418335,
      "rewards/rejected": -4.9553117752075195,
      "step": 390
    },
    {
      "epoch": 0.8426666666666667,
      "grad_norm": 2.013122081756592,
      "learning_rate": 4.226586119311291e-06,
      "logits/chosen": -1.8408855199813843,
      "logits/rejected": -1.7532079219818115,
      "logps/chosen": -255.99557495117188,
      "logps/rejected": -339.9315185546875,
      "loss": 0.1723,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.5134051442146301,
      "rewards/margins": 4.4296488761901855,
      "rewards/rejected": -4.94305419921875,
      "step": 395
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 2.0674853324890137,
      "learning_rate": 4.205650861904782e-06,
      "logits/chosen": -1.8111393451690674,
      "logits/rejected": -1.7262649536132812,
      "logps/chosen": -218.2287139892578,
      "logps/rejected": -331.74493408203125,
      "loss": 0.2054,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.650185763835907,
      "rewards/margins": 4.551926612854004,
      "rewards/rejected": -5.202112674713135,
      "step": 400
    },
    {
      "epoch": 0.864,
      "grad_norm": 3.9873046875,
      "learning_rate": 4.184489402544486e-06,
      "logits/chosen": -1.906991720199585,
      "logits/rejected": -1.7338151931762695,
      "logps/chosen": -248.287109375,
      "logps/rejected": -324.5978088378906,
      "loss": 0.2191,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.7245001196861267,
      "rewards/margins": 4.4610748291015625,
      "rewards/rejected": -5.185575485229492,
      "step": 405
    },
    {
      "epoch": 0.8746666666666667,
      "grad_norm": 3.678515672683716,
      "learning_rate": 4.1631045476449865e-06,
      "logits/chosen": -1.8973890542984009,
      "logits/rejected": -1.7549943923950195,
      "logps/chosen": -232.38192749023438,
      "logps/rejected": -343.0477294921875,
      "loss": 0.1735,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.3311890959739685,
      "rewards/margins": 4.941264629364014,
      "rewards/rejected": -5.272453308105469,
      "step": 410
    },
    {
      "epoch": 0.8853333333333333,
      "grad_norm": 3.474243402481079,
      "learning_rate": 4.141499133247391e-06,
      "logits/chosen": -1.747066855430603,
      "logits/rejected": -1.6317088603973389,
      "logps/chosen": -238.07205200195312,
      "logps/rejected": -329.851318359375,
      "loss": 0.1708,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.11001911014318466,
      "rewards/margins": 4.652035713195801,
      "rewards/rejected": -4.762054920196533,
      "step": 415
    },
    {
      "epoch": 0.896,
      "grad_norm": 3.526038885116577,
      "learning_rate": 4.119676024643217e-06,
      "logits/chosen": -1.7861535549163818,
      "logits/rejected": -1.6475368738174438,
      "logps/chosen": -242.25662231445312,
      "logps/rejected": -335.34637451171875,
      "loss": 0.1491,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.10151362419128418,
      "rewards/margins": 4.927655220031738,
      "rewards/rejected": -5.029168128967285,
      "step": 420
    },
    {
      "epoch": 0.9066666666666666,
      "grad_norm": 4.218128681182861,
      "learning_rate": 4.0976381159944055e-06,
      "logits/chosen": -1.8683736324310303,
      "logits/rejected": -1.7600526809692383,
      "logps/chosen": -248.19558715820312,
      "logps/rejected": -363.2998046875,
      "loss": 0.1956,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.1459302008152008,
      "rewards/margins": 4.895400047302246,
      "rewards/rejected": -5.041329383850098,
      "step": 425
    },
    {
      "epoch": 0.9173333333333333,
      "grad_norm": 2.3034024238586426,
      "learning_rate": 4.07538832994949e-06,
      "logits/chosen": -1.8572527170181274,
      "logits/rejected": -1.6589491367340088,
      "logps/chosen": -243.2412109375,
      "logps/rejected": -366.75897216796875,
      "loss": 0.1447,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.39440393447875977,
      "rewards/margins": 5.091270446777344,
      "rewards/rejected": -5.4856743812561035,
      "step": 430
    },
    {
      "epoch": 0.928,
      "grad_norm": 3.796588897705078,
      "learning_rate": 4.0529296172560055e-06,
      "logits/chosen": -1.8980152606964111,
      "logits/rejected": -1.6751664876937866,
      "logps/chosen": -242.3210906982422,
      "logps/rejected": -351.9512634277344,
      "loss": 0.1234,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.45586246252059937,
      "rewards/margins": 5.15492582321167,
      "rewards/rejected": -5.6107892990112305,
      "step": 435
    },
    {
      "epoch": 0.9386666666666666,
      "grad_norm": 3.219355344772339,
      "learning_rate": 4.030264956369158e-06,
      "logits/chosen": -1.819602608680725,
      "logits/rejected": -1.7250854969024658,
      "logps/chosen": -240.2069549560547,
      "logps/rejected": -333.0482482910156,
      "loss": 0.1515,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6956228017807007,
      "rewards/margins": 4.75858211517334,
      "rewards/rejected": -5.454204559326172,
      "step": 440
    },
    {
      "epoch": 0.9493333333333334,
      "grad_norm": 3.2204010486602783,
      "learning_rate": 4.0073973530568274e-06,
      "logits/chosen": -1.8022804260253906,
      "logits/rejected": -1.6240564584732056,
      "logps/chosen": -263.3968811035156,
      "logps/rejected": -343.99139404296875,
      "loss": 0.1268,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.5587592720985413,
      "rewards/margins": 5.209686279296875,
      "rewards/rejected": -5.7684454917907715,
      "step": 445
    },
    {
      "epoch": 0.96,
      "grad_norm": 2.604736089706421,
      "learning_rate": 3.9843298400009435e-06,
      "logits/chosen": -1.856161117553711,
      "logits/rejected": -1.790837049484253,
      "logps/chosen": -248.9732666015625,
      "logps/rejected": -353.9434814453125,
      "loss": 0.1207,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.5389277935028076,
      "rewards/margins": 5.065027713775635,
      "rewards/rejected": -5.603955268859863,
      "step": 450
    },
    {
      "epoch": 0.9706666666666667,
      "grad_norm": 3.975954532623291,
      "learning_rate": 3.961065476395296e-06,
      "logits/chosen": -1.830162763595581,
      "logits/rejected": -1.6826388835906982,
      "logps/chosen": -268.0751953125,
      "logps/rejected": -363.59967041015625,
      "loss": 0.1978,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.8818696737289429,
      "rewards/margins": 5.387181758880615,
      "rewards/rejected": -6.269051551818848,
      "step": 455
    },
    {
      "epoch": 0.9813333333333333,
      "grad_norm": 4.976337432861328,
      "learning_rate": 3.937607347539823e-06,
      "logits/chosen": -1.814556360244751,
      "logits/rejected": -1.7047951221466064,
      "logps/chosen": -282.71575927734375,
      "logps/rejected": -367.3832092285156,
      "loss": 0.148,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1829640865325928,
      "rewards/margins": 4.825137138366699,
      "rewards/rejected": -6.008100986480713,
      "step": 460
    },
    {
      "epoch": 0.992,
      "grad_norm": 2.259762763977051,
      "learning_rate": 3.913958564431447e-06,
      "logits/chosen": -1.8797647953033447,
      "logits/rejected": -1.752611756324768,
      "logps/chosen": -252.342041015625,
      "logps/rejected": -379.76861572265625,
      "loss": 0.1244,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.859488844871521,
      "rewards/margins": 6.077948570251465,
      "rewards/rejected": -6.937437534332275,
      "step": 465
    },
    {
      "epoch": 1.0021333333333333,
      "grad_norm": 1.6464236974716187,
      "learning_rate": 3.89012226335149e-06,
      "logits/chosen": -1.7027655839920044,
      "logits/rejected": -1.6129655838012695,
      "logps/chosen": -256.5233459472656,
      "logps/rejected": -357.3712158203125,
      "loss": 0.0864,
      "rewards/accuracies": 0.9736841917037964,
      "rewards/chosen": -0.9277030825614929,
      "rewards/margins": 5.774387836456299,
      "rewards/rejected": -6.702090740203857,
      "step": 470
    },
    {
      "epoch": 1.0128,
      "grad_norm": 1.6793190240859985,
      "learning_rate": 3.86610160544975e-06,
      "logits/chosen": -1.8415664434432983,
      "logits/rejected": -1.6897361278533936,
      "logps/chosen": -247.79385375976562,
      "logps/rejected": -364.9015197753906,
      "loss": 0.0841,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.8070721626281738,
      "rewards/margins": 6.2195940017700195,
      "rewards/rejected": -7.026666164398193,
      "step": 475
    },
    {
      "epoch": 1.0234666666666667,
      "grad_norm": 2.209942579269409,
      "learning_rate": 3.841899776325267e-06,
      "logits/chosen": -1.8175064325332642,
      "logits/rejected": -1.6627798080444336,
      "logps/chosen": -276.42974853515625,
      "logps/rejected": -348.6553955078125,
      "loss": 0.1084,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.6729779243469238,
      "rewards/margins": 5.814656734466553,
      "rewards/rejected": -6.487635135650635,
      "step": 480
    },
    {
      "epoch": 1.0341333333333333,
      "grad_norm": 5.6867594718933105,
      "learning_rate": 3.817519985603855e-06,
      "logits/chosen": -1.950702428817749,
      "logits/rejected": -1.8375511169433594,
      "logps/chosen": -237.49368286132812,
      "logps/rejected": -373.88739013671875,
      "loss": 0.122,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.1298967599868774,
      "rewards/margins": 6.234266757965088,
      "rewards/rejected": -7.364163398742676,
      "step": 485
    },
    {
      "epoch": 1.0448,
      "grad_norm": 3.8766770362854004,
      "learning_rate": 3.792965466512437e-06,
      "logits/chosen": -1.8295539617538452,
      "logits/rejected": -1.6258299350738525,
      "logps/chosen": -284.9169616699219,
      "logps/rejected": -360.6707763671875,
      "loss": 0.1105,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.3740360736846924,
      "rewards/margins": 6.0007429122924805,
      "rewards/rejected": -7.37477970123291,
      "step": 490
    },
    {
      "epoch": 1.0554666666666668,
      "grad_norm": 1.0155181884765625,
      "learning_rate": 3.7682394754502687e-06,
      "logits/chosen": -1.8263795375823975,
      "logits/rejected": -1.7287553548812866,
      "logps/chosen": -253.3537139892578,
      "logps/rejected": -351.697265625,
      "loss": 0.1145,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.1872411966323853,
      "rewards/margins": 6.306341648101807,
      "rewards/rejected": -7.493583679199219,
      "step": 495
    },
    {
      "epoch": 1.0661333333333334,
      "grad_norm": 2.8762567043304443,
      "learning_rate": 3.7433452915570685e-06,
      "logits/chosen": -1.8811756372451782,
      "logits/rejected": -1.8007991313934326,
      "logps/chosen": -265.2466735839844,
      "logps/rejected": -367.4648132324219,
      "loss": 0.1216,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.0806666612625122,
      "rewards/margins": 6.194316864013672,
      "rewards/rejected": -7.274983882904053,
      "step": 500
    },
    {
      "epoch": 1.0768,
      "grad_norm": 2.1295065879821777,
      "learning_rate": 3.718286216278143e-06,
      "logits/chosen": -1.882246732711792,
      "logits/rejected": -1.761561393737793,
      "logps/chosen": -302.8157958984375,
      "logps/rejected": -370.31353759765625,
      "loss": 0.1144,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.217259407043457,
      "rewards/margins": 6.504574775695801,
      "rewards/rejected": -7.7218337059021,
      "step": 505
    },
    {
      "epoch": 1.0874666666666666,
      "grad_norm": 4.344186782836914,
      "learning_rate": 3.6930655729265557e-06,
      "logits/chosen": -1.8296124935150146,
      "logits/rejected": -1.650610327720642,
      "logps/chosen": -282.95611572265625,
      "logps/rejected": -352.1752014160156,
      "loss": 0.1132,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.343949317932129,
      "rewards/margins": 6.264041900634766,
      "rewards/rejected": -7.6079912185668945,
      "step": 510
    },
    {
      "epoch": 1.0981333333333334,
      "grad_norm": 2.932526111602783,
      "learning_rate": 3.667686706242386e-06,
      "logits/chosen": -1.7797019481658936,
      "logits/rejected": -1.6947786808013916,
      "logps/chosen": -257.07012939453125,
      "logps/rejected": -363.6566467285156,
      "loss": 0.1203,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.1053550243377686,
      "rewards/margins": 6.050717830657959,
      "rewards/rejected": -7.156073093414307,
      "step": 515
    },
    {
      "epoch": 1.1088,
      "grad_norm": 4.214327812194824,
      "learning_rate": 3.6421529819491576e-06,
      "logits/chosen": -1.9010741710662842,
      "logits/rejected": -1.8004162311553955,
      "logps/chosen": -292.2718811035156,
      "logps/rejected": -375.58453369140625,
      "loss": 0.1276,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.4249348640441895,
      "rewards/margins": 5.854917049407959,
      "rewards/rejected": -7.279852390289307,
      "step": 520
    },
    {
      "epoch": 1.1194666666666666,
      "grad_norm": 2.3777670860290527,
      "learning_rate": 3.6164677863074783e-06,
      "logits/chosen": -1.8030182123184204,
      "logits/rejected": -1.6846275329589844,
      "logps/chosen": -272.50238037109375,
      "logps/rejected": -366.3551330566406,
      "loss": 0.1121,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.5246198177337646,
      "rewards/margins": 6.520844459533691,
      "rewards/rejected": -8.045464515686035,
      "step": 525
    },
    {
      "epoch": 1.1301333333333332,
      "grad_norm": 4.471464157104492,
      "learning_rate": 3.590634525665955e-06,
      "logits/chosen": -1.7948955297470093,
      "logits/rejected": -1.6809746026992798,
      "logps/chosen": -262.9297180175781,
      "logps/rejected": -381.04901123046875,
      "loss": 0.0936,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.556490182876587,
      "rewards/margins": 6.692111015319824,
      "rewards/rejected": -8.248600959777832,
      "step": 530
    },
    {
      "epoch": 1.1408,
      "grad_norm": 3.911820888519287,
      "learning_rate": 3.564656626009446e-06,
      "logits/chosen": -1.8893358707427979,
      "logits/rejected": -1.7485568523406982,
      "logps/chosen": -268.07177734375,
      "logps/rejected": -390.9849853515625,
      "loss": 0.1555,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.9269845485687256,
      "rewards/margins": 6.015888690948486,
      "rewards/rejected": -7.942873477935791,
      "step": 535
    },
    {
      "epoch": 1.1514666666666666,
      "grad_norm": 3.294266700744629,
      "learning_rate": 3.5385375325047167e-06,
      "logits/chosen": -1.8690032958984375,
      "logits/rejected": -1.7063426971435547,
      "logps/chosen": -267.24578857421875,
      "logps/rejected": -384.7957458496094,
      "loss": 0.1166,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6315892934799194,
      "rewards/margins": 6.775505065917969,
      "rewards/rejected": -8.407094955444336,
      "step": 540
    },
    {
      "epoch": 1.1621333333333332,
      "grad_norm": 4.33017110824585,
      "learning_rate": 3.512280709043534e-06,
      "logits/chosen": -1.851406455039978,
      "logits/rejected": -1.6782381534576416,
      "logps/chosen": -262.625,
      "logps/rejected": -372.85479736328125,
      "loss": 0.1376,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.9270426034927368,
      "rewards/margins": 6.284356594085693,
      "rewards/rejected": -7.211399078369141,
      "step": 545
    },
    {
      "epoch": 1.1728,
      "grad_norm": 3.753199815750122,
      "learning_rate": 3.4858896377832966e-06,
      "logits/chosen": -1.8479499816894531,
      "logits/rejected": -1.7503430843353271,
      "logps/chosen": -239.7978057861328,
      "logps/rejected": -369.9670104980469,
      "loss": 0.0983,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.9708150625228882,
      "rewards/margins": 6.7487077713012695,
      "rewards/rejected": -7.719522953033447,
      "step": 550
    },
    {
      "epoch": 1.1834666666666667,
      "grad_norm": 3.7350525856018066,
      "learning_rate": 3.459367818685231e-06,
      "logits/chosen": -1.8815902471542358,
      "logits/rejected": -1.7400764226913452,
      "logps/chosen": -243.23226928710938,
      "logps/rejected": -369.14349365234375,
      "loss": 0.1243,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.9012893438339233,
      "rewards/margins": 6.555959224700928,
      "rewards/rejected": -7.457249641418457,
      "step": 555
    },
    {
      "epoch": 1.1941333333333333,
      "grad_norm": 3.1958982944488525,
      "learning_rate": 3.432718769050228e-06,
      "logits/chosen": -1.764378547668457,
      "logits/rejected": -1.6634269952774048,
      "logps/chosen": -248.8746337890625,
      "logps/rejected": -346.23834228515625,
      "loss": 0.1185,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.120922565460205,
      "rewards/margins": 5.680094242095947,
      "rewards/rejected": -6.801016807556152,
      "step": 560
    },
    {
      "epoch": 1.2048,
      "grad_norm": 1.215667486190796,
      "learning_rate": 3.405946023052386e-06,
      "logits/chosen": -1.8708391189575195,
      "logits/rejected": -1.7209943532943726,
      "logps/chosen": -268.7168884277344,
      "logps/rejected": -383.05194091796875,
      "loss": 0.1217,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.5790942907333374,
      "rewards/margins": 6.199840068817139,
      "rewards/rejected": -7.778934478759766,
      "step": 565
    },
    {
      "epoch": 1.2154666666666667,
      "grad_norm": 4.288374900817871,
      "learning_rate": 3.3790531312703055e-06,
      "logits/chosen": -1.7612943649291992,
      "logits/rejected": -1.6972544193267822,
      "logps/chosen": -243.02767944335938,
      "logps/rejected": -364.6146545410156,
      "loss": 0.1226,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6340267658233643,
      "rewards/margins": 5.973841667175293,
      "rewards/rejected": -7.6078691482543945,
      "step": 570
    },
    {
      "epoch": 1.2261333333333333,
      "grad_norm": 2.426377058029175,
      "learning_rate": 3.35204366021622e-06,
      "logits/chosen": -1.701094388961792,
      "logits/rejected": -1.6422580480575562,
      "logps/chosen": -262.212646484375,
      "logps/rejected": -371.54376220703125,
      "loss": 0.1251,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.1702404022216797,
      "rewards/margins": 6.165692329406738,
      "rewards/rejected": -7.33593225479126,
      "step": 575
    },
    {
      "epoch": 1.2368000000000001,
      "grad_norm": 4.914599895477295,
      "learning_rate": 3.3249211918630055e-06,
      "logits/chosen": -1.8257882595062256,
      "logits/rejected": -1.7253891229629517,
      "logps/chosen": -248.6946563720703,
      "logps/rejected": -377.42041015625,
      "loss": 0.1123,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.544409155845642,
      "rewards/margins": 6.707772731781006,
      "rewards/rejected": -8.252182006835938,
      "step": 580
    },
    {
      "epoch": 1.2474666666666667,
      "grad_norm": 3.5044515132904053,
      "learning_rate": 3.2976893231691424e-06,
      "logits/chosen": -1.656166672706604,
      "logits/rejected": -1.6489145755767822,
      "logps/chosen": -270.85125732421875,
      "logps/rejected": -362.0047302246094,
      "loss": 0.1183,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.5041134357452393,
      "rewards/margins": 6.453446865081787,
      "rewards/rejected": -7.957561492919922,
      "step": 585
    },
    {
      "epoch": 1.2581333333333333,
      "grad_norm": 2.836451292037964,
      "learning_rate": 3.270351665601691e-06,
      "logits/chosen": -1.7868537902832031,
      "logits/rejected": -1.6743824481964111,
      "logps/chosen": -255.75405883789062,
      "logps/rejected": -383.26068115234375,
      "loss": 0.088,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.5279412269592285,
      "rewards/margins": 7.09237003326416,
      "rewards/rejected": -8.62031078338623,
      "step": 590
    },
    {
      "epoch": 1.2688,
      "grad_norm": 5.035298824310303,
      "learning_rate": 3.2429118446573417e-06,
      "logits/chosen": -1.7914812564849854,
      "logits/rejected": -1.6376540660858154,
      "logps/chosen": -252.78878784179688,
      "logps/rejected": -383.7055969238281,
      "loss": 0.1018,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.580556869506836,
      "rewards/margins": 6.985851287841797,
      "rewards/rejected": -8.566408157348633,
      "step": 595
    },
    {
      "epoch": 1.2794666666666665,
      "grad_norm": 4.154091835021973,
      "learning_rate": 3.215373499381602e-06,
      "logits/chosen": -1.7791035175323486,
      "logits/rejected": -1.624894142150879,
      "logps/chosen": -251.47994995117188,
      "logps/rejected": -339.7424621582031,
      "loss": 0.1252,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.2945369482040405,
      "rewards/margins": 5.928402900695801,
      "rewards/rejected": -7.222939968109131,
      "step": 600
    },
    {
      "epoch": 1.2901333333333334,
      "grad_norm": 1.5163425207138062,
      "learning_rate": 3.1877402818861954e-06,
      "logits/chosen": -1.8631162643432617,
      "logits/rejected": -1.7690770626068115,
      "logps/chosen": -246.5840606689453,
      "logps/rejected": -398.3299255371094,
      "loss": 0.0679,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.8003944158554077,
      "rewards/margins": 7.669053554534912,
      "rewards/rejected": -9.46944808959961,
      "step": 605
    },
    {
      "epoch": 1.3008,
      "grad_norm": 1.9956570863723755,
      "learning_rate": 3.1600158568647145e-06,
      "logits/chosen": -1.7449201345443726,
      "logits/rejected": -1.6165313720703125,
      "logps/chosen": -282.90399169921875,
      "logps/rejected": -367.9050598144531,
      "loss": 0.1269,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.6516921520233154,
      "rewards/margins": 6.830259799957275,
      "rewards/rejected": -8.481951713562012,
      "step": 610
    },
    {
      "epoch": 1.3114666666666666,
      "grad_norm": 1.5468382835388184,
      "learning_rate": 3.1322039011066172e-06,
      "logits/chosen": -1.780417799949646,
      "logits/rejected": -1.7057693004608154,
      "logps/chosen": -257.28753662109375,
      "logps/rejected": -378.251708984375,
      "loss": 0.0976,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.1922212839126587,
      "rewards/margins": 7.120926856994629,
      "rewards/rejected": -8.313148498535156,
      "step": 615
    },
    {
      "epoch": 1.3221333333333334,
      "grad_norm": 2.0375499725341797,
      "learning_rate": 3.1043081030096125e-06,
      "logits/chosen": -1.7060388326644897,
      "logits/rejected": -1.641692876815796,
      "logps/chosen": -254.7395782470703,
      "logps/rejected": -357.2270202636719,
      "loss": 0.1211,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.581194281578064,
      "rewards/margins": 6.09962272644043,
      "rewards/rejected": -7.680817604064941,
      "step": 620
    },
    {
      "epoch": 1.3328,
      "grad_norm": 3.9574337005615234,
      "learning_rate": 3.0763321620905072e-06,
      "logits/chosen": -1.736730933189392,
      "logits/rejected": -1.6614354848861694,
      "logps/chosen": -255.10400390625,
      "logps/rejected": -376.260498046875,
      "loss": 0.1029,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.3230445384979248,
      "rewards/margins": 6.803250312805176,
      "rewards/rejected": -8.12629508972168,
      "step": 625
    },
    {
      "epoch": 1.3434666666666666,
      "grad_norm": 2.847869634628296,
      "learning_rate": 3.048279788494583e-06,
      "logits/chosen": -1.808293342590332,
      "logits/rejected": -1.6874719858169556,
      "logps/chosen": -241.78811645507812,
      "logps/rejected": -371.5953674316406,
      "loss": 0.104,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.4912164211273193,
      "rewards/margins": 6.666079521179199,
      "rewards/rejected": -8.157295227050781,
      "step": 630
    },
    {
      "epoch": 1.3541333333333334,
      "grad_norm": 3.1436240673065186,
      "learning_rate": 3.0201547025035553e-06,
      "logits/chosen": -1.786981225013733,
      "logits/rejected": -1.7256486415863037,
      "logps/chosen": -251.4448699951172,
      "logps/rejected": -379.3356628417969,
      "loss": 0.0956,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.5715595483779907,
      "rewards/margins": 6.413365840911865,
      "rewards/rejected": -7.98492431640625,
      "step": 635
    },
    {
      "epoch": 1.3648,
      "grad_norm": 3.5846307277679443,
      "learning_rate": 2.9919606340421963e-06,
      "logits/chosen": -1.6872005462646484,
      "logits/rejected": -1.5978786945343018,
      "logps/chosen": -247.89767456054688,
      "logps/rejected": -378.5216369628906,
      "loss": 0.0886,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.7142194509506226,
      "rewards/margins": 6.703309535980225,
      "rewards/rejected": -8.417529106140137,
      "step": 640
    },
    {
      "epoch": 1.3754666666666666,
      "grad_norm": 3.123239278793335,
      "learning_rate": 2.9637013221836723e-06,
      "logits/chosen": -1.7665446996688843,
      "logits/rejected": -1.66081964969635,
      "logps/chosen": -263.37200927734375,
      "logps/rejected": -377.9620666503906,
      "loss": 0.1191,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.6348273754119873,
      "rewards/margins": 6.643343925476074,
      "rewards/rejected": -8.278172492980957,
      "step": 645
    },
    {
      "epoch": 1.3861333333333334,
      "grad_norm": 5.442415714263916,
      "learning_rate": 2.9353805146536762e-06,
      "logits/chosen": -1.7114347219467163,
      "logits/rejected": -1.5927168130874634,
      "logps/chosen": -258.96282958984375,
      "logps/rejected": -360.3382873535156,
      "loss": 0.0935,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.6617997884750366,
      "rewards/margins": 6.874170780181885,
      "rewards/rejected": -8.535970687866211,
      "step": 650
    },
    {
      "epoch": 1.3968,
      "grad_norm": 4.683470726013184,
      "learning_rate": 2.907001967333401e-06,
      "logits/chosen": -1.748187780380249,
      "logits/rejected": -1.7318449020385742,
      "logps/chosen": -271.8290710449219,
      "logps/rejected": -402.3279724121094,
      "loss": 0.0957,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.7431981563568115,
      "rewards/margins": 7.089310646057129,
      "rewards/rejected": -8.83250904083252,
      "step": 655
    },
    {
      "epoch": 1.4074666666666666,
      "grad_norm": 2.5879671573638916,
      "learning_rate": 2.878569443761442e-06,
      "logits/chosen": -1.695862054824829,
      "logits/rejected": -1.6406151056289673,
      "logps/chosen": -258.796875,
      "logps/rejected": -352.96966552734375,
      "loss": 0.154,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.9192339181900024,
      "rewards/margins": 6.030402660369873,
      "rewards/rejected": -7.949636936187744,
      "step": 660
    },
    {
      "epoch": 1.4181333333333335,
      "grad_norm": 3.4974851608276367,
      "learning_rate": 2.85008671463468e-06,
      "logits/chosen": -1.7247854471206665,
      "logits/rejected": -1.613422155380249,
      "logps/chosen": -246.9770965576172,
      "logps/rejected": -371.6976623535156,
      "loss": 0.0889,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.660942792892456,
      "rewards/margins": 7.5227766036987305,
      "rewards/rejected": -9.18371868133545,
      "step": 665
    },
    {
      "epoch": 1.4288,
      "grad_norm": 6.057355880737305,
      "learning_rate": 2.8215575573082127e-06,
      "logits/chosen": -1.791477918624878,
      "logits/rejected": -1.7090383768081665,
      "logps/chosen": -240.56173706054688,
      "logps/rejected": -391.3064880371094,
      "loss": 0.1342,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9196621179580688,
      "rewards/margins": 7.553940773010254,
      "rewards/rejected": -9.473602294921875,
      "step": 670
    },
    {
      "epoch": 1.4394666666666667,
      "grad_norm": 3.2657501697540283,
      "learning_rate": 2.792985755294404e-06,
      "logits/chosen": -1.745305061340332,
      "logits/rejected": -1.6773016452789307,
      "logps/chosen": -256.55780029296875,
      "logps/rejected": -385.68304443359375,
      "loss": 0.1057,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.1826651096343994,
      "rewards/margins": 6.950257301330566,
      "rewards/rejected": -9.132922172546387,
      "step": 675
    },
    {
      "epoch": 1.4501333333333333,
      "grad_norm": 1.1689438819885254,
      "learning_rate": 2.764375097761122e-06,
      "logits/chosen": -1.6467416286468506,
      "logits/rejected": -1.5601441860198975,
      "logps/chosen": -241.9093475341797,
      "logps/rejected": -382.58197021484375,
      "loss": 0.1232,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.4370450973510742,
      "rewards/margins": 6.910675048828125,
      "rewards/rejected": -8.3477201461792,
      "step": 680
    },
    {
      "epoch": 1.4607999999999999,
      "grad_norm": 2.4693634510040283,
      "learning_rate": 2.73572937902922e-06,
      "logits/chosen": -1.7208999395370483,
      "logits/rejected": -1.5330959558486938,
      "logps/chosen": -260.446044921875,
      "logps/rejected": -351.178466796875,
      "loss": 0.0977,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.2097041606903076,
      "rewards/margins": 7.18826961517334,
      "rewards/rejected": -8.397974014282227,
      "step": 685
    },
    {
      "epoch": 1.4714666666666667,
      "grad_norm": 0.7693854570388794,
      "learning_rate": 2.7070523980693362e-06,
      "logits/chosen": -1.5774312019348145,
      "logits/rejected": -1.5146809816360474,
      "logps/chosen": -255.57992553710938,
      "logps/rejected": -353.2482604980469,
      "loss": 0.1025,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.336712121963501,
      "rewards/margins": 6.85360860824585,
      "rewards/rejected": -8.19032096862793,
      "step": 690
    },
    {
      "epoch": 1.4821333333333333,
      "grad_norm": 1.9660614728927612,
      "learning_rate": 2.678347957998081e-06,
      "logits/chosen": -1.7294142246246338,
      "logits/rejected": -1.6881201267242432,
      "logps/chosen": -254.1973876953125,
      "logps/rejected": -371.61785888671875,
      "loss": 0.1127,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7192710638046265,
      "rewards/margins": 7.066683769226074,
      "rewards/rejected": -8.785955429077148,
      "step": 695
    },
    {
      "epoch": 1.4928,
      "grad_norm": 3.160243511199951,
      "learning_rate": 2.649619865573666e-06,
      "logits/chosen": -1.8685362339019775,
      "logits/rejected": -1.6781858205795288,
      "logps/chosen": -250.9998016357422,
      "logps/rejected": -389.24212646484375,
      "loss": 0.0503,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.2655198574066162,
      "rewards/margins": 8.069536209106445,
      "rewards/rejected": -9.335058212280273,
      "step": 700
    },
    {
      "epoch": 1.5034666666666667,
      "grad_norm": 2.4785704612731934,
      "learning_rate": 2.620871930691058e-06,
      "logits/chosen": -1.7562789916992188,
      "logits/rejected": -1.6890990734100342,
      "logps/chosen": -257.45574951171875,
      "logps/rejected": -376.5826110839844,
      "loss": 0.1266,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.5733845233917236,
      "rewards/margins": 6.933058261871338,
      "rewards/rejected": -8.506443977355957,
      "step": 705
    },
    {
      "epoch": 1.5141333333333333,
      "grad_norm": 2.3427517414093018,
      "learning_rate": 2.5921079658767152e-06,
      "logits/chosen": -1.7983629703521729,
      "logits/rejected": -1.6955991983413696,
      "logps/chosen": -265.74200439453125,
      "logps/rejected": -360.1547546386719,
      "loss": 0.144,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.7915146350860596,
      "rewards/margins": 6.563783168792725,
      "rewards/rejected": -8.355298042297363,
      "step": 710
    },
    {
      "epoch": 1.5248,
      "grad_norm": 3.470958948135376,
      "learning_rate": 2.56333178578297e-06,
      "logits/chosen": -1.779236078262329,
      "logits/rejected": -1.6289777755737305,
      "logps/chosen": -253.560546875,
      "logps/rejected": -373.86187744140625,
      "loss": 0.1072,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.766411542892456,
      "rewards/margins": 7.2796630859375,
      "rewards/rejected": -9.046075820922852,
      "step": 715
    },
    {
      "epoch": 1.5354666666666668,
      "grad_norm": 6.400219440460205,
      "learning_rate": 2.534547206682136e-06,
      "logits/chosen": -1.7732305526733398,
      "logits/rejected": -1.6621010303497314,
      "logps/chosen": -248.0487060546875,
      "logps/rejected": -365.96014404296875,
      "loss": 0.1459,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.613825798034668,
      "rewards/margins": 7.0502424240112305,
      "rewards/rejected": -8.664068222045898,
      "step": 720
    },
    {
      "epoch": 1.5461333333333334,
      "grad_norm": 2.4263100624084473,
      "learning_rate": 2.5057580459603947e-06,
      "logits/chosen": -1.8636562824249268,
      "logits/rejected": -1.7396777868270874,
      "logps/chosen": -259.3055419921875,
      "logps/rejected": -364.75115966796875,
      "loss": 0.084,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.641998529434204,
      "rewards/margins": 7.215865135192871,
      "rewards/rejected": -8.857863426208496,
      "step": 725
    },
    {
      "epoch": 1.5568,
      "grad_norm": 2.6077945232391357,
      "learning_rate": 2.4769681216115384e-06,
      "logits/chosen": -1.8251991271972656,
      "logits/rejected": -1.752423882484436,
      "logps/chosen": -272.037841796875,
      "logps/rejected": -376.82769775390625,
      "loss": 0.106,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.859642744064331,
      "rewards/margins": 6.8842644691467285,
      "rewards/rejected": -8.74390697479248,
      "step": 730
    },
    {
      "epoch": 1.5674666666666668,
      "grad_norm": 2.8333311080932617,
      "learning_rate": 2.448181251730632e-06,
      "logits/chosen": -1.7708256244659424,
      "logits/rejected": -1.621558427810669,
      "logps/chosen": -251.92745971679688,
      "logps/rejected": -365.28289794921875,
      "loss": 0.1074,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.313032627105713,
      "rewards/margins": 6.848665714263916,
      "rewards/rejected": -8.161698341369629,
      "step": 735
    },
    {
      "epoch": 1.5781333333333334,
      "grad_norm": 3.1226139068603516,
      "learning_rate": 2.419401254007658e-06,
      "logits/chosen": -1.7166410684585571,
      "logits/rejected": -1.6209323406219482,
      "logps/chosen": -248.4964599609375,
      "logps/rejected": -359.0329895019531,
      "loss": 0.083,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.2056635618209839,
      "rewards/margins": 7.018212795257568,
      "rewards/rejected": -8.223875999450684,
      "step": 740
    },
    {
      "epoch": 1.5888,
      "grad_norm": 4.352267742156982,
      "learning_rate": 2.390631945221221e-06,
      "logits/chosen": -1.7971668243408203,
      "logits/rejected": -1.6970182657241821,
      "logps/chosen": -256.62530517578125,
      "logps/rejected": -390.9517517089844,
      "loss": 0.1439,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6711896657943726,
      "rewards/margins": 6.6160407066345215,
      "rewards/rejected": -8.287230491638184,
      "step": 745
    },
    {
      "epoch": 1.5994666666666668,
      "grad_norm": 0.7765277028083801,
      "learning_rate": 2.361877140732366e-06,
      "logits/chosen": -1.834469199180603,
      "logits/rejected": -1.6532119512557983,
      "logps/chosen": -249.960693359375,
      "logps/rejected": -363.2095642089844,
      "loss": 0.063,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.540730357170105,
      "rewards/margins": 7.1370530128479,
      "rewards/rejected": -8.67778205871582,
      "step": 750
    },
    {
      "epoch": 1.6101333333333332,
      "grad_norm": 3.875185489654541,
      "learning_rate": 2.333140653978592e-06,
      "logits/chosen": -1.7678000926971436,
      "logits/rejected": -1.6562210321426392,
      "logps/chosen": -254.3054962158203,
      "logps/rejected": -370.5506286621094,
      "loss": 0.1142,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.564232587814331,
      "rewards/margins": 7.189335823059082,
      "rewards/rejected": -8.753567695617676,
      "step": 755
    },
    {
      "epoch": 1.6208,
      "grad_norm": 1.7597862482070923,
      "learning_rate": 2.3044262959681156e-06,
      "logits/chosen": -1.7677427530288696,
      "logits/rejected": -1.6805483102798462,
      "logps/chosen": -238.32080078125,
      "logps/rejected": -368.8369140625,
      "loss": 0.0859,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.3052327632904053,
      "rewards/margins": 6.70794677734375,
      "rewards/rejected": -8.013179779052734,
      "step": 760
    },
    {
      "epoch": 1.6314666666666666,
      "grad_norm": 3.3619678020477295,
      "learning_rate": 2.2757378747744575e-06,
      "logits/chosen": -1.8301753997802734,
      "logits/rejected": -1.6926501989364624,
      "logps/chosen": -242.58230590820312,
      "logps/rejected": -367.21893310546875,
      "loss": 0.0896,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.6891601085662842,
      "rewards/margins": 7.6020050048828125,
      "rewards/rejected": -9.291166305541992,
      "step": 765
    },
    {
      "epoch": 1.6421333333333332,
      "grad_norm": 12.008077621459961,
      "learning_rate": 2.24707919503142e-06,
      "logits/chosen": -1.776662826538086,
      "logits/rejected": -1.733825922012329,
      "logps/chosen": -255.0137939453125,
      "logps/rejected": -397.6699523925781,
      "loss": 0.1343,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.5180081129074097,
      "rewards/margins": 7.4235711097717285,
      "rewards/rejected": -8.941579818725586,
      "step": 770
    },
    {
      "epoch": 1.6528,
      "grad_norm": 2.1331827640533447,
      "learning_rate": 2.2184540574285213e-06,
      "logits/chosen": -1.7663053274154663,
      "logits/rejected": -1.6731045246124268,
      "logps/chosen": -277.5382995605469,
      "logps/rejected": -404.40435791015625,
      "loss": 0.0919,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.5203996896743774,
      "rewards/margins": 7.110515594482422,
      "rewards/rejected": -8.630915641784668,
      "step": 775
    },
    {
      "epoch": 1.6634666666666666,
      "grad_norm": 4.992430210113525,
      "learning_rate": 2.1898662582069486e-06,
      "logits/chosen": -1.7861846685409546,
      "logits/rejected": -1.666831612586975,
      "logps/chosen": -245.0485076904297,
      "logps/rejected": -373.1742858886719,
      "loss": 0.0901,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.7309143543243408,
      "rewards/margins": 7.291200160980225,
      "rewards/rejected": -9.022114753723145,
      "step": 780
    },
    {
      "epoch": 1.6741333333333333,
      "grad_norm": 1.1797524690628052,
      "learning_rate": 2.1613195886561044e-06,
      "logits/chosen": -1.7284221649169922,
      "logits/rejected": -1.6566905975341797,
      "logps/chosen": -259.3564453125,
      "logps/rejected": -381.2314758300781,
      "loss": 0.1172,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.002119302749634,
      "rewards/margins": 7.27401876449585,
      "rewards/rejected": -9.276138305664062,
      "step": 785
    },
    {
      "epoch": 1.6848,
      "grad_norm": 4.15262508392334,
      "learning_rate": 2.132817834610811e-06,
      "logits/chosen": -1.7999082803726196,
      "logits/rejected": -1.6664059162139893,
      "logps/chosen": -256.9659118652344,
      "logps/rejected": -378.0249938964844,
      "loss": 0.1458,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.0210254192352295,
      "rewards/margins": 7.168332099914551,
      "rewards/rejected": -9.18935775756836,
      "step": 790
    },
    {
      "epoch": 1.6954666666666667,
      "grad_norm": 1.3209813833236694,
      "learning_rate": 2.1043647759492338e-06,
      "logits/chosen": -1.6264126300811768,
      "logits/rejected": -1.6061350107192993,
      "logps/chosen": -264.5497131347656,
      "logps/rejected": -382.03759765625,
      "loss": 0.1261,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.736513376235962,
      "rewards/margins": 6.7935075759887695,
      "rewards/rejected": -8.530019760131836,
      "step": 795
    },
    {
      "epoch": 1.7061333333333333,
      "grad_norm": 4.297145366668701,
      "learning_rate": 2.075964186091599e-06,
      "logits/chosen": -1.7351667881011963,
      "logits/rejected": -1.7021973133087158,
      "logps/chosen": -252.9200897216797,
      "logps/rejected": -394.8159484863281,
      "loss": 0.1009,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.8241618871688843,
      "rewards/margins": 7.219931125640869,
      "rewards/rejected": -9.044093132019043,
      "step": 800
    },
    {
      "epoch": 1.7168,
      "grad_norm": 1.3593531847000122,
      "learning_rate": 2.047619831499765e-06,
      "logits/chosen": -1.6939481496810913,
      "logits/rejected": -1.6624157428741455,
      "logps/chosen": -256.53021240234375,
      "logps/rejected": -387.21490478515625,
      "loss": 0.1098,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.6262582540512085,
      "rewards/margins": 7.0810866355896,
      "rewards/rejected": -8.707345962524414,
      "step": 805
    },
    {
      "epoch": 1.7274666666666667,
      "grad_norm": 2.174813747406006,
      "learning_rate": 2.0193354711777155e-06,
      "logits/chosen": -1.8526443243026733,
      "logits/rejected": -1.7142467498779297,
      "logps/chosen": -262.6146240234375,
      "logps/rejected": -372.4858093261719,
      "loss": 0.0899,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.491632103919983,
      "rewards/margins": 7.115251064300537,
      "rewards/rejected": -8.606884956359863,
      "step": 810
    },
    {
      "epoch": 1.7381333333333333,
      "grad_norm": 2.587124824523926,
      "learning_rate": 1.991114856173048e-06,
      "logits/chosen": -1.8616607189178467,
      "logits/rejected": -1.7974239587783813,
      "logps/chosen": -251.6106414794922,
      "logps/rejected": -384.2275390625,
      "loss": 0.0772,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.016730308532715,
      "rewards/margins": 7.789799690246582,
      "rewards/rejected": -9.806529998779297,
      "step": 815
    },
    {
      "epoch": 1.7488000000000001,
      "grad_norm": 1.9651814699172974,
      "learning_rate": 1.9629617290795115e-06,
      "logits/chosen": -1.8186252117156982,
      "logits/rejected": -1.7092021703720093,
      "logps/chosen": -244.533447265625,
      "logps/rejected": -407.6358642578125,
      "loss": 0.0777,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.0436220169067383,
      "rewards/margins": 7.583238124847412,
      "rewards/rejected": -9.626859664916992,
      "step": 820
    },
    {
      "epoch": 1.7594666666666665,
      "grad_norm": 7.782098770141602,
      "learning_rate": 1.934879823540663e-06,
      "logits/chosen": -1.8047682046890259,
      "logits/rejected": -1.6448373794555664,
      "logps/chosen": -243.6357879638672,
      "logps/rejected": -371.1468811035156,
      "loss": 0.1504,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.050574779510498,
      "rewards/margins": 7.807431697845459,
      "rewards/rejected": -9.85800552368164,
      "step": 825
    },
    {
      "epoch": 1.7701333333333333,
      "grad_norm": 5.324383735656738,
      "learning_rate": 1.9068728637547195e-06,
      "logits/chosen": -1.7081286907196045,
      "logits/rejected": -1.611659049987793,
      "logps/chosen": -254.0673370361328,
      "logps/rejected": -399.0706481933594,
      "loss": 0.1823,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.228820323944092,
      "rewards/margins": 7.0853753089904785,
      "rewards/rejected": -9.31419563293457,
      "step": 830
    },
    {
      "epoch": 1.7808000000000002,
      "grad_norm": 4.415861129760742,
      "learning_rate": 1.878944563980658e-06,
      "logits/chosen": -1.754969596862793,
      "logits/rejected": -1.5842684507369995,
      "logps/chosen": -267.50372314453125,
      "logps/rejected": -385.69244384765625,
      "loss": 0.117,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.268184185028076,
      "rewards/margins": 7.348923683166504,
      "rewards/rejected": -9.617107391357422,
      "step": 835
    },
    {
      "epoch": 1.7914666666666665,
      "grad_norm": 4.7538299560546875,
      "learning_rate": 1.85109862804563e-06,
      "logits/chosen": -1.7412681579589844,
      "logits/rejected": -1.6989322900772095,
      "logps/chosen": -267.0843811035156,
      "logps/rejected": -400.9412536621094,
      "loss": 0.0858,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.5771751403808594,
      "rewards/margins": 8.0297212600708,
      "rewards/rejected": -9.60689640045166,
      "step": 840
    },
    {
      "epoch": 1.8021333333333334,
      "grad_norm": 2.8075060844421387,
      "learning_rate": 1.823338748853766e-06,
      "logits/chosen": -1.7989814281463623,
      "logits/rejected": -1.695749282836914,
      "logps/chosen": -267.8409118652344,
      "logps/rejected": -406.75860595703125,
      "loss": 0.1053,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.9596891403198242,
      "rewards/margins": 7.2939252853393555,
      "rewards/rejected": -9.25361442565918,
      "step": 845
    },
    {
      "epoch": 1.8128,
      "grad_norm": 2.450321912765503,
      "learning_rate": 1.7956686078964257e-06,
      "logits/chosen": -1.8007376194000244,
      "logits/rejected": -1.696488618850708,
      "logps/chosen": -268.1695251464844,
      "logps/rejected": -390.95697021484375,
      "loss": 0.0814,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.6625934839248657,
      "rewards/margins": 7.488409519195557,
      "rewards/rejected": -9.151002883911133,
      "step": 850
    },
    {
      "epoch": 1.8234666666666666,
      "grad_norm": 4.618243217468262,
      "learning_rate": 1.7680918747639608e-06,
      "logits/chosen": -1.7959873676300049,
      "logits/rejected": -1.7379106283187866,
      "logps/chosen": -280.7197265625,
      "logps/rejected": -414.0787048339844,
      "loss": 0.0978,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.2764201164245605,
      "rewards/margins": 7.298665523529053,
      "rewards/rejected": -9.57508659362793,
      "step": 855
    },
    {
      "epoch": 1.8341333333333334,
      "grad_norm": 4.439633369445801,
      "learning_rate": 1.740612206659057e-06,
      "logits/chosen": -1.786678671836853,
      "logits/rejected": -1.6649478673934937,
      "logps/chosen": -255.8995819091797,
      "logps/rejected": -389.19049072265625,
      "loss": 0.1015,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.0884041786193848,
      "rewards/margins": 7.275248050689697,
      "rewards/rejected": -9.363652229309082,
      "step": 860
    },
    {
      "epoch": 1.8448,
      "grad_norm": 3.4084367752075195,
      "learning_rate": 1.7132332479117204e-06,
      "logits/chosen": -1.7992115020751953,
      "logits/rejected": -1.6410239934921265,
      "logps/chosen": -272.89288330078125,
      "logps/rejected": -402.8624572753906,
      "loss": 0.0714,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.0090274810791016,
      "rewards/margins": 7.744027614593506,
      "rewards/rejected": -9.75305461883545,
      "step": 865
    },
    {
      "epoch": 1.8554666666666666,
      "grad_norm": 3.4071297645568848,
      "learning_rate": 1.6859586294959666e-06,
      "logits/chosen": -1.8045650720596313,
      "logits/rejected": -1.6810834407806396,
      "logps/chosen": -263.1116638183594,
      "logps/rejected": -397.77880859375,
      "loss": 0.0751,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.172431468963623,
      "rewards/margins": 8.27795124053955,
      "rewards/rejected": -10.450382232666016,
      "step": 870
    },
    {
      "epoch": 1.8661333333333334,
      "grad_norm": 6.4309983253479,
      "learning_rate": 1.6587919685482884e-06,
      "logits/chosen": -1.8300575017929077,
      "logits/rejected": -1.7437467575073242,
      "logps/chosen": -253.6339874267578,
      "logps/rejected": -413.36407470703125,
      "loss": 0.1226,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.1895852088928223,
      "rewards/margins": 8.25765323638916,
      "rewards/rejected": -10.44723892211914,
      "step": 875
    },
    {
      "epoch": 1.8768,
      "grad_norm": 4.335732460021973,
      "learning_rate": 1.6317368678879497e-06,
      "logits/chosen": -1.7067629098892212,
      "logits/rejected": -1.6405712366104126,
      "logps/chosen": -249.4864501953125,
      "logps/rejected": -369.77691650390625,
      "loss": 0.1302,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.5186713933944702,
      "rewards/margins": 6.943307399749756,
      "rewards/rejected": -8.4619779586792,
      "step": 880
    },
    {
      "epoch": 1.8874666666666666,
      "grad_norm": 6.803175926208496,
      "learning_rate": 1.6047969155391856e-06,
      "logits/chosen": -1.7294635772705078,
      "logits/rejected": -1.6601709127426147,
      "logps/chosen": -276.130859375,
      "logps/rejected": -373.820556640625,
      "loss": 0.1253,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.0530483722686768,
      "rewards/margins": 6.646698951721191,
      "rewards/rejected": -8.699747085571289,
      "step": 885
    },
    {
      "epoch": 1.8981333333333335,
      "grad_norm": 4.3072590827941895,
      "learning_rate": 1.5779756842553617e-06,
      "logits/chosen": -1.7643864154815674,
      "logits/rejected": -1.7279653549194336,
      "logps/chosen": -278.6283874511719,
      "logps/rejected": -399.0856018066406,
      "loss": 0.115,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.922265648841858,
      "rewards/margins": 7.855748176574707,
      "rewards/rejected": -9.778013229370117,
      "step": 890
    },
    {
      "epoch": 1.9088,
      "grad_norm": 2.2582664489746094,
      "learning_rate": 1.551276731045157e-06,
      "logits/chosen": -1.7211596965789795,
      "logits/rejected": -1.6444860696792603,
      "logps/chosen": -290.8036804199219,
      "logps/rejected": -395.46099853515625,
      "loss": 0.1081,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.7245696783065796,
      "rewards/margins": 7.342646598815918,
      "rewards/rejected": -9.067216873168945,
      "step": 895
    },
    {
      "epoch": 1.9194666666666667,
      "grad_norm": 2.4317190647125244,
      "learning_rate": 1.5247035967008369e-06,
      "logits/chosen": -1.6402801275253296,
      "logits/rejected": -1.6369024515151978,
      "logps/chosen": -248.6401824951172,
      "logps/rejected": -388.771484375,
      "loss": 0.1485,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.689244270324707,
      "rewards/margins": 7.255883693695068,
      "rewards/rejected": -8.945128440856934,
      "step": 900
    },
    {
      "epoch": 1.9301333333333335,
      "grad_norm": 1.7278972864151,
      "learning_rate": 1.498259805328678e-06,
      "logits/chosen": -1.799878716468811,
      "logits/rejected": -1.6546272039413452,
      "logps/chosen": -253.69113159179688,
      "logps/rejected": -377.49237060546875,
      "loss": 0.1227,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.6651760339736938,
      "rewards/margins": 7.9041266441345215,
      "rewards/rejected": -9.569303512573242,
      "step": 905
    },
    {
      "epoch": 1.9407999999999999,
      "grad_norm": 4.5444016456604,
      "learning_rate": 1.471948863881601e-06,
      "logits/chosen": -1.7796027660369873,
      "logits/rejected": -1.633963942527771,
      "logps/chosen": -284.4118347167969,
      "logps/rejected": -395.56317138671875,
      "loss": 0.0978,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.3895740509033203,
      "rewards/margins": 7.175274848937988,
      "rewards/rejected": -8.564848899841309,
      "step": 910
    },
    {
      "epoch": 1.9514666666666667,
      "grad_norm": 3.3118534088134766,
      "learning_rate": 1.4457742616940864e-06,
      "logits/chosen": -1.6823219060897827,
      "logits/rejected": -1.6117699146270752,
      "logps/chosen": -263.32958984375,
      "logps/rejected": -377.5818786621094,
      "loss": 0.1023,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.4369984865188599,
      "rewards/margins": 7.370998382568359,
      "rewards/rejected": -8.80799674987793,
      "step": 915
    },
    {
      "epoch": 1.9621333333333333,
      "grad_norm": 3.7233338356018066,
      "learning_rate": 1.4197394700194165e-06,
      "logits/chosen": -1.7843736410140991,
      "logits/rejected": -1.6962296962738037,
      "logps/chosen": -260.7303161621094,
      "logps/rejected": -370.4283142089844,
      "loss": 0.0852,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.220878005027771,
      "rewards/margins": 7.065885066986084,
      "rewards/rejected": -8.286763191223145,
      "step": 920
    },
    {
      "epoch": 1.9727999999999999,
      "grad_norm": 3.0080010890960693,
      "learning_rate": 1.3938479415693244e-06,
      "logits/chosen": -1.8749253749847412,
      "logits/rejected": -1.7267560958862305,
      "logps/chosen": -239.6318359375,
      "logps/rejected": -353.81134033203125,
      "loss": 0.0997,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0718796253204346,
      "rewards/margins": 6.565346717834473,
      "rewards/rejected": -7.637226104736328,
      "step": 925
    },
    {
      "epoch": 1.9834666666666667,
      "grad_norm": 3.3940622806549072,
      "learning_rate": 1.368103110056096e-06,
      "logits/chosen": -1.7393932342529297,
      "logits/rejected": -1.6677579879760742,
      "logps/chosen": -267.26727294921875,
      "logps/rejected": -375.4582824707031,
      "loss": 0.0921,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.6906468868255615,
      "rewards/margins": 7.26543664932251,
      "rewards/rejected": -8.956083297729492,
      "step": 930
    },
    {
      "epoch": 1.9941333333333333,
      "grad_norm": 1.297263503074646,
      "learning_rate": 1.3425083897371983e-06,
      "logits/chosen": -1.652087926864624,
      "logits/rejected": -1.6215932369232178,
      "logps/chosen": -275.397216796875,
      "logps/rejected": -385.1170349121094,
      "loss": 0.099,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.3324906826019287,
      "rewards/margins": 7.011690616607666,
      "rewards/rejected": -8.344181060791016,
      "step": 935
    },
    {
      "epoch": 2.0042666666666666,
      "grad_norm": 2.640092372894287,
      "learning_rate": 1.317067174962478e-06,
      "logits/chosen": -1.716870903968811,
      "logits/rejected": -1.6423840522766113,
      "logps/chosen": -280.1388244628906,
      "logps/rejected": -405.33685302734375,
      "loss": 0.0804,
      "rewards/accuracies": 0.9868420958518982,
      "rewards/chosen": -1.5010406970977783,
      "rewards/margins": 7.210644245147705,
      "rewards/rejected": -8.711686134338379,
      "step": 940
    },
    {
      "epoch": 2.0149333333333335,
      "grad_norm": 0.6266006231307983,
      "learning_rate": 1.291782839724009e-06,
      "logits/chosen": -1.8108909130096436,
      "logits/rejected": -1.6904157400131226,
      "logps/chosen": -283.57354736328125,
      "logps/rejected": -391.6214294433594,
      "loss": 0.0755,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.4528306722640991,
      "rewards/margins": 7.529816627502441,
      "rewards/rejected": -8.982647895812988,
      "step": 945
    },
    {
      "epoch": 2.0256,
      "grad_norm": 1.9588918685913086,
      "learning_rate": 1.2666587372086352e-06,
      "logits/chosen": -1.6926864385604858,
      "logits/rejected": -1.6244741678237915,
      "logps/chosen": -250.9656524658203,
      "logps/rejected": -378.7623596191406,
      "loss": 0.085,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.433120608329773,
      "rewards/margins": 6.887550354003906,
      "rewards/rejected": -8.320670127868652,
      "step": 950
    },
    {
      "epoch": 2.0362666666666667,
      "grad_norm": 2.0722038745880127,
      "learning_rate": 1.241698199353275e-06,
      "logits/chosen": -1.8123573064804077,
      "logits/rejected": -1.7618974447250366,
      "logps/chosen": -253.0042266845703,
      "logps/rejected": -365.7357482910156,
      "loss": 0.0788,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.5714794397354126,
      "rewards/margins": 6.544591426849365,
      "rewards/rejected": -8.116071701049805,
      "step": 955
    },
    {
      "epoch": 2.0469333333333335,
      "grad_norm": 1.5139915943145752,
      "learning_rate": 1.2169045364030404e-06,
      "logits/chosen": -1.8418400287628174,
      "logits/rejected": -1.7378206253051758,
      "logps/chosen": -247.4741973876953,
      "logps/rejected": -366.4799499511719,
      "loss": 0.072,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.658636450767517,
      "rewards/margins": 7.499236106872559,
      "rewards/rejected": -9.157873153686523,
      "step": 960
    },
    {
      "epoch": 2.0576,
      "grad_norm": 6.052288055419922,
      "learning_rate": 1.1922810364722402e-06,
      "logits/chosen": -1.7987949848175049,
      "logits/rejected": -1.6627559661865234,
      "logps/chosen": -270.3972473144531,
      "logps/rejected": -379.8498840332031,
      "loss": 0.084,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.3009984493255615,
      "rewards/margins": 6.970292568206787,
      "rewards/rejected": -8.271291732788086,
      "step": 965
    },
    {
      "epoch": 2.0682666666666667,
      "grad_norm": 1.6238378286361694,
      "learning_rate": 1.1678309651083055e-06,
      "logits/chosen": -1.8293536901474,
      "logits/rejected": -1.7384620904922485,
      "logps/chosen": -265.1159973144531,
      "logps/rejected": -396.7872314453125,
      "loss": 0.0514,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.7698032855987549,
      "rewards/margins": 7.758294582366943,
      "rewards/rejected": -9.528097152709961,
      "step": 970
    },
    {
      "epoch": 2.0789333333333335,
      "grad_norm": 1.211134910583496,
      "learning_rate": 1.1435575648587224e-06,
      "logits/chosen": -1.6530580520629883,
      "logits/rejected": -1.649542212486267,
      "logps/chosen": -239.8114776611328,
      "logps/rejected": -357.4482116699219,
      "loss": 0.0854,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.353614091873169,
      "rewards/margins": 7.633919715881348,
      "rewards/rejected": -8.987533569335938,
      "step": 975
    },
    {
      "epoch": 2.0896,
      "grad_norm": 3.313692331314087,
      "learning_rate": 1.1194640548410039e-06,
      "logits/chosen": -1.7463849782943726,
      "logits/rejected": -1.6572465896606445,
      "logps/chosen": -272.10955810546875,
      "logps/rejected": -365.974609375,
      "loss": 0.1135,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.6518981456756592,
      "rewards/margins": 7.179516792297363,
      "rewards/rejected": -8.831415176391602,
      "step": 980
    },
    {
      "epoch": 2.1002666666666667,
      "grad_norm": 2.2295451164245605,
      "learning_rate": 1.095553630315774e-06,
      "logits/chosen": -1.735437035560608,
      "logits/rejected": -1.6388800144195557,
      "logps/chosen": -255.56564331054688,
      "logps/rejected": -370.2838439941406,
      "loss": 0.0804,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.7196842432022095,
      "rewards/margins": 7.343177795410156,
      "rewards/rejected": -9.06286334991455,
      "step": 985
    },
    {
      "epoch": 2.1109333333333336,
      "grad_norm": 1.403108835220337,
      "learning_rate": 1.0718294622630188e-06,
      "logits/chosen": -1.8256213665008545,
      "logits/rejected": -1.7383081912994385,
      "logps/chosen": -246.55419921875,
      "logps/rejected": -404.0504150390625,
      "loss": 0.0593,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.499764084815979,
      "rewards/margins": 7.549830436706543,
      "rewards/rejected": -9.049592971801758,
      "step": 990
    },
    {
      "epoch": 2.1216,
      "grad_norm": 3.562825918197632,
      "learning_rate": 1.0482946969615495e-06,
      "logits/chosen": -1.859043836593628,
      "logits/rejected": -1.6865688562393188,
      "logps/chosen": -258.09228515625,
      "logps/rejected": -418.1453552246094,
      "loss": 0.0666,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.222257614135742,
      "rewards/margins": 8.348030090332031,
      "rewards/rejected": -10.570287704467773,
      "step": 995
    },
    {
      "epoch": 2.1322666666666668,
      "grad_norm": 0.6572063565254211,
      "learning_rate": 1.0249524555717461e-06,
      "logits/chosen": -1.804359793663025,
      "logits/rejected": -1.5746873617172241,
      "logps/chosen": -233.9191436767578,
      "logps/rejected": -369.0513000488281,
      "loss": 0.0785,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.3012584447860718,
      "rewards/margins": 7.778643608093262,
      "rewards/rejected": -9.079902648925781,
      "step": 1000
    },
    {
      "epoch": 2.142933333333333,
      "grad_norm": 2.282438278198242,
      "learning_rate": 1.0018058337216327e-06,
      "logits/chosen": -1.6505435705184937,
      "logits/rejected": -1.6292928457260132,
      "logps/chosen": -265.0,
      "logps/rejected": -377.8451232910156,
      "loss": 0.0635,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9961382150650024,
      "rewards/margins": 7.445557594299316,
      "rewards/rejected": -9.441695213317871,
      "step": 1005
    },
    {
      "epoch": 2.1536,
      "grad_norm": 3.273324966430664,
      "learning_rate": 9.788579010963387e-07,
      "logits/chosen": -1.7554762363433838,
      "logits/rejected": -1.622506856918335,
      "logps/chosen": -276.3446960449219,
      "logps/rejected": -387.22259521484375,
      "loss": 0.0746,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.0051329135894775,
      "rewards/margins": 8.116923332214355,
      "rewards/rejected": -10.122055053710938,
      "step": 1010
    },
    {
      "epoch": 2.164266666666667,
      "grad_norm": 2.056083917617798,
      "learning_rate": 9.56111701030998e-07,
      "logits/chosen": -1.777017593383789,
      "logits/rejected": -1.634634256362915,
      "logps/chosen": -257.826171875,
      "logps/rejected": -375.39349365234375,
      "loss": 0.0912,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.8016389608383179,
      "rewards/margins": 7.549478054046631,
      "rewards/rejected": -9.351117134094238,
      "step": 1015
    },
    {
      "epoch": 2.174933333333333,
      "grad_norm": 2.441992998123169,
      "learning_rate": 9.335702501071459e-07,
      "logits/chosen": -1.7044284343719482,
      "logits/rejected": -1.6332372426986694,
      "logps/chosen": -265.9286193847656,
      "logps/rejected": -397.9439697265625,
      "loss": 0.0764,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.5722075700759888,
      "rewards/margins": 7.702206611633301,
      "rewards/rejected": -9.274413108825684,
      "step": 1020
    },
    {
      "epoch": 2.1856,
      "grad_norm": 3.191629409790039,
      "learning_rate": 9.112365377526636e-07,
      "logits/chosen": -1.8316318988800049,
      "logits/rejected": -1.7240879535675049,
      "logps/chosen": -234.20425415039062,
      "logps/rejected": -386.95831298828125,
      "loss": 0.0628,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.1653170585632324,
      "rewards/margins": 8.376472473144531,
      "rewards/rejected": -10.541789054870605,
      "step": 1025
    },
    {
      "epoch": 2.196266666666667,
      "grad_norm": 0.986142635345459,
      "learning_rate": 8.891135258453195e-07,
      "logits/chosen": -1.6836477518081665,
      "logits/rejected": -1.6255674362182617,
      "logps/chosen": -262.45831298828125,
      "logps/rejected": -398.49090576171875,
      "loss": 0.0681,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.9247242212295532,
      "rewards/margins": 8.067194938659668,
      "rewards/rejected": -9.99191951751709,
      "step": 1030
    },
    {
      "epoch": 2.206933333333333,
      "grad_norm": 3.276202917098999,
      "learning_rate": 8.672041483199709e-07,
      "logits/chosen": -1.7350730895996094,
      "logits/rejected": -1.6049894094467163,
      "logps/chosen": -275.9724426269531,
      "logps/rejected": -408.84161376953125,
      "loss": 0.1033,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.006324291229248,
      "rewards/margins": 7.502695560455322,
      "rewards/rejected": -9.50901985168457,
      "step": 1035
    },
    {
      "epoch": 2.2176,
      "grad_norm": 1.4491831064224243,
      "learning_rate": 8.455113107794652e-07,
      "logits/chosen": -1.7363901138305664,
      "logits/rejected": -1.6504329442977905,
      "logps/chosen": -250.92138671875,
      "logps/rejected": -378.4014587402344,
      "loss": 0.053,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.7011101245880127,
      "rewards/margins": 8.367371559143066,
      "rewards/rejected": -10.068480491638184,
      "step": 1040
    },
    {
      "epoch": 2.228266666666667,
      "grad_norm": 2.495983362197876,
      "learning_rate": 8.240378901093035e-07,
      "logits/chosen": -1.7328039407730103,
      "logits/rejected": -1.6397279500961304,
      "logps/chosen": -277.71942138671875,
      "logps/rejected": -401.9482116699219,
      "loss": 0.0764,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.0400569438934326,
      "rewards/margins": 8.178812980651855,
      "rewards/rejected": -10.218870162963867,
      "step": 1045
    },
    {
      "epoch": 2.238933333333333,
      "grad_norm": 0.955857515335083,
      "learning_rate": 8.027867340961113e-07,
      "logits/chosen": -1.6360746622085571,
      "logits/rejected": -1.591421365737915,
      "logps/chosen": -275.9684143066406,
      "logps/rejected": -408.9102478027344,
      "loss": 0.0648,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.0029473304748535,
      "rewards/margins": 8.104959487915039,
      "rewards/rejected": -10.107905387878418,
      "step": 1050
    },
    {
      "epoch": 2.2496,
      "grad_norm": 3.9517617225646973,
      "learning_rate": 7.817606610499639e-07,
      "logits/chosen": -1.7221180200576782,
      "logits/rejected": -1.6445471048355103,
      "logps/chosen": -281.46209716796875,
      "logps/rejected": -402.68359375,
      "loss": 0.0544,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.2298858165740967,
      "rewards/margins": 8.064582824707031,
      "rewards/rejected": -10.294468879699707,
      "step": 1055
    },
    {
      "epoch": 2.2602666666666664,
      "grad_norm": 0.5854415893554688,
      "learning_rate": 7.609624594306278e-07,
      "logits/chosen": -1.8364423513412476,
      "logits/rejected": -1.717710256576538,
      "logps/chosen": -272.9776306152344,
      "logps/rejected": -412.51751708984375,
      "loss": 0.0728,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.3371310234069824,
      "rewards/margins": 8.450685501098633,
      "rewards/rejected": -10.787816047668457,
      "step": 1060
    },
    {
      "epoch": 2.2709333333333332,
      "grad_norm": 1.2468198537826538,
      "learning_rate": 7.403948874777556e-07,
      "logits/chosen": -1.7665348052978516,
      "logits/rejected": -1.694647192955017,
      "logps/chosen": -252.20999145507812,
      "logps/rejected": -381.0401916503906,
      "loss": 0.0656,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.9755653142929077,
      "rewards/margins": 8.150426864624023,
      "rewards/rejected": -10.125993728637695,
      "step": 1065
    },
    {
      "epoch": 2.2816,
      "grad_norm": 0.46930360794067383,
      "learning_rate": 7.20060672845091e-07,
      "logits/chosen": -1.7220458984375,
      "logits/rejected": -1.6614139080047607,
      "logps/chosen": -274.89935302734375,
      "logps/rejected": -415.0008239746094,
      "loss": 0.0554,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.264709949493408,
      "rewards/margins": 8.847898483276367,
      "rewards/rejected": -11.112607955932617,
      "step": 1070
    },
    {
      "epoch": 2.2922666666666665,
      "grad_norm": 0.8241863250732422,
      "learning_rate": 6.99962512238733e-07,
      "logits/chosen": -1.6326186656951904,
      "logits/rejected": -1.541085958480835,
      "logps/chosen": -257.67852783203125,
      "logps/rejected": -377.48297119140625,
      "loss": 0.0646,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.0227603912353516,
      "rewards/margins": 7.802756309509277,
      "rewards/rejected": -9.825517654418945,
      "step": 1075
    },
    {
      "epoch": 2.3029333333333333,
      "grad_norm": 2.2578377723693848,
      "learning_rate": 6.801030710594949e-07,
      "logits/chosen": -1.6156307458877563,
      "logits/rejected": -1.5336544513702393,
      "logps/chosen": -271.2424011230469,
      "logps/rejected": -407.70916748046875,
      "loss": 0.0802,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.3306591510772705,
      "rewards/margins": 8.529387474060059,
      "rewards/rejected": -10.860045433044434,
      "step": 1080
    },
    {
      "epoch": 2.3136,
      "grad_norm": 5.054652214050293,
      "learning_rate": 6.604849830494264e-07,
      "logits/chosen": -1.7466853857040405,
      "logits/rejected": -1.686531662940979,
      "logps/chosen": -259.6241760253906,
      "logps/rejected": -385.6200256347656,
      "loss": 0.0746,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.436797618865967,
      "rewards/margins": 8.553865432739258,
      "rewards/rejected": -10.990663528442383,
      "step": 1085
    },
    {
      "epoch": 2.3242666666666665,
      "grad_norm": 0.7833718061447144,
      "learning_rate": 6.411108499425253e-07,
      "logits/chosen": -1.7405407428741455,
      "logits/rejected": -1.6250250339508057,
      "logps/chosen": -270.8252258300781,
      "logps/rejected": -394.97406005859375,
      "loss": 0.065,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.308990001678467,
      "rewards/margins": 8.283408164978027,
      "rewards/rejected": -10.592397689819336,
      "step": 1090
    },
    {
      "epoch": 2.3349333333333333,
      "grad_norm": 3.1654927730560303,
      "learning_rate": 6.219832411197013e-07,
      "logits/chosen": -1.6691102981567383,
      "logits/rejected": -1.56028151512146,
      "logps/chosen": -267.05072021484375,
      "logps/rejected": -386.01934814453125,
      "loss": 0.0988,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.091330051422119,
      "rewards/margins": 8.160810470581055,
      "rewards/rejected": -10.2521390914917,
      "step": 1095
    },
    {
      "epoch": 2.3456,
      "grad_norm": 0.874729335308075,
      "learning_rate": 6.031046932680229e-07,
      "logits/chosen": -1.7366691827774048,
      "logits/rejected": -1.633628249168396,
      "logps/chosen": -263.6493225097656,
      "logps/rejected": -395.6670227050781,
      "loss": 0.061,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.301298141479492,
      "rewards/margins": 8.136720657348633,
      "rewards/rejected": -10.438018798828125,
      "step": 1100
    },
    {
      "epoch": 2.3562666666666665,
      "grad_norm": 2.866297483444214,
      "learning_rate": 5.844777100443091e-07,
      "logits/chosen": -1.7248958349227905,
      "logits/rejected": -1.6350904703140259,
      "logps/chosen": -286.45098876953125,
      "logps/rejected": -419.8631286621094,
      "loss": 0.0935,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.488067388534546,
      "rewards/margins": 8.331172943115234,
      "rewards/rejected": -10.81924057006836,
      "step": 1105
    },
    {
      "epoch": 2.3669333333333333,
      "grad_norm": 1.4425195455551147,
      "learning_rate": 5.66104761743092e-07,
      "logits/chosen": -1.8552663326263428,
      "logits/rejected": -1.7886766195297241,
      "logps/chosen": -256.6435241699219,
      "logps/rejected": -435.21502685546875,
      "loss": 0.0306,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.3535618782043457,
      "rewards/margins": 9.712784767150879,
      "rewards/rejected": -12.066347122192383,
      "step": 1110
    },
    {
      "epoch": 2.3776,
      "grad_norm": 5.315279960632324,
      "learning_rate": 5.479882849690102e-07,
      "logits/chosen": -1.6617796421051025,
      "logits/rejected": -1.6377900838851929,
      "logps/chosen": -238.333740234375,
      "logps/rejected": -358.6871032714844,
      "loss": 0.0983,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.9535621404647827,
      "rewards/margins": 7.863764762878418,
      "rewards/rejected": -9.817326545715332,
      "step": 1115
    },
    {
      "epoch": 2.3882666666666665,
      "grad_norm": 4.480486869812012,
      "learning_rate": 5.301306823136687e-07,
      "logits/chosen": -1.693786382675171,
      "logits/rejected": -1.5776554346084595,
      "logps/chosen": -259.76934814453125,
      "logps/rejected": -386.46551513671875,
      "loss": 0.0925,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.3971920013427734,
      "rewards/margins": 7.947735786437988,
      "rewards/rejected": -10.344927787780762,
      "step": 1120
    },
    {
      "epoch": 2.3989333333333334,
      "grad_norm": 2.4435911178588867,
      "learning_rate": 5.125343220370088e-07,
      "logits/chosen": -1.7341457605361938,
      "logits/rejected": -1.6198333501815796,
      "logps/chosen": -264.7253112792969,
      "logps/rejected": -387.65765380859375,
      "loss": 0.0634,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.605968952178955,
      "rewards/margins": 8.447062492370605,
      "rewards/rejected": -11.053030967712402,
      "step": 1125
    },
    {
      "epoch": 2.4096,
      "grad_norm": 0.4779101014137268,
      "learning_rate": 4.952015377532329e-07,
      "logits/chosen": -1.763282060623169,
      "logits/rejected": -1.6295171976089478,
      "logps/chosen": -272.1696472167969,
      "logps/rejected": -430.5567932128906,
      "loss": 0.0397,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.594783306121826,
      "rewards/margins": 8.7913818359375,
      "rewards/rejected": -11.386165618896484,
      "step": 1130
    },
    {
      "epoch": 2.4202666666666666,
      "grad_norm": 3.4868571758270264,
      "learning_rate": 4.781346281213204e-07,
      "logits/chosen": -1.7784852981567383,
      "logits/rejected": -1.7182724475860596,
      "logps/chosen": -253.816650390625,
      "logps/rejected": -415.3417053222656,
      "loss": 0.0583,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.5662384033203125,
      "rewards/margins": 9.169561386108398,
      "rewards/rejected": -11.735798835754395,
      "step": 1135
    },
    {
      "epoch": 2.4309333333333334,
      "grad_norm": 1.0860471725463867,
      "learning_rate": 4.613358565401818e-07,
      "logits/chosen": -1.735467553138733,
      "logits/rejected": -1.5868717432022095,
      "logps/chosen": -281.07281494140625,
      "logps/rejected": -403.81378173828125,
      "loss": 0.0974,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.6185271739959717,
      "rewards/margins": 8.223875045776367,
      "rewards/rejected": -10.84240436553955,
      "step": 1140
    },
    {
      "epoch": 2.4416,
      "grad_norm": 2.8024306297302246,
      "learning_rate": 4.448074508484906e-07,
      "logits/chosen": -1.7682955265045166,
      "logits/rejected": -1.626478910446167,
      "logps/chosen": -276.26904296875,
      "logps/rejected": -389.8146057128906,
      "loss": 0.0643,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.2558016777038574,
      "rewards/margins": 7.577065467834473,
      "rewards/rejected": -9.832866668701172,
      "step": 1145
    },
    {
      "epoch": 2.4522666666666666,
      "grad_norm": 4.152536869049072,
      "learning_rate": 4.2855160302922777e-07,
      "logits/chosen": -1.692818284034729,
      "logits/rejected": -1.6406266689300537,
      "logps/chosen": -266.7679138183594,
      "logps/rejected": -426.93035888671875,
      "loss": 0.0647,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.1690187454223633,
      "rewards/margins": 9.473871231079102,
      "rewards/rejected": -11.642890930175781,
      "step": 1150
    },
    {
      "epoch": 2.4629333333333334,
      "grad_norm": 2.232576608657837,
      "learning_rate": 4.125704689189819e-07,
      "logits/chosen": -1.71976637840271,
      "logits/rejected": -1.531394362449646,
      "logps/chosen": -311.14019775390625,
      "logps/rejected": -393.14801025390625,
      "loss": 0.0587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7771999835968018,
      "rewards/margins": 8.577513694763184,
      "rewards/rejected": -10.354714393615723,
      "step": 1155
    },
    {
      "epoch": 2.4736000000000002,
      "grad_norm": 0.38305553793907166,
      "learning_rate": 3.9686616792204677e-07,
      "logits/chosen": -1.7162336111068726,
      "logits/rejected": -1.6649755239486694,
      "logps/chosen": -254.1268310546875,
      "logps/rejected": -379.88067626953125,
      "loss": 0.0335,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.962572693824768,
      "rewards/margins": 8.603181838989258,
      "rewards/rejected": -10.565755844116211,
      "step": 1160
    },
    {
      "epoch": 2.4842666666666666,
      "grad_norm": 1.2464972734451294,
      "learning_rate": 3.8144078272934676e-07,
      "logits/chosen": -1.740289330482483,
      "logits/rejected": -1.6422460079193115,
      "logps/chosen": -276.12921142578125,
      "logps/rejected": -404.734130859375,
      "loss": 0.0628,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.208197832107544,
      "rewards/margins": 8.343481063842773,
      "rewards/rejected": -10.551678657531738,
      "step": 1165
    },
    {
      "epoch": 2.4949333333333334,
      "grad_norm": 1.7205370664596558,
      "learning_rate": 3.6629635904222943e-07,
      "logits/chosen": -1.7128251791000366,
      "logits/rejected": -1.6331779956817627,
      "logps/chosen": -248.5833282470703,
      "logps/rejected": -392.1952819824219,
      "loss": 0.0736,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.1602909564971924,
      "rewards/margins": 8.458908081054688,
      "rewards/rejected": -10.619199752807617,
      "step": 1170
    },
    {
      "epoch": 2.5056000000000003,
      "grad_norm": 2.6853506565093994,
      "learning_rate": 3.514349053011698e-07,
      "logits/chosen": -1.6488935947418213,
      "logits/rejected": -1.5202906131744385,
      "logps/chosen": -284.69122314453125,
      "logps/rejected": -407.3353576660156,
      "loss": 0.1129,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.177077531814575,
      "rewards/margins": 8.398216247558594,
      "rewards/rejected": -10.575294494628906,
      "step": 1175
    },
    {
      "epoch": 2.5162666666666667,
      "grad_norm": 3.8878862857818604,
      "learning_rate": 3.3685839241941133e-07,
      "logits/chosen": -1.6800882816314697,
      "logits/rejected": -1.6675255298614502,
      "logps/chosen": -240.10922241210938,
      "logps/rejected": -375.624267578125,
      "loss": 0.0799,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.040280818939209,
      "rewards/margins": 8.62706184387207,
      "rewards/rejected": -10.667341232299805,
      "step": 1180
    },
    {
      "epoch": 2.5269333333333335,
      "grad_norm": 1.2384514808654785,
      "learning_rate": 3.2256875352158435e-07,
      "logits/chosen": -1.7015600204467773,
      "logits/rejected": -1.6122848987579346,
      "logps/chosen": -279.6901550292969,
      "logps/rejected": -418.06622314453125,
      "loss": 0.0642,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.619711399078369,
      "rewards/margins": 8.104384422302246,
      "rewards/rejected": -10.72409439086914,
      "step": 1185
    },
    {
      "epoch": 2.5376,
      "grad_norm": 1.836496114730835,
      "learning_rate": 3.085678836873399e-07,
      "logits/chosen": -1.7416712045669556,
      "logits/rejected": -1.635202169418335,
      "logps/chosen": -263.14776611328125,
      "logps/rejected": -392.9107971191406,
      "loss": 0.0629,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.4201416969299316,
      "rewards/margins": 8.608360290527344,
      "rewards/rejected": -11.028502464294434,
      "step": 1190
    },
    {
      "epoch": 2.5482666666666667,
      "grad_norm": 2.9101390838623047,
      "learning_rate": 2.9485763970002106e-07,
      "logits/chosen": -1.7004315853118896,
      "logits/rejected": -1.5977933406829834,
      "logps/chosen": -273.3360595703125,
      "logps/rejected": -400.28741455078125,
      "loss": 0.0519,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.7801859378814697,
      "rewards/margins": 8.464853286743164,
      "rewards/rejected": -11.245038986206055,
      "step": 1195
    },
    {
      "epoch": 2.558933333333333,
      "grad_norm": 2.5379421710968018,
      "learning_rate": 2.8143983980042174e-07,
      "logits/chosen": -1.78177011013031,
      "logits/rejected": -1.6268895864486694,
      "logps/chosen": -261.5065002441406,
      "logps/rejected": -413.2015686035156,
      "loss": 0.0682,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.6103978157043457,
      "rewards/margins": 8.457944869995117,
      "rewards/rejected": -11.068342208862305,
      "step": 1200
    },
    {
      "epoch": 2.5696,
      "grad_norm": 1.3991801738739014,
      "learning_rate": 2.683162634456515e-07,
      "logits/chosen": -1.585166335105896,
      "logits/rejected": -1.547821283340454,
      "logps/chosen": -265.8369445800781,
      "logps/rejected": -381.36395263671875,
      "loss": 0.058,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.0374863147735596,
      "rewards/margins": 8.10401725769043,
      "rewards/rejected": -10.141504287719727,
      "step": 1205
    },
    {
      "epoch": 2.5802666666666667,
      "grad_norm": 1.468181848526001,
      "learning_rate": 2.5548865107314606e-07,
      "logits/chosen": -1.6954666376113892,
      "logits/rejected": -1.5682141780853271,
      "logps/chosen": -300.32867431640625,
      "logps/rejected": -409.02191162109375,
      "loss": 0.0738,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.4429965019226074,
      "rewards/margins": 8.085245132446289,
      "rewards/rejected": -10.528241157531738,
      "step": 1210
    },
    {
      "epoch": 2.590933333333333,
      "grad_norm": 1.9603363275527954,
      "learning_rate": 2.429587038698511e-07,
      "logits/chosen": -1.6946890354156494,
      "logits/rejected": -1.6248438358306885,
      "logps/chosen": -250.2937774658203,
      "logps/rejected": -393.65191650390625,
      "loss": 0.0931,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.27023983001709,
      "rewards/margins": 8.228813171386719,
      "rewards/rejected": -10.499053001403809,
      "step": 1215
    },
    {
      "epoch": 2.6016,
      "grad_norm": 2.1774659156799316,
      "learning_rate": 2.307280835466144e-07,
      "logits/chosen": -1.6663267612457275,
      "logits/rejected": -1.6113440990447998,
      "logps/chosen": -252.66366577148438,
      "logps/rejected": -393.9212646484375,
      "loss": 0.0488,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.4190704822540283,
      "rewards/margins": 8.709858894348145,
      "rewards/rejected": -11.128928184509277,
      "step": 1220
    },
    {
      "epoch": 2.6122666666666667,
      "grad_norm": 2.0370330810546875,
      "learning_rate": 2.1879841211780856e-07,
      "logits/chosen": -1.6466172933578491,
      "logits/rejected": -1.5969603061676025,
      "logps/chosen": -271.2088623046875,
      "logps/rejected": -395.32940673828125,
      "loss": 0.1036,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.6539084911346436,
      "rewards/margins": 8.313491821289062,
      "rewards/rejected": -10.967400550842285,
      "step": 1225
    },
    {
      "epoch": 2.622933333333333,
      "grad_norm": 1.2966225147247314,
      "learning_rate": 2.0717127168622202e-07,
      "logits/chosen": -1.7209373712539673,
      "logits/rejected": -1.6275761127471924,
      "logps/chosen": -269.34234619140625,
      "logps/rejected": -388.9476623535156,
      "loss": 0.0974,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.696427822113037,
      "rewards/margins": 8.036722183227539,
      "rewards/rejected": -10.733149528503418,
      "step": 1230
    },
    {
      "epoch": 2.6336,
      "grad_norm": 2.5588059425354004,
      "learning_rate": 1.958482042332424e-07,
      "logits/chosen": -1.6694538593292236,
      "logits/rejected": -1.5491987466812134,
      "logps/chosen": -286.1966552734375,
      "logps/rejected": -389.8074951171875,
      "loss": 0.1192,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.7213234901428223,
      "rewards/margins": 8.268573760986328,
      "rewards/rejected": -10.989896774291992,
      "step": 1235
    },
    {
      "epoch": 2.6442666666666668,
      "grad_norm": 3.664574384689331,
      "learning_rate": 1.8483071141435937e-07,
      "logits/chosen": -1.6609315872192383,
      "logits/rejected": -1.603724718093872,
      "logps/chosen": -282.61090087890625,
      "logps/rejected": -388.0448913574219,
      "loss": 0.0811,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.6346335411071777,
      "rewards/margins": 7.658135890960693,
      "rewards/rejected": -10.292769432067871,
      "step": 1240
    },
    {
      "epoch": 2.654933333333333,
      "grad_norm": 6.937290191650391,
      "learning_rate": 1.741202543600176e-07,
      "logits/chosen": -1.7843129634857178,
      "logits/rejected": -1.6339008808135986,
      "logps/chosen": -261.72698974609375,
      "logps/rejected": -393.80853271484375,
      "loss": 0.1399,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.6746315956115723,
      "rewards/margins": 7.8194074630737305,
      "rewards/rejected": -10.494039535522461,
      "step": 1245
    },
    {
      "epoch": 2.6656,
      "grad_norm": 1.8684526681900024,
      "learning_rate": 1.6371825348184155e-07,
      "logits/chosen": -1.6750285625457764,
      "logits/rejected": -1.6143300533294678,
      "logps/chosen": -280.13720703125,
      "logps/rejected": -439.83770751953125,
      "loss": 0.0874,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.390502452850342,
      "rewards/margins": 9.048379898071289,
      "rewards/rejected": -11.438881874084473,
      "step": 1250
    },
    {
      "epoch": 2.676266666666667,
      "grad_norm": 3.4125890731811523,
      "learning_rate": 1.5362608828426246e-07,
      "logits/chosen": -1.692704200744629,
      "logits/rejected": -1.614297866821289,
      "logps/chosen": -273.0475158691406,
      "logps/rejected": -390.3589172363281,
      "loss": 0.105,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.143181800842285,
      "rewards/margins": 7.783265113830566,
      "rewards/rejected": -9.926446914672852,
      "step": 1255
    },
    {
      "epoch": 2.686933333333333,
      "grad_norm": 6.792511940002441,
      "learning_rate": 1.438450971815686e-07,
      "logits/chosen": -1.777464509010315,
      "logits/rejected": -1.7350318431854248,
      "logps/chosen": -249.89877319335938,
      "logps/rejected": -405.6546630859375,
      "loss": 0.0653,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.4070210456848145,
      "rewards/margins": 8.65117359161377,
      "rewards/rejected": -11.058195114135742,
      "step": 1260
    },
    {
      "epoch": 2.6976,
      "grad_norm": 4.641789436340332,
      "learning_rate": 1.3437657732040783e-07,
      "logits/chosen": -1.6777312755584717,
      "logits/rejected": -1.5757746696472168,
      "logps/chosen": -279.8283996582031,
      "logps/rejected": -402.06903076171875,
      "loss": 0.0696,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.8644824028015137,
      "rewards/margins": 8.306800842285156,
      "rewards/rejected": -11.171284675598145,
      "step": 1265
    },
    {
      "epoch": 2.708266666666667,
      "grad_norm": 3.1361374855041504,
      "learning_rate": 1.2522178440776022e-07,
      "logits/chosen": -1.72237229347229,
      "logits/rejected": -1.6854991912841797,
      "logps/chosen": -258.05682373046875,
      "logps/rejected": -367.5188903808594,
      "loss": 0.0931,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.406632661819458,
      "rewards/margins": 7.722708225250244,
      "rewards/rejected": -10.129342079162598,
      "step": 1270
    },
    {
      "epoch": 2.718933333333333,
      "grad_norm": 0.9646216630935669,
      "learning_rate": 1.1638193254440655e-07,
      "logits/chosen": -1.7326161861419678,
      "logits/rejected": -1.64824640750885,
      "logps/chosen": -266.62567138671875,
      "logps/rejected": -391.803466796875,
      "loss": 0.072,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.726926803588867,
      "rewards/margins": 7.8639631271362305,
      "rewards/rejected": -10.590889930725098,
      "step": 1275
    },
    {
      "epoch": 2.7296,
      "grad_norm": 1.7434144020080566,
      "learning_rate": 1.0785819406391562e-07,
      "logits/chosen": -1.652780532836914,
      "logits/rejected": -1.623931646347046,
      "logps/chosen": -256.96063232421875,
      "logps/rejected": -414.2125549316406,
      "loss": 0.0834,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.3380179405212402,
      "rewards/margins": 9.076375007629395,
      "rewards/rejected": -11.414393424987793,
      "step": 1280
    },
    {
      "epoch": 2.740266666666667,
      "grad_norm": 0.6346848607063293,
      "learning_rate": 9.965169937717095e-08,
      "logits/chosen": -1.7070586681365967,
      "logits/rejected": -1.6493428945541382,
      "logps/chosen": -265.99652099609375,
      "logps/rejected": -411.8456115722656,
      "loss": 0.0846,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.9297215938568115,
      "rewards/margins": 8.40210247039795,
      "rewards/rejected": -11.331823348999023,
      "step": 1285
    },
    {
      "epoch": 2.7509333333333332,
      "grad_norm": 3.758061647415161,
      "learning_rate": 9.176353682245676e-08,
      "logits/chosen": -1.7081968784332275,
      "logits/rejected": -1.6483770608901978,
      "logps/chosen": -249.968994140625,
      "logps/rejected": -401.0195617675781,
      "loss": 0.0668,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.510201930999756,
      "rewards/margins": 8.664570808410645,
      "rewards/rejected": -11.174773216247559,
      "step": 1290
    },
    {
      "epoch": 2.7616,
      "grad_norm": 2.5584027767181396,
      "learning_rate": 8.41947525211223e-08,
      "logits/chosen": -1.6588901281356812,
      "logits/rejected": -1.6179072856903076,
      "logps/chosen": -270.0908203125,
      "logps/rejected": -404.5058898925781,
      "loss": 0.0701,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.3527674674987793,
      "rewards/margins": 8.176429748535156,
      "rewards/rejected": -10.529196739196777,
      "step": 1295
    },
    {
      "epoch": 2.772266666666667,
      "grad_norm": 0.7949896454811096,
      "learning_rate": 7.694635023884789e-08,
      "logits/chosen": -1.6635520458221436,
      "logits/rejected": -1.633556604385376,
      "logps/chosen": -273.2143249511719,
      "logps/rejected": -411.2735900878906,
      "loss": 0.0689,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.589845895767212,
      "rewards/margins": 8.327569007873535,
      "rewards/rejected": -10.917413711547852,
      "step": 1300
    },
    {
      "epoch": 2.7829333333333333,
      "grad_norm": 1.5558366775512695,
      "learning_rate": 7.00192912525241e-08,
      "logits/chosen": -1.7068121433258057,
      "logits/rejected": -1.60091233253479,
      "logps/chosen": -275.8150939941406,
      "logps/rejected": -400.9371643066406,
      "loss": 0.0792,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.610804557800293,
      "rewards/margins": 8.357553482055664,
      "rewards/rejected": -10.968358993530273,
      "step": 1305
    },
    {
      "epoch": 2.7936,
      "grad_norm": 3.2928590774536133,
      "learning_rate": 6.341449422277047e-08,
      "logits/chosen": -1.673627495765686,
      "logits/rejected": -1.665616750717163,
      "logps/chosen": -286.53314208984375,
      "logps/rejected": -410.33099365234375,
      "loss": 0.0881,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.5637083053588867,
      "rewards/margins": 7.853028774261475,
      "rewards/rejected": -10.416738510131836,
      "step": 1310
    },
    {
      "epoch": 2.804266666666667,
      "grad_norm": 2.260819911956787,
      "learning_rate": 5.7132835072101486e-08,
      "logits/chosen": -1.6774076223373413,
      "logits/rejected": -1.676124930381775,
      "logps/chosen": -273.211669921875,
      "logps/rejected": -410.07977294921875,
      "loss": 0.058,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.765542507171631,
      "rewards/margins": 8.562185287475586,
      "rewards/rejected": -11.327728271484375,
      "step": 1315
    },
    {
      "epoch": 2.8149333333333333,
      "grad_norm": 3.5438122749328613,
      "learning_rate": 5.117514686876379e-08,
      "logits/chosen": -1.6422332525253296,
      "logits/rejected": -1.5864102840423584,
      "logps/chosen": -244.44949340820312,
      "logps/rejected": -404.03802490234375,
      "loss": 0.0853,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.003594160079956,
      "rewards/margins": 8.986794471740723,
      "rewards/rejected": -10.990389823913574,
      "step": 1320
    },
    {
      "epoch": 2.8256,
      "grad_norm": 2.5405335426330566,
      "learning_rate": 4.5542219716255274e-08,
      "logits/chosen": -1.7319309711456299,
      "logits/rejected": -1.6355736255645752,
      "logps/chosen": -290.3944396972656,
      "logps/rejected": -438.39697265625,
      "loss": 0.0431,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.727210760116577,
      "rewards/margins": 9.366616249084473,
      "rewards/rejected": -12.093827247619629,
      "step": 1325
    },
    {
      "epoch": 2.836266666666667,
      "grad_norm": 0.5261633992195129,
      "learning_rate": 4.023480064854174e-08,
      "logits/chosen": -1.783346176147461,
      "logits/rejected": -1.6483442783355713,
      "logps/chosen": -250.863525390625,
      "logps/rejected": -378.8450927734375,
      "loss": 0.0649,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.479001522064209,
      "rewards/margins": 8.179192543029785,
      "rewards/rejected": -10.658193588256836,
      "step": 1330
    },
    {
      "epoch": 2.8469333333333333,
      "grad_norm": 0.4898112714290619,
      "learning_rate": 3.525359353098584e-08,
      "logits/chosen": -1.7240718603134155,
      "logits/rejected": -1.593396544456482,
      "logps/chosen": -263.9686584472656,
      "logps/rejected": -387.5354309082031,
      "loss": 0.0767,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.9066507816314697,
      "rewards/margins": 8.4071044921875,
      "rewards/rejected": -10.31375503540039,
      "step": 1335
    },
    {
      "epoch": 2.8576,
      "grad_norm": 5.789946556091309,
      "learning_rate": 3.059925896700094e-08,
      "logits/chosen": -1.6261345148086548,
      "logits/rejected": -1.5245052576065063,
      "logps/chosen": -279.90728759765625,
      "logps/rejected": -391.10687255859375,
      "loss": 0.1376,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.2622745037078857,
      "rewards/margins": 8.004220962524414,
      "rewards/rejected": -10.266495704650879,
      "step": 1340
    },
    {
      "epoch": 2.8682666666666665,
      "grad_norm": 4.288834571838379,
      "learning_rate": 2.6272414210443964e-08,
      "logits/chosen": -1.6331889629364014,
      "logits/rejected": -1.5616645812988281,
      "logps/chosen": -285.0086669921875,
      "logps/rejected": -380.7625427246094,
      "loss": 0.0822,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.7788119316101074,
      "rewards/margins": 8.204727172851562,
      "rewards/rejected": -10.983540534973145,
      "step": 1345
    },
    {
      "epoch": 2.8789333333333333,
      "grad_norm": 2.1750404834747314,
      "learning_rate": 2.2273633083753632e-08,
      "logits/chosen": -1.6630783081054688,
      "logits/rejected": -1.647499680519104,
      "logps/chosen": -246.02957153320312,
      "logps/rejected": -382.4586486816406,
      "loss": 0.0665,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.2752842903137207,
      "rewards/margins": 8.254133224487305,
      "rewards/rejected": -10.5294189453125,
      "step": 1350
    },
    {
      "epoch": 2.8895999999999997,
      "grad_norm": 1.8180460929870605,
      "learning_rate": 1.8603445901851647e-08,
      "logits/chosen": -1.7276430130004883,
      "logits/rejected": -1.6383469104766846,
      "logps/chosen": -262.91595458984375,
      "logps/rejected": -414.97955322265625,
      "loss": 0.0761,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.588278293609619,
      "rewards/margins": 8.917512893676758,
      "rewards/rejected": -11.505790710449219,
      "step": 1355
    },
    {
      "epoch": 2.9002666666666665,
      "grad_norm": 3.9006402492523193,
      "learning_rate": 1.5262339401813375e-08,
      "logits/chosen": -1.7000020742416382,
      "logits/rejected": -1.6428985595703125,
      "logps/chosen": -253.78414916992188,
      "logps/rejected": -418.7207946777344,
      "loss": 0.0732,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.5373544692993164,
      "rewards/margins": 8.902445793151855,
      "rewards/rejected": -11.439800262451172,
      "step": 1360
    },
    {
      "epoch": 2.9109333333333334,
      "grad_norm": 4.6902689933776855,
      "learning_rate": 1.2250756678315601e-08,
      "logits/chosen": -1.743619680404663,
      "logits/rejected": -1.6624910831451416,
      "logps/chosen": -287.44415283203125,
      "logps/rejected": -420.8153381347656,
      "loss": 0.0475,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.5243899822235107,
      "rewards/margins": 8.417469024658203,
      "rewards/rejected": -10.941858291625977,
      "step": 1365
    },
    {
      "epoch": 2.9215999999999998,
      "grad_norm": 3.476588726043701,
      "learning_rate": 9.569097124875193e-09,
      "logits/chosen": -1.6620032787322998,
      "logits/rejected": -1.669245958328247,
      "logps/chosen": -270.68609619140625,
      "logps/rejected": -412.59857177734375,
      "loss": 0.105,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.7364907264709473,
      "rewards/margins": 8.751927375793457,
      "rewards/rejected": -11.488417625427246,
      "step": 1370
    },
    {
      "epoch": 2.9322666666666666,
      "grad_norm": 2.2454278469085693,
      "learning_rate": 7.217716380881479e-09,
      "logits/chosen": -1.6574100255966187,
      "logits/rejected": -1.654077172279358,
      "logps/chosen": -282.48870849609375,
      "logps/rejected": -396.88031005859375,
      "loss": 0.0765,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.570164203643799,
      "rewards/margins": 8.327180862426758,
      "rewards/rejected": -10.897344589233398,
      "step": 1375
    },
    {
      "epoch": 2.9429333333333334,
      "grad_norm": 1.074591040611267,
      "learning_rate": 5.196926284430359e-09,
      "logits/chosen": -1.6404346227645874,
      "logits/rejected": -1.5578677654266357,
      "logps/chosen": -256.85791015625,
      "logps/rejected": -386.71575927734375,
      "loss": 0.0563,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.627209424972534,
      "rewards/margins": 7.910440921783447,
      "rewards/rejected": -10.537651062011719,
      "step": 1380
    },
    {
      "epoch": 2.9536,
      "grad_norm": 1.59853994846344,
      "learning_rate": 3.5069948309712775e-09,
      "logits/chosen": -1.651651382446289,
      "logits/rejected": -1.5908164978027344,
      "logps/chosen": -271.5132141113281,
      "logps/rejected": -393.08148193359375,
      "loss": 0.0829,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.2630515098571777,
      "rewards/margins": 7.777309417724609,
      "rewards/rejected": -10.040361404418945,
      "step": 1385
    },
    {
      "epoch": 2.9642666666666666,
      "grad_norm": 1.6438560485839844,
      "learning_rate": 2.1481461377634294e-09,
      "logits/chosen": -1.6835439205169678,
      "logits/rejected": -1.664738416671753,
      "logps/chosen": -273.68212890625,
      "logps/rejected": -414.2811584472656,
      "loss": 0.0402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.379215955734253,
      "rewards/margins": 9.452581405639648,
      "rewards/rejected": -11.831796646118164,
      "step": 1390
    },
    {
      "epoch": 2.9749333333333334,
      "grad_norm": 2.484712600708008,
      "learning_rate": 1.120560414154259e-09,
      "logits/chosen": -1.7078514099121094,
      "logits/rejected": -1.516570806503296,
      "logps/chosen": -251.81283569335938,
      "logps/rejected": -385.6812744140625,
      "loss": 0.0671,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.3888099193573,
      "rewards/margins": 8.810364723205566,
      "rewards/rejected": -11.199174880981445,
      "step": 1395
    },
    {
      "epoch": 2.9856,
      "grad_norm": 1.5737336874008179,
      "learning_rate": 4.2437393768079983e-10,
      "logits/chosen": -1.609192132949829,
      "logits/rejected": -1.5943609476089478,
      "logps/chosen": -266.55340576171875,
      "logps/rejected": -393.8858947753906,
      "loss": 0.0634,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.4015676975250244,
      "rewards/margins": 8.4376859664917,
      "rewards/rejected": -10.839254379272461,
      "step": 1400
    },
    {
      "epoch": 2.9962666666666666,
      "grad_norm": 2.8753819465637207,
      "learning_rate": 5.967903599635171e-11,
      "logits/chosen": -1.6768028736114502,
      "logits/rejected": -1.5742844343185425,
      "logps/chosen": -267.65020751953125,
      "logps/rejected": -438.10845947265625,
      "loss": 0.0667,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.351564884185791,
      "rewards/margins": 9.339567184448242,
      "rewards/rejected": -11.691132545471191,
      "step": 1405
    }
  ],
  "logging_steps": 5,
  "max_steps": 1407,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}