{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 1875,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.008,
      "grad_norm": 3.09834361076355,
      "learning_rate": 3.5087719298245616e-07,
      "logits/chosen": -1.2909044027328491,
      "logits/rejected": -1.2319936752319336,
      "logps/chosen": -232.99636840820312,
      "logps/rejected": -240.0132293701172,
      "loss": 0.693,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -1.5878351860010298e-06,
      "rewards/margins": 0.0003593349829316139,
      "rewards/rejected": -0.00036092280060984194,
      "step": 5
    },
    {
      "epoch": 0.016,
      "grad_norm": 2.9631168842315674,
      "learning_rate": 7.894736842105263e-07,
      "logits/chosen": -1.2395670413970947,
      "logits/rejected": -1.1628472805023193,
      "logps/chosen": -226.4220428466797,
      "logps/rejected": -245.43753051757812,
      "loss": 0.6909,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0036325000692158937,
      "rewards/margins": 0.004563910886645317,
      "rewards/rejected": -0.0009314107592217624,
      "step": 10
    },
    {
      "epoch": 0.024,
      "grad_norm": 3.6685404777526855,
      "learning_rate": 1.2280701754385965e-06,
      "logits/chosen": -1.2391703128814697,
      "logits/rejected": -1.15793776512146,
      "logps/chosen": -224.14266967773438,
      "logps/rejected": -234.9234619140625,
      "loss": 0.6851,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.01015846710652113,
      "rewards/margins": 0.016449259594082832,
      "rewards/rejected": -0.006290791090577841,
      "step": 15
    },
    {
      "epoch": 0.032,
      "grad_norm": 2.7006146907806396,
      "learning_rate": 1.6666666666666667e-06,
      "logits/chosen": -1.122662901878357,
      "logits/rejected": -1.1195858716964722,
      "logps/chosen": -234.66134643554688,
      "logps/rejected": -241.30050659179688,
      "loss": 0.6757,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.0218190997838974,
      "rewards/margins": 0.03629850223660469,
      "rewards/rejected": -0.014479401521384716,
      "step": 20
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.9753403663635254,
      "learning_rate": 2.105263157894737e-06,
      "logits/chosen": -1.18116295337677,
      "logits/rejected": -1.2049816846847534,
      "logps/chosen": -215.8043212890625,
      "logps/rejected": -255.64266967773438,
      "loss": 0.668,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.032455407083034515,
      "rewards/margins": 0.05338490009307861,
      "rewards/rejected": -0.0209294892847538,
      "step": 25
    },
    {
      "epoch": 0.048,
      "grad_norm": 3.15077543258667,
      "learning_rate": 2.5438596491228075e-06,
      "logits/chosen": -1.1878756284713745,
      "logits/rejected": -1.193882703781128,
      "logps/chosen": -217.12744140625,
      "logps/rejected": -248.48825073242188,
      "loss": 0.6515,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.056911785155534744,
      "rewards/margins": 0.0923050194978714,
      "rewards/rejected": -0.03539323806762695,
      "step": 30
    },
    {
      "epoch": 0.056,
      "grad_norm": 2.79844069480896,
      "learning_rate": 2.9824561403508774e-06,
      "logits/chosen": -1.183730125427246,
      "logits/rejected": -1.173300862312317,
      "logps/chosen": -231.12142944335938,
      "logps/rejected": -251.42605590820312,
      "loss": 0.6357,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 0.07155142724514008,
      "rewards/margins": 0.13084718585014343,
      "rewards/rejected": -0.05929575487971306,
      "step": 35
    },
    {
      "epoch": 0.064,
      "grad_norm": 2.861743450164795,
      "learning_rate": 3.421052631578948e-06,
      "logits/chosen": -1.2913212776184082,
      "logits/rejected": -1.1513500213623047,
      "logps/chosen": -223.00509643554688,
      "logps/rejected": -215.6059112548828,
      "loss": 0.627,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.09407249093055725,
      "rewards/margins": 0.15628233551979065,
      "rewards/rejected": -0.0622098445892334,
      "step": 40
    },
    {
      "epoch": 0.072,
      "grad_norm": 2.4972827434539795,
      "learning_rate": 3.859649122807018e-06,
      "logits/chosen": -1.1650872230529785,
      "logits/rejected": -1.0375068187713623,
      "logps/chosen": -228.2639617919922,
      "logps/rejected": -230.9693145751953,
      "loss": 0.5934,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.1547546088695526,
      "rewards/margins": 0.26725736260414124,
      "rewards/rejected": -0.11250276863574982,
      "step": 45
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.606940746307373,
      "learning_rate": 4.298245614035088e-06,
      "logits/chosen": -1.171435832977295,
      "logits/rejected": -1.1679085493087769,
      "logps/chosen": -247.4260711669922,
      "logps/rejected": -261.73394775390625,
      "loss": 0.5431,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.25676974654197693,
      "rewards/margins": 0.4268248677253723,
      "rewards/rejected": -0.1700551062822342,
      "step": 50
    },
    {
      "epoch": 0.088,
      "grad_norm": 2.8133466243743896,
      "learning_rate": 4.736842105263158e-06,
      "logits/chosen": -1.1443212032318115,
      "logits/rejected": -1.1487455368041992,
      "logps/chosen": -217.5292510986328,
      "logps/rejected": -251.6279296875,
      "loss": 0.5568,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22170846164226532,
      "rewards/margins": 0.42460116744041443,
      "rewards/rejected": -0.20289269089698792,
      "step": 55
    },
    {
      "epoch": 0.096,
      "grad_norm": 2.8709912300109863,
      "learning_rate": 4.999985069252202e-06,
      "logits/chosen": -1.2766786813735962,
      "logits/rejected": -1.2180373668670654,
      "logps/chosen": -235.04623413085938,
      "logps/rejected": -271.3938293457031,
      "loss": 0.5274,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 0.2577889561653137,
      "rewards/margins": 0.5365679264068604,
      "rewards/rejected": -0.27877891063690186,
      "step": 60
    },
    {
      "epoch": 0.104,
      "grad_norm": 2.896838903427124,
      "learning_rate": 4.999817100387611e-06,
      "logits/chosen": -1.275023102760315,
      "logits/rejected": -1.273930311203003,
      "logps/chosen": -218.29830932617188,
      "logps/rejected": -236.113525390625,
      "loss": 0.5154,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.18201565742492676,
      "rewards/margins": 0.5186901092529297,
      "rewards/rejected": -0.33667445182800293,
      "step": 65
    },
    {
      "epoch": 0.112,
      "grad_norm": 2.841238021850586,
      "learning_rate": 4.99946251180492e-06,
      "logits/chosen": -1.3415954113006592,
      "logits/rejected": -1.3333144187927246,
      "logps/chosen": -240.9877471923828,
      "logps/rejected": -274.54022216796875,
      "loss": 0.5186,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.14453572034835815,
      "rewards/margins": 0.6278968453407288,
      "rewards/rejected": -0.4833610951900482,
      "step": 70
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.8624284267425537,
      "learning_rate": 4.998921329975353e-06,
      "logits/chosen": -1.3028557300567627,
      "logits/rejected": -1.198012113571167,
      "logps/chosen": -213.59176635742188,
      "logps/rejected": -253.2032470703125,
      "loss": 0.5051,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.1062125712633133,
      "rewards/margins": 0.6504680514335632,
      "rewards/rejected": -0.5442554950714111,
      "step": 75
    },
    {
      "epoch": 0.128,
      "grad_norm": 3.075573682785034,
      "learning_rate": 4.998193595299945e-06,
      "logits/chosen": -1.4071934223175049,
      "logits/rejected": -1.2867439985275269,
      "logps/chosen": -223.06655883789062,
      "logps/rejected": -265.8553771972656,
      "loss": 0.4527,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.2724152207374573,
      "rewards/margins": 1.0262385606765747,
      "rewards/rejected": -0.7538233995437622,
      "step": 80
    },
    {
      "epoch": 0.136,
      "grad_norm": 2.3981781005859375,
      "learning_rate": 4.997279362106528e-06,
      "logits/chosen": -1.3612762689590454,
      "logits/rejected": -1.279844045639038,
      "logps/chosen": -212.7271270751953,
      "logps/rejected": -265.90081787109375,
      "loss": 0.4299,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.16929033398628235,
      "rewards/margins": 1.0672941207885742,
      "rewards/rejected": -0.8980037569999695,
      "step": 85
    },
    {
      "epoch": 0.144,
      "grad_norm": 2.9816811084747314,
      "learning_rate": 4.9961786986456705e-06,
      "logits/chosen": -1.3840779066085815,
      "logits/rejected": -1.4094316959381104,
      "logps/chosen": -213.59060668945312,
      "logps/rejected": -260.33258056640625,
      "loss": 0.4511,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.11396338790655136,
      "rewards/margins": 1.0669071674346924,
      "rewards/rejected": -0.9529438018798828,
      "step": 90
    },
    {
      "epoch": 0.152,
      "grad_norm": 2.093193769454956,
      "learning_rate": 4.994891687085587e-06,
      "logits/chosen": -1.332556128501892,
      "logits/rejected": -1.307656168937683,
      "logps/chosen": -216.29476928710938,
      "logps/rejected": -251.3971710205078,
      "loss": 0.3924,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.09258659183979034,
      "rewards/margins": 1.270951747894287,
      "rewards/rejected": -1.1783651113510132,
      "step": 95
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.8752410411834717,
      "learning_rate": 4.9934184235060005e-06,
      "logits/chosen": -1.5339454412460327,
      "logits/rejected": -1.3785521984100342,
      "logps/chosen": -239.418701171875,
      "logps/rejected": -255.296875,
      "loss": 0.416,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.05207740142941475,
      "rewards/margins": 1.1651036739349365,
      "rewards/rejected": -1.1130263805389404,
      "step": 100
    },
    {
      "epoch": 0.168,
      "grad_norm": 3.262073040008545,
      "learning_rate": 4.9917590178909695e-06,
      "logits/chosen": -1.3999998569488525,
      "logits/rejected": -1.367429494857788,
      "logps/chosen": -226.4331817626953,
      "logps/rejected": -264.509521484375,
      "loss": 0.4265,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.008939206600189209,
      "rewards/margins": 1.2716344594955444,
      "rewards/rejected": -1.2805736064910889,
      "step": 105
    },
    {
      "epoch": 0.176,
      "grad_norm": 2.390469551086426,
      "learning_rate": 4.989913594120682e-06,
      "logits/chosen": -1.4643319845199585,
      "logits/rejected": -1.393023133277893,
      "logps/chosen": -249.9105987548828,
      "logps/rejected": -269.37591552734375,
      "loss": 0.4418,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.04763683304190636,
      "rewards/margins": 1.3064950704574585,
      "rewards/rejected": -1.354131817817688,
      "step": 110
    },
    {
      "epoch": 0.184,
      "grad_norm": 3.1575868129730225,
      "learning_rate": 4.987882289962201e-06,
      "logits/chosen": -1.3801255226135254,
      "logits/rejected": -1.2841880321502686,
      "logps/chosen": -244.7362823486328,
      "logps/rejected": -268.76727294921875,
      "loss": 0.3836,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.015139693394303322,
      "rewards/margins": 1.4781547784805298,
      "rewards/rejected": -1.4630149602890015,
      "step": 115
    },
    {
      "epoch": 0.192,
      "grad_norm": 2.9875669479370117,
      "learning_rate": 4.9856652570591865e-06,
      "logits/chosen": -1.4804288148880005,
      "logits/rejected": -1.4396897554397583,
      "logps/chosen": -223.47317504882812,
      "logps/rejected": -274.21429443359375,
      "loss": 0.4113,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02329862117767334,
      "rewards/margins": 1.192557454109192,
      "rewards/rejected": -1.2158560752868652,
      "step": 120
    },
    {
      "epoch": 0.2,
      "grad_norm": 11.367328643798828,
      "learning_rate": 4.983262660920567e-06,
      "logits/chosen": -1.5214124917984009,
      "logits/rejected": -1.478496789932251,
      "logps/chosen": -284.20745849609375,
      "logps/rejected": -264.2999267578125,
      "loss": 0.3841,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.17885152995586395,
      "rewards/margins": 1.4730366468429565,
      "rewards/rejected": -1.2941851615905762,
      "step": 125
    },
    {
      "epoch": 0.208,
      "grad_norm": 2.774146795272827,
      "learning_rate": 4.980674680908192e-06,
      "logits/chosen": -1.523785948753357,
      "logits/rejected": -1.4983959197998047,
      "logps/chosen": -229.0098876953125,
      "logps/rejected": -248.9108123779297,
      "loss": 0.3995,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.1047826036810875,
      "rewards/margins": 1.3212553262710571,
      "rewards/rejected": -1.216472864151001,
      "step": 130
    },
    {
      "epoch": 0.216,
      "grad_norm": 3.826845645904541,
      "learning_rate": 4.9779015102234354e-06,
      "logits/chosen": -1.429242491722107,
      "logits/rejected": -1.4095516204833984,
      "logps/chosen": -210.46408081054688,
      "logps/rejected": -243.783935546875,
      "loss": 0.4262,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.008653700351715088,
      "rewards/margins": 1.4706300497055054,
      "rewards/rejected": -1.461976408958435,
      "step": 135
    },
    {
      "epoch": 0.224,
      "grad_norm": 3.50723934173584,
      "learning_rate": 4.9749433558927755e-06,
      "logits/chosen": -1.4507659673690796,
      "logits/rejected": -1.4398767948150635,
      "logps/chosen": -236.15261840820312,
      "logps/rejected": -261.0746154785156,
      "loss": 0.461,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.05946246534585953,
      "rewards/margins": 1.1294766664505005,
      "rewards/rejected": -1.188939094543457,
      "step": 140
    },
    {
      "epoch": 0.232,
      "grad_norm": 2.5489661693573,
      "learning_rate": 4.971800438752339e-06,
      "logits/chosen": -1.422628402709961,
      "logits/rejected": -1.389582633972168,
      "logps/chosen": -224.6511688232422,
      "logps/rejected": -267.15472412109375,
      "loss": 0.3153,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.26400551199913025,
      "rewards/margins": 1.8636376857757568,
      "rewards/rejected": -1.5996323823928833,
      "step": 145
    },
    {
      "epoch": 0.24,
      "grad_norm": 3.3601999282836914,
      "learning_rate": 4.968472993431418e-06,
      "logits/chosen": -1.5419387817382812,
      "logits/rejected": -1.5043963193893433,
      "logps/chosen": -214.88528442382812,
      "logps/rejected": -248.4092254638672,
      "loss": 0.3794,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.07963865250349045,
      "rewards/margins": 1.558951497077942,
      "rewards/rejected": -1.479312777519226,
      "step": 150
    },
    {
      "epoch": 0.248,
      "grad_norm": 3.492460012435913,
      "learning_rate": 4.964961268334947e-06,
      "logits/chosen": -1.407315731048584,
      "logits/rejected": -1.3751634359359741,
      "logps/chosen": -226.08737182617188,
      "logps/rejected": -245.1667938232422,
      "loss": 0.4431,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.07714741677045822,
      "rewards/margins": 1.2392067909240723,
      "rewards/rejected": -1.1620593070983887,
      "step": 155
    },
    {
      "epoch": 0.256,
      "grad_norm": 2.7841556072235107,
      "learning_rate": 4.961265525624965e-06,
      "logits/chosen": -1.5322622060775757,
      "logits/rejected": -1.4266688823699951,
      "logps/chosen": -217.79513549804688,
      "logps/rejected": -250.97207641601562,
      "loss": 0.3799,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.034860264509916306,
      "rewards/margins": 1.4884862899780273,
      "rewards/rejected": -1.5233466625213623,
      "step": 160
    },
    {
      "epoch": 0.264,
      "grad_norm": 3.2082886695861816,
      "learning_rate": 4.957386041201043e-06,
      "logits/chosen": -1.5398249626159668,
      "logits/rejected": -1.465033769607544,
      "logps/chosen": -228.68197631835938,
      "logps/rejected": -235.70950317382812,
      "loss": 0.3659,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 0.12753064930438995,
      "rewards/margins": 1.6183111667633057,
      "rewards/rejected": -1.4907804727554321,
      "step": 165
    },
    {
      "epoch": 0.272,
      "grad_norm": 3.9094443321228027,
      "learning_rate": 4.953323104679683e-06,
      "logits/chosen": -1.416504979133606,
      "logits/rejected": -1.3369841575622559,
      "logps/chosen": -224.4219512939453,
      "logps/rejected": -256.59808349609375,
      "loss": 0.373,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.28729182481765747,
      "rewards/margins": 1.6720726490020752,
      "rewards/rejected": -1.384780764579773,
      "step": 170
    },
    {
      "epoch": 0.28,
      "grad_norm": 2.4305460453033447,
      "learning_rate": 4.949077019372704e-06,
      "logits/chosen": -1.4719858169555664,
      "logits/rejected": -1.369411587715149,
      "logps/chosen": -203.30699157714844,
      "logps/rejected": -248.2655792236328,
      "loss": 0.3418,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.168064683675766,
      "rewards/margins": 1.683985948562622,
      "rewards/rejected": -1.5159209966659546,
      "step": 175
    },
    {
      "epoch": 0.288,
      "grad_norm": 3.1803479194641113,
      "learning_rate": 4.944648102264594e-06,
      "logits/chosen": -1.4664204120635986,
      "logits/rejected": -1.4561368227005005,
      "logps/chosen": -235.5093994140625,
      "logps/rejected": -286.78265380859375,
      "loss": 0.3208,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2242468148469925,
      "rewards/margins": 1.8115650415420532,
      "rewards/rejected": -1.5873181819915771,
      "step": 180
    },
    {
      "epoch": 0.296,
      "grad_norm": 2.9232826232910156,
      "learning_rate": 4.940036683988847e-06,
      "logits/chosen": -1.6076456308364868,
      "logits/rejected": -1.520662546157837,
      "logps/chosen": -232.9312286376953,
      "logps/rejected": -275.1305236816406,
      "loss": 0.3502,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 0.04518136382102966,
      "rewards/margins": 1.9395564794540405,
      "rewards/rejected": -1.894375205039978,
      "step": 185
    },
    {
      "epoch": 0.304,
      "grad_norm": 3.534334182739258,
      "learning_rate": 4.9352431088032785e-06,
      "logits/chosen": -1.5013444423675537,
      "logits/rejected": -1.4597349166870117,
      "logps/chosen": -248.06784057617188,
      "logps/rejected": -275.9266052246094,
      "loss": 0.3619,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": 0.07667362689971924,
      "rewards/margins": 1.9556586742401123,
      "rewards/rejected": -1.878985047340393,
      "step": 190
    },
    {
      "epoch": 0.312,
      "grad_norm": 2.96315336227417,
      "learning_rate": 4.9302677345643316e-06,
      "logits/chosen": -1.4589729309082031,
      "logits/rejected": -1.4188172817230225,
      "logps/chosen": -241.571044921875,
      "logps/rejected": -264.0114440917969,
      "loss": 0.3528,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": 0.13401958346366882,
      "rewards/margins": 2.0827994346618652,
      "rewards/rejected": -1.9487800598144531,
      "step": 195
    },
    {
      "epoch": 0.32,
      "grad_norm": 3.3248233795166016,
      "learning_rate": 4.925110932700353e-06,
      "logits/chosen": -1.4326791763305664,
      "logits/rejected": -1.393992304801941,
      "logps/chosen": -243.5026397705078,
      "logps/rejected": -279.20361328125,
      "loss": 0.2832,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 0.26788315176963806,
      "rewards/margins": 2.33567476272583,
      "rewards/rejected": -2.067791700363159,
      "step": 200
    },
    {
      "epoch": 0.328,
      "grad_norm": 2.753101348876953,
      "learning_rate": 4.919773088183873e-06,
      "logits/chosen": -1.4298564195632935,
      "logits/rejected": -1.3792085647583008,
      "logps/chosen": -232.48947143554688,
      "logps/rejected": -265.4277648925781,
      "loss": 0.3057,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.29389330744743347,
      "rewards/margins": 2.062680721282959,
      "rewards/rejected": -1.768787145614624,
      "step": 205
    },
    {
      "epoch": 0.336,
      "grad_norm": 3.2286505699157715,
      "learning_rate": 4.914254599502859e-06,
      "logits/chosen": -1.6037689447402954,
      "logits/rejected": -1.5768611431121826,
      "logps/chosen": -221.110595703125,
      "logps/rejected": -259.8968200683594,
      "loss": 0.3139,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.1620008498430252,
      "rewards/margins": 2.205214262008667,
      "rewards/rejected": -2.043213367462158,
      "step": 210
    },
    {
      "epoch": 0.344,
      "grad_norm": 3.4793143272399902,
      "learning_rate": 4.908555878630973e-06,
      "logits/chosen": -1.410553216934204,
      "logits/rejected": -1.416738510131836,
      "logps/chosen": -198.44369506835938,
      "logps/rejected": -268.19049072265625,
      "loss": 0.286,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 0.47130662202835083,
      "rewards/margins": 2.549254894256592,
      "rewards/rejected": -2.0779480934143066,
      "step": 215
    },
    {
      "epoch": 0.352,
      "grad_norm": 2.717116355895996,
      "learning_rate": 4.902677350996812e-06,
      "logits/chosen": -1.5396114587783813,
      "logits/rejected": -1.4347336292266846,
      "logps/chosen": -226.77890014648438,
      "logps/rejected": -249.70449829101562,
      "loss": 0.3559,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 0.35638585686683655,
      "rewards/margins": 2.339998960494995,
      "rewards/rejected": -1.9836132526397705,
      "step": 220
    },
    {
      "epoch": 0.36,
      "grad_norm": 6.953813552856445,
      "learning_rate": 4.896619455452149e-06,
      "logits/chosen": -1.4302887916564941,
      "logits/rejected": -1.3422778844833374,
      "logps/chosen": -226.4547576904297,
      "logps/rejected": -247.8949737548828,
      "loss": 0.3402,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.4336090087890625,
      "rewards/margins": 2.148460865020752,
      "rewards/rejected": -1.714851975440979,
      "step": 225
    },
    {
      "epoch": 0.368,
      "grad_norm": 4.407371520996094,
      "learning_rate": 4.890382644239176e-06,
      "logits/chosen": -1.5161406993865967,
      "logits/rejected": -1.4464107751846313,
      "logps/chosen": -225.75973510742188,
      "logps/rejected": -264.4316711425781,
      "loss": 0.3814,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": 0.34316977858543396,
      "rewards/margins": 1.8401553630828857,
      "rewards/rejected": -1.496985673904419,
      "step": 230
    },
    {
      "epoch": 0.376,
      "grad_norm": 4.450686931610107,
      "learning_rate": 4.8839673829567345e-06,
      "logits/chosen": -1.5348962545394897,
      "logits/rejected": -1.4852139949798584,
      "logps/chosen": -223.89297485351562,
      "logps/rejected": -273.54058837890625,
      "loss": 0.346,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.3010365962982178,
      "rewards/margins": 2.130527973175049,
      "rewards/rejected": -1.8294916152954102,
      "step": 235
    },
    {
      "epoch": 0.384,
      "grad_norm": 3.7453107833862305,
      "learning_rate": 4.877374150525563e-06,
      "logits/chosen": -1.524099349975586,
      "logits/rejected": -1.4040982723236084,
      "logps/chosen": -218.1953125,
      "logps/rejected": -271.7344665527344,
      "loss": 0.3333,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": 0.542862057685852,
      "rewards/margins": 2.4943270683288574,
      "rewards/rejected": -1.9514648914337158,
      "step": 240
    },
    {
      "epoch": 0.392,
      "grad_norm": 3.430736541748047,
      "learning_rate": 4.870603439152544e-06,
      "logits/chosen": -1.4076268672943115,
      "logits/rejected": -1.3688628673553467,
      "logps/chosen": -238.49276733398438,
      "logps/rejected": -286.45098876953125,
      "loss": 0.2818,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.3373296856880188,
      "rewards/margins": 2.4323055744171143,
      "rewards/rejected": -2.0949759483337402,
      "step": 245
    },
    {
      "epoch": 0.4,
      "grad_norm": 3.630533218383789,
      "learning_rate": 4.863655754293953e-06,
      "logits/chosen": -1.5296860933303833,
      "logits/rejected": -1.5080636739730835,
      "logps/chosen": -240.5646514892578,
      "logps/rejected": -269.9687805175781,
      "loss": 0.3282,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 0.3692770302295685,
      "rewards/margins": 2.3015217781066895,
      "rewards/rejected": -1.932244896888733,
      "step": 250
    },
    {
      "epoch": 0.408,
      "grad_norm": 3.693142890930176,
      "learning_rate": 4.856531614617734e-06,
      "logits/chosen": -1.6221294403076172,
      "logits/rejected": -1.5056837797164917,
      "logps/chosen": -217.1576690673828,
      "logps/rejected": -270.16436767578125,
      "loss": 0.2793,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 0.34854626655578613,
      "rewards/margins": 2.3167076110839844,
      "rewards/rejected": -1.9681613445281982,
      "step": 255
    },
    {
      "epoch": 0.416,
      "grad_norm": 3.790282726287842,
      "learning_rate": 4.849231551964771e-06,
      "logits/chosen": -1.5149017572402954,
      "logits/rejected": -1.4539215564727783,
      "logps/chosen": -216.43994140625,
      "logps/rejected": -245.41830444335938,
      "loss": 0.3103,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.6397329568862915,
      "rewards/margins": 2.408543109893799,
      "rewards/rejected": -1.7688102722167969,
      "step": 260
    },
    {
      "epoch": 0.424,
      "grad_norm": 2.5389716625213623,
      "learning_rate": 4.841756111309188e-06,
      "logits/chosen": -1.5661799907684326,
      "logits/rejected": -1.412497639656067,
      "logps/chosen": -233.9298858642578,
      "logps/rejected": -276.6418762207031,
      "loss": 0.2872,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 0.543883204460144,
      "rewards/margins": 2.6593985557556152,
      "rewards/rejected": -2.1155149936676025,
      "step": 265
    },
    {
      "epoch": 0.432,
      "grad_norm": 3.379960060119629,
      "learning_rate": 4.834105850717664e-06,
      "logits/chosen": -1.558663010597229,
      "logits/rejected": -1.4975134134292603,
      "logps/chosen": -216.4776611328125,
      "logps/rejected": -252.5549774169922,
      "loss": 0.337,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.36121705174446106,
      "rewards/margins": 2.108269214630127,
      "rewards/rejected": -1.7470521926879883,
      "step": 270
    },
    {
      "epoch": 0.44,
      "grad_norm": 3.4550364017486572,
      "learning_rate": 4.826281341307772e-06,
      "logits/chosen": -1.5177628993988037,
      "logits/rejected": -1.4909249544143677,
      "logps/chosen": -211.82522583007812,
      "logps/rejected": -275.4071960449219,
      "loss": 0.2673,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.43875017762184143,
      "rewards/margins": 2.2563743591308594,
      "rewards/rejected": -1.8176243305206299,
      "step": 275
    },
    {
      "epoch": 0.448,
      "grad_norm": 2.3109264373779297,
      "learning_rate": 4.818283167205341e-06,
      "logits/chosen": -1.4933243989944458,
      "logits/rejected": -1.476614236831665,
      "logps/chosen": -211.17626953125,
      "logps/rejected": -257.49737548828125,
      "loss": 0.284,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.6001291275024414,
      "rewards/margins": 2.5724263191223145,
      "rewards/rejected": -1.9722973108291626,
      "step": 280
    },
    {
      "epoch": 0.456,
      "grad_norm": 2.1601908206939697,
      "learning_rate": 4.8101119255008555e-06,
      "logits/chosen": -1.6040397882461548,
      "logits/rejected": -1.5376040935516357,
      "logps/chosen": -216.3963165283203,
      "logps/rejected": -285.90106201171875,
      "loss": 0.2571,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.6661025881767273,
      "rewards/margins": 2.8501198291778564,
      "rewards/rejected": -2.1840171813964844,
      "step": 285
    },
    {
      "epoch": 0.464,
      "grad_norm": 2.8724215030670166,
      "learning_rate": 4.801768226204871e-06,
      "logits/chosen": -1.4982980489730835,
      "logits/rejected": -1.5107654333114624,
      "logps/chosen": -197.4161376953125,
      "logps/rejected": -261.05780029296875,
      "loss": 0.3096,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.3717041313648224,
      "rewards/margins": 2.240090847015381,
      "rewards/rejected": -1.8683868646621704,
      "step": 290
    },
    {
      "epoch": 0.472,
      "grad_norm": 4.054579734802246,
      "learning_rate": 4.7932526922024835e-06,
      "logits/chosen": -1.58639395236969,
      "logits/rejected": -1.514135479927063,
      "logps/chosen": -224.87435913085938,
      "logps/rejected": -274.8253479003906,
      "loss": 0.2683,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.2706889510154724,
      "rewards/margins": 2.824399471282959,
      "rewards/rejected": -2.553710460662842,
      "step": 295
    },
    {
      "epoch": 0.48,
      "grad_norm": 2.7186853885650635,
      "learning_rate": 4.784565959206822e-06,
      "logits/chosen": -1.6050167083740234,
      "logits/rejected": -1.5349472761154175,
      "logps/chosen": -233.95419311523438,
      "logps/rejected": -278.12042236328125,
      "loss": 0.2701,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 0.08019473403692245,
      "rewards/margins": 2.7215523719787598,
      "rewards/rejected": -2.641357421875,
      "step": 300
    },
    {
      "epoch": 0.488,
      "grad_norm": 2.830869436264038,
      "learning_rate": 4.775708675711597e-06,
      "logits/chosen": -1.6498082876205444,
      "logits/rejected": -1.54813551902771,
      "logps/chosen": -216.80239868164062,
      "logps/rejected": -275.505126953125,
      "loss": 0.2549,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 0.19069986045360565,
      "rewards/margins": 2.954810380935669,
      "rewards/rejected": -2.764110565185547,
      "step": 305
    },
    {
      "epoch": 0.496,
      "grad_norm": 4.789516925811768,
      "learning_rate": 4.766681502942681e-06,
      "logits/chosen": -1.5943660736083984,
      "logits/rejected": -1.6171715259552002,
      "logps/chosen": -214.1531219482422,
      "logps/rejected": -270.62957763671875,
      "loss": 0.2825,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 0.21599332988262177,
      "rewards/margins": 2.9400386810302734,
      "rewards/rejected": -2.7240452766418457,
      "step": 310
    },
    {
      "epoch": 0.504,
      "grad_norm": 3.499081611633301,
      "learning_rate": 4.757485114808755e-06,
      "logits/chosen": -1.6780385971069336,
      "logits/rejected": -1.5276051759719849,
      "logps/chosen": -226.2732391357422,
      "logps/rejected": -281.07330322265625,
      "loss": 0.2719,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.1714359074831009,
      "rewards/margins": 2.7567970752716064,
      "rewards/rejected": -2.5853612422943115,
      "step": 315
    },
    {
      "epoch": 0.512,
      "grad_norm": 3.7923743724823,
      "learning_rate": 4.748120197850989e-06,
      "logits/chosen": -1.5633914470672607,
      "logits/rejected": -1.542341947555542,
      "logps/chosen": -240.93814086914062,
      "logps/rejected": -263.87066650390625,
      "loss": 0.2843,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3062919080257416,
      "rewards/margins": 2.8927149772644043,
      "rewards/rejected": -2.586423397064209,
      "step": 320
    },
    {
      "epoch": 0.52,
      "grad_norm": 2.524531841278076,
      "learning_rate": 4.738587451191796e-06,
      "logits/chosen": -1.5463718175888062,
      "logits/rejected": -1.4387627840042114,
      "logps/chosen": -236.073486328125,
      "logps/rejected": -272.9033203125,
      "loss": 0.2858,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 0.3569238483905792,
      "rewards/margins": 2.676743268966675,
      "rewards/rejected": -2.319819450378418,
      "step": 325
    },
    {
      "epoch": 0.528,
      "grad_norm": 4.112205505371094,
      "learning_rate": 4.728887586482638e-06,
      "logits/chosen": -1.5366756916046143,
      "logits/rejected": -1.4600120782852173,
      "logps/chosen": -245.8846435546875,
      "logps/rejected": -273.06353759765625,
      "loss": 0.2451,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.3740922808647156,
      "rewards/margins": 2.8029963970184326,
      "rewards/rejected": -2.4289040565490723,
      "step": 330
    },
    {
      "epoch": 0.536,
      "grad_norm": 2.5791103839874268,
      "learning_rate": 4.719021327850898e-06,
      "logits/chosen": -1.4344203472137451,
      "logits/rejected": -1.4434177875518799,
      "logps/chosen": -225.71279907226562,
      "logps/rejected": -262.5049743652344,
      "loss": 0.2681,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1705799102783203,
      "rewards/margins": 2.7896885871887207,
      "rewards/rejected": -2.6191084384918213,
      "step": 335
    },
    {
      "epoch": 0.544,
      "grad_norm": 3.0821750164031982,
      "learning_rate": 4.708989411845826e-06,
      "logits/chosen": -1.6164867877960205,
      "logits/rejected": -1.4977359771728516,
      "logps/chosen": -228.20065307617188,
      "logps/rejected": -264.6740417480469,
      "loss": 0.2676,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.2144383192062378,
      "rewards/margins": 2.7585530281066895,
      "rewards/rejected": -2.544114589691162,
      "step": 340
    },
    {
      "epoch": 0.552,
      "grad_norm": 3.665308713912964,
      "learning_rate": 4.698792587383545e-06,
      "logits/chosen": -1.679888129234314,
      "logits/rejected": -1.5331531763076782,
      "logps/chosen": -228.595947265625,
      "logps/rejected": -261.4881896972656,
      "loss": 0.2912,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.19902127981185913,
      "rewards/margins": 2.8159170150756836,
      "rewards/rejected": -2.616895914077759,
      "step": 345
    },
    {
      "epoch": 0.56,
      "grad_norm": 2.2218050956726074,
      "learning_rate": 4.68843161569115e-06,
      "logits/chosen": -1.6025232076644897,
      "logits/rejected": -1.6274888515472412,
      "logps/chosen": -221.0915985107422,
      "logps/rejected": -275.3337707519531,
      "loss": 0.2624,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23736414313316345,
      "rewards/margins": 2.8529913425445557,
      "rewards/rejected": -2.6156270503997803,
      "step": 350
    },
    {
      "epoch": 0.568,
      "grad_norm": 3.0726404190063477,
      "learning_rate": 4.6779072702498765e-06,
      "logits/chosen": -1.496206521987915,
      "logits/rejected": -1.482466459274292,
      "logps/chosen": -219.1163330078125,
      "logps/rejected": -258.2294006347656,
      "loss": 0.2819,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 0.44380149245262146,
      "rewards/margins": 2.6352720260620117,
      "rewards/rejected": -2.1914706230163574,
      "step": 355
    },
    {
      "epoch": 0.576,
      "grad_norm": 3.823976516723633,
      "learning_rate": 4.667220336737355e-06,
      "logits/chosen": -1.5284912586212158,
      "logits/rejected": -1.4588353633880615,
      "logps/chosen": -224.3351287841797,
      "logps/rejected": -264.51531982421875,
      "loss": 0.2491,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.42943182587623596,
      "rewards/margins": 2.9441678524017334,
      "rewards/rejected": -2.5147361755371094,
      "step": 360
    },
    {
      "epoch": 0.584,
      "grad_norm": 4.347540855407715,
      "learning_rate": 4.656371612968961e-06,
      "logits/chosen": -1.55275559425354,
      "logits/rejected": -1.5454585552215576,
      "logps/chosen": -213.6840057373047,
      "logps/rejected": -282.0579528808594,
      "loss": 0.2327,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.4197406768798828,
      "rewards/margins": 3.0559322834014893,
      "rewards/rejected": -2.6361916065216064,
      "step": 365
    },
    {
      "epoch": 0.592,
      "grad_norm": 5.5594611167907715,
      "learning_rate": 4.645361908838255e-06,
      "logits/chosen": -1.6658909320831299,
      "logits/rejected": -1.5418522357940674,
      "logps/chosen": -224.8914794921875,
      "logps/rejected": -266.9699401855469,
      "loss": 0.3323,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.01982409879565239,
      "rewards/margins": 2.640495777130127,
      "rewards/rejected": -2.6206717491149902,
      "step": 370
    },
    {
      "epoch": 0.6,
      "grad_norm": 2.3482155799865723,
      "learning_rate": 4.634192046256521e-06,
      "logits/chosen": -1.545961618423462,
      "logits/rejected": -1.5492537021636963,
      "logps/chosen": -228.63076782226562,
      "logps/rejected": -293.1945495605469,
      "loss": 0.2174,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.24786663055419922,
      "rewards/margins": 3.227076292037964,
      "rewards/rejected": -2.9792098999023438,
      "step": 375
    },
    {
      "epoch": 0.608,
      "grad_norm": 2.7039475440979004,
      "learning_rate": 4.622862859091407e-06,
      "logits/chosen": -1.5828018188476562,
      "logits/rejected": -1.5519826412200928,
      "logps/chosen": -224.43521118164062,
      "logps/rejected": -289.30133056640625,
      "loss": 0.2281,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.2096446454524994,
      "rewards/margins": 3.253903865814209,
      "rewards/rejected": -3.044259548187256,
      "step": 380
    },
    {
      "epoch": 0.616,
      "grad_norm": 3.9861464500427246,
      "learning_rate": 4.611375193104674e-06,
      "logits/chosen": -1.641060471534729,
      "logits/rejected": -1.563591718673706,
      "logps/chosen": -218.3530731201172,
      "logps/rejected": -275.87127685546875,
      "loss": 0.2379,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.006566214375197887,
      "rewards/margins": 3.3008251190185547,
      "rewards/rejected": -3.2942588329315186,
      "step": 385
    },
    {
      "epoch": 0.624,
      "grad_norm": 2.931182622909546,
      "learning_rate": 4.59972990588906e-06,
      "logits/chosen": -1.462402105331421,
      "logits/rejected": -1.4504549503326416,
      "logps/chosen": -237.58432006835938,
      "logps/rejected": -280.5559997558594,
      "loss": 0.2608,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.006480635609477758,
      "rewards/margins": 2.936253786087036,
      "rewards/rejected": -2.9297728538513184,
      "step": 390
    },
    {
      "epoch": 0.632,
      "grad_norm": 6.512454032897949,
      "learning_rate": 4.5879278668042525e-06,
      "logits/chosen": -1.635387659072876,
      "logits/rejected": -1.543980360031128,
      "logps/chosen": -225.7213897705078,
      "logps/rejected": -257.0222473144531,
      "loss": 0.2732,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.037258487194776535,
      "rewards/margins": 2.8508574962615967,
      "rewards/rejected": -2.8881163597106934,
      "step": 395
    },
    {
      "epoch": 0.64,
      "grad_norm": 2.6228084564208984,
      "learning_rate": 4.575969956911994e-06,
      "logits/chosen": -1.600145697593689,
      "logits/rejected": -1.466813325881958,
      "logps/chosen": -230.7875213623047,
      "logps/rejected": -274.317138671875,
      "loss": 0.2125,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0649040937423706,
      "rewards/margins": 3.4587504863739014,
      "rewards/rejected": -3.5236542224884033,
      "step": 400
    },
    {
      "epoch": 0.648,
      "grad_norm": 2.864257335662842,
      "learning_rate": 4.563857068910303e-06,
      "logits/chosen": -1.520639181137085,
      "logits/rejected": -1.5240384340286255,
      "logps/chosen": -219.7958221435547,
      "logps/rejected": -283.594482421875,
      "loss": 0.1655,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.11086788028478622,
      "rewards/margins": 3.5582213401794434,
      "rewards/rejected": -3.4473540782928467,
      "step": 405
    },
    {
      "epoch": 0.656,
      "grad_norm": 4.094531536102295,
      "learning_rate": 4.551590107066833e-06,
      "logits/chosen": -1.6114097833633423,
      "logits/rejected": -1.581969976425171,
      "logps/chosen": -270.63824462890625,
      "logps/rejected": -291.4254150390625,
      "loss": 0.2999,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 0.051137734204530716,
      "rewards/margins": 3.479069232940674,
      "rewards/rejected": -3.427931547164917,
      "step": 410
    },
    {
      "epoch": 0.664,
      "grad_norm": 1.4797621965408325,
      "learning_rate": 4.5391699871513675e-06,
      "logits/chosen": -1.6318258047103882,
      "logits/rejected": -1.512136459350586,
      "logps/chosen": -241.17367553710938,
      "logps/rejected": -287.22613525390625,
      "loss": 0.2384,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.08122474700212479,
      "rewards/margins": 3.71380877494812,
      "rewards/rejected": -3.6325843334198,
      "step": 415
    },
    {
      "epoch": 0.672,
      "grad_norm": 3.8422563076019287,
      "learning_rate": 4.526597636367449e-06,
      "logits/chosen": -1.7238775491714478,
      "logits/rejected": -1.6469764709472656,
      "logps/chosen": -243.2270965576172,
      "logps/rejected": -272.0723876953125,
      "loss": 0.2496,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.0937575250864029,
      "rewards/margins": 3.0788111686706543,
      "rewards/rejected": -3.1725687980651855,
      "step": 420
    },
    {
      "epoch": 0.68,
      "grad_norm": 4.859435558319092,
      "learning_rate": 4.5138739932831674e-06,
      "logits/chosen": -1.600683569908142,
      "logits/rejected": -1.515218734741211,
      "logps/chosen": -241.6292724609375,
      "logps/rejected": -278.32427978515625,
      "loss": 0.2226,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.06983254849910736,
      "rewards/margins": 3.874213457107544,
      "rewards/rejected": -3.8043811321258545,
      "step": 425
    },
    {
      "epoch": 0.688,
      "grad_norm": 3.252007007598877,
      "learning_rate": 4.501000007761091e-06,
      "logits/chosen": -1.6320215463638306,
      "logits/rejected": -1.5925445556640625,
      "logps/chosen": -229.80099487304688,
      "logps/rejected": -283.92095947265625,
      "loss": 0.2189,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.1597936749458313,
      "rewards/margins": 3.3600659370422363,
      "rewards/rejected": -3.2002720832824707,
      "step": 430
    },
    {
      "epoch": 0.696,
      "grad_norm": 3.6351332664489746,
      "learning_rate": 4.487976640887349e-06,
      "logits/chosen": -1.576635718345642,
      "logits/rejected": -1.564374327659607,
      "logps/chosen": -223.580322265625,
      "logps/rejected": -283.73175048828125,
      "loss": 0.2519,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.35887137055397034,
      "rewards/margins": 3.7305588722229004,
      "rewards/rejected": -3.3716869354248047,
      "step": 435
    },
    {
      "epoch": 0.704,
      "grad_norm": 4.0519700050354,
      "learning_rate": 4.474804864899895e-06,
      "logits/chosen": -1.527977705001831,
      "logits/rejected": -1.4295679330825806,
      "logps/chosen": -207.36312866210938,
      "logps/rejected": -259.65740966796875,
      "loss": 0.238,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.36125385761260986,
      "rewards/margins": 3.2277350425720215,
      "rewards/rejected": -2.866481304168701,
      "step": 440
    },
    {
      "epoch": 0.712,
      "grad_norm": 4.860156536102295,
      "learning_rate": 4.461485663115915e-06,
      "logits/chosen": -1.6169579029083252,
      "logits/rejected": -1.5100561380386353,
      "logps/chosen": -230.5595245361328,
      "logps/rejected": -272.85198974609375,
      "loss": 0.3059,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 0.08551453053951263,
      "rewards/margins": 3.0663275718688965,
      "rewards/rejected": -2.980813503265381,
      "step": 445
    },
    {
      "epoch": 0.72,
      "grad_norm": 2.4698750972747803,
      "learning_rate": 4.448020029858427e-06,
      "logits/chosen": -1.5895106792449951,
      "logits/rejected": -1.5054939985275269,
      "logps/chosen": -215.32870483398438,
      "logps/rejected": -267.3359375,
      "loss": 0.2383,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.29027166962623596,
      "rewards/margins": 3.742069959640503,
      "rewards/rejected": -3.451798677444458,
      "step": 450
    },
    {
      "epoch": 0.728,
      "grad_norm": 4.764713764190674,
      "learning_rate": 4.4344089703820495e-06,
      "logits/chosen": -1.5860201120376587,
      "logits/rejected": -1.52178156375885,
      "logps/chosen": -224.6458740234375,
      "logps/rejected": -301.4439392089844,
      "loss": 0.2253,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.401482492685318,
      "rewards/margins": 3.844498872756958,
      "rewards/rejected": -3.4430160522460938,
      "step": 455
    },
    {
      "epoch": 0.736,
      "grad_norm": 3.805908441543579,
      "learning_rate": 4.420653500797952e-06,
      "logits/chosen": -1.4725072383880615,
      "logits/rejected": -1.4182043075561523,
      "logps/chosen": -240.71923828125,
      "logps/rejected": -270.241943359375,
      "loss": 0.2165,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.3455933928489685,
      "rewards/margins": 3.414217710494995,
      "rewards/rejected": -3.068624258041382,
      "step": 460
    },
    {
      "epoch": 0.744,
      "grad_norm": 2.8406822681427,
      "learning_rate": 4.406754647998005e-06,
      "logits/chosen": -1.6234924793243408,
      "logits/rejected": -1.4973393678665161,
      "logps/chosen": -218.62863159179688,
      "logps/rejected": -281.8992004394531,
      "loss": 0.1714,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.40566954016685486,
      "rewards/margins": 3.8715057373046875,
      "rewards/rejected": -3.4658360481262207,
      "step": 465
    },
    {
      "epoch": 0.752,
      "grad_norm": 6.241238594055176,
      "learning_rate": 4.3927134495781155e-06,
      "logits/chosen": -1.5617748498916626,
      "logits/rejected": -1.5487487316131592,
      "logps/chosen": -248.231689453125,
      "logps/rejected": -287.02960205078125,
      "loss": 0.2686,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 0.0831371322274208,
      "rewards/margins": 3.679551601409912,
      "rewards/rejected": -3.596414566040039,
      "step": 470
    },
    {
      "epoch": 0.76,
      "grad_norm": 4.16778039932251,
      "learning_rate": 4.378530953760768e-06,
      "logits/chosen": -1.4174320697784424,
      "logits/rejected": -1.4294613599777222,
      "logps/chosen": -226.9671173095703,
      "logps/rejected": -263.05389404296875,
      "loss": 0.2885,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 0.02728237584233284,
      "rewards/margins": 2.947277545928955,
      "rewards/rejected": -2.919994831085205,
      "step": 475
    },
    {
      "epoch": 0.768,
      "grad_norm": 5.187206268310547,
      "learning_rate": 4.364208219316771e-06,
      "logits/chosen": -1.5073765516281128,
      "logits/rejected": -1.5238683223724365,
      "logps/chosen": -211.14846801757812,
      "logps/rejected": -280.1331787109375,
      "loss": 0.2781,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.08308421075344086,
      "rewards/margins": 3.373748302459717,
      "rewards/rejected": -3.4568328857421875,
      "step": 480
    },
    {
      "epoch": 0.776,
      "grad_norm": 3.847803831100464,
      "learning_rate": 4.349746315486214e-06,
      "logits/chosen": -1.4829554557800293,
      "logits/rejected": -1.4054362773895264,
      "logps/chosen": -224.22915649414062,
      "logps/rejected": -282.7549133300781,
      "loss": 0.242,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.16652753949165344,
      "rewards/margins": 3.6406772136688232,
      "rewards/rejected": -3.474149703979492,
      "step": 485
    },
    {
      "epoch": 0.784,
      "grad_norm": 6.660366058349609,
      "learning_rate": 4.335146321898651e-06,
      "logits/chosen": -1.597735047340393,
      "logits/rejected": -1.5115840435028076,
      "logps/chosen": -221.6646270751953,
      "logps/rejected": -284.57452392578125,
      "loss": 0.3001,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.21003933250904083,
      "rewards/margins": 3.47546124458313,
      "rewards/rejected": -3.6855006217956543,
      "step": 490
    },
    {
      "epoch": 0.792,
      "grad_norm": 3.7531986236572266,
      "learning_rate": 4.320409328492495e-06,
      "logits/chosen": -1.5909736156463623,
      "logits/rejected": -1.5427086353302002,
      "logps/chosen": -235.1774139404297,
      "logps/rejected": -293.44940185546875,
      "loss": 0.2523,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 0.25116807222366333,
      "rewards/margins": 3.825078248977661,
      "rewards/rejected": -3.573909282684326,
      "step": 495
    },
    {
      "epoch": 0.8,
      "grad_norm": 4.075142860412598,
      "learning_rate": 4.305536435433656e-06,
      "logits/chosen": -1.5668506622314453,
      "logits/rejected": -1.5530762672424316,
      "logps/chosen": -222.394775390625,
      "logps/rejected": -293.10736083984375,
      "loss": 0.2185,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.26374197006225586,
      "rewards/margins": 3.8278470039367676,
      "rewards/rejected": -3.5641047954559326,
      "step": 500
    },
    {
      "epoch": 0.808,
      "grad_norm": 3.839890956878662,
      "learning_rate": 4.290528753033411e-06,
      "logits/chosen": -1.5763019323349,
      "logits/rejected": -1.5077393054962158,
      "logps/chosen": -232.4560546875,
      "logps/rejected": -289.55499267578125,
      "loss": 0.2744,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.025719815865159035,
      "rewards/margins": 3.2603039741516113,
      "rewards/rejected": -3.2860240936279297,
      "step": 505
    },
    {
      "epoch": 0.816,
      "grad_norm": 5.276324272155762,
      "learning_rate": 4.275387401665506e-06,
      "logits/chosen": -1.667394995689392,
      "logits/rejected": -1.5322387218475342,
      "logps/chosen": -217.12167358398438,
      "logps/rejected": -276.91119384765625,
      "loss": 0.2641,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.012757187709212303,
      "rewards/margins": 3.3533809185028076,
      "rewards/rejected": -3.3406238555908203,
      "step": 510
    },
    {
      "epoch": 0.824,
      "grad_norm": 2.3465888500213623,
      "learning_rate": 4.2601135116825295e-06,
      "logits/chosen": -1.5679190158843994,
      "logits/rejected": -1.5460660457611084,
      "logps/chosen": -230.0165252685547,
      "logps/rejected": -277.22540283203125,
      "loss": 0.2604,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.008122218772768974,
      "rewards/margins": 3.359611988067627,
      "rewards/rejected": -3.351489543914795,
      "step": 515
    },
    {
      "epoch": 0.832,
      "grad_norm": 5.303627014160156,
      "learning_rate": 4.244708223331519e-06,
      "logits/chosen": -1.5283758640289307,
      "logits/rejected": -1.4631974697113037,
      "logps/chosen": -239.413818359375,
      "logps/rejected": -262.42071533203125,
      "loss": 0.2643,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.04543369263410568,
      "rewards/margins": 3.1686339378356934,
      "rewards/rejected": -3.2140679359436035,
      "step": 520
    },
    {
      "epoch": 0.84,
      "grad_norm": 4.114057540893555,
      "learning_rate": 4.229172686668838e-06,
      "logits/chosen": -1.6217492818832397,
      "logits/rejected": -1.57975435256958,
      "logps/chosen": -229.3615264892578,
      "logps/rejected": -274.1919250488281,
      "loss": 0.2762,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3959481418132782,
      "rewards/margins": 3.1732304096221924,
      "rewards/rejected": -3.569178819656372,
      "step": 525
    },
    {
      "epoch": 0.848,
      "grad_norm": 6.136798858642578,
      "learning_rate": 4.213508061474327e-06,
      "logits/chosen": -1.7025260925292969,
      "logits/rejected": -1.5809694528579712,
      "logps/chosen": -205.9241180419922,
      "logps/rejected": -291.3397521972656,
      "loss": 0.2371,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.26137492060661316,
      "rewards/margins": 3.5369045734405518,
      "rewards/rejected": -3.798279285430908,
      "step": 530
    },
    {
      "epoch": 0.856,
      "grad_norm": 2.6232376098632812,
      "learning_rate": 4.197715517164716e-06,
      "logits/chosen": -1.638715147972107,
      "logits/rejected": -1.539113163948059,
      "logps/chosen": -238.2532196044922,
      "logps/rejected": -268.07415771484375,
      "loss": 0.2408,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.18916237354278564,
      "rewards/margins": 3.497271776199341,
      "rewards/rejected": -3.686434268951416,
      "step": 535
    },
    {
      "epoch": 0.864,
      "grad_norm": 3.9021799564361572,
      "learning_rate": 4.181796232706322e-06,
      "logits/chosen": -1.6269233226776123,
      "logits/rejected": -1.5677905082702637,
      "logps/chosen": -219.751708984375,
      "logps/rejected": -256.9467468261719,
      "loss": 0.2363,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.3333657681941986,
      "rewards/margins": 3.4952011108398438,
      "rewards/rejected": -3.828566789627075,
      "step": 540
    },
    {
      "epoch": 0.872,
      "grad_norm": 3.2338857650756836,
      "learning_rate": 4.165751396527044e-06,
      "logits/chosen": -1.5140718221664429,
      "logits/rejected": -1.5093212127685547,
      "logps/chosen": -235.4116668701172,
      "logps/rejected": -303.4933166503906,
      "loss": 0.2068,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.029620448127388954,
      "rewards/margins": 3.950321674346924,
      "rewards/rejected": -3.9799418449401855,
      "step": 545
    },
    {
      "epoch": 0.88,
      "grad_norm": 3.0233776569366455,
      "learning_rate": 4.149582206427634e-06,
      "logits/chosen": -1.6049916744232178,
      "logits/rejected": -1.614729642868042,
      "logps/chosen": -235.3531494140625,
      "logps/rejected": -303.1663513183594,
      "loss": 0.2307,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.004267203621566296,
      "rewards/margins": 3.4955551624298096,
      "rewards/rejected": -3.49128794670105,
      "step": 550
    },
    {
      "epoch": 0.888,
      "grad_norm": 4.327015399932861,
      "learning_rate": 4.133289869492281e-06,
      "logits/chosen": -1.655649185180664,
      "logits/rejected": -1.6161320209503174,
      "logps/chosen": -228.37173461914062,
      "logps/rejected": -289.50726318359375,
      "loss": 0.2433,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.171146959066391,
      "rewards/margins": 3.6598048210144043,
      "rewards/rejected": -3.8309521675109863,
      "step": 555
    },
    {
      "epoch": 0.896,
      "grad_norm": 2.6060330867767334,
      "learning_rate": 4.116875601998499e-06,
      "logits/chosen": -1.5099356174468994,
      "logits/rejected": -1.533095359802246,
      "logps/chosen": -209.5348663330078,
      "logps/rejected": -287.14080810546875,
      "loss": 0.1912,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.30249691009521484,
      "rewards/margins": 3.736576557159424,
      "rewards/rejected": -3.434079647064209,
      "step": 560
    },
    {
      "epoch": 0.904,
      "grad_norm": 3.044135093688965,
      "learning_rate": 4.100340629326324e-06,
      "logits/chosen": -1.59171724319458,
      "logits/rejected": -1.4727497100830078,
      "logps/chosen": -231.43966674804688,
      "logps/rejected": -269.6001281738281,
      "loss": 0.2462,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.26826876401901245,
      "rewards/margins": 3.4600913524627686,
      "rewards/rejected": -3.1918225288391113,
      "step": 565
    },
    {
      "epoch": 0.912,
      "grad_norm": 2.689462900161743,
      "learning_rate": 4.083686185866841e-06,
      "logits/chosen": -1.5678335428237915,
      "logits/rejected": -1.5138065814971924,
      "logps/chosen": -239.888671875,
      "logps/rejected": -290.61932373046875,
      "loss": 0.1873,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.33111506700515747,
      "rewards/margins": 3.883434772491455,
      "rewards/rejected": -3.5523197650909424,
      "step": 570
    },
    {
      "epoch": 0.92,
      "grad_norm": 2.54464054107666,
      "learning_rate": 4.0669135149300274e-06,
      "logits/chosen": -1.5594810247421265,
      "logits/rejected": -1.4758410453796387,
      "logps/chosen": -217.28890991210938,
      "logps/rejected": -288.6588134765625,
      "loss": 0.2231,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.2762870788574219,
      "rewards/margins": 3.7494559288024902,
      "rewards/rejected": -3.4731688499450684,
      "step": 575
    },
    {
      "epoch": 0.928,
      "grad_norm": 3.501030206680298,
      "learning_rate": 4.050023868651938e-06,
      "logits/chosen": -1.7346251010894775,
      "logits/rejected": -1.639661431312561,
      "logps/chosen": -203.6880340576172,
      "logps/rejected": -278.46197509765625,
      "loss": 0.2775,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.0705312117934227,
      "rewards/margins": 3.4696056842803955,
      "rewards/rejected": -3.3990745544433594,
      "step": 580
    },
    {
      "epoch": 0.936,
      "grad_norm": 3.608591318130493,
      "learning_rate": 4.033018507901229e-06,
      "logits/chosen": -1.57565438747406,
      "logits/rejected": -1.4335561990737915,
      "logps/chosen": -250.42318725585938,
      "logps/rejected": -307.19134521484375,
      "loss": 0.2373,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.0479436032474041,
      "rewards/margins": 3.5765857696533203,
      "rewards/rejected": -3.52864146232605,
      "step": 585
    },
    {
      "epoch": 0.944,
      "grad_norm": 4.200185298919678,
      "learning_rate": 4.015898702185028e-06,
      "logits/chosen": -1.4519469738006592,
      "logits/rejected": -1.4604626893997192,
      "logps/chosen": -236.66989135742188,
      "logps/rejected": -282.889892578125,
      "loss": 0.1888,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.17684012651443481,
      "rewards/margins": 3.7582874298095703,
      "rewards/rejected": -3.5814476013183594,
      "step": 590
    },
    {
      "epoch": 0.952,
      "grad_norm": 4.1930975914001465,
      "learning_rate": 3.998665729554165e-06,
      "logits/chosen": -1.4375026226043701,
      "logits/rejected": -1.3792469501495361,
      "logps/chosen": -226.0066680908203,
      "logps/rejected": -287.548828125,
      "loss": 0.2448,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 0.20100080966949463,
      "rewards/margins": 4.001578330993652,
      "rewards/rejected": -3.8005776405334473,
      "step": 595
    },
    {
      "epoch": 0.96,
      "grad_norm": 1.7681405544281006,
      "learning_rate": 3.9813208765077575e-06,
      "logits/chosen": -1.5724737644195557,
      "logits/rejected": -1.559664249420166,
      "logps/chosen": -230.7184600830078,
      "logps/rejected": -300.39178466796875,
      "loss": 0.2479,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.04428700730204582,
      "rewards/margins": 3.71527099609375,
      "rewards/rejected": -3.7595577239990234,
      "step": 600
    },
    {
      "epoch": 0.968,
      "grad_norm": 2.4093072414398193,
      "learning_rate": 3.96386543789717e-06,
      "logits/chosen": -1.5758655071258545,
      "logits/rejected": -1.526741623878479,
      "logps/chosen": -222.5027618408203,
      "logps/rejected": -270.0567626953125,
      "loss": 0.2558,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.05063873529434204,
      "rewards/margins": 3.403191089630127,
      "rewards/rejected": -3.4538300037384033,
      "step": 605
    },
    {
      "epoch": 0.976,
      "grad_norm": 4.265599727630615,
      "learning_rate": 3.946300716829352e-06,
      "logits/chosen": -1.5326379537582397,
      "logits/rejected": -1.479724645614624,
      "logps/chosen": -223.6086883544922,
      "logps/rejected": -298.55877685546875,
      "loss": 0.2146,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.2664863169193268,
      "rewards/margins": 4.003918647766113,
      "rewards/rejected": -3.7374324798583984,
      "step": 610
    },
    {
      "epoch": 0.984,
      "grad_norm": 6.780034065246582,
      "learning_rate": 3.9286280245695526e-06,
      "logits/chosen": -1.5027611255645752,
      "logits/rejected": -1.3937050104141235,
      "logps/chosen": -216.69552612304688,
      "logps/rejected": -277.9105224609375,
      "loss": 0.2438,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 0.20606736838817596,
      "rewards/margins": 3.780785083770752,
      "rewards/rejected": -3.5747172832489014,
      "step": 615
    },
    {
      "epoch": 0.992,
      "grad_norm": 3.8066725730895996,
      "learning_rate": 3.910848680443432e-06,
      "logits/chosen": -1.543540596961975,
      "logits/rejected": -1.5942132472991943,
      "logps/chosen": -218.4105682373047,
      "logps/rejected": -291.05810546875,
      "loss": 0.2594,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.14096443355083466,
      "rewards/margins": 3.5304324626922607,
      "rewards/rejected": -3.6713974475860596,
      "step": 620
    },
    {
      "epoch": 1.0,
      "grad_norm": 4.143672466278076,
      "learning_rate": 3.892964011738571e-06,
      "logits/chosen": -1.5397661924362183,
      "logits/rejected": -1.476738691329956,
      "logps/chosen": -229.1673126220703,
      "logps/rejected": -285.22662353515625,
      "loss": 0.2476,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.1350642591714859,
      "rewards/margins": 3.7622408866882324,
      "rewards/rejected": -3.627176284790039,
      "step": 625
    },
    {
      "epoch": 1.008,
      "grad_norm": 2.3031466007232666,
      "learning_rate": 3.874975353605382e-06,
      "logits/chosen": -1.4658385515213013,
      "logits/rejected": -1.352839469909668,
      "logps/chosen": -234.90890502929688,
      "logps/rejected": -282.998046875,
      "loss": 0.2004,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.3079695403575897,
      "rewards/margins": 3.906391143798828,
      "rewards/rejected": -3.598421573638916,
      "step": 630
    },
    {
      "epoch": 1.016,
      "grad_norm": 3.11726975440979,
      "learning_rate": 3.856884048957435e-06,
      "logits/chosen": -1.4668641090393066,
      "logits/rejected": -1.4360945224761963,
      "logps/chosen": -215.73709106445312,
      "logps/rejected": -278.59356689453125,
      "loss": 0.2069,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.2725924849510193,
      "rewards/margins": 3.960139513015747,
      "rewards/rejected": -3.687546968460083,
      "step": 635
    },
    {
      "epoch": 1.024,
      "grad_norm": 3.6341516971588135,
      "learning_rate": 3.8386914483712085e-06,
      "logits/chosen": -1.6500327587127686,
      "logits/rejected": -1.5192759037017822,
      "logps/chosen": -222.8524627685547,
      "logps/rejected": -287.26336669921875,
      "loss": 0.2025,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.19326825439929962,
      "rewards/margins": 3.847665786743164,
      "rewards/rejected": -3.6543972492218018,
      "step": 640
    },
    {
      "epoch": 1.032,
      "grad_norm": 3.898862600326538,
      "learning_rate": 3.82039890998526e-06,
      "logits/chosen": -1.578674077987671,
      "logits/rejected": -1.521864652633667,
      "logps/chosen": -231.91354370117188,
      "logps/rejected": -285.1756896972656,
      "loss": 0.202,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.09104955941438675,
      "rewards/margins": 3.9102611541748047,
      "rewards/rejected": -3.81921124458313,
      "step": 645
    },
    {
      "epoch": 1.04,
      "grad_norm": 3.565178394317627,
      "learning_rate": 3.8020077993988392e-06,
      "logits/chosen": -1.6293882131576538,
      "logits/rejected": -1.5812406539916992,
      "logps/chosen": -216.76052856445312,
      "logps/rejected": -261.8741149902344,
      "loss": 0.2163,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.07969091832637787,
      "rewards/margins": 3.7176177501678467,
      "rewards/rejected": -3.6379265785217285,
      "step": 650
    },
    {
      "epoch": 1.048,
      "grad_norm": 4.372779369354248,
      "learning_rate": 3.783519489569941e-06,
      "logits/chosen": -1.6112524271011353,
      "logits/rejected": -1.5277847051620483,
      "logps/chosen": -206.58047485351562,
      "logps/rejected": -293.06402587890625,
      "loss": 0.1932,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.17493240535259247,
      "rewards/margins": 4.243735313415527,
      "rewards/rejected": -4.068803310394287,
      "step": 655
    },
    {
      "epoch": 1.056,
      "grad_norm": 3.313710927963257,
      "learning_rate": 3.76493536071281e-06,
      "logits/chosen": -1.6428502798080444,
      "logits/rejected": -1.5417510271072388,
      "logps/chosen": -219.9541473388672,
      "logps/rejected": -294.42877197265625,
      "loss": 0.2107,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.21960565447807312,
      "rewards/margins": 3.8889222145080566,
      "rewards/rejected": -3.669316053390503,
      "step": 660
    },
    {
      "epoch": 1.064,
      "grad_norm": 2.9225704669952393,
      "learning_rate": 3.7462568001948998e-06,
      "logits/chosen": -1.4473567008972168,
      "logits/rejected": -1.4605478048324585,
      "logps/chosen": -243.60775756835938,
      "logps/rejected": -293.059814453125,
      "loss": 0.1868,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.031231069937348366,
      "rewards/margins": 4.10404109954834,
      "rewards/rejected": -4.07280969619751,
      "step": 665
    },
    {
      "epoch": 1.072,
      "grad_norm": 5.222462177276611,
      "learning_rate": 3.727485202433306e-06,
      "logits/chosen": -1.565495491027832,
      "logits/rejected": -1.5146604776382446,
      "logps/chosen": -225.4569854736328,
      "logps/rejected": -277.9839172363281,
      "loss": 0.2113,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.06461232900619507,
      "rewards/margins": 4.261273384094238,
      "rewards/rejected": -4.325885772705078,
      "step": 670
    },
    {
      "epoch": 1.08,
      "grad_norm": 3.0117835998535156,
      "learning_rate": 3.708621968790664e-06,
      "logits/chosen": -1.4770100116729736,
      "logits/rejected": -1.4104207754135132,
      "logps/chosen": -217.6276397705078,
      "logps/rejected": -272.0361633300781,
      "loss": 0.1713,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.24164548516273499,
      "rewards/margins": 4.098926067352295,
      "rewards/rejected": -3.857280731201172,
      "step": 675
    },
    {
      "epoch": 1.088,
      "grad_norm": 3.177833318710327,
      "learning_rate": 3.689668507470538e-06,
      "logits/chosen": -1.500281572341919,
      "logits/rejected": -1.4820690155029297,
      "logps/chosen": -220.1053466796875,
      "logps/rejected": -289.69903564453125,
      "loss": 0.1829,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.013184416107833385,
      "rewards/margins": 4.457877159118652,
      "rewards/rejected": -4.444692611694336,
      "step": 680
    },
    {
      "epoch": 1.096,
      "grad_norm": 4.731311321258545,
      "learning_rate": 3.670626233412286e-06,
      "logits/chosen": -1.5912206172943115,
      "logits/rejected": -1.5220589637756348,
      "logps/chosen": -201.251708984375,
      "logps/rejected": -270.2958984375,
      "loss": 0.2014,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.341642826795578,
      "rewards/margins": 4.2412543296813965,
      "rewards/rejected": -4.582897186279297,
      "step": 685
    },
    {
      "epoch": 1.104,
      "grad_norm": 2.3776421546936035,
      "learning_rate": 3.651496568185437e-06,
      "logits/chosen": -1.5553052425384521,
      "logits/rejected": -1.594269037246704,
      "logps/chosen": -232.0573272705078,
      "logps/rejected": -281.67572021484375,
      "loss": 0.1584,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.1348535120487213,
      "rewards/margins": 4.210672855377197,
      "rewards/rejected": -4.075818061828613,
      "step": 690
    },
    {
      "epoch": 1.112,
      "grad_norm": 2.743382215499878,
      "learning_rate": 3.6322809398835633e-06,
      "logits/chosen": -1.56232488155365,
      "logits/rejected": -1.6043574810028076,
      "logps/chosen": -231.5035400390625,
      "logps/rejected": -310.3698425292969,
      "loss": 0.1365,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.3358309268951416,
      "rewards/margins": 4.229272365570068,
      "rewards/rejected": -4.565102577209473,
      "step": 695
    },
    {
      "epoch": 1.12,
      "grad_norm": 5.52133846282959,
      "learning_rate": 3.6129807830176677e-06,
      "logits/chosen": -1.5213687419891357,
      "logits/rejected": -1.427746295928955,
      "logps/chosen": -221.33627319335938,
      "logps/rejected": -293.8585510253906,
      "loss": 0.1724,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.30416998267173767,
      "rewards/margins": 5.004787445068359,
      "rewards/rejected": -5.308957099914551,
      "step": 700
    },
    {
      "epoch": 1.1280000000000001,
      "grad_norm": 2.9546468257904053,
      "learning_rate": 3.593597538409095e-06,
      "logits/chosen": -1.595198154449463,
      "logits/rejected": -1.5737626552581787,
      "logps/chosen": -228.0746307373047,
      "logps/rejected": -294.04345703125,
      "loss": 0.2306,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.3983309268951416,
      "rewards/margins": 4.113828659057617,
      "rewards/rejected": -4.512159824371338,
      "step": 705
    },
    {
      "epoch": 1.1360000000000001,
      "grad_norm": 4.587896347045898,
      "learning_rate": 3.574132653081966e-06,
      "logits/chosen": -1.594580888748169,
      "logits/rejected": -1.5994797945022583,
      "logps/chosen": -263.15087890625,
      "logps/rejected": -298.9627990722656,
      "loss": 0.2129,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.20374688506126404,
      "rewards/margins": 4.656205654144287,
      "rewards/rejected": -4.859952449798584,
      "step": 710
    },
    {
      "epoch": 1.144,
      "grad_norm": 2.239849090576172,
      "learning_rate": 3.554587580155156e-06,
      "logits/chosen": -1.5059843063354492,
      "logits/rejected": -1.4877843856811523,
      "logps/chosen": -234.9763641357422,
      "logps/rejected": -300.68017578125,
      "loss": 0.1998,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.6419249176979065,
      "rewards/margins": 4.179691791534424,
      "rewards/rejected": -4.821616172790527,
      "step": 715
    },
    {
      "epoch": 1.152,
      "grad_norm": 2.508899211883545,
      "learning_rate": 3.534963778733813e-06,
      "logits/chosen": -1.5398582220077515,
      "logits/rejected": -1.4734686613082886,
      "logps/chosen": -233.4082489013672,
      "logps/rejected": -288.84893798828125,
      "loss": 0.228,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.31519871950149536,
      "rewards/margins": 4.3696699142456055,
      "rewards/rejected": -4.684868812561035,
      "step": 720
    },
    {
      "epoch": 1.16,
      "grad_norm": 4.6924543380737305,
      "learning_rate": 3.5152627138004307e-06,
      "logits/chosen": -1.5623716115951538,
      "logits/rejected": -1.501668930053711,
      "logps/chosen": -218.7466583251953,
      "logps/rejected": -302.3932800292969,
      "loss": 0.1949,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.20108382403850555,
      "rewards/margins": 4.127780914306641,
      "rewards/rejected": -4.328865051269531,
      "step": 725
    },
    {
      "epoch": 1.168,
      "grad_norm": 1.3646442890167236,
      "learning_rate": 3.495485856105483e-06,
      "logits/chosen": -1.5090452432632446,
      "logits/rejected": -1.4874252080917358,
      "logps/chosen": -222.24996948242188,
      "logps/rejected": -288.701171875,
      "loss": 0.1532,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.22113752365112305,
      "rewards/margins": 5.008795261383057,
      "rewards/rejected": -5.2299323081970215,
      "step": 730
    },
    {
      "epoch": 1.176,
      "grad_norm": 2.9527175426483154,
      "learning_rate": 3.4756346820576254e-06,
      "logits/chosen": -1.5279325246810913,
      "logits/rejected": -1.4767868518829346,
      "logps/chosen": -224.994140625,
      "logps/rejected": -298.9396057128906,
      "loss": 0.1771,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.06507287174463272,
      "rewards/margins": 5.222357273101807,
      "rewards/rejected": -5.287430286407471,
      "step": 735
    },
    {
      "epoch": 1.184,
      "grad_norm": 3.2858221530914307,
      "learning_rate": 3.455710673613481e-06,
      "logits/chosen": -1.5745080709457397,
      "logits/rejected": -1.5177898406982422,
      "logps/chosen": -242.11489868164062,
      "logps/rejected": -295.5982360839844,
      "loss": 0.176,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.3049863874912262,
      "rewards/margins": 4.537248611450195,
      "rewards/rejected": -4.8422346115112305,
      "step": 740
    },
    {
      "epoch": 1.192,
      "grad_norm": 3.9363486766815186,
      "learning_rate": 3.4357153181670026e-06,
      "logits/chosen": -1.469070315361023,
      "logits/rejected": -1.4067113399505615,
      "logps/chosen": -249.7563018798828,
      "logps/rejected": -298.33514404296875,
      "loss": 0.1697,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.2381099909543991,
      "rewards/margins": 4.4300432205200195,
      "rewards/rejected": -4.668152809143066,
      "step": 745
    },
    {
      "epoch": 1.2,
      "grad_norm": 4.966811180114746,
      "learning_rate": 3.4156501084384366e-06,
      "logits/chosen": -1.6039005517959595,
      "logits/rejected": -1.4729602336883545,
      "logps/chosen": -273.5932312011719,
      "logps/rejected": -294.7056579589844,
      "loss": 0.1801,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.16059216856956482,
      "rewards/margins": 5.157106876373291,
      "rewards/rejected": -4.996514797210693,
      "step": 750
    },
    {
      "epoch": 1.208,
      "grad_norm": 4.185051441192627,
      "learning_rate": 3.3955165423628837e-06,
      "logits/chosen": -1.477455496788025,
      "logits/rejected": -1.4620649814605713,
      "logps/chosen": -234.0481414794922,
      "logps/rejected": -306.4354553222656,
      "loss": 0.1811,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.3949986398220062,
      "rewards/margins": 5.021819114685059,
      "rewards/rejected": -5.416817665100098,
      "step": 755
    },
    {
      "epoch": 1.216,
      "grad_norm": 2.5614964962005615,
      "learning_rate": 3.375316122978477e-06,
      "logits/chosen": -1.4299887418746948,
      "logits/rejected": -1.5205398797988892,
      "logps/chosen": -221.0004119873047,
      "logps/rejected": -303.17156982421875,
      "loss": 0.2033,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.15307825803756714,
      "rewards/margins": 4.517648696899414,
      "rewards/rejected": -4.670726776123047,
      "step": 760
    },
    {
      "epoch": 1.224,
      "grad_norm": 4.064622402191162,
      "learning_rate": 3.3550503583141726e-06,
      "logits/chosen": -1.52488374710083,
      "logits/rejected": -1.53218674659729,
      "logps/chosen": -217.2790069580078,
      "logps/rejected": -300.43756103515625,
      "loss": 0.1503,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.1853751242160797,
      "rewards/margins": 4.725203037261963,
      "rewards/rejected": -4.910577774047852,
      "step": 765
    },
    {
      "epoch": 1.232,
      "grad_norm": 3.794024705886841,
      "learning_rate": 3.3347207612771685e-06,
      "logits/chosen": -1.5807452201843262,
      "logits/rejected": -1.4922082424163818,
      "logps/chosen": -228.3516387939453,
      "logps/rejected": -306.33966064453125,
      "loss": 0.1985,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.4110232889652252,
      "rewards/margins": 4.793760776519775,
      "rewards/rejected": -5.204783916473389,
      "step": 770
    },
    {
      "epoch": 1.24,
      "grad_norm": 2.162344217300415,
      "learning_rate": 3.3143288495399663e-06,
      "logits/chosen": -1.5615044832229614,
      "logits/rejected": -1.5205762386322021,
      "logps/chosen": -224.5255126953125,
      "logps/rejected": -294.60888671875,
      "loss": 0.1701,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.17508749663829803,
      "rewards/margins": 4.54577112197876,
      "rewards/rejected": -4.720858573913574,
      "step": 775
    },
    {
      "epoch": 1.248,
      "grad_norm": 3.240323781967163,
      "learning_rate": 3.2938761454270652e-06,
      "logits/chosen": -1.5861835479736328,
      "logits/rejected": -1.5375291109085083,
      "logps/chosen": -216.03616333007812,
      "logps/rejected": -306.78656005859375,
      "loss": 0.1846,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.41480645537376404,
      "rewards/margins": 4.762531280517578,
      "rewards/rejected": -5.177338123321533,
      "step": 780
    },
    {
      "epoch": 1.256,
      "grad_norm": 6.329495429992676,
      "learning_rate": 3.2733641758013224e-06,
      "logits/chosen": -1.5894248485565186,
      "logits/rejected": -1.486215591430664,
      "logps/chosen": -236.5580291748047,
      "logps/rejected": -283.81109619140625,
      "loss": 0.227,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3073071837425232,
      "rewards/margins": 4.490199089050293,
      "rewards/rejected": -4.797506809234619,
      "step": 785
    },
    {
      "epoch": 1.264,
      "grad_norm": 3.0770888328552246,
      "learning_rate": 3.2527944719499614e-06,
      "logits/chosen": -1.6199833154678345,
      "logits/rejected": -1.5427840948104858,
      "logps/chosen": -226.6359405517578,
      "logps/rejected": -277.6859130859375,
      "loss": 0.1648,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.3684183955192566,
      "rewards/margins": 4.242851734161377,
      "rewards/rejected": -4.611270427703857,
      "step": 790
    },
    {
      "epoch": 1.272,
      "grad_norm": 3.9166150093078613,
      "learning_rate": 3.2321685694702604e-06,
      "logits/chosen": -1.5863549709320068,
      "logits/rejected": -1.5562946796417236,
      "logps/chosen": -221.8000030517578,
      "logps/rejected": -295.9337158203125,
      "loss": 0.1567,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.2886640429496765,
      "rewards/margins": 4.546719551086426,
      "rewards/rejected": -4.835383892059326,
      "step": 795
    },
    {
      "epoch": 1.28,
      "grad_norm": 3.533409595489502,
      "learning_rate": 3.211488008154913e-06,
      "logits/chosen": -1.5623531341552734,
      "logits/rejected": -1.5109331607818604,
      "logps/chosen": -241.4813690185547,
      "logps/rejected": -311.42047119140625,
      "loss": 0.167,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.16028550267219543,
      "rewards/margins": 4.911408424377441,
      "rewards/rejected": -5.071693420410156,
      "step": 800
    },
    {
      "epoch": 1.288,
      "grad_norm": 4.332025527954102,
      "learning_rate": 3.190754331877076e-06,
      "logits/chosen": -1.6541258096694946,
      "logits/rejected": -1.5064847469329834,
      "logps/chosen": -229.9279327392578,
      "logps/rejected": -264.9905700683594,
      "loss": 0.1983,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.24898216128349304,
      "rewards/margins": 4.498307704925537,
      "rewards/rejected": -4.747290134429932,
      "step": 805
    },
    {
      "epoch": 1.296,
      "grad_norm": 3.0773584842681885,
      "learning_rate": 3.1699690884751205e-06,
      "logits/chosen": -1.5797970294952393,
      "logits/rejected": -1.5334302186965942,
      "logps/chosen": -223.06967163085938,
      "logps/rejected": -284.0609436035156,
      "loss": 0.1923,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.35305580496788025,
      "rewards/margins": 4.535957336425781,
      "rewards/rejected": -4.889012336730957,
      "step": 810
    },
    {
      "epoch": 1.304,
      "grad_norm": 4.666840553283691,
      "learning_rate": 3.149133829637072e-06,
      "logits/chosen": -1.4840573072433472,
      "logits/rejected": -1.4880752563476562,
      "logps/chosen": -220.59378051757812,
      "logps/rejected": -295.93572998046875,
      "loss": 0.1604,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5703553557395935,
      "rewards/margins": 4.865350246429443,
      "rewards/rejected": -5.435705661773682,
      "step": 815
    },
    {
      "epoch": 1.312,
      "grad_norm": 3.065351963043213,
      "learning_rate": 3.128250110784778e-06,
      "logits/chosen": -1.6565717458724976,
      "logits/rejected": -1.481231451034546,
      "logps/chosen": -255.04312133789062,
      "logps/rejected": -284.08197021484375,
      "loss": 0.1835,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.38553112745285034,
      "rewards/margins": 4.551497459411621,
      "rewards/rejected": -4.937028884887695,
      "step": 820
    },
    {
      "epoch": 1.32,
      "grad_norm": 3.423949956893921,
      "learning_rate": 3.107319490957787e-06,
      "logits/chosen": -1.566410779953003,
      "logits/rejected": -1.5307550430297852,
      "logps/chosen": -235.0450897216797,
      "logps/rejected": -312.58770751953125,
      "loss": 0.186,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.5675615668296814,
      "rewards/margins": 4.500984191894531,
      "rewards/rejected": -5.068546295166016,
      "step": 825
    },
    {
      "epoch": 1.328,
      "grad_norm": 1.9515182971954346,
      "learning_rate": 3.0863435326969636e-06,
      "logits/chosen": -1.5598112344741821,
      "logits/rejected": -1.480304479598999,
      "logps/chosen": -249.81643676757812,
      "logps/rejected": -297.97186279296875,
      "loss": 0.1759,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.40629854798316956,
      "rewards/margins": 4.989799499511719,
      "rewards/rejected": -5.3960981369018555,
      "step": 830
    },
    {
      "epoch": 1.336,
      "grad_norm": 3.0678558349609375,
      "learning_rate": 3.065323801927837e-06,
      "logits/chosen": -1.6347713470458984,
      "logits/rejected": -1.5678890943527222,
      "logps/chosen": -239.1768035888672,
      "logps/rejected": -304.45159912109375,
      "loss": 0.1399,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4652630686759949,
      "rewards/margins": 4.925078392028809,
      "rewards/rejected": -5.390341281890869,
      "step": 835
    },
    {
      "epoch": 1.3439999999999999,
      "grad_norm": 1.6803035736083984,
      "learning_rate": 3.044261867843702e-06,
      "logits/chosen": -1.600610375404358,
      "logits/rejected": -1.547503113746643,
      "logps/chosen": -242.60928344726562,
      "logps/rejected": -299.3851013183594,
      "loss": 0.1305,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.09166824072599411,
      "rewards/margins": 4.836394309997559,
      "rewards/rejected": -4.92806339263916,
      "step": 840
    },
    {
      "epoch": 1.3519999999999999,
      "grad_norm": 3.992915153503418,
      "learning_rate": 3.0231593027884713e-06,
      "logits/chosen": -1.715815544128418,
      "logits/rejected": -1.7157901525497437,
      "logps/chosen": -217.67306518554688,
      "logps/rejected": -314.8918151855469,
      "loss": 0.2032,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.4734416902065277,
      "rewards/margins": 4.529386043548584,
      "rewards/rejected": -5.0028276443481445,
      "step": 845
    },
    {
      "epoch": 1.3599999999999999,
      "grad_norm": 2.1506552696228027,
      "learning_rate": 3.0020176821392965e-06,
      "logits/chosen": -1.5611059665679932,
      "logits/rejected": -1.5400519371032715,
      "logps/chosen": -217.276611328125,
      "logps/rejected": -293.49066162109375,
      "loss": 0.1648,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.31788164377212524,
      "rewards/margins": 4.510266304016113,
      "rewards/rejected": -4.828147888183594,
      "step": 850
    },
    {
      "epoch": 1.3679999999999999,
      "grad_norm": 5.309419631958008,
      "learning_rate": 2.9808385841889588e-06,
      "logits/chosen": -1.4734315872192383,
      "logits/rejected": -1.484628438949585,
      "logps/chosen": -231.23458862304688,
      "logps/rejected": -303.7582092285156,
      "loss": 0.2078,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.34313979744911194,
      "rewards/margins": 4.747766494750977,
      "rewards/rejected": -5.090906620025635,
      "step": 855
    },
    {
      "epoch": 1.376,
      "grad_norm": 4.189775466918945,
      "learning_rate": 2.9596235900280463e-06,
      "logits/chosen": -1.59661865234375,
      "logits/rejected": -1.595775842666626,
      "logps/chosen": -241.6797637939453,
      "logps/rejected": -294.72186279296875,
      "loss": 0.1996,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.4572688937187195,
      "rewards/margins": 4.40045166015625,
      "rewards/rejected": -4.857720851898193,
      "step": 860
    },
    {
      "epoch": 1.384,
      "grad_norm": 3.6436808109283447,
      "learning_rate": 2.9383742834269195e-06,
      "logits/chosen": -1.5166704654693604,
      "logits/rejected": -1.5094910860061646,
      "logps/chosen": -213.38858032226562,
      "logps/rejected": -299.029541015625,
      "loss": 0.2195,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.3637376129627228,
      "rewards/margins": 4.762623310089111,
      "rewards/rejected": -5.126360893249512,
      "step": 865
    },
    {
      "epoch": 1.392,
      "grad_norm": 3.5518014430999756,
      "learning_rate": 2.917092250717478e-06,
      "logits/chosen": -1.548153042793274,
      "logits/rejected": -1.4777790307998657,
      "logps/chosen": -221.06396484375,
      "logps/rejected": -300.5973205566406,
      "loss": 0.1963,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.16941462457180023,
      "rewards/margins": 5.223523139953613,
      "rewards/rejected": -5.392938137054443,
      "step": 870
    },
    {
      "epoch": 1.4,
      "grad_norm": 4.208786487579346,
      "learning_rate": 2.895779080674734e-06,
      "logits/chosen": -1.496821403503418,
      "logits/rejected": -1.44963538646698,
      "logps/chosen": -246.3695068359375,
      "logps/rejected": -278.2579345703125,
      "loss": 0.238,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.2623414099216461,
      "rewards/margins": 3.9486358165740967,
      "rewards/rejected": -4.210977077484131,
      "step": 875
    },
    {
      "epoch": 1.408,
      "grad_norm": 5.761550426483154,
      "learning_rate": 2.8744363643982043e-06,
      "logits/chosen": -1.5691251754760742,
      "logits/rejected": -1.5343207120895386,
      "logps/chosen": -227.2958526611328,
      "logps/rejected": -284.8408203125,
      "loss": 0.2237,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.3449009656906128,
      "rewards/margins": 4.538619518280029,
      "rewards/rejected": -4.88352108001709,
      "step": 880
    },
    {
      "epoch": 1.416,
      "grad_norm": 3.3380019664764404,
      "learning_rate": 2.853065695193135e-06,
      "logits/chosen": -1.4605835676193237,
      "logits/rejected": -1.4047877788543701,
      "logps/chosen": -232.16897583007812,
      "logps/rejected": -295.48626708984375,
      "loss": 0.1473,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.09790012985467911,
      "rewards/margins": 4.885996341705322,
      "rewards/rejected": -4.788095951080322,
      "step": 885
    },
    {
      "epoch": 1.424,
      "grad_norm": 2.6631267070770264,
      "learning_rate": 2.831668668451549e-06,
      "logits/chosen": -1.511093258857727,
      "logits/rejected": -1.4666423797607422,
      "logps/chosen": -232.2217559814453,
      "logps/rejected": -291.5470275878906,
      "loss": 0.1595,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.36790722608566284,
      "rewards/margins": 4.524458885192871,
      "rewards/rejected": -4.892366409301758,
      "step": 890
    },
    {
      "epoch": 1.432,
      "grad_norm": 5.322598457336426,
      "learning_rate": 2.8102468815331487e-06,
      "logits/chosen": -1.5681586265563965,
      "logits/rejected": -1.5516602993011475,
      "logps/chosen": -214.8710174560547,
      "logps/rejected": -306.7196044921875,
      "loss": 0.2156,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.6621019244194031,
      "rewards/margins": 4.608248710632324,
      "rewards/rejected": -5.270350933074951,
      "step": 895
    },
    {
      "epoch": 1.44,
      "grad_norm": 3.846139907836914,
      "learning_rate": 2.788801933646066e-06,
      "logits/chosen": -1.535198450088501,
      "logits/rejected": -1.4371068477630615,
      "logps/chosen": -225.10507202148438,
      "logps/rejected": -300.16339111328125,
      "loss": 0.1716,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.22050288319587708,
      "rewards/margins": 4.9329986572265625,
      "rewards/rejected": -5.153501987457275,
      "step": 900
    },
    {
      "epoch": 1.448,
      "grad_norm": 4.434758186340332,
      "learning_rate": 2.7673354257274775e-06,
      "logits/chosen": -1.6025463342666626,
      "logits/rejected": -1.5076074600219727,
      "logps/chosen": -225.4650421142578,
      "logps/rejected": -309.60272216796875,
      "loss": 0.185,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.4188717305660248,
      "rewards/margins": 4.646775722503662,
      "rewards/rejected": -5.065647125244141,
      "step": 905
    },
    {
      "epoch": 1.456,
      "grad_norm": 2.5944976806640625,
      "learning_rate": 2.7458489603240873e-06,
      "logits/chosen": -1.6403672695159912,
      "logits/rejected": -1.5778125524520874,
      "logps/chosen": -216.8976593017578,
      "logps/rejected": -304.27142333984375,
      "loss": 0.1484,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.30255550146102905,
      "rewards/margins": 5.206709861755371,
      "rewards/rejected": -5.509265422821045,
      "step": 910
    },
    {
      "epoch": 1.464,
      "grad_norm": 5.241691589355469,
      "learning_rate": 2.724344141472493e-06,
      "logits/chosen": -1.429987907409668,
      "logits/rejected": -1.3562849760055542,
      "logps/chosen": -255.3536834716797,
      "logps/rejected": -304.39044189453125,
      "loss": 0.2206,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.3938491940498352,
      "rewards/margins": 4.941486358642578,
      "rewards/rejected": -5.335335731506348,
      "step": 915
    },
    {
      "epoch": 1.472,
      "grad_norm": 1.7611676454544067,
      "learning_rate": 2.702822574579439e-06,
      "logits/chosen": -1.5172083377838135,
      "logits/rejected": -1.4978201389312744,
      "logps/chosen": -227.9757843017578,
      "logps/rejected": -301.53466796875,
      "loss": 0.148,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.3590165376663208,
      "rewards/margins": 5.5135297775268555,
      "rewards/rejected": -5.872546195983887,
      "step": 920
    },
    {
      "epoch": 1.48,
      "grad_norm": 2.4033844470977783,
      "learning_rate": 2.681285866301967e-06,
      "logits/chosen": -1.4651075601577759,
      "logits/rejected": -1.4009742736816406,
      "logps/chosen": -262.92706298828125,
      "logps/rejected": -274.20477294921875,
      "loss": 0.1876,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.12432646751403809,
      "rewards/margins": 4.453019618988037,
      "rewards/rejected": -4.577345848083496,
      "step": 925
    },
    {
      "epoch": 1.488,
      "grad_norm": 2.467533826828003,
      "learning_rate": 2.6597356244274723e-06,
      "logits/chosen": -1.4565765857696533,
      "logits/rejected": -1.4273284673690796,
      "logps/chosen": -222.59634399414062,
      "logps/rejected": -284.24267578125,
      "loss": 0.2066,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.47027140855789185,
      "rewards/margins": 4.3873701095581055,
      "rewards/rejected": -4.85764217376709,
      "step": 930
    },
    {
      "epoch": 1.496,
      "grad_norm": 1.8973395824432373,
      "learning_rate": 2.6381734577536816e-06,
      "logits/chosen": -1.6343520879745483,
      "logits/rejected": -1.5240951776504517,
      "logps/chosen": -237.08358764648438,
      "logps/rejected": -299.1141662597656,
      "loss": 0.1203,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.15792664885520935,
      "rewards/margins": 5.232786655426025,
      "rewards/rejected": -5.390713214874268,
      "step": 935
    },
    {
      "epoch": 1.504,
      "grad_norm": 2.850900650024414,
      "learning_rate": 2.616600975968544e-06,
      "logits/chosen": -1.5603736639022827,
      "logits/rejected": -1.4937515258789062,
      "logps/chosen": -229.7916717529297,
      "logps/rejected": -306.2135925292969,
      "loss": 0.1522,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.43655306100845337,
      "rewards/margins": 5.207942962646484,
      "rewards/rejected": -5.644496440887451,
      "step": 940
    },
    {
      "epoch": 1.512,
      "grad_norm": 3.7755420207977295,
      "learning_rate": 2.59501978953007e-06,
      "logits/chosen": -1.605474829673767,
      "logits/rejected": -1.5499731302261353,
      "logps/chosen": -236.4785919189453,
      "logps/rejected": -303.67742919921875,
      "loss": 0.1512,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.48815402388572693,
      "rewards/margins": 5.069121360778809,
      "rewards/rejected": -5.557275772094727,
      "step": 945
    },
    {
      "epoch": 1.52,
      "grad_norm": 7.098239421844482,
      "learning_rate": 2.5734315095460983e-06,
      "logits/chosen": -1.5143400430679321,
      "logits/rejected": -1.5596923828125,
      "logps/chosen": -232.8845672607422,
      "logps/rejected": -296.623779296875,
      "loss": 0.2305,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.8665415048599243,
      "rewards/margins": 4.796109676361084,
      "rewards/rejected": -5.662651062011719,
      "step": 950
    },
    {
      "epoch": 1.528,
      "grad_norm": 3.9633948802948,
      "learning_rate": 2.551837747654027e-06,
      "logits/chosen": -1.5993857383728027,
      "logits/rejected": -1.555641531944275,
      "logps/chosen": -242.03060913085938,
      "logps/rejected": -328.293701171875,
      "loss": 0.1351,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.8731464147567749,
      "rewards/margins": 5.36746072769165,
      "rewards/rejected": -6.240607261657715,
      "step": 955
    },
    {
      "epoch": 1.536,
      "grad_norm": 4.446506977081299,
      "learning_rate": 2.530240115900494e-06,
      "logits/chosen": -1.533271312713623,
      "logits/rejected": -1.5505766868591309,
      "logps/chosen": -222.93154907226562,
      "logps/rejected": -297.92071533203125,
      "loss": 0.1835,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.8138089179992676,
      "rewards/margins": 4.758518218994141,
      "rewards/rejected": -5.572327136993408,
      "step": 960
    },
    {
      "epoch": 1.544,
      "grad_norm": 4.153780937194824,
      "learning_rate": 2.5086402266210403e-06,
      "logits/chosen": -1.5660960674285889,
      "logits/rejected": -1.5074437856674194,
      "logps/chosen": -239.93869018554688,
      "logps/rejected": -301.6562805175781,
      "loss": 0.1765,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.2847996652126312,
      "rewards/margins": 4.576987266540527,
      "rewards/rejected": -4.861786842346191,
      "step": 965
    },
    {
      "epoch": 1.552,
      "grad_norm": 2.6983306407928467,
      "learning_rate": 2.4870396923197347e-06,
      "logits/chosen": -1.6386797428131104,
      "logits/rejected": -1.539115071296692,
      "logps/chosen": -241.8653564453125,
      "logps/rejected": -293.84979248046875,
      "loss": 0.2475,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.6399244070053101,
      "rewards/margins": 3.9493603706359863,
      "rewards/rejected": -4.589285373687744,
      "step": 970
    },
    {
      "epoch": 1.56,
      "grad_norm": 2.8271076679229736,
      "learning_rate": 2.4654401255487993e-06,
      "logits/chosen": -1.5707069635391235,
      "logits/rejected": -1.557492971420288,
      "logps/chosen": -221.89389038085938,
      "logps/rejected": -299.2413024902344,
      "loss": 0.1839,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5429846048355103,
      "rewards/margins": 4.966402530670166,
      "rewards/rejected": -5.509387016296387,
      "step": 975
    },
    {
      "epoch": 1.568,
      "grad_norm": 5.555903911590576,
      "learning_rate": 2.443843138788228e-06,
      "logits/chosen": -1.6090481281280518,
      "logits/rejected": -1.4825258255004883,
      "logps/chosen": -220.1716766357422,
      "logps/rejected": -300.88458251953125,
      "loss": 0.157,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.31384724378585815,
      "rewards/margins": 5.2531962394714355,
      "rewards/rejected": -5.567043781280518,
      "step": 980
    },
    {
      "epoch": 1.576,
      "grad_norm": 3.2215209007263184,
      "learning_rate": 2.4222503443254084e-06,
      "logits/chosen": -1.5508859157562256,
      "logits/rejected": -1.5698188543319702,
      "logps/chosen": -239.60537719726562,
      "logps/rejected": -307.3664245605469,
      "loss": 0.2068,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.189010351896286,
      "rewards/margins": 4.920119285583496,
      "rewards/rejected": -5.109129905700684,
      "step": 985
    },
    {
      "epoch": 1.584,
      "grad_norm": 2.4060542583465576,
      "learning_rate": 2.400663354134758e-06,
      "logits/chosen": -1.5376317501068115,
      "logits/rejected": -1.4998356103897095,
      "logps/chosen": -243.26486206054688,
      "logps/rejected": -289.48065185546875,
      "loss": 0.1764,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.5606673359870911,
      "rewards/margins": 4.382869720458984,
      "rewards/rejected": -4.943536758422852,
      "step": 990
    },
    {
      "epoch": 1.592,
      "grad_norm": 4.755422592163086,
      "learning_rate": 2.3790837797573867e-06,
      "logits/chosen": -1.4841296672821045,
      "logits/rejected": -1.4592382907867432,
      "logps/chosen": -235.3893585205078,
      "logps/rejected": -284.45806884765625,
      "loss": 0.1904,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.288729190826416,
      "rewards/margins": 4.829131126403809,
      "rewards/rejected": -5.117860317230225,
      "step": 995
    },
    {
      "epoch": 1.6,
      "grad_norm": 3.5496389865875244,
      "learning_rate": 2.357513232180788e-06,
      "logits/chosen": -1.6013635396957397,
      "logits/rejected": -1.4790610074996948,
      "logps/chosen": -240.17324829101562,
      "logps/rejected": -303.7333068847656,
      "loss": 0.1618,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.15540306270122528,
      "rewards/margins": 5.00370454788208,
      "rewards/rejected": -5.159107208251953,
      "step": 1000
    },
    {
      "epoch": 1.608,
      "grad_norm": 4.431177616119385,
      "learning_rate": 2.335953321718578e-06,
      "logits/chosen": -1.6200017929077148,
      "logits/rejected": -1.538055658340454,
      "logps/chosen": -253.2138671875,
      "logps/rejected": -305.25054931640625,
      "loss": 0.1785,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.22772517800331116,
      "rewards/margins": 4.754574775695801,
      "rewards/rejected": -4.982300281524658,
      "step": 1005
    },
    {
      "epoch": 1.616,
      "grad_norm": 4.100212574005127,
      "learning_rate": 2.3144056578902713e-06,
      "logits/chosen": -1.4878506660461426,
      "logits/rejected": -1.4349610805511475,
      "logps/chosen": -240.9538116455078,
      "logps/rejected": -279.4476013183594,
      "loss": 0.1915,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.35914379358291626,
      "rewards/margins": 4.77241325378418,
      "rewards/rejected": -5.131556987762451,
      "step": 1010
    },
    {
      "epoch": 1.624,
      "grad_norm": 4.914249420166016,
      "learning_rate": 2.292871849301135e-06,
      "logits/chosen": -1.5851023197174072,
      "logits/rejected": -1.4445489645004272,
      "logps/chosen": -245.83212280273438,
      "logps/rejected": -300.76226806640625,
      "loss": 0.2035,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.06682246923446655,
      "rewards/margins": 4.696481227874756,
      "rewards/rejected": -4.763303756713867,
      "step": 1015
    },
    {
      "epoch": 1.6320000000000001,
      "grad_norm": 5.474418640136719,
      "learning_rate": 2.271353503522092e-06,
      "logits/chosen": -1.4620692729949951,
      "logits/rejected": -1.474820852279663,
      "logps/chosen": -234.1145477294922,
      "logps/rejected": -303.3814392089844,
      "loss": 0.231,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.4707890450954437,
      "rewards/margins": 4.186692237854004,
      "rewards/rejected": -4.6574811935424805,
      "step": 1020
    },
    {
      "epoch": 1.6400000000000001,
      "grad_norm": 2.9608800411224365,
      "learning_rate": 2.2498522269697193e-06,
      "logits/chosen": -1.4659358263015747,
      "logits/rejected": -1.5075595378875732,
      "logps/chosen": -234.145751953125,
      "logps/rejected": -303.431396484375,
      "loss": 0.1632,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.674322247505188,
      "rewards/margins": 5.1520586013793945,
      "rewards/rejected": -5.826380729675293,
      "step": 1025
    },
    {
      "epoch": 1.6480000000000001,
      "grad_norm": 3.0162484645843506,
      "learning_rate": 2.228369624786314e-06,
      "logits/chosen": -1.5545270442962646,
      "logits/rejected": -1.4817333221435547,
      "logps/chosen": -243.4051971435547,
      "logps/rejected": -318.2030334472656,
      "loss": 0.1455,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.2738165259361267,
      "rewards/margins": 5.5686564445495605,
      "rewards/rejected": -5.842472553253174,
      "step": 1030
    },
    {
      "epoch": 1.6560000000000001,
      "grad_norm": 1.2202463150024414,
      "learning_rate": 2.2069073007200735e-06,
      "logits/chosen": -1.5296119451522827,
      "logits/rejected": -1.4982463121414185,
      "logps/chosen": -248.0770721435547,
      "logps/rejected": -293.5713806152344,
      "loss": 0.1328,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.10386035591363907,
      "rewards/margins": 5.480264186859131,
      "rewards/rejected": -5.584124565124512,
      "step": 1035
    },
    {
      "epoch": 1.6640000000000001,
      "grad_norm": 4.070965766906738,
      "learning_rate": 2.1854668570053617e-06,
      "logits/chosen": -1.4422088861465454,
      "logits/rejected": -1.336922287940979,
      "logps/chosen": -244.10546875,
      "logps/rejected": -291.4113464355469,
      "loss": 0.162,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.21176743507385254,
      "rewards/margins": 5.223017692565918,
      "rewards/rejected": -5.434784889221191,
      "step": 1040
    },
    {
      "epoch": 1.6720000000000002,
      "grad_norm": 2.6828646659851074,
      "learning_rate": 2.164049894243106e-06,
      "logits/chosen": -1.521547555923462,
      "logits/rejected": -1.5329920053482056,
      "logps/chosen": -220.31912231445312,
      "logps/rejected": -300.8257141113281,
      "loss": 0.1597,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8682439923286438,
      "rewards/margins": 4.75205135345459,
      "rewards/rejected": -5.620295524597168,
      "step": 1045
    },
    {
      "epoch": 1.6800000000000002,
      "grad_norm": 5.221405506134033,
      "learning_rate": 2.1426580112812963e-06,
      "logits/chosen": -1.4651761054992676,
      "logits/rejected": -1.4989073276519775,
      "logps/chosen": -225.73898315429688,
      "logps/rejected": -306.9569091796875,
      "loss": 0.1811,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.521946370601654,
      "rewards/margins": 4.900145530700684,
      "rewards/rejected": -5.422091960906982,
      "step": 1050
    },
    {
      "epoch": 1.688,
      "grad_norm": 4.7056684494018555,
      "learning_rate": 2.1212928050956363e-06,
      "logits/chosen": -1.4704397916793823,
      "logits/rejected": -1.3621515035629272,
      "logps/chosen": -261.0582580566406,
      "logps/rejected": -309.2656555175781,
      "loss": 0.1629,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.46547871828079224,
      "rewards/margins": 5.209592342376709,
      "rewards/rejected": -5.6750712394714355,
      "step": 1055
    },
    {
      "epoch": 1.696,
      "grad_norm": 6.003643989562988,
      "learning_rate": 2.0999558706703156e-06,
      "logits/chosen": -1.5227447748184204,
      "logits/rejected": -1.4205873012542725,
      "logps/chosen": -242.269287109375,
      "logps/rejected": -301.63250732421875,
      "loss": 0.1872,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.2986319959163666,
      "rewards/margins": 4.871571063995361,
      "rewards/rejected": -5.17020320892334,
      "step": 1060
    },
    {
      "epoch": 1.704,
      "grad_norm": 3.101877212524414,
      "learning_rate": 2.0786488008789452e-06,
      "logits/chosen": -1.5970118045806885,
      "logits/rejected": -1.5494978427886963,
      "logps/chosen": -249.52725219726562,
      "logps/rejected": -304.41656494140625,
      "loss": 0.1266,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.39330193400382996,
      "rewards/margins": 5.001396179199219,
      "rewards/rejected": -5.394698143005371,
      "step": 1065
    },
    {
      "epoch": 1.712,
      "grad_norm": 1.7850382328033447,
      "learning_rate": 2.0573731863656378e-06,
      "logits/chosen": -1.5234953165054321,
      "logits/rejected": -1.4536514282226562,
      "logps/chosen": -234.9643096923828,
      "logps/rejected": -293.68804931640625,
      "loss": 0.1331,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.3854447305202484,
      "rewards/margins": 5.071932315826416,
      "rewards/rejected": -5.457376956939697,
      "step": 1070
    },
    {
      "epoch": 1.72,
      "grad_norm": 4.341036796569824,
      "learning_rate": 2.036130615426267e-06,
      "logits/chosen": -1.541611671447754,
      "logits/rejected": -1.4438472986221313,
      "logps/chosen": -226.50082397460938,
      "logps/rejected": -294.1707458496094,
      "loss": 0.1548,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.2821238040924072,
      "rewards/margins": 4.956992149353027,
      "rewards/rejected": -5.239116191864014,
      "step": 1075
    },
    {
      "epoch": 1.728,
      "grad_norm": 1.9874736070632935,
      "learning_rate": 2.0149226738898923e-06,
      "logits/chosen": -1.6126731634140015,
      "logits/rejected": -1.5012925863265991,
      "logps/chosen": -241.3867950439453,
      "logps/rejected": -312.0802917480469,
      "loss": 0.1447,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.49232107400894165,
      "rewards/margins": 5.413763523101807,
      "rewards/rejected": -5.906083583831787,
      "step": 1080
    },
    {
      "epoch": 1.736,
      "grad_norm": 4.81574821472168,
      "learning_rate": 1.9937509450003715e-06,
      "logits/chosen": -1.4321324825286865,
      "logits/rejected": -1.4526867866516113,
      "logps/chosen": -215.76388549804688,
      "logps/rejected": -297.8123474121094,
      "loss": 0.1816,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.575035572052002,
      "rewards/margins": 5.639148712158203,
      "rewards/rejected": -6.214184761047363,
      "step": 1085
    },
    {
      "epoch": 1.744,
      "grad_norm": 3.8950352668762207,
      "learning_rate": 1.9726170092981676e-06,
      "logits/chosen": -1.5303950309753418,
      "logits/rejected": -1.5153299570083618,
      "logps/chosen": -237.6295928955078,
      "logps/rejected": -315.077880859375,
      "loss": 0.1566,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.65621018409729,
      "rewards/margins": 5.208491802215576,
      "rewards/rejected": -5.864701747894287,
      "step": 1090
    },
    {
      "epoch": 1.752,
      "grad_norm": 3.6696534156799316,
      "learning_rate": 1.9515224445023553e-06,
      "logits/chosen": -1.5249470472335815,
      "logits/rejected": -1.4677118062973022,
      "logps/chosen": -226.087646484375,
      "logps/rejected": -301.89190673828125,
      "loss": 0.1377,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.3383384346961975,
      "rewards/margins": 5.166504859924316,
      "rewards/rejected": -5.504843711853027,
      "step": 1095
    },
    {
      "epoch": 1.76,
      "grad_norm": 3.5852746963500977,
      "learning_rate": 1.9304688253928398e-06,
      "logits/chosen": -1.533934235572815,
      "logits/rejected": -1.5655229091644287,
      "logps/chosen": -235.65298461914062,
      "logps/rejected": -301.34503173828125,
      "loss": 0.1833,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.8953931927680969,
      "rewards/margins": 5.259232044219971,
      "rewards/rejected": -6.154624938964844,
      "step": 1100
    },
    {
      "epoch": 1.768,
      "grad_norm": 5.908403396606445,
      "learning_rate": 1.909457723692794e-06,
      "logits/chosen": -1.5542901754379272,
      "logits/rejected": -1.5842716693878174,
      "logps/chosen": -244.9988250732422,
      "logps/rejected": -300.6830139160156,
      "loss": 0.2367,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.088773488998413,
      "rewards/margins": 5.178781032562256,
      "rewards/rejected": -6.267554759979248,
      "step": 1105
    },
    {
      "epoch": 1.776,
      "grad_norm": 4.3570661544799805,
      "learning_rate": 1.8884907079513227e-06,
      "logits/chosen": -1.5214910507202148,
      "logits/rejected": -1.4828593730926514,
      "logps/chosen": -231.094970703125,
      "logps/rejected": -313.3832702636719,
      "loss": 0.1663,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.30087193846702576,
      "rewards/margins": 5.7574782371521,
      "rewards/rejected": -6.058350086212158,
      "step": 1110
    },
    {
      "epoch": 1.784,
      "grad_norm": 3.1484177112579346,
      "learning_rate": 1.8675693434263652e-06,
      "logits/chosen": -1.5721497535705566,
      "logits/rejected": -1.4944632053375244,
      "logps/chosen": -221.1279296875,
      "logps/rejected": -300.9421081542969,
      "loss": 0.1515,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.32752472162246704,
      "rewards/margins": 5.808013439178467,
      "rewards/rejected": -6.135538101196289,
      "step": 1115
    },
    {
      "epoch": 1.792,
      "grad_norm": 3.595839500427246,
      "learning_rate": 1.8466951919678491e-06,
      "logits/chosen": -1.59939444065094,
      "logits/rejected": -1.5375818014144897,
      "logps/chosen": -230.9847412109375,
      "logps/rejected": -309.09027099609375,
      "loss": 0.1364,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.29122036695480347,
      "rewards/margins": 5.625614643096924,
      "rewards/rejected": -5.916835308074951,
      "step": 1120
    },
    {
      "epoch": 1.8,
      "grad_norm": 3.8204920291900635,
      "learning_rate": 1.8258698119010832e-06,
      "logits/chosen": -1.5291309356689453,
      "logits/rejected": -1.4495049715042114,
      "logps/chosen": -232.0759735107422,
      "logps/rejected": -281.865478515625,
      "loss": 0.1622,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.2713313102722168,
      "rewards/margins": 4.578837871551514,
      "rewards/rejected": -4.850168704986572,
      "step": 1125
    },
    {
      "epoch": 1.808,
      "grad_norm": 2.033203363418579,
      "learning_rate": 1.8050947579104325e-06,
      "logits/chosen": -1.5407826900482178,
      "logits/rejected": -1.5142061710357666,
      "logps/chosen": -227.7032470703125,
      "logps/rejected": -311.5636291503906,
      "loss": 0.1278,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.25752535462379456,
      "rewards/margins": 5.367326736450195,
      "rewards/rejected": -5.624852180480957,
      "step": 1130
    },
    {
      "epoch": 1.8159999999999998,
      "grad_norm": 4.951417922973633,
      "learning_rate": 1.7843715809232486e-06,
      "logits/chosen": -1.4634641408920288,
      "logits/rejected": -1.4312846660614014,
      "logps/chosen": -229.4953155517578,
      "logps/rejected": -293.6320495605469,
      "loss": 0.2149,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.5195713639259338,
      "rewards/margins": 4.903964519500732,
      "rewards/rejected": -5.4235358238220215,
      "step": 1135
    },
    {
      "epoch": 1.8239999999999998,
      "grad_norm": 2.566675901412964,
      "learning_rate": 1.763701827994095e-06,
      "logits/chosen": -1.4973647594451904,
      "logits/rejected": -1.424039602279663,
      "logps/chosen": -219.69979858398438,
      "logps/rejected": -298.1719665527344,
      "loss": 0.1902,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.3790625035762787,
      "rewards/margins": 5.123688697814941,
      "rewards/rejected": -5.50275182723999,
      "step": 1140
    },
    {
      "epoch": 1.8319999999999999,
      "grad_norm": 5.082289218902588,
      "learning_rate": 1.7430870421892459e-06,
      "logits/chosen": -1.5290133953094482,
      "logits/rejected": -1.4534924030303955,
      "logps/chosen": -231.0949249267578,
      "logps/rejected": -288.7142639160156,
      "loss": 0.1439,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.3327312469482422,
      "rewards/margins": 5.081864356994629,
      "rewards/rejected": -5.414595603942871,
      "step": 1145
    },
    {
      "epoch": 1.8399999999999999,
      "grad_norm": 2.2908127307891846,
      "learning_rate": 1.7225287624714987e-06,
      "logits/chosen": -1.557173728942871,
      "logits/rejected": -1.451888918876648,
      "logps/chosen": -237.3270263671875,
      "logps/rejected": -302.59124755859375,
      "loss": 0.1289,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.10339298099279404,
      "rewards/margins": 5.4684553146362305,
      "rewards/rejected": -5.571847915649414,
      "step": 1150
    },
    {
      "epoch": 1.8479999999999999,
      "grad_norm": 3.687828540802002,
      "learning_rate": 1.7020285235852824e-06,
      "logits/chosen": -1.483363151550293,
      "logits/rejected": -1.4083940982818604,
      "logps/chosen": -229.43991088867188,
      "logps/rejected": -289.58795166015625,
      "loss": 0.1625,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.04203609377145767,
      "rewards/margins": 5.190605640411377,
      "rewards/rejected": -5.23264217376709,
      "step": 1155
    },
    {
      "epoch": 1.8559999999999999,
      "grad_norm": 3.409076690673828,
      "learning_rate": 1.6815878559420806e-06,
      "logits/chosen": -1.637976050376892,
      "logits/rejected": -1.5942111015319824,
      "logps/chosen": -233.52560424804688,
      "logps/rejected": -307.1108093261719,
      "loss": 0.1353,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.40837258100509644,
      "rewards/margins": 4.940521240234375,
      "rewards/rejected": -5.348894119262695,
      "step": 1160
    },
    {
      "epoch": 1.8639999999999999,
      "grad_norm": 3.731051206588745,
      "learning_rate": 1.661208285506188e-06,
      "logits/chosen": -1.5421288013458252,
      "logits/rejected": -1.4871435165405273,
      "logps/chosen": -214.5398712158203,
      "logps/rejected": -283.8194885253906,
      "loss": 0.2104,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.5290651321411133,
      "rewards/margins": 4.5201005935668945,
      "rewards/rejected": -5.04916524887085,
      "step": 1165
    },
    {
      "epoch": 1.8719999999999999,
      "grad_norm": 4.392094612121582,
      "learning_rate": 1.6408913336807835e-06,
      "logits/chosen": -1.7092958688735962,
      "logits/rejected": -1.677384376525879,
      "logps/chosen": -239.9275665283203,
      "logps/rejected": -326.7081604003906,
      "loss": 0.201,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.9028335809707642,
      "rewards/margins": 5.153952598571777,
      "rewards/rejected": -6.05678653717041,
      "step": 1170
    },
    {
      "epoch": 1.88,
      "grad_norm": 3.3208742141723633,
      "learning_rate": 1.6206385171943628e-06,
      "logits/chosen": -1.5975421667099,
      "logits/rejected": -1.5135239362716675,
      "logps/chosen": -221.99978637695312,
      "logps/rejected": -312.03778076171875,
      "loss": 0.1347,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.614351212978363,
      "rewards/margins": 5.365405559539795,
      "rewards/rejected": -5.979756832122803,
      "step": 1175
    },
    {
      "epoch": 1.888,
      "grad_norm": 4.675105571746826,
      "learning_rate": 1.6004513479874975e-06,
      "logits/chosen": -1.5107734203338623,
      "logits/rejected": -1.4795351028442383,
      "logps/chosen": -221.3353729248047,
      "logps/rejected": -302.6809997558594,
      "loss": 0.1506,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.6793597936630249,
      "rewards/margins": 5.78696870803833,
      "rewards/rejected": -6.466329097747803,
      "step": 1180
    },
    {
      "epoch": 1.896,
      "grad_norm": 3.0278146266937256,
      "learning_rate": 1.580331333099976e-06,
      "logits/chosen": -1.589797854423523,
      "logits/rejected": -1.5239031314849854,
      "logps/chosen": -228.4359893798828,
      "logps/rejected": -315.303955078125,
      "loss": 0.1711,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.9097194671630859,
      "rewards/margins": 5.420597076416016,
      "rewards/rejected": -6.330316543579102,
      "step": 1185
    },
    {
      "epoch": 1.904,
      "grad_norm": 5.7446513175964355,
      "learning_rate": 1.5602799745582859e-06,
      "logits/chosen": -1.5522780418395996,
      "logits/rejected": -1.491706132888794,
      "logps/chosen": -253.501708984375,
      "logps/rejected": -303.25872802734375,
      "loss": 0.1788,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.768315315246582,
      "rewards/margins": 5.147302150726318,
      "rewards/rejected": -5.915616989135742,
      "step": 1190
    },
    {
      "epoch": 1.912,
      "grad_norm": 2.9774773120880127,
      "learning_rate": 1.5402987692634941e-06,
      "logits/chosen": -1.564814567565918,
      "logits/rejected": -1.5448888540267944,
      "logps/chosen": -217.634033203125,
      "logps/rejected": -321.3653869628906,
      "loss": 0.1247,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.9152569770812988,
      "rewards/margins": 5.907143592834473,
      "rewards/rejected": -6.822400093078613,
      "step": 1195
    },
    {
      "epoch": 1.92,
      "grad_norm": 3.8479833602905273,
      "learning_rate": 1.5203892088794888e-06,
      "logits/chosen": -1.5153319835662842,
      "logits/rejected": -1.5472588539123535,
      "logps/chosen": -241.7740020751953,
      "logps/rejected": -319.2223205566406,
      "loss": 0.1756,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.827950119972229,
      "rewards/margins": 5.2503790855407715,
      "rewards/rejected": -6.078328609466553,
      "step": 1200
    },
    {
      "epoch": 1.928,
      "grad_norm": 7.263859748840332,
      "learning_rate": 1.5005527797216284e-06,
      "logits/chosen": -1.5309240818023682,
      "logits/rejected": -1.5484015941619873,
      "logps/chosen": -245.1676483154297,
      "logps/rejected": -309.8975524902344,
      "loss": 0.206,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.322698175907135,
      "rewards/margins": 5.289670944213867,
      "rewards/rejected": -5.612368583679199,
      "step": 1205
    },
    {
      "epoch": 1.936,
      "grad_norm": 4.652385711669922,
      "learning_rate": 1.4807909626457785e-06,
      "logits/chosen": -1.603051781654358,
      "logits/rejected": -1.4821627140045166,
      "logps/chosen": -224.2364044189453,
      "logps/rejected": -321.984375,
      "loss": 0.172,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.7000328302383423,
      "rewards/margins": 5.369967937469482,
      "rewards/rejected": -6.070000648498535,
      "step": 1210
    },
    {
      "epoch": 1.944,
      "grad_norm": 2.9950015544891357,
      "learning_rate": 1.4611052329377629e-06,
      "logits/chosen": -1.6220756769180298,
      "logits/rejected": -1.5894696712493896,
      "logps/chosen": -225.9698486328125,
      "logps/rejected": -324.79254150390625,
      "loss": 0.1683,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.7203431129455566,
      "rewards/margins": 5.393028736114502,
      "rewards/rejected": -6.1133713722229,
      "step": 1215
    },
    {
      "epoch": 1.952,
      "grad_norm": 2.781445026397705,
      "learning_rate": 1.4414970602032301e-06,
      "logits/chosen": -1.4824979305267334,
      "logits/rejected": -1.455690622329712,
      "logps/chosen": -222.3482666015625,
      "logps/rejected": -307.96856689453125,
      "loss": 0.1571,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.7416940927505493,
      "rewards/margins": 5.335463523864746,
      "rewards/rejected": -6.077157974243164,
      "step": 1220
    },
    {
      "epoch": 1.96,
      "grad_norm": 4.076323986053467,
      "learning_rate": 1.4219679082579384e-06,
      "logits/chosen": -1.558056354522705,
      "logits/rejected": -1.5473124980926514,
      "logps/chosen": -253.4956512451172,
      "logps/rejected": -309.33221435546875,
      "loss": 0.187,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.9098089337348938,
      "rewards/margins": 5.028618812561035,
      "rewards/rejected": -5.938427925109863,
      "step": 1225
    },
    {
      "epoch": 1.968,
      "grad_norm": 4.632643699645996,
      "learning_rate": 1.4025192350184847e-06,
      "logits/chosen": -1.5654726028442383,
      "logits/rejected": -1.52810800075531,
      "logps/chosen": -239.4210968017578,
      "logps/rejected": -316.1085510253906,
      "loss": 0.1592,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7853416204452515,
      "rewards/margins": 5.446610450744629,
      "rewards/rejected": -6.231952667236328,
      "step": 1230
    },
    {
      "epoch": 1.976,
      "grad_norm": 2.3505971431732178,
      "learning_rate": 1.3831524923934525e-06,
      "logits/chosen": -1.4985238313674927,
      "logits/rejected": -1.4004181623458862,
      "logps/chosen": -241.896240234375,
      "logps/rejected": -305.42242431640625,
      "loss": 0.0944,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.4821149706840515,
      "rewards/margins": 5.794647216796875,
      "rewards/rejected": -6.27676248550415,
      "step": 1235
    },
    {
      "epoch": 1.984,
      "grad_norm": 4.241216659545898,
      "learning_rate": 1.3638691261750402e-06,
      "logits/chosen": -1.5663890838623047,
      "logits/rejected": -1.4924695491790771,
      "logps/chosen": -228.1782989501953,
      "logps/rejected": -331.6749572753906,
      "loss": 0.1288,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.5289267897605896,
      "rewards/margins": 5.913106441497803,
      "rewards/rejected": -6.442032814025879,
      "step": 1240
    },
    {
      "epoch": 1.992,
      "grad_norm": 4.635298728942871,
      "learning_rate": 1.3446705759311087e-06,
      "logits/chosen": -1.6584011316299438,
      "logits/rejected": -1.5797560214996338,
      "logps/chosen": -245.9219207763672,
      "logps/rejected": -326.98760986328125,
      "loss": 0.1045,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.826259970664978,
      "rewards/margins": 5.950412273406982,
      "rewards/rejected": -6.77667236328125,
      "step": 1245
    },
    {
      "epoch": 2.0,
      "grad_norm": 2.418168783187866,
      "learning_rate": 1.325558274897732e-06,
      "logits/chosen": -1.4989831447601318,
      "logits/rejected": -1.4638856649398804,
      "logps/chosen": -236.1473846435547,
      "logps/rejected": -303.3345031738281,
      "loss": 0.1753,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.45226964354515076,
      "rewards/margins": 5.467932224273682,
      "rewards/rejected": -5.920201778411865,
      "step": 1250
    },
    {
      "epoch": 2.008,
      "grad_norm": 2.998203754425049,
      "learning_rate": 1.306533649872182e-06,
      "logits/chosen": -1.517118215560913,
      "logits/rejected": -1.4960505962371826,
      "logps/chosen": -224.80142211914062,
      "logps/rejected": -304.97857666015625,
      "loss": 0.1276,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.8061330914497375,
      "rewards/margins": 5.22979736328125,
      "rewards/rejected": -6.0359296798706055,
      "step": 1255
    },
    {
      "epoch": 2.016,
      "grad_norm": 3.6085073947906494,
      "learning_rate": 1.287598121106432e-06,
      "logits/chosen": -1.4973828792572021,
      "logits/rejected": -1.430394172668457,
      "logps/chosen": -224.1341094970703,
      "logps/rejected": -306.37847900390625,
      "loss": 0.1382,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.8516316413879395,
      "rewards/margins": 5.601286888122559,
      "rewards/rejected": -6.45291805267334,
      "step": 1260
    },
    {
      "epoch": 2.024,
      "grad_norm": 5.166800498962402,
      "learning_rate": 1.2687531022011146e-06,
      "logits/chosen": -1.5529485940933228,
      "logits/rejected": -1.5082321166992188,
      "logps/chosen": -231.6868438720703,
      "logps/rejected": -289.4835205078125,
      "loss": 0.2142,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.7622639536857605,
      "rewards/margins": 5.081564903259277,
      "rewards/rejected": -5.843829154968262,
      "step": 1265
    },
    {
      "epoch": 2.032,
      "grad_norm": 2.8838765621185303,
      "learning_rate": 1.2500000000000007e-06,
      "logits/chosen": -1.5373793840408325,
      "logits/rejected": -1.491275668144226,
      "logps/chosen": -218.83438110351562,
      "logps/rejected": -298.385986328125,
      "loss": 0.1204,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.4552576541900635,
      "rewards/margins": 5.728938102722168,
      "rewards/rejected": -6.184195518493652,
      "step": 1270
    },
    {
      "epoch": 2.04,
      "grad_norm": 3.6019749641418457,
      "learning_rate": 1.2313402144849714e-06,
      "logits/chosen": -1.6223344802856445,
      "logits/rejected": -1.564382791519165,
      "logps/chosen": -247.25927734375,
      "logps/rejected": -321.1236877441406,
      "loss": 0.1288,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.5351258516311646,
      "rewards/margins": 5.776213645935059,
      "rewards/rejected": -6.311339378356934,
      "step": 1275
    },
    {
      "epoch": 2.048,
      "grad_norm": 2.6691808700561523,
      "learning_rate": 1.2127751386715054e-06,
      "logits/chosen": -1.5619357824325562,
      "logits/rejected": -1.4246995449066162,
      "logps/chosen": -250.17330932617188,
      "logps/rejected": -289.0823669433594,
      "loss": 0.1265,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.5506324768066406,
      "rewards/margins": 5.429201126098633,
      "rewards/rejected": -5.979834079742432,
      "step": 1280
    },
    {
      "epoch": 2.056,
      "grad_norm": 4.023916244506836,
      "learning_rate": 1.194306158504683e-06,
      "logits/chosen": -1.542194128036499,
      "logits/rejected": -1.5818068981170654,
      "logps/chosen": -224.98544311523438,
      "logps/rejected": -313.40020751953125,
      "loss": 0.1482,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.6270204782485962,
      "rewards/margins": 5.459203243255615,
      "rewards/rejected": -6.08622407913208,
      "step": 1285
    },
    {
      "epoch": 2.064,
      "grad_norm": 3.2508156299591064,
      "learning_rate": 1.1759346527557247e-06,
      "logits/chosen": -1.5900170803070068,
      "logits/rejected": -1.5209767818450928,
      "logps/chosen": -239.06289672851562,
      "logps/rejected": -296.8851623535156,
      "loss": 0.1824,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.7839146852493286,
      "rewards/margins": 5.1801862716674805,
      "rewards/rejected": -5.964101314544678,
      "step": 1290
    },
    {
      "epoch": 2.072,
      "grad_norm": 3.451742172241211,
      "learning_rate": 1.157661992919058e-06,
      "logits/chosen": -1.497745156288147,
      "logits/rejected": -1.4466352462768555,
      "logps/chosen": -246.4713592529297,
      "logps/rejected": -291.977294921875,
      "loss": 0.1708,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.40274477005004883,
      "rewards/margins": 5.257174491882324,
      "rewards/rejected": -5.659919261932373,
      "step": 1295
    },
    {
      "epoch": 2.08,
      "grad_norm": 3.3363468647003174,
      "learning_rate": 1.1394895431099315e-06,
      "logits/chosen": -1.462850570678711,
      "logits/rejected": -1.4387768507003784,
      "logps/chosen": -214.2821807861328,
      "logps/rejected": -312.31256103515625,
      "loss": 0.0987,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.3282471299171448,
      "rewards/margins": 6.343912601470947,
      "rewards/rejected": -6.672159671783447,
      "step": 1300
    },
    {
      "epoch": 2.088,
      "grad_norm": 2.328258514404297,
      "learning_rate": 1.1214186599625835e-06,
      "logits/chosen": -1.402923583984375,
      "logits/rejected": -1.4672176837921143,
      "logps/chosen": -240.6547393798828,
      "logps/rejected": -320.3161926269531,
      "loss": 0.1345,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.8091422319412231,
      "rewards/margins": 5.365943908691406,
      "rewards/rejected": -6.17508602142334,
      "step": 1305
    },
    {
      "epoch": 2.096,
      "grad_norm": 3.9099833965301514,
      "learning_rate": 1.103450692528954e-06,
      "logits/chosen": -1.4001861810684204,
      "logits/rejected": -1.4040039777755737,
      "logps/chosen": -253.8073272705078,
      "logps/rejected": -298.1487121582031,
      "loss": 0.1378,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.8334349393844604,
      "rewards/margins": 5.020366191864014,
      "rewards/rejected": -5.8538007736206055,
      "step": 1310
    },
    {
      "epoch": 2.104,
      "grad_norm": 4.8871541023254395,
      "learning_rate": 1.0855869821779883e-06,
      "logits/chosen": -1.5196176767349243,
      "logits/rejected": -1.50358247756958,
      "logps/chosen": -228.2346649169922,
      "logps/rejected": -301.0938415527344,
      "loss": 0.1661,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.9291476011276245,
      "rewards/margins": 5.45242977142334,
      "rewards/rejected": -6.381577968597412,
      "step": 1315
    },
    {
      "epoch": 2.112,
      "grad_norm": 4.641075134277344,
      "learning_rate": 1.0678288624954836e-06,
      "logits/chosen": -1.5506222248077393,
      "logits/rejected": -1.538140892982483,
      "logps/chosen": -275.78460693359375,
      "logps/rejected": -298.97076416015625,
      "loss": 0.1659,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.011925202794373035,
      "rewards/margins": 5.404875755310059,
      "rewards/rejected": -5.392950057983398,
      "step": 1320
    },
    {
      "epoch": 2.12,
      "grad_norm": 3.5139901638031006,
      "learning_rate": 1.0501776591845472e-06,
      "logits/chosen": -1.521786093711853,
      "logits/rejected": -1.509730577468872,
      "logps/chosen": -223.95529174804688,
      "logps/rejected": -306.37933349609375,
      "loss": 0.1303,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5522927641868591,
      "rewards/margins": 5.402201175689697,
      "rewards/rejected": -5.954493522644043,
      "step": 1325
    },
    {
      "epoch": 2.128,
      "grad_norm": 3.935023069381714,
      "learning_rate": 1.0326346899666143e-06,
      "logits/chosen": -1.5349467992782593,
      "logits/rejected": -1.4667823314666748,
      "logps/chosen": -227.30422973632812,
      "logps/rejected": -298.933349609375,
      "loss": 0.1265,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.4221525192260742,
      "rewards/margins": 5.432391166687012,
      "rewards/rejected": -5.854543209075928,
      "step": 1330
    },
    {
      "epoch": 2.136,
      "grad_norm": 3.250990152359009,
      "learning_rate": 1.0152012644830897e-06,
      "logits/chosen": -1.5740466117858887,
      "logits/rejected": -1.6187541484832764,
      "logps/chosen": -225.9763641357422,
      "logps/rejected": -285.09185791015625,
      "loss": 0.1386,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.5500262379646301,
      "rewards/margins": 5.309136867523193,
      "rewards/rejected": -5.859163284301758,
      "step": 1335
    },
    {
      "epoch": 2.144,
      "grad_norm": 3.963165521621704,
      "learning_rate": 9.978786841975647e-07,
      "logits/chosen": -1.5362905263900757,
      "logits/rejected": -1.494563341140747,
      "logps/chosen": -238.83505249023438,
      "logps/rejected": -307.7918701171875,
      "loss": 0.1578,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.7797427177429199,
      "rewards/margins": 5.327605247497559,
      "rewards/rejected": -6.107347011566162,
      "step": 1340
    },
    {
      "epoch": 2.152,
      "grad_norm": 4.277246952056885,
      "learning_rate": 9.806682422986716e-07,
      "logits/chosen": -1.4467289447784424,
      "logits/rejected": -1.4939007759094238,
      "logps/chosen": -235.6219024658203,
      "logps/rejected": -302.406982421875,
      "loss": 0.1333,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5323159098625183,
      "rewards/margins": 5.708221912384033,
      "rewards/rejected": -6.240537166595459,
      "step": 1345
    },
    {
      "epoch": 2.16,
      "grad_norm": 3.088682174682617,
      "learning_rate": 9.635712236035305e-07,
      "logits/chosen": -1.4703865051269531,
      "logits/rejected": -1.4268938302993774,
      "logps/chosen": -239.4414825439453,
      "logps/rejected": -310.94677734375,
      "loss": 0.1358,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.5711382031440735,
      "rewards/margins": 5.83718729019165,
      "rewards/rejected": -6.408326148986816,
      "step": 1350
    },
    {
      "epoch": 2.168,
      "grad_norm": 2.9433059692382812,
      "learning_rate": 9.465889044618463e-07,
      "logits/chosen": -1.531912922859192,
      "logits/rejected": -1.5826823711395264,
      "logps/chosen": -226.4390869140625,
      "logps/rejected": -294.73016357421875,
      "loss": 0.158,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0183833837509155,
      "rewards/margins": 4.735301971435547,
      "rewards/rejected": -5.753684997558594,
      "step": 1355
    },
    {
      "epoch": 2.176,
      "grad_norm": 3.169440507888794,
      "learning_rate": 9.297225526606105e-07,
      "logits/chosen": -1.4751956462860107,
      "logits/rejected": -1.4470738172531128,
      "logps/chosen": -252.1983642578125,
      "logps/rejected": -320.13812255859375,
      "loss": 0.1171,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.7644977569580078,
      "rewards/margins": 5.812073707580566,
      "rewards/rejected": -6.576571464538574,
      "step": 1360
    },
    {
      "epoch": 2.184,
      "grad_norm": 4.91829252243042,
      "learning_rate": 9.129734273294718e-07,
      "logits/chosen": -1.552290439605713,
      "logits/rejected": -1.4868769645690918,
      "logps/chosen": -230.2556610107422,
      "logps/rejected": -291.697998046875,
      "loss": 0.1257,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.046489953994751,
      "rewards/margins": 5.263110160827637,
      "rewards/rejected": -6.309600830078125,
      "step": 1365
    },
    {
      "epoch": 2.192,
      "grad_norm": 2.0276551246643066,
      "learning_rate": 8.963427788467241e-07,
      "logits/chosen": -1.545638084411621,
      "logits/rejected": -1.466631531715393,
      "logps/chosen": -244.7844696044922,
      "logps/rejected": -315.06695556640625,
      "loss": 0.1714,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.9884885549545288,
      "rewards/margins": 5.733240604400635,
      "rewards/rejected": -6.721728324890137,
      "step": 1370
    },
    {
      "epoch": 2.2,
      "grad_norm": 5.268774509429932,
      "learning_rate": 8.79831848745972e-07,
      "logits/chosen": -1.597069501876831,
      "logits/rejected": -1.5245897769927979,
      "logps/chosen": -248.6392059326172,
      "logps/rejected": -304.13348388671875,
      "loss": 0.1752,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.6362749338150024,
      "rewards/margins": 5.747987270355225,
      "rewards/rejected": -6.384262561798096,
      "step": 1375
    },
    {
      "epoch": 2.208,
      "grad_norm": 4.227468013763428,
      "learning_rate": 8.634418696234403e-07,
      "logits/chosen": -1.6100578308105469,
      "logits/rejected": -1.5145816802978516,
      "logps/chosen": -260.8665771484375,
      "logps/rejected": -310.1653747558594,
      "loss": 0.1859,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.0051674842834473,
      "rewards/margins": 5.3024678230285645,
      "rewards/rejected": -6.307635307312012,
      "step": 1380
    },
    {
      "epoch": 2.216,
      "grad_norm": 3.3430335521698,
      "learning_rate": 8.471740650459582e-07,
      "logits/chosen": -1.4245984554290771,
      "logits/rejected": -1.4522405862808228,
      "logps/chosen": -240.0626983642578,
      "logps/rejected": -317.7297058105469,
      "loss": 0.1313,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.5361242294311523,
      "rewards/margins": 5.816006660461426,
      "rewards/rejected": -6.352130889892578,
      "step": 1385
    },
    {
      "epoch": 2.224,
      "grad_norm": 3.8488659858703613,
      "learning_rate": 8.310296494596159e-07,
      "logits/chosen": -1.4325063228607178,
      "logits/rejected": -1.4604406356811523,
      "logps/chosen": -232.338134765625,
      "logps/rejected": -300.801025390625,
      "loss": 0.1457,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.9337561726570129,
      "rewards/margins": 5.740461349487305,
      "rewards/rejected": -6.674217224121094,
      "step": 1390
    },
    {
      "epoch": 2.232,
      "grad_norm": 3.3606622219085693,
      "learning_rate": 8.150098280991029e-07,
      "logits/chosen": -1.61603581905365,
      "logits/rejected": -1.4698944091796875,
      "logps/chosen": -201.53733825683594,
      "logps/rejected": -290.5418395996094,
      "loss": 0.1196,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.5469184517860413,
      "rewards/margins": 5.2674431800842285,
      "rewards/rejected": -5.814361572265625,
      "step": 1395
    },
    {
      "epoch": 2.24,
      "grad_norm": 4.717289447784424,
      "learning_rate": 7.991157968977322e-07,
      "logits/chosen": -1.5912104845046997,
      "logits/rejected": -1.5626710653305054,
      "logps/chosen": -228.1504669189453,
      "logps/rejected": -326.4696350097656,
      "loss": 0.1317,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.556774914264679,
      "rewards/margins": 6.516512393951416,
      "rewards/rejected": -7.0732879638671875,
      "step": 1400
    },
    {
      "epoch": 2.248,
      "grad_norm": 2.843961477279663,
      "learning_rate": 7.833487423981581e-07,
      "logits/chosen": -1.484765887260437,
      "logits/rejected": -1.4198788404464722,
      "logps/chosen": -228.2584991455078,
      "logps/rejected": -315.5864562988281,
      "loss": 0.1104,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.5225152969360352,
      "rewards/margins": 6.307417869567871,
      "rewards/rejected": -6.829933166503906,
      "step": 1405
    },
    {
      "epoch": 2.2560000000000002,
      "grad_norm": 5.61122465133667,
      "learning_rate": 7.677098416638037e-07,
      "logits/chosen": -1.5627809762954712,
      "logits/rejected": -1.561023235321045,
      "logps/chosen": -237.8609619140625,
      "logps/rejected": -313.85479736328125,
      "loss": 0.1684,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.603573203086853,
      "rewards/margins": 5.463527202606201,
      "rewards/rejected": -6.067100524902344,
      "step": 1410
    },
    {
      "epoch": 2.2640000000000002,
      "grad_norm": 3.832843542098999,
      "learning_rate": 7.522002621909785e-07,
      "logits/chosen": -1.5515294075012207,
      "logits/rejected": -1.5249848365783691,
      "logps/chosen": -210.19668579101562,
      "logps/rejected": -317.81121826171875,
      "loss": 0.1313,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.8036311864852905,
      "rewards/margins": 5.415702819824219,
      "rewards/rejected": -6.219333648681641,
      "step": 1415
    },
    {
      "epoch": 2.2720000000000002,
      "grad_norm": 3.504119396209717,
      "learning_rate": 7.368211618217325e-07,
      "logits/chosen": -1.482468843460083,
      "logits/rejected": -1.4689217805862427,
      "logps/chosen": -237.25418090820312,
      "logps/rejected": -307.9796447753906,
      "loss": 0.1392,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.6051761507987976,
      "rewards/margins": 5.398853302001953,
      "rewards/rejected": -6.004029273986816,
      "step": 1420
    },
    {
      "epoch": 2.2800000000000002,
      "grad_norm": 4.0742316246032715,
      "learning_rate": 7.21573688657408e-07,
      "logits/chosen": -1.5382550954818726,
      "logits/rejected": -1.5822103023529053,
      "logps/chosen": -234.6659698486328,
      "logps/rejected": -320.8883972167969,
      "loss": 0.1264,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.8635808825492859,
      "rewards/margins": 5.727967262268066,
      "rewards/rejected": -6.591547966003418,
      "step": 1425
    },
    {
      "epoch": 2.288,
      "grad_norm": 3.2900421619415283,
      "learning_rate": 7.064589809729416e-07,
      "logits/chosen": -1.6166538000106812,
      "logits/rejected": -1.5299408435821533,
      "logps/chosen": -240.4681396484375,
      "logps/rejected": -300.2424011230469,
      "loss": 0.1137,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.8580220937728882,
      "rewards/margins": 5.814321041107178,
      "rewards/rejected": -6.6723432540893555,
      "step": 1430
    },
    {
      "epoch": 2.296,
      "grad_norm": 4.590370178222656,
      "learning_rate": 6.914781671318757e-07,
      "logits/chosen": -1.5603392124176025,
      "logits/rejected": -1.5085687637329102,
      "logps/chosen": -240.8662872314453,
      "logps/rejected": -297.9051208496094,
      "loss": 0.1545,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.9626073837280273,
      "rewards/margins": 5.510343551635742,
      "rewards/rejected": -6.4729509353637695,
      "step": 1435
    },
    {
      "epoch": 2.304,
      "grad_norm": 1.9299755096435547,
      "learning_rate": 6.766323655021356e-07,
      "logits/chosen": -1.4613937139511108,
      "logits/rejected": -1.464815616607666,
      "logps/chosen": -219.6249542236328,
      "logps/rejected": -291.37982177734375,
      "loss": 0.144,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.7187808752059937,
      "rewards/margins": 6.116919040679932,
      "rewards/rejected": -6.835699558258057,
      "step": 1440
    },
    {
      "epoch": 2.312,
      "grad_norm": 1.7471683025360107,
      "learning_rate": 6.619226843725266e-07,
      "logits/chosen": -1.518546462059021,
      "logits/rejected": -1.4668691158294678,
      "logps/chosen": -213.170166015625,
      "logps/rejected": -317.2686462402344,
      "loss": 0.1152,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.5413722395896912,
      "rewards/margins": 6.646666526794434,
      "rewards/rejected": -7.188039302825928,
      "step": 1445
    },
    {
      "epoch": 2.32,
      "grad_norm": 4.943786144256592,
      "learning_rate": 6.473502218700081e-07,
      "logits/chosen": -1.5848660469055176,
      "logits/rejected": -1.5765035152435303,
      "logps/chosen": -231.70864868164062,
      "logps/rejected": -317.9561767578125,
      "loss": 0.1248,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.9403325319290161,
      "rewards/margins": 5.609653949737549,
      "rewards/rejected": -6.549986362457275,
      "step": 1450
    },
    {
      "epoch": 2.328,
      "grad_norm": 6.270254611968994,
      "learning_rate": 6.329160658777062e-07,
      "logits/chosen": -1.6108720302581787,
      "logits/rejected": -1.5802674293518066,
      "logps/chosen": -233.30233764648438,
      "logps/rejected": -310.03814697265625,
      "loss": 0.1612,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.1865160465240479,
      "rewards/margins": 6.1157379150390625,
      "rewards/rejected": -7.302254676818848,
      "step": 1455
    },
    {
      "epoch": 2.336,
      "grad_norm": 5.8520379066467285,
      "learning_rate": 6.18621293953704e-07,
      "logits/chosen": -1.5015791654586792,
      "logits/rejected": -1.458094835281372,
      "logps/chosen": -245.0795135498047,
      "logps/rejected": -312.7300109863281,
      "loss": 0.1316,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.6283895373344421,
      "rewards/margins": 5.754814147949219,
      "rewards/rejected": -6.383203983306885,
      "step": 1460
    },
    {
      "epoch": 2.344,
      "grad_norm": 4.313529014587402,
      "learning_rate": 6.044669732505968e-07,
      "logits/chosen": -1.4273951053619385,
      "logits/rejected": -1.4270398616790771,
      "logps/chosen": -232.2421417236328,
      "logps/rejected": -308.38275146484375,
      "loss": 0.157,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.1415029764175415,
      "rewards/margins": 5.1364593505859375,
      "rewards/rejected": -6.277962684631348,
      "step": 1465
    },
    {
      "epoch": 2.352,
      "grad_norm": 3.829115629196167,
      "learning_rate": 5.904541604358257e-07,
      "logits/chosen": -1.5568525791168213,
      "logits/rejected": -1.4775985479354858,
      "logps/chosen": -231.17343139648438,
      "logps/rejected": -319.28338623046875,
      "loss": 0.1103,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.6411401033401489,
      "rewards/margins": 6.239230632781982,
      "rewards/rejected": -6.880370140075684,
      "step": 1470
    },
    {
      "epoch": 2.36,
      "grad_norm": 3.9916186332702637,
      "learning_rate": 5.765839016127953e-07,
      "logits/chosen": -1.601308822631836,
      "logits/rejected": -1.556640386581421,
      "logps/chosen": -247.08151245117188,
      "logps/rejected": -326.2941589355469,
      "loss": 0.0974,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.1435987949371338,
      "rewards/margins": 6.010471343994141,
      "rewards/rejected": -7.1540703773498535,
      "step": 1475
    },
    {
      "epoch": 2.368,
      "grad_norm": 4.295379638671875,
      "learning_rate": 5.628572322427755e-07,
      "logits/chosen": -1.4031133651733398,
      "logits/rejected": -1.4401230812072754,
      "logps/chosen": -246.1109161376953,
      "logps/rejected": -311.51519775390625,
      "loss": 0.1096,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.9082221984863281,
      "rewards/margins": 6.309326648712158,
      "rewards/rejected": -7.217548370361328,
      "step": 1480
    },
    {
      "epoch": 2.376,
      "grad_norm": 5.539330005645752,
      "learning_rate": 5.492751770676042e-07,
      "logits/chosen": -1.4508850574493408,
      "logits/rejected": -1.484483003616333,
      "logps/chosen": -248.3240966796875,
      "logps/rejected": -311.86114501953125,
      "loss": 0.1696,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.7610406875610352,
      "rewards/margins": 5.5662055015563965,
      "rewards/rejected": -6.327246189117432,
      "step": 1485
    },
    {
      "epoch": 2.384,
      "grad_norm": 2.899242639541626,
      "learning_rate": 5.358387500331844e-07,
      "logits/chosen": -1.550159215927124,
      "logits/rejected": -1.5097694396972656,
      "logps/chosen": -220.2747039794922,
      "logps/rejected": -304.1933288574219,
      "loss": 0.1453,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.11124587059021,
      "rewards/margins": 5.5194854736328125,
      "rewards/rejected": -6.630731105804443,
      "step": 1490
    },
    {
      "epoch": 2.392,
      "grad_norm": 4.426836967468262,
      "learning_rate": 5.22548954213791e-07,
      "logits/chosen": -1.5275931358337402,
      "logits/rejected": -1.5510616302490234,
      "logps/chosen": -265.9932556152344,
      "logps/rejected": -331.5732116699219,
      "loss": 0.1398,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.0064342021942139,
      "rewards/margins": 6.052700996398926,
      "rewards/rejected": -7.059134483337402,
      "step": 1495
    },
    {
      "epoch": 2.4,
      "grad_norm": 2.684803009033203,
      "learning_rate": 5.094067817371878e-07,
      "logits/chosen": -1.4548851251602173,
      "logits/rejected": -1.4815185070037842,
      "logps/chosen": -228.2082977294922,
      "logps/rejected": -318.86248779296875,
      "loss": 0.1197,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.7952719926834106,
      "rewards/margins": 6.384288787841797,
      "rewards/rejected": -7.179561614990234,
      "step": 1500
    },
    {
      "epoch": 2.408,
      "grad_norm": 2.261495590209961,
      "learning_rate": 4.964132137105617e-07,
      "logits/chosen": -1.4797651767730713,
      "logits/rejected": -1.440944790840149,
      "logps/chosen": -236.2601776123047,
      "logps/rejected": -323.2953186035156,
      "loss": 0.1415,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.6942367553710938,
      "rewards/margins": 5.794956207275391,
      "rewards/rejected": -6.489192962646484,
      "step": 1505
    },
    {
      "epoch": 2.416,
      "grad_norm": 3.655900716781616,
      "learning_rate": 4.835692201472794e-07,
      "logits/chosen": -1.5585110187530518,
      "logits/rejected": -1.4639651775360107,
      "logps/chosen": -221.63577270507812,
      "logps/rejected": -317.4681396484375,
      "loss": 0.1161,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.9286325573921204,
      "rewards/margins": 5.880457878112793,
      "rewards/rejected": -6.809090614318848,
      "step": 1510
    },
    {
      "epoch": 2.424,
      "grad_norm": 4.601859092712402,
      "learning_rate": 4.7087575989447353e-07,
      "logits/chosen": -1.5128328800201416,
      "logits/rejected": -1.4157400131225586,
      "logps/chosen": -252.9822235107422,
      "logps/rejected": -314.6139221191406,
      "loss": 0.1445,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.8977339863777161,
      "rewards/margins": 5.779736518859863,
      "rewards/rejected": -6.6774702072143555,
      "step": 1515
    },
    {
      "epoch": 2.432,
      "grad_norm": 4.9197869300842285,
      "learning_rate": 4.583337805614602e-07,
      "logits/chosen": -1.4999001026153564,
      "logits/rejected": -1.4343955516815186,
      "logps/chosen": -229.39651489257812,
      "logps/rejected": -317.10858154296875,
      "loss": 0.1504,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8339191675186157,
      "rewards/margins": 5.6455535888671875,
      "rewards/rejected": -6.479473114013672,
      "step": 1520
    },
    {
      "epoch": 2.44,
      "grad_norm": 4.315069198608398,
      "learning_rate": 4.4594421844899844e-07,
      "logits/chosen": -1.4863693714141846,
      "logits/rejected": -1.4807065725326538,
      "logps/chosen": -227.49496459960938,
      "logps/rejected": -310.77850341796875,
      "loss": 0.1039,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.9351770281791687,
      "rewards/margins": 6.132818698883057,
      "rewards/rejected": -7.067995548248291,
      "step": 1525
    },
    {
      "epoch": 2.448,
      "grad_norm": 4.87631368637085,
      "learning_rate": 4.33707998479391e-07,
      "logits/chosen": -1.582213282585144,
      "logits/rejected": -1.6060279607772827,
      "logps/chosen": -299.22735595703125,
      "logps/rejected": -332.704833984375,
      "loss": 0.2081,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.0234695672988892,
      "rewards/margins": 5.895220756530762,
      "rewards/rejected": -6.9186906814575195,
      "step": 1530
    },
    {
      "epoch": 2.456,
      "grad_norm": 3.5169260501861572,
      "learning_rate": 4.2162603412743585e-07,
      "logits/chosen": -1.4920437335968018,
      "logits/rejected": -1.474491834640503,
      "logps/chosen": -229.24575805664062,
      "logps/rejected": -305.94171142578125,
      "loss": 0.1227,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.8777539134025574,
      "rewards/margins": 5.8880133628845215,
      "rewards/rejected": -6.765768527984619,
      "step": 1535
    },
    {
      "epoch": 2.464,
      "grad_norm": 3.1716701984405518,
      "learning_rate": 4.0969922735223296e-07,
      "logits/chosen": -1.5159512758255005,
      "logits/rejected": -1.4225298166275024,
      "logps/chosen": -221.02584838867188,
      "logps/rejected": -322.4046630859375,
      "loss": 0.126,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.9317782521247864,
      "rewards/margins": 6.3952813148498535,
      "rewards/rejected": -7.327059268951416,
      "step": 1540
    },
    {
      "epoch": 2.472,
      "grad_norm": 1.5373523235321045,
      "learning_rate": 3.979284685298493e-07,
      "logits/chosen": -1.5270391702651978,
      "logits/rejected": -1.4918533563613892,
      "logps/chosen": -231.8376922607422,
      "logps/rejected": -322.9443359375,
      "loss": 0.0849,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.8065363764762878,
      "rewards/margins": 6.097052574157715,
      "rewards/rejected": -6.903588771820068,
      "step": 1545
    },
    {
      "epoch": 2.48,
      "grad_norm": 5.075746536254883,
      "learning_rate": 3.863146363868495e-07,
      "logits/chosen": -1.5676982402801514,
      "logits/rejected": -1.4815164804458618,
      "logps/chosen": -226.5551300048828,
      "logps/rejected": -311.35284423828125,
      "loss": 0.1273,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.0178006887435913,
      "rewards/margins": 6.11976957321167,
      "rewards/rejected": -7.137570381164551,
      "step": 1550
    },
    {
      "epoch": 2.488,
      "grad_norm": 4.667273998260498,
      "learning_rate": 3.7485859793469636e-07,
      "logits/chosen": -1.4717382192611694,
      "logits/rejected": -1.4203176498413086,
      "logps/chosen": -251.059814453125,
      "logps/rejected": -314.29156494140625,
      "loss": 0.1622,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6194905042648315,
      "rewards/margins": 5.366819381713867,
      "rewards/rejected": -5.986310005187988,
      "step": 1555
    },
    {
      "epoch": 2.496,
      "grad_norm": 4.541383266448975,
      "learning_rate": 3.6356120840502553e-07,
      "logits/chosen": -1.598649024963379,
      "logits/rejected": -1.6106332540512085,
      "logps/chosen": -227.8941650390625,
      "logps/rejected": -312.3360290527344,
      "loss": 0.1947,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.984524130821228,
      "rewards/margins": 5.360861778259277,
      "rewards/rejected": -6.345385551452637,
      "step": 1560
    },
    {
      "epoch": 2.504,
      "grad_norm": 3.541747570037842,
      "learning_rate": 3.52423311185798e-07,
      "logits/chosen": -1.4794892072677612,
      "logits/rejected": -1.4599707126617432,
      "logps/chosen": -223.511962890625,
      "logps/rejected": -308.583984375,
      "loss": 0.143,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.2889810800552368,
      "rewards/margins": 5.206359386444092,
      "rewards/rejected": -6.495339870452881,
      "step": 1565
    },
    {
      "epoch": 2.512,
      "grad_norm": 4.896753311157227,
      "learning_rate": 3.414457377583413e-07,
      "logits/chosen": -1.512119174003601,
      "logits/rejected": -1.4517587423324585,
      "logps/chosen": -250.6962127685547,
      "logps/rejected": -331.4056701660156,
      "loss": 0.1387,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.8094693422317505,
      "rewards/margins": 6.361907958984375,
      "rewards/rejected": -7.171376705169678,
      "step": 1570
    },
    {
      "epoch": 2.52,
      "grad_norm": 5.767399787902832,
      "learning_rate": 3.30629307635274e-07,
      "logits/chosen": -1.5372960567474365,
      "logits/rejected": -1.5459281206130981,
      "logps/chosen": -222.1968994140625,
      "logps/rejected": -318.656005859375,
      "loss": 0.1104,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.6385549306869507,
      "rewards/margins": 6.328636169433594,
      "rewards/rejected": -6.967191219329834,
      "step": 1575
    },
    {
      "epoch": 2.528,
      "grad_norm": 2.8931169509887695,
      "learning_rate": 3.199748282993273e-07,
      "logits/chosen": -1.5647623538970947,
      "logits/rejected": -1.4885820150375366,
      "logps/chosen": -230.0967559814453,
      "logps/rejected": -316.1094665527344,
      "loss": 0.1245,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.6826328039169312,
      "rewards/margins": 6.465601921081543,
      "rewards/rejected": -7.148235321044922,
      "step": 1580
    },
    {
      "epoch": 2.536,
      "grad_norm": 3.3651797771453857,
      "learning_rate": 3.094830951430644e-07,
      "logits/chosen": -1.4947978258132935,
      "logits/rejected": -1.4919826984405518,
      "logps/chosen": -247.35824584960938,
      "logps/rejected": -329.46697998046875,
      "loss": 0.1327,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.8214322924613953,
      "rewards/margins": 6.117218017578125,
      "rewards/rejected": -6.938650608062744,
      "step": 1585
    },
    {
      "epoch": 2.544,
      "grad_norm": 3.0831105709075928,
      "learning_rate": 2.9915489140950076e-07,
      "logits/chosen": -1.534934401512146,
      "logits/rejected": -1.5211741924285889,
      "logps/chosen": -238.5296173095703,
      "logps/rejected": -291.08160400390625,
      "loss": 0.1872,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.9827302694320679,
      "rewards/margins": 5.219890117645264,
      "rewards/rejected": -6.202620506286621,
      "step": 1590
    },
    {
      "epoch": 2.552,
      "grad_norm": 3.8352949619293213,
      "learning_rate": 2.889909881336328e-07,
      "logits/chosen": -1.5629241466522217,
      "logits/rejected": -1.5492066144943237,
      "logps/chosen": -246.606201171875,
      "logps/rejected": -315.3060607910156,
      "loss": 0.1732,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.1226515769958496,
      "rewards/margins": 5.307157039642334,
      "rewards/rejected": -6.4298095703125,
      "step": 1595
    },
    {
      "epoch": 2.56,
      "grad_norm": 3.378814458847046,
      "learning_rate": 2.789921440848778e-07,
      "logits/chosen": -1.693416953086853,
      "logits/rejected": -1.556857705116272,
      "logps/chosen": -233.92196655273438,
      "logps/rejected": -321.92962646484375,
      "loss": 0.1014,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.9284855127334595,
      "rewards/margins": 6.132410049438477,
      "rewards/rejected": -7.0608954429626465,
      "step": 1600
    },
    {
      "epoch": 2.568,
      "grad_norm": 4.796236038208008,
      "learning_rate": 2.691591057104287e-07,
      "logits/chosen": -1.5096356868743896,
      "logits/rejected": -1.532952070236206,
      "logps/chosen": -236.99169921875,
      "logps/rejected": -332.6849060058594,
      "loss": 0.1642,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.5096395611763,
      "rewards/margins": 5.59578800201416,
      "rewards/rejected": -6.1054277420043945,
      "step": 1605
    },
    {
      "epoch": 2.576,
      "grad_norm": 3.768148899078369,
      "learning_rate": 2.594926070795295e-07,
      "logits/chosen": -1.5512020587921143,
      "logits/rejected": -1.5035789012908936,
      "logps/chosen": -243.709716796875,
      "logps/rejected": -319.43084716796875,
      "loss": 0.1359,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.9202622175216675,
      "rewards/margins": 5.834639072418213,
      "rewards/rejected": -6.754900932312012,
      "step": 1610
    },
    {
      "epoch": 2.584,
      "grad_norm": 4.090240001678467,
      "learning_rate": 2.4999336982867496e-07,
      "logits/chosen": -1.4447572231292725,
      "logits/rejected": -1.467057228088379,
      "logps/chosen": -246.8234405517578,
      "logps/rejected": -334.2413330078125,
      "loss": 0.141,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.9848871231079102,
      "rewards/margins": 5.94996452331543,
      "rewards/rejected": -6.934852600097656,
      "step": 1615
    },
    {
      "epoch": 2.592,
      "grad_norm": 3.5356523990631104,
      "learning_rate": 2.4066210310773776e-07,
      "logits/chosen": -1.4008734226226807,
      "logits/rejected": -1.4272077083587646,
      "logps/chosen": -227.53982543945312,
      "logps/rejected": -310.771240234375,
      "loss": 0.1112,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.43771782517433167,
      "rewards/margins": 5.975172996520996,
      "rewards/rejected": -6.412891387939453,
      "step": 1620
    },
    {
      "epoch": 2.6,
      "grad_norm": 1.1229864358901978,
      "learning_rate": 2.3149950352702787e-07,
      "logits/chosen": -1.575382947921753,
      "logits/rejected": -1.5440458059310913,
      "logps/chosen": -227.5854034423828,
      "logps/rejected": -332.78656005859375,
      "loss": 0.0806,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.9323962926864624,
      "rewards/margins": 6.606331825256348,
      "rewards/rejected": -7.538727760314941,
      "step": 1625
    },
    {
      "epoch": 2.608,
      "grad_norm": 2.3300342559814453,
      "learning_rate": 2.2250625510528783e-07,
      "logits/chosen": -1.48783278465271,
      "logits/rejected": -1.398735761642456,
      "logps/chosen": -235.2317657470703,
      "logps/rejected": -308.76959228515625,
      "loss": 0.1099,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5314217805862427,
      "rewards/margins": 6.1185760498046875,
      "rewards/rejected": -6.649998664855957,
      "step": 1630
    },
    {
      "epoch": 2.616,
      "grad_norm": 3.574950933456421,
      "learning_rate": 2.136830292186298e-07,
      "logits/chosen": -1.4672324657440186,
      "logits/rejected": -1.4376590251922607,
      "logps/chosen": -249.235107421875,
      "logps/rejected": -325.9244384765625,
      "loss": 0.1538,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7882421612739563,
      "rewards/margins": 5.851679801940918,
      "rewards/rejected": -6.63992166519165,
      "step": 1635
    },
    {
      "epoch": 2.624,
      "grad_norm": 1.543809413909912,
      "learning_rate": 2.050304845504139e-07,
      "logits/chosen": -1.525334119796753,
      "logits/rejected": -1.3860852718353271,
      "logps/chosen": -259.4710693359375,
      "logps/rejected": -307.67340087890625,
      "loss": 0.1104,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.8463247418403625,
      "rewards/margins": 5.842609405517578,
      "rewards/rejected": -6.688934326171875,
      "step": 1640
    },
    {
      "epoch": 2.632,
      "grad_norm": 2.3976762294769287,
      "learning_rate": 1.9654926704207627e-07,
      "logits/chosen": -1.4831944704055786,
      "logits/rejected": -1.4159176349639893,
      "logps/chosen": -228.45761108398438,
      "logps/rejected": -317.77362060546875,
      "loss": 0.1252,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.6478134393692017,
      "rewards/margins": 5.69929313659668,
      "rewards/rejected": -6.347106456756592,
      "step": 1645
    },
    {
      "epoch": 2.64,
      "grad_norm": 2.3702242374420166,
      "learning_rate": 1.882400098449058e-07,
      "logits/chosen": -1.5494359731674194,
      "logits/rejected": -1.5194289684295654,
      "logps/chosen": -249.11532592773438,
      "logps/rejected": -323.60675048828125,
      "loss": 0.1229,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.8728028535842896,
      "rewards/margins": 5.897127628326416,
      "rewards/rejected": -6.769930839538574,
      "step": 1650
    },
    {
      "epoch": 2.648,
      "grad_norm": 5.607757091522217,
      "learning_rate": 1.8010333327277947e-07,
      "logits/chosen": -1.5770350694656372,
      "logits/rejected": -1.5135689973831177,
      "logps/chosen": -245.56640625,
      "logps/rejected": -333.17462158203125,
      "loss": 0.145,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.2801220417022705,
      "rewards/margins": 5.944527626037598,
      "rewards/rejected": -7.2246503829956055,
      "step": 1655
    },
    {
      "epoch": 2.656,
      "grad_norm": 6.220284461975098,
      "learning_rate": 1.7213984475585143e-07,
      "logits/chosen": -1.499732255935669,
      "logits/rejected": -1.4527981281280518,
      "logps/chosen": -209.24880981445312,
      "logps/rejected": -297.71746826171875,
      "loss": 0.1497,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.8399412035942078,
      "rewards/margins": 5.599820613861084,
      "rewards/rejected": -6.439762115478516,
      "step": 1660
    },
    {
      "epoch": 2.664,
      "grad_norm": 6.647799491882324,
      "learning_rate": 1.6435013879520844e-07,
      "logits/chosen": -1.5817861557006836,
      "logits/rejected": -1.5299128293991089,
      "logps/chosen": -240.9619598388672,
      "logps/rejected": -315.337158203125,
      "loss": 0.1264,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.8431243896484375,
      "rewards/margins": 5.834400177001953,
      "rewards/rejected": -6.677524566650391,
      "step": 1665
    },
    {
      "epoch": 2.672,
      "grad_norm": 3.162820816040039,
      "learning_rate": 1.5673479691848736e-07,
      "logits/chosen": -1.538938045501709,
      "logits/rejected": -1.5266144275665283,
      "logps/chosen": -255.1186065673828,
      "logps/rejected": -306.49041748046875,
      "loss": 0.116,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.8456069827079773,
      "rewards/margins": 6.127451419830322,
      "rewards/rejected": -6.973057746887207,
      "step": 1670
    },
    {
      "epoch": 2.68,
      "grad_norm": 3.5515389442443848,
      "learning_rate": 1.492943876364611e-07,
      "logits/chosen": -1.576812744140625,
      "logits/rejected": -1.541210412979126,
      "logps/chosen": -245.21884155273438,
      "logps/rejected": -319.4032897949219,
      "loss": 0.122,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.8992266654968262,
      "rewards/margins": 5.852095603942871,
      "rewards/rejected": -6.7513227462768555,
      "step": 1675
    },
    {
      "epoch": 2.6879999999999997,
      "grad_norm": 3.37399959564209,
      "learning_rate": 1.4202946640060138e-07,
      "logits/chosen": -1.4246790409088135,
      "logits/rejected": -1.4544460773468018,
      "logps/chosen": -229.1632843017578,
      "logps/rejected": -313.3404846191406,
      "loss": 0.1375,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.7347529530525208,
      "rewards/margins": 5.777396202087402,
      "rewards/rejected": -6.512149810791016,
      "step": 1680
    },
    {
      "epoch": 2.6959999999999997,
      "grad_norm": 2.765533685684204,
      "learning_rate": 1.3494057556160567e-07,
      "logits/chosen": -1.5078999996185303,
      "logits/rejected": -1.4551175832748413,
      "logps/chosen": -241.14511108398438,
      "logps/rejected": -319.65020751953125,
      "loss": 0.1085,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.8278571367263794,
      "rewards/margins": 5.884353160858154,
      "rewards/rejected": -6.712210655212402,
      "step": 1685
    },
    {
      "epoch": 2.7039999999999997,
      "grad_norm": 5.154428482055664,
      "learning_rate": 1.2802824432891658e-07,
      "logits/chosen": -1.5341852903366089,
      "logits/rejected": -1.4802993535995483,
      "logps/chosen": -264.14093017578125,
      "logps/rejected": -332.47259521484375,
      "loss": 0.1298,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.7193272113800049,
      "rewards/margins": 5.93912410736084,
      "rewards/rejected": -6.658452033996582,
      "step": 1690
    },
    {
      "epoch": 2.7119999999999997,
      "grad_norm": 5.017862319946289,
      "learning_rate": 1.2129298873120753e-07,
      "logits/chosen": -1.5598084926605225,
      "logits/rejected": -1.4870166778564453,
      "logps/chosen": -231.30032348632812,
      "logps/rejected": -324.11279296875,
      "loss": 0.1291,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.8461719751358032,
      "rewards/margins": 6.092933654785156,
      "rewards/rejected": -6.939105987548828,
      "step": 1695
    },
    {
      "epoch": 2.7199999999999998,
      "grad_norm": 2.480734348297119,
      "learning_rate": 1.14735311577866e-07,
      "logits/chosen": -1.5418061017990112,
      "logits/rejected": -1.558050274848938,
      "logps/chosen": -249.1891632080078,
      "logps/rejected": -333.8502502441406,
      "loss": 0.1441,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1636663675308228,
      "rewards/margins": 6.228588104248047,
      "rewards/rejected": -7.392254829406738,
      "step": 1700
    },
    {
      "epoch": 2.7279999999999998,
      "grad_norm": 4.858855724334717,
      "learning_rate": 1.0835570242145072e-07,
      "logits/chosen": -1.6297658681869507,
      "logits/rejected": -1.5588557720184326,
      "logps/chosen": -239.9254913330078,
      "logps/rejected": -313.71563720703125,
      "loss": 0.1529,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.8668500781059265,
      "rewards/margins": 5.69160795211792,
      "rewards/rejected": -6.558457851409912,
      "step": 1705
    },
    {
      "epoch": 2.7359999999999998,
      "grad_norm": 3.9852185249328613,
      "learning_rate": 1.0215463752115101e-07,
      "logits/chosen": -1.5099657773971558,
      "logits/rejected": -1.4762407541275024,
      "logps/chosen": -220.3232421875,
      "logps/rejected": -319.54681396484375,
      "loss": 0.1223,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.7317541241645813,
      "rewards/margins": 6.199631214141846,
      "rewards/rejected": -6.931385040283203,
      "step": 1710
    },
    {
      "epoch": 2.7439999999999998,
      "grad_norm": 1.988715410232544,
      "learning_rate": 9.613257980722685e-08,
      "logits/chosen": -1.5078150033950806,
      "logits/rejected": -1.4630861282348633,
      "logps/chosen": -242.2015838623047,
      "logps/rejected": -299.0679931640625,
      "loss": 0.0811,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5184379816055298,
      "rewards/margins": 5.81303071975708,
      "rewards/rejected": -6.33146858215332,
      "step": 1715
    },
    {
      "epoch": 2.752,
      "grad_norm": 4.002151966094971,
      "learning_rate": 9.028997884645535e-08,
      "logits/chosen": -1.4705991744995117,
      "logits/rejected": -1.4686193466186523,
      "logps/chosen": -226.935791015625,
      "logps/rejected": -305.57562255859375,
      "loss": 0.1356,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.1761690378189087,
      "rewards/margins": 5.607525825500488,
      "rewards/rejected": -6.783695220947266,
      "step": 1720
    },
    {
      "epoch": 2.76,
      "grad_norm": 1.376519799232483,
      "learning_rate": 8.462727080856243e-08,
      "logits/chosen": -1.5141279697418213,
      "logits/rejected": -1.448982834815979,
      "logps/chosen": -237.4274444580078,
      "logps/rejected": -339.56768798828125,
      "loss": 0.1158,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.5523132085800171,
      "rewards/margins": 6.665265083312988,
      "rewards/rejected": -7.2175774574279785,
      "step": 1725
    },
    {
      "epoch": 2.768,
      "grad_norm": 2.3542063236236572,
      "learning_rate": 7.914487843366875e-08,
      "logits/chosen": -1.5205110311508179,
      "logits/rejected": -1.4258232116699219,
      "logps/chosen": -232.4739227294922,
      "logps/rejected": -308.33062744140625,
      "loss": 0.1493,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.8891655206680298,
      "rewards/margins": 5.613663673400879,
      "rewards/rejected": -6.502829074859619,
      "step": 1730
    },
    {
      "epoch": 2.776,
      "grad_norm": 1.9824843406677246,
      "learning_rate": 7.384321100072339e-08,
      "logits/chosen": -1.5084909200668335,
      "logits/rejected": -1.4419305324554443,
      "logps/chosen": -244.23532104492188,
      "logps/rejected": -316.4294128417969,
      "loss": 0.1651,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.8520871996879578,
      "rewards/margins": 6.0770697593688965,
      "rewards/rejected": -6.9291582107543945,
      "step": 1735
    },
    {
      "epoch": 2.784,
      "grad_norm": 4.31235408782959,
      "learning_rate": 6.872266429695518e-08,
      "logits/chosen": -1.5736935138702393,
      "logits/rejected": -1.4683692455291748,
      "logps/chosen": -243.3426513671875,
      "logps/rejected": -316.1590576171875,
      "loss": 0.1086,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.9821755290031433,
      "rewards/margins": 5.919834136962891,
      "rewards/rejected": -6.9020094871521,
      "step": 1740
    },
    {
      "epoch": 2.792,
      "grad_norm": 6.580287456512451,
      "learning_rate": 6.37836205883241e-08,
      "logits/chosen": -1.4502264261245728,
      "logits/rejected": -1.4019014835357666,
      "logps/chosen": -237.04660034179688,
      "logps/rejected": -304.73858642578125,
      "loss": 0.1768,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6265815496444702,
      "rewards/margins": 5.869175910949707,
      "rewards/rejected": -6.495758056640625,
      "step": 1745
    },
    {
      "epoch": 2.8,
      "grad_norm": 3.6616616249084473,
      "learning_rate": 5.9026448590982497e-08,
      "logits/chosen": -1.5317808389663696,
      "logits/rejected": -1.4841121435165405,
      "logps/chosen": -234.8379669189453,
      "logps/rejected": -316.34454345703125,
      "loss": 0.1134,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.7818082571029663,
      "rewards/margins": 6.074831962585449,
      "rewards/rejected": -6.856640815734863,
      "step": 1750
    },
    {
      "epoch": 2.808,
      "grad_norm": 2.6014156341552734,
      "learning_rate": 5.4451503443751774e-08,
      "logits/chosen": -1.4140145778656006,
      "logits/rejected": -1.4126405715942383,
      "logps/chosen": -236.1201629638672,
      "logps/rejected": -313.41497802734375,
      "loss": 0.0918,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.9264975786209106,
      "rewards/margins": 6.3627729415893555,
      "rewards/rejected": -7.289270877838135,
      "step": 1755
    },
    {
      "epoch": 2.816,
      "grad_norm": 4.039976119995117,
      "learning_rate": 5.0059126681607226e-08,
      "logits/chosen": -1.4606797695159912,
      "logits/rejected": -1.4615904092788696,
      "logps/chosen": -233.7847137451172,
      "logps/rejected": -324.12091064453125,
      "loss": 0.121,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.6958805322647095,
      "rewards/margins": 6.251741886138916,
      "rewards/rejected": -6.9476213455200195,
      "step": 1760
    },
    {
      "epoch": 2.824,
      "grad_norm": 3.214773654937744,
      "learning_rate": 4.5849646210184837e-08,
      "logits/chosen": -1.4053151607513428,
      "logits/rejected": -1.4549171924591064,
      "logps/chosen": -231.1645965576172,
      "logps/rejected": -330.7877197265625,
      "loss": 0.1145,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.609539806842804,
      "rewards/margins": 6.281281471252441,
      "rewards/rejected": -6.8908209800720215,
      "step": 1765
    },
    {
      "epoch": 2.832,
      "grad_norm": 1.2576606273651123,
      "learning_rate": 4.1823376281297515e-08,
      "logits/chosen": -1.3523794412612915,
      "logits/rejected": -1.399296522140503,
      "logps/chosen": -261.13092041015625,
      "logps/rejected": -322.9598083496094,
      "loss": 0.1283,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.6175804734230042,
      "rewards/margins": 6.264570713043213,
      "rewards/rejected": -6.882150173187256,
      "step": 1770
    },
    {
      "epoch": 2.84,
      "grad_norm": 2.263697385787964,
      "learning_rate": 3.798061746947995e-08,
      "logits/chosen": -1.4639291763305664,
      "logits/rejected": -1.4517295360565186,
      "logps/chosen": -242.2957763671875,
      "logps/rejected": -311.65545654296875,
      "loss": 0.1298,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6775432825088501,
      "rewards/margins": 5.772078990936279,
      "rewards/rejected": -6.449623107910156,
      "step": 1775
    },
    {
      "epoch": 2.848,
      "grad_norm": 5.509952545166016,
      "learning_rate": 3.432165664954551e-08,
      "logits/chosen": -1.5598200559616089,
      "logits/rejected": -1.474449872970581,
      "logps/chosen": -258.42950439453125,
      "logps/rejected": -309.0158386230469,
      "loss": 0.1269,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.9016896486282349,
      "rewards/margins": 5.584256649017334,
      "rewards/rejected": -6.4859466552734375,
      "step": 1780
    },
    {
      "epoch": 2.856,
      "grad_norm": 5.17020845413208,
      "learning_rate": 3.0846766975173583e-08,
      "logits/chosen": -1.5272047519683838,
      "logits/rejected": -1.4640551805496216,
      "logps/chosen": -224.8867645263672,
      "logps/rejected": -301.6604919433594,
      "loss": 0.1449,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.9279806017875671,
      "rewards/margins": 5.865192890167236,
      "rewards/rejected": -6.793173313140869,
      "step": 1785
    },
    {
      "epoch": 2.864,
      "grad_norm": 5.173901081085205,
      "learning_rate": 2.7556207858514262e-08,
      "logits/chosen": -1.5167121887207031,
      "logits/rejected": -1.535841464996338,
      "logps/chosen": -249.1966552734375,
      "logps/rejected": -297.0778503417969,
      "loss": 0.1406,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.1131664514541626,
      "rewards/margins": 5.719702243804932,
      "rewards/rejected": -6.832869052886963,
      "step": 1790
    },
    {
      "epoch": 2.872,
      "grad_norm": 2.5215299129486084,
      "learning_rate": 2.4450224950826607e-08,
      "logits/chosen": -1.543928623199463,
      "logits/rejected": -1.4860496520996094,
      "logps/chosen": -220.95565795898438,
      "logps/rejected": -311.44158935546875,
      "loss": 0.1465,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.9338169097900391,
      "rewards/margins": 5.805626392364502,
      "rewards/rejected": -6.739443302154541,
      "step": 1795
    },
    {
      "epoch": 2.88,
      "grad_norm": 2.975391387939453,
      "learning_rate": 2.15290501241347e-08,
      "logits/chosen": -1.5811344385147095,
      "logits/rejected": -1.5057846307754517,
      "logps/chosen": -236.87191772460938,
      "logps/rejected": -321.338134765625,
      "loss": 0.1335,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7364934682846069,
      "rewards/margins": 6.344755172729492,
      "rewards/rejected": -7.081249237060547,
      "step": 1800
    },
    {
      "epoch": 2.888,
      "grad_norm": 3.126237154006958,
      "learning_rate": 1.8792901453923428e-08,
      "logits/chosen": -1.5378950834274292,
      "logits/rejected": -1.516510248184204,
      "logps/chosen": -237.1420135498047,
      "logps/rejected": -311.3038635253906,
      "loss": 0.1549,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.9023133516311646,
      "rewards/margins": 5.623795032501221,
      "rewards/rejected": -6.526108741760254,
      "step": 1805
    },
    {
      "epoch": 2.896,
      "grad_norm": 4.797680854797363,
      "learning_rate": 1.624198320285264e-08,
      "logits/chosen": -1.5249507427215576,
      "logits/rejected": -1.4735972881317139,
      "logps/chosen": -234.7667999267578,
      "logps/rejected": -326.597412109375,
      "loss": 0.1449,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.2055195569992065,
      "rewards/margins": 5.8126044273376465,
      "rewards/rejected": -7.018124580383301,
      "step": 1810
    },
    {
      "epoch": 2.904,
      "grad_norm": 1.3551347255706787,
      "learning_rate": 1.3876485805513762e-08,
      "logits/chosen": -1.5904200077056885,
      "logits/rejected": -1.6041524410247803,
      "logps/chosen": -225.4705810546875,
      "logps/rejected": -329.77874755859375,
      "loss": 0.118,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.1419427394866943,
      "rewards/margins": 6.027765274047852,
      "rewards/rejected": -7.169708251953125,
      "step": 1815
    },
    {
      "epoch": 2.912,
      "grad_norm": 3.9451160430908203,
      "learning_rate": 1.169658585420813e-08,
      "logits/chosen": -1.4940643310546875,
      "logits/rejected": -1.5199902057647705,
      "logps/chosen": -231.20944213867188,
      "logps/rejected": -320.60662841796875,
      "loss": 0.1453,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.0270144939422607,
      "rewards/margins": 6.1724395751953125,
      "rewards/rejected": -7.199454307556152,
      "step": 1820
    },
    {
      "epoch": 2.92,
      "grad_norm": 2.1993324756622314,
      "learning_rate": 9.702446085768091e-09,
      "logits/chosen": -1.514919400215149,
      "logits/rejected": -1.537259817123413,
      "logps/chosen": -227.4259033203125,
      "logps/rejected": -331.2008361816406,
      "loss": 0.1286,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.0264793634414673,
      "rewards/margins": 6.270672798156738,
      "rewards/rejected": -7.297152519226074,
      "step": 1825
    },
    {
      "epoch": 2.928,
      "grad_norm": 5.0035080909729,
      "learning_rate": 7.894215369405322e-09,
      "logits/chosen": -1.4085063934326172,
      "logits/rejected": -1.3085895776748657,
      "logps/chosen": -240.96902465820312,
      "logps/rejected": -309.6005859375,
      "loss": 0.1369,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.6670558452606201,
      "rewards/margins": 5.737823486328125,
      "rewards/rejected": -6.404879570007324,
      "step": 1830
    },
    {
      "epoch": 2.936,
      "grad_norm": 2.903502941131592,
      "learning_rate": 6.272028695598065e-09,
      "logits/chosen": -1.4993667602539062,
      "logits/rejected": -1.482932209968567,
      "logps/chosen": -225.55258178710938,
      "logps/rejected": -322.9141540527344,
      "loss": 0.1414,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.6842883825302124,
      "rewards/margins": 6.101261615753174,
      "rewards/rejected": -6.785550117492676,
      "step": 1835
    },
    {
      "epoch": 2.944,
      "grad_norm": 6.005549907684326,
      "learning_rate": 4.836007166014179e-09,
      "logits/chosen": -1.5267841815948486,
      "logits/rejected": -1.5386269092559814,
      "logps/chosen": -230.11038208007812,
      "logps/rejected": -331.4931335449219,
      "loss": 0.151,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1418485641479492,
      "rewards/margins": 6.155372619628906,
      "rewards/rejected": -7.297222137451172,
      "step": 1840
    },
    {
      "epoch": 2.952,
      "grad_norm": 3.431831121444702,
      "learning_rate": 3.5862579844697652e-09,
      "logits/chosen": -1.5528335571289062,
      "logits/rejected": -1.5695273876190186,
      "logps/chosen": -237.58261108398438,
      "logps/rejected": -302.52191162109375,
      "loss": 0.1869,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.3783166408538818,
      "rewards/margins": 5.019209861755371,
      "rewards/rejected": -6.397526264190674,
      "step": 1845
    },
    {
      "epoch": 2.96,
      "grad_norm": 3.1080453395843506,
      "learning_rate": 2.522874448926682e-09,
      "logits/chosen": -1.5317953824996948,
      "logits/rejected": -1.4420945644378662,
      "logps/chosen": -249.25967407226562,
      "logps/rejected": -301.2223815917969,
      "loss": 0.1357,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.8196800947189331,
      "rewards/margins": 5.4106550216674805,
      "rewards/rejected": -6.230335235595703,
      "step": 1850
    },
    {
      "epoch": 2.968,
      "grad_norm": 2.7016687393188477,
      "learning_rate": 1.645935944526722e-09,
      "logits/chosen": -1.4864146709442139,
      "logits/rejected": -1.4250588417053223,
      "logps/chosen": -239.615478515625,
      "logps/rejected": -288.51947021484375,
      "loss": 0.1743,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.9490318298339844,
      "rewards/margins": 5.670099258422852,
      "rewards/rejected": -6.619131565093994,
      "step": 1855
    },
    {
      "epoch": 2.976,
      "grad_norm": 3.5190517902374268,
      "learning_rate": 9.555079376663556e-10,
      "logits/chosen": -1.4785500764846802,
      "logits/rejected": -1.411423921585083,
      "logps/chosen": -247.8288116455078,
      "logps/rejected": -309.9002990722656,
      "loss": 0.1295,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.6508281230926514,
      "rewards/margins": 5.810454368591309,
      "rewards/rejected": -6.461281776428223,
      "step": 1860
    },
    {
      "epoch": 2.984,
      "grad_norm": 3.1977896690368652,
      "learning_rate": 4.5164197110786346e-10,
      "logits/chosen": -1.5095391273498535,
      "logits/rejected": -1.4798269271850586,
      "logps/chosen": -240.112548828125,
      "logps/rejected": -312.7066650390625,
      "loss": 0.1405,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.9202483892440796,
      "rewards/margins": 5.801078796386719,
      "rewards/rejected": -6.7213263511657715,
      "step": 1865
    },
    {
      "epoch": 2.992,
      "grad_norm": 2.8568241596221924,
      "learning_rate": 1.3437566013380087e-10,
      "logits/chosen": -1.5050697326660156,
      "logits/rejected": -1.49628746509552,
      "logps/chosen": -222.0975341796875,
      "logps/rejected": -315.3607482910156,
      "loss": 0.0995,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.5805636048316956,
      "rewards/margins": 6.518218994140625,
      "rewards/rejected": -7.098783016204834,
      "step": 1870
    },
    {
      "epoch": 3.0,
      "grad_norm": 4.118714809417725,
      "learning_rate": 3.732689736191119e-12,
      "logits/chosen": -1.6023108959197998,
      "logits/rejected": -1.5270345211029053,
      "logps/chosen": -222.2280731201172,
      "logps/rejected": -314.62908935546875,
      "loss": 0.1354,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.7832231521606445,
      "rewards/margins": 5.982799530029297,
      "rewards/rejected": -6.7660231590271,
      "step": 1875
    }
  ],
  "logging_steps": 5,
  "max_steps": 1875,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}