{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.2809913726867605,
  "eval_steps": 100,
  "global_step": 3200,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0008780980396461265,
      "grad_norm": 6.670812129974365,
      "learning_rate": 1.0000000000000001e-07,
      "logits/chosen": 4.2431488037109375,
      "logits/rejected": 4.231738567352295,
      "logps/chosen": -9.991304397583008,
      "logps/rejected": -10.524327278137207,
      "loss": 3.0309,
      "nll_loss": 2.3641905784606934,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -2.99739146232605,
      "rewards/margins": 0.15990665555000305,
      "rewards/rejected": -3.1572983264923096,
      "step": 10
    },
    {
      "epoch": 0.001756196079292253,
      "grad_norm": 10.682082176208496,
      "learning_rate": 2.1111111111111113e-07,
      "logits/chosen": 4.309535980224609,
      "logits/rejected": 4.361606597900391,
      "logps/chosen": -9.786101341247559,
      "logps/rejected": -10.518722534179688,
      "loss": 3.1598,
      "nll_loss": 2.5160868167877197,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.9358303546905518,
      "rewards/margins": 0.2197863757610321,
      "rewards/rejected": -3.1556167602539062,
      "step": 20
    },
    {
      "epoch": 0.0026342941189383797,
      "grad_norm": 15.04028034210205,
      "learning_rate": 3.2222222222222227e-07,
      "logits/chosen": 4.225083351135254,
      "logits/rejected": 4.215059757232666,
      "logps/chosen": -9.968446731567383,
      "logps/rejected": -10.634344100952148,
      "loss": 2.9674,
      "nll_loss": 2.318664073944092,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -2.9905343055725098,
      "rewards/margins": 0.19976934790611267,
      "rewards/rejected": -3.1903038024902344,
      "step": 30
    },
    {
      "epoch": 0.003512392158584506,
      "grad_norm": 10.793362617492676,
      "learning_rate": 4.333333333333334e-07,
      "logits/chosen": 4.328730583190918,
      "logits/rejected": 4.321128845214844,
      "logps/chosen": -9.762226104736328,
      "logps/rejected": -10.319639205932617,
      "loss": 2.7628,
      "nll_loss": 2.110645294189453,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.928668260574341,
      "rewards/margins": 0.1672237515449524,
      "rewards/rejected": -3.0958914756774902,
      "step": 40
    },
    {
      "epoch": 0.004390490198230633,
      "grad_norm": 6.863041400909424,
      "learning_rate": 5.444444444444444e-07,
      "logits/chosen": 4.535063743591309,
      "logits/rejected": 4.555140495300293,
      "logps/chosen": -10.173809051513672,
      "logps/rejected": -10.436089515686035,
      "loss": 2.7587,
      "nll_loss": 2.0572845935821533,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -3.052142858505249,
      "rewards/margins": 0.07868396490812302,
      "rewards/rejected": -3.130826711654663,
      "step": 50
    },
    {
      "epoch": 0.005268588237876759,
      "grad_norm": 11.63379955291748,
      "learning_rate": 6.555555555555556e-07,
      "logits/chosen": 4.210951805114746,
      "logits/rejected": 4.193416595458984,
      "logps/chosen": -9.710775375366211,
      "logps/rejected": -10.588180541992188,
      "loss": 3.023,
      "nll_loss": 2.409853458404541,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.9132332801818848,
      "rewards/margins": 0.2632210850715637,
      "rewards/rejected": -3.1764540672302246,
      "step": 60
    },
    {
      "epoch": 0.006146686277522885,
      "grad_norm": 4.952016830444336,
      "learning_rate": 7.666666666666667e-07,
      "logits/chosen": 4.56010627746582,
      "logits/rejected": 4.571717739105225,
      "logps/chosen": -9.788490295410156,
      "logps/rejected": -10.214717864990234,
      "loss": 2.6203,
      "nll_loss": 1.9469432830810547,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.936546802520752,
      "rewards/margins": 0.12786847352981567,
      "rewards/rejected": -3.064415454864502,
      "step": 70
    },
    {
      "epoch": 0.007024784317169012,
      "grad_norm": 10.809489250183105,
      "learning_rate": 8.777777777777778e-07,
      "logits/chosen": 4.2120256423950195,
      "logits/rejected": 4.222228050231934,
      "logps/chosen": -9.806170463562012,
      "logps/rejected": -10.296957969665527,
      "loss": 3.3042,
      "nll_loss": 2.644357681274414,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -2.9418509006500244,
      "rewards/margins": 0.1472366452217102,
      "rewards/rejected": -3.08908748626709,
      "step": 80
    },
    {
      "epoch": 0.007902882356815138,
      "grad_norm": 9.27618408203125,
      "learning_rate": 9.88888888888889e-07,
      "logits/chosen": 4.336479187011719,
      "logits/rejected": 4.3232035636901855,
      "logps/chosen": -9.828582763671875,
      "logps/rejected": -10.485132217407227,
      "loss": 3.1223,
      "nll_loss": 2.4759140014648438,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.948575496673584,
      "rewards/margins": 0.1969645768404007,
      "rewards/rejected": -3.1455399990081787,
      "step": 90
    },
    {
      "epoch": 0.008780980396461266,
      "grad_norm": 19.710798263549805,
      "learning_rate": 1.1e-06,
      "logits/chosen": 4.149864673614502,
      "logits/rejected": 4.147267818450928,
      "logps/chosen": -9.967988014221191,
      "logps/rejected": -10.663251876831055,
      "loss": 2.9038,
      "nll_loss": 2.2608590126037598,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.990396499633789,
      "rewards/margins": 0.20857906341552734,
      "rewards/rejected": -3.1989755630493164,
      "step": 100
    },
    {
      "epoch": 0.009659078436107391,
      "grad_norm": 10.258548736572266,
      "learning_rate": 1.2111111111111111e-06,
      "logits/chosen": 4.241273880004883,
      "logits/rejected": 4.249630928039551,
      "logps/chosen": -9.708308219909668,
      "logps/rejected": -10.221686363220215,
      "loss": 3.2698,
      "nll_loss": 2.603203535079956,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.912492513656616,
      "rewards/margins": 0.15401321649551392,
      "rewards/rejected": -3.0665059089660645,
      "step": 110
    },
    {
      "epoch": 0.010537176475753519,
      "grad_norm": 3.6609084606170654,
      "learning_rate": 1.3222222222222222e-06,
      "logits/chosen": 4.420263290405273,
      "logits/rejected": 4.413485527038574,
      "logps/chosen": -9.733232498168945,
      "logps/rejected": -10.436403274536133,
      "loss": 2.5418,
      "nll_loss": 1.9018337726593018,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.9199700355529785,
      "rewards/margins": 0.21095120906829834,
      "rewards/rejected": -3.1309211254119873,
      "step": 120
    },
    {
      "epoch": 0.011415274515399644,
      "grad_norm": 10.068683624267578,
      "learning_rate": 1.4333333333333335e-06,
      "logits/chosen": 4.42335319519043,
      "logits/rejected": 4.493862152099609,
      "logps/chosen": -9.73039722442627,
      "logps/rejected": -10.323850631713867,
      "loss": 2.5285,
      "nll_loss": 1.8774116039276123,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -2.919119358062744,
      "rewards/margins": 0.17803625762462616,
      "rewards/rejected": -3.0971553325653076,
      "step": 130
    },
    {
      "epoch": 0.01229337255504577,
      "grad_norm": 11.362488746643066,
      "learning_rate": 1.5444444444444446e-06,
      "logits/chosen": 4.332414150238037,
      "logits/rejected": 4.329668045043945,
      "logps/chosen": -9.279766082763672,
      "logps/rejected": -9.99905014038086,
      "loss": 2.7181,
      "nll_loss": 2.0923380851745605,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.7839295864105225,
      "rewards/margins": 0.21578574180603027,
      "rewards/rejected": -2.9997153282165527,
      "step": 140
    },
    {
      "epoch": 0.013171470594691898,
      "grad_norm": 14.254817008972168,
      "learning_rate": 1.6555555555555559e-06,
      "logits/chosen": 4.319321155548096,
      "logits/rejected": 4.321578502655029,
      "logps/chosen": -9.059564590454102,
      "logps/rejected": -9.602411270141602,
      "loss": 2.8514,
      "nll_loss": 2.1910147666931152,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -2.717869758605957,
      "rewards/margins": 0.16285373270511627,
      "rewards/rejected": -2.880723476409912,
      "step": 150
    },
    {
      "epoch": 0.014049568634338023,
      "grad_norm": 4.008646488189697,
      "learning_rate": 1.7666666666666668e-06,
      "logits/chosen": 4.2461042404174805,
      "logits/rejected": 4.25100040435791,
      "logps/chosen": -8.857942581176758,
      "logps/rejected": -9.37680435180664,
      "loss": 2.5006,
      "nll_loss": 1.839271903038025,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -2.6573829650878906,
      "rewards/margins": 0.15565846860408783,
      "rewards/rejected": -2.8130412101745605,
      "step": 160
    },
    {
      "epoch": 0.01492766667398415,
      "grad_norm": 10.644233703613281,
      "learning_rate": 1.8777777777777778e-06,
      "logits/chosen": 4.304837226867676,
      "logits/rejected": 4.3417558670043945,
      "logps/chosen": -8.620405197143555,
      "logps/rejected": -9.385229110717773,
      "loss": 2.5802,
      "nll_loss": 1.932861328125,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.5861220359802246,
      "rewards/margins": 0.22944733500480652,
      "rewards/rejected": -2.8155694007873535,
      "step": 170
    },
    {
      "epoch": 0.015805764713630276,
      "grad_norm": 9.301039695739746,
      "learning_rate": 1.988888888888889e-06,
      "logits/chosen": 4.26754903793335,
      "logits/rejected": 4.271862030029297,
      "logps/chosen": -8.370404243469238,
      "logps/rejected": -8.689592361450195,
      "loss": 2.4808,
      "nll_loss": 1.7635695934295654,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.5111212730407715,
      "rewards/margins": 0.09575649350881577,
      "rewards/rejected": -2.6068778038024902,
      "step": 180
    },
    {
      "epoch": 0.016683862753276404,
      "grad_norm": 6.643105506896973,
      "learning_rate": 2.1000000000000002e-06,
      "logits/chosen": 4.255660533905029,
      "logits/rejected": 4.268857955932617,
      "logps/chosen": -7.950819492340088,
      "logps/rejected": -8.541925430297852,
      "loss": 2.2629,
      "nll_loss": 1.6152054071426392,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -2.3852455615997314,
      "rewards/margins": 0.17733201384544373,
      "rewards/rejected": -2.562577724456787,
      "step": 190
    },
    {
      "epoch": 0.01756196079292253,
      "grad_norm": 9.348048210144043,
      "learning_rate": 2.2111111111111113e-06,
      "logits/chosen": 4.450976848602295,
      "logits/rejected": 4.4492387771606445,
      "logps/chosen": -7.3747453689575195,
      "logps/rejected": -8.266874313354492,
      "loss": 2.0158,
      "nll_loss": 1.3951395750045776,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -2.212423801422119,
      "rewards/margins": 0.2676388621330261,
      "rewards/rejected": -2.480062484741211,
      "step": 200
    },
    {
      "epoch": 0.018440058832568655,
      "grad_norm": 5.35267972946167,
      "learning_rate": 2.3222222222222224e-06,
      "logits/chosen": 4.387726783752441,
      "logits/rejected": 4.407253265380859,
      "logps/chosen": -6.376626491546631,
      "logps/rejected": -7.535942077636719,
      "loss": 1.7794,
      "nll_loss": 1.1858270168304443,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.9129879474639893,
      "rewards/margins": 0.3477945923805237,
      "rewards/rejected": -2.2607827186584473,
      "step": 210
    },
    {
      "epoch": 0.019318156872214783,
      "grad_norm": 8.461145401000977,
      "learning_rate": 2.4333333333333335e-06,
      "logits/chosen": 4.57470703125,
      "logits/rejected": 4.573002815246582,
      "logps/chosen": -5.478797912597656,
      "logps/rejected": -6.914730072021484,
      "loss": 1.4017,
      "nll_loss": 0.8511344194412231,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.643639326095581,
      "rewards/margins": 0.4307795464992523,
      "rewards/rejected": -2.0744190216064453,
      "step": 220
    },
    {
      "epoch": 0.02019625491186091,
      "grad_norm": 5.639391899108887,
      "learning_rate": 2.5444444444444446e-06,
      "logits/chosen": 4.498848915100098,
      "logits/rejected": 4.52827262878418,
      "logps/chosen": -4.859742164611816,
      "logps/rejected": -5.578665733337402,
      "loss": 1.4217,
      "nll_loss": 0.754524827003479,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.4579226970672607,
      "rewards/margins": 0.21567705273628235,
      "rewards/rejected": -1.6735999584197998,
      "step": 230
    },
    {
      "epoch": 0.021074352951507037,
      "grad_norm": 5.855747699737549,
      "learning_rate": 2.6555555555555556e-06,
      "logits/chosen": 4.4441728591918945,
      "logits/rejected": 4.421013832092285,
      "logps/chosen": -3.958739757537842,
      "logps/rejected": -4.544581413269043,
      "loss": 1.3529,
      "nll_loss": 0.6709738969802856,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.1876219511032104,
      "rewards/margins": 0.17575237154960632,
      "rewards/rejected": -1.3633743524551392,
      "step": 240
    },
    {
      "epoch": 0.02195245099115316,
      "grad_norm": 5.383121490478516,
      "learning_rate": 2.766666666666667e-06,
      "logits/chosen": 4.187775611877441,
      "logits/rejected": 4.246646404266357,
      "logps/chosen": -2.965056896209717,
      "logps/rejected": -3.8153586387634277,
      "loss": 1.0879,
      "nll_loss": 0.45148134231567383,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.8895170092582703,
      "rewards/margins": 0.255090594291687,
      "rewards/rejected": -1.144607663154602,
      "step": 250
    },
    {
      "epoch": 0.02283054903079929,
      "grad_norm": 4.042531967163086,
      "learning_rate": 2.8777777777777782e-06,
      "logits/chosen": 4.055316925048828,
      "logits/rejected": 4.1006879806518555,
      "logps/chosen": -2.189060688018799,
      "logps/rejected": -2.7986176013946533,
      "loss": 0.9665,
      "nll_loss": 0.3139139711856842,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.6567181348800659,
      "rewards/margins": 0.1828671246767044,
      "rewards/rejected": -0.8395851850509644,
      "step": 260
    },
    {
      "epoch": 0.023708647070445416,
      "grad_norm": 3.6113719940185547,
      "learning_rate": 2.988888888888889e-06,
      "logits/chosen": 4.224070072174072,
      "logits/rejected": 4.2020182609558105,
      "logps/chosen": -1.5525212287902832,
      "logps/rejected": -2.3060789108276367,
      "loss": 0.7876,
      "nll_loss": 0.17524096369743347,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.465756356716156,
      "rewards/margins": 0.22606734931468964,
      "rewards/rejected": -0.6918237805366516,
      "step": 270
    },
    {
      "epoch": 0.02458674511009154,
      "grad_norm": 2.20628023147583,
      "learning_rate": 3.1000000000000004e-06,
      "logits/chosen": 3.9985511302948,
      "logits/rejected": 3.9572558403015137,
      "logps/chosen": -1.0324242115020752,
      "logps/rejected": -1.434692621231079,
      "loss": 0.8117,
      "nll_loss": 0.1578795313835144,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.309727281332016,
      "rewards/margins": 0.12068048864603043,
      "rewards/rejected": -0.4304077625274658,
      "step": 280
    },
    {
      "epoch": 0.025464843149737668,
      "grad_norm": 2.605509042739868,
      "learning_rate": 3.2111111111111115e-06,
      "logits/chosen": 3.9703261852264404,
      "logits/rejected": 3.978794574737549,
      "logps/chosen": -1.302750587463379,
      "logps/rejected": -1.4635370969772339,
      "loss": 0.8424,
      "nll_loss": 0.15038228034973145,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.39082518219947815,
      "rewards/margins": 0.048235934227705,
      "rewards/rejected": -0.43906116485595703,
      "step": 290
    },
    {
      "epoch": 0.026342941189383795,
      "grad_norm": 4.225305080413818,
      "learning_rate": 3.322222222222222e-06,
      "logits/chosen": 3.872316360473633,
      "logits/rejected": 3.909350872039795,
      "logps/chosen": -0.7655197381973267,
      "logps/rejected": -1.3384641408920288,
      "loss": 0.7091,
      "nll_loss": 0.07751598209142685,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2296559065580368,
      "rewards/margins": 0.17188331484794617,
      "rewards/rejected": -0.40153923630714417,
      "step": 300
    },
    {
      "epoch": 0.027221039229029922,
      "grad_norm": 2.167863607406616,
      "learning_rate": 3.4333333333333336e-06,
      "logits/chosen": 3.8423914909362793,
      "logits/rejected": 3.8470401763916016,
      "logps/chosen": -0.781024158000946,
      "logps/rejected": -1.1648194789886475,
      "loss": 0.7324,
      "nll_loss": 0.07609061896800995,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.23430728912353516,
      "rewards/margins": 0.11513856798410416,
      "rewards/rejected": -0.3494458794593811,
      "step": 310
    },
    {
      "epoch": 0.028099137268676046,
      "grad_norm": 2.662125587463379,
      "learning_rate": 3.5444444444444447e-06,
      "logits/chosen": 3.844832181930542,
      "logits/rejected": 3.897855758666992,
      "logps/chosen": -0.8712307214736938,
      "logps/rejected": -1.2118942737579346,
      "loss": 0.754,
      "nll_loss": 0.09477487206459045,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.2613692581653595,
      "rewards/margins": 0.10219905525445938,
      "rewards/rejected": -0.3635682463645935,
      "step": 320
    },
    {
      "epoch": 0.028977235308322174,
      "grad_norm": 3.6492230892181396,
      "learning_rate": 3.6555555555555562e-06,
      "logits/chosen": 3.791374683380127,
      "logits/rejected": 3.823376417160034,
      "logps/chosen": -0.8908417820930481,
      "logps/rejected": -1.1247615814208984,
      "loss": 0.7571,
      "nll_loss": 0.07855098694562912,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.26725253462791443,
      "rewards/margins": 0.07017592340707779,
      "rewards/rejected": -0.337428480386734,
      "step": 330
    },
    {
      "epoch": 0.0298553333479683,
      "grad_norm": 4.195188999176025,
      "learning_rate": 3.766666666666667e-06,
      "logits/chosen": 3.7930731773376465,
      "logits/rejected": 3.771570920944214,
      "logps/chosen": -0.5707345008850098,
      "logps/rejected": -1.0306470394134521,
      "loss": 0.7241,
      "nll_loss": 0.07002006471157074,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1712203472852707,
      "rewards/margins": 0.13797374069690704,
      "rewards/rejected": -0.30919408798217773,
      "step": 340
    },
    {
      "epoch": 0.03073343138761443,
      "grad_norm": 6.730047225952148,
      "learning_rate": 3.877777777777778e-06,
      "logits/chosen": 3.7914137840270996,
      "logits/rejected": 3.830873966217041,
      "logps/chosen": -0.48989325761795044,
      "logps/rejected": -1.329006552696228,
      "loss": 0.6549,
      "nll_loss": 0.04915159195661545,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.14696797728538513,
      "rewards/margins": 0.2517339587211609,
      "rewards/rejected": -0.3987019658088684,
      "step": 350
    },
    {
      "epoch": 0.03161152942726055,
      "grad_norm": 5.190159320831299,
      "learning_rate": 3.9888888888888895e-06,
      "logits/chosen": 3.5701937675476074,
      "logits/rejected": 3.623018264770508,
      "logps/chosen": -1.0031934976577759,
      "logps/rejected": -1.3071503639221191,
      "loss": 0.7632,
      "nll_loss": 0.07658834755420685,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.3009580075740814,
      "rewards/margins": 0.09118713438510895,
      "rewards/rejected": -0.39214515686035156,
      "step": 360
    },
    {
      "epoch": 0.03248962746690668,
      "grad_norm": 6.694168567657471,
      "learning_rate": 4.1e-06,
      "logits/chosen": 3.823359727859497,
      "logits/rejected": 3.852163314819336,
      "logps/chosen": -0.6341744661331177,
      "logps/rejected": -1.3019847869873047,
      "loss": 0.7057,
      "nll_loss": 0.07171504944562912,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.19025234878063202,
      "rewards/margins": 0.20034310221672058,
      "rewards/rejected": -0.3905954957008362,
      "step": 370
    },
    {
      "epoch": 0.03336772550655281,
      "grad_norm": 7.252128601074219,
      "learning_rate": 4.211111111111112e-06,
      "logits/chosen": 3.7956886291503906,
      "logits/rejected": 3.795431137084961,
      "logps/chosen": -0.7523115277290344,
      "logps/rejected": -1.489225149154663,
      "loss": 0.6923,
      "nll_loss": 0.07404422760009766,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.2256934642791748,
      "rewards/margins": 0.22107413411140442,
      "rewards/rejected": -0.4467676281929016,
      "step": 380
    },
    {
      "epoch": 0.034245823546198935,
      "grad_norm": 2.945425271987915,
      "learning_rate": 4.322222222222223e-06,
      "logits/chosen": 3.5031909942626953,
      "logits/rejected": 3.5364387035369873,
      "logps/chosen": -0.6282280087471008,
      "logps/rejected": -1.3442871570587158,
      "loss": 0.7103,
      "nll_loss": 0.06170845031738281,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.18846839666366577,
      "rewards/margins": 0.21481776237487793,
      "rewards/rejected": -0.4032861292362213,
      "step": 390
    },
    {
      "epoch": 0.03512392158584506,
      "grad_norm": 6.988142490386963,
      "learning_rate": 4.433333333333334e-06,
      "logits/chosen": 3.9105467796325684,
      "logits/rejected": 3.931438446044922,
      "logps/chosen": -0.9674631357192993,
      "logps/rejected": -1.5149281024932861,
      "loss": 0.7768,
      "nll_loss": 0.11233203113079071,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.2902389466762543,
      "rewards/margins": 0.1642395257949829,
      "rewards/rejected": -0.45447850227355957,
      "step": 400
    },
    {
      "epoch": 0.03600201962549118,
      "grad_norm": 0.2611992359161377,
      "learning_rate": 4.544444444444445e-06,
      "logits/chosen": 3.6991469860076904,
      "logits/rejected": 3.726545810699463,
      "logps/chosen": -0.5881733894348145,
      "logps/rejected": -1.2820765972137451,
      "loss": 0.6929,
      "nll_loss": 0.06608637422323227,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.17645201086997986,
      "rewards/margins": 0.20817098021507263,
      "rewards/rejected": -0.38462305068969727,
      "step": 410
    },
    {
      "epoch": 0.03688011766513731,
      "grad_norm": 3.019066095352173,
      "learning_rate": 4.655555555555556e-06,
      "logits/chosen": 3.4919254779815674,
      "logits/rejected": 3.5151939392089844,
      "logps/chosen": -0.5710722804069519,
      "logps/rejected": -1.1787471771240234,
      "loss": 0.712,
      "nll_loss": 0.05282425880432129,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17132170498371124,
      "rewards/margins": 0.18230250477790833,
      "rewards/rejected": -0.353624165058136,
      "step": 420
    },
    {
      "epoch": 0.03775821570478344,
      "grad_norm": 2.8214099407196045,
      "learning_rate": 4.766666666666667e-06,
      "logits/chosen": 3.7750792503356934,
      "logits/rejected": 3.7456068992614746,
      "logps/chosen": -0.7161394357681274,
      "logps/rejected": -1.8331083059310913,
      "loss": 0.6897,
      "nll_loss": 0.09056379646062851,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.2148418426513672,
      "rewards/margins": 0.335090696811676,
      "rewards/rejected": -0.5499325394630432,
      "step": 430
    },
    {
      "epoch": 0.038636313744429565,
      "grad_norm": 5.72930383682251,
      "learning_rate": 4.877777777777778e-06,
      "logits/chosen": 3.5365283489227295,
      "logits/rejected": 3.5479636192321777,
      "logps/chosen": -0.7414464950561523,
      "logps/rejected": -1.5791943073272705,
      "loss": 0.6933,
      "nll_loss": 0.0765593945980072,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.22243395447731018,
      "rewards/margins": 0.2513243556022644,
      "rewards/rejected": -0.4737583100795746,
      "step": 440
    },
    {
      "epoch": 0.03951441178407569,
      "grad_norm": 2.2949461936950684,
      "learning_rate": 4.988888888888889e-06,
      "logits/chosen": 3.2915852069854736,
      "logits/rejected": 3.309730052947998,
      "logps/chosen": -0.5450859069824219,
      "logps/rejected": -0.948569118976593,
      "loss": 0.7334,
      "nll_loss": 0.0632125660777092,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1635257601737976,
      "rewards/margins": 0.12104494869709015,
      "rewards/rejected": -0.28457072377204895,
      "step": 450
    },
    {
      "epoch": 0.04039250982372182,
      "grad_norm": 7.028234481811523,
      "learning_rate": 5.1e-06,
      "logits/chosen": 3.5347137451171875,
      "logits/rejected": 3.542628049850464,
      "logps/chosen": -0.6212174296379089,
      "logps/rejected": -1.3661630153656006,
      "loss": 0.7226,
      "nll_loss": 0.08348599821329117,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.18636523187160492,
      "rewards/margins": 0.2234836369752884,
      "rewards/rejected": -0.4098488688468933,
      "step": 460
    },
    {
      "epoch": 0.04127060786336795,
      "grad_norm": 5.250203609466553,
      "learning_rate": 5.211111111111111e-06,
      "logits/chosen": 3.496631622314453,
      "logits/rejected": 3.5331413745880127,
      "logps/chosen": -0.6912875175476074,
      "logps/rejected": -1.6311848163604736,
      "loss": 0.6796,
      "nll_loss": 0.062180064618587494,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.20738628506660461,
      "rewards/margins": 0.2819691598415375,
      "rewards/rejected": -0.4893553853034973,
      "step": 470
    },
    {
      "epoch": 0.042148705903014075,
      "grad_norm": 2.3006033897399902,
      "learning_rate": 5.322222222222223e-06,
      "logits/chosen": 3.731518268585205,
      "logits/rejected": 3.775359630584717,
      "logps/chosen": -0.8064204454421997,
      "logps/rejected": -1.6423746347427368,
      "loss": 0.7282,
      "nll_loss": 0.09897418320178986,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.2419261485338211,
      "rewards/margins": 0.25078627467155457,
      "rewards/rejected": -0.49271243810653687,
      "step": 480
    },
    {
      "epoch": 0.043026803942660195,
      "grad_norm": 6.259355545043945,
      "learning_rate": 5.4333333333333335e-06,
      "logits/chosen": 3.3239219188690186,
      "logits/rejected": 3.3287899494171143,
      "logps/chosen": -0.623904824256897,
      "logps/rejected": -1.5192670822143555,
      "loss": 0.7155,
      "nll_loss": 0.07001027464866638,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.18717142939567566,
      "rewards/margins": 0.2686087191104889,
      "rewards/rejected": -0.4557802081108093,
      "step": 490
    },
    {
      "epoch": 0.04390490198230632,
      "grad_norm": 4.574368000030518,
      "learning_rate": 5.544444444444445e-06,
      "logits/chosen": 3.667168378829956,
      "logits/rejected": 3.707645893096924,
      "logps/chosen": -0.7253153920173645,
      "logps/rejected": -1.8622252941131592,
      "loss": 0.6561,
      "nll_loss": 0.06381665915250778,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.21759465336799622,
      "rewards/margins": 0.34107303619384766,
      "rewards/rejected": -0.5586676597595215,
      "step": 500
    },
    {
      "epoch": 0.04478300002195245,
      "grad_norm": 36.264381408691406,
      "learning_rate": 5.6555555555555566e-06,
      "logits/chosen": 3.398568630218506,
      "logits/rejected": 3.468022108078003,
      "logps/chosen": -0.5565214157104492,
      "logps/rejected": -1.1638367176055908,
      "loss": 0.7078,
      "nll_loss": 0.06747711449861526,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.16695642471313477,
      "rewards/margins": 0.18219462037086487,
      "rewards/rejected": -0.34915101528167725,
      "step": 510
    },
    {
      "epoch": 0.04566109806159858,
      "grad_norm": 4.252997398376465,
      "learning_rate": 5.766666666666667e-06,
      "logits/chosen": 3.5346503257751465,
      "logits/rejected": 3.526895046234131,
      "logps/chosen": -0.9578359723091125,
      "logps/rejected": -1.617163896560669,
      "loss": 0.7567,
      "nll_loss": 0.11074657738208771,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.2873508036136627,
      "rewards/margins": 0.19779837131500244,
      "rewards/rejected": -0.4851491451263428,
      "step": 520
    },
    {
      "epoch": 0.046539196101244705,
      "grad_norm": 10.661053657531738,
      "learning_rate": 5.877777777777778e-06,
      "logits/chosen": 3.5897374153137207,
      "logits/rejected": 3.599020481109619,
      "logps/chosen": -0.8656774759292603,
      "logps/rejected": -1.2782808542251587,
      "loss": 0.7624,
      "nll_loss": 0.09282848984003067,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.25970324873924255,
      "rewards/margins": 0.12378102540969849,
      "rewards/rejected": -0.38348424434661865,
      "step": 530
    },
    {
      "epoch": 0.04741729414089083,
      "grad_norm": 2.3466947078704834,
      "learning_rate": 5.98888888888889e-06,
      "logits/chosen": 3.41766619682312,
      "logits/rejected": 3.4891743659973145,
      "logps/chosen": -0.69093918800354,
      "logps/rejected": -2.0744166374206543,
      "loss": 0.6747,
      "nll_loss": 0.0662418007850647,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.20728178322315216,
      "rewards/margins": 0.41504326462745667,
      "rewards/rejected": -0.6223250031471252,
      "step": 540
    },
    {
      "epoch": 0.04829539218053696,
      "grad_norm": 4.315151214599609,
      "learning_rate": 6.1e-06,
      "logits/chosen": 3.4109902381896973,
      "logits/rejected": 3.5040442943573,
      "logps/chosen": -0.5460541248321533,
      "logps/rejected": -1.9011294841766357,
      "loss": 0.653,
      "nll_loss": 0.06273610144853592,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.16381624341011047,
      "rewards/margins": 0.40652260184288025,
      "rewards/rejected": -0.5703388452529907,
      "step": 550
    },
    {
      "epoch": 0.04917349022018308,
      "grad_norm": 4.520711898803711,
      "learning_rate": 6.211111111111111e-06,
      "logits/chosen": 3.562473773956299,
      "logits/rejected": 3.6386642456054688,
      "logps/chosen": -0.7312324047088623,
      "logps/rejected": -1.6271352767944336,
      "loss": 0.6966,
      "nll_loss": 0.06128234788775444,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.21936972439289093,
      "rewards/margins": 0.26877090334892273,
      "rewards/rejected": -0.48814067244529724,
      "step": 560
    },
    {
      "epoch": 0.05005158825982921,
      "grad_norm": 7.277171611785889,
      "learning_rate": 6.322222222222223e-06,
      "logits/chosen": 3.472989559173584,
      "logits/rejected": 3.443589687347412,
      "logps/chosen": -0.7062110900878906,
      "logps/rejected": -2.1304116249084473,
      "loss": 0.6869,
      "nll_loss": 0.09942348301410675,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21186332404613495,
      "rewards/margins": 0.42726022005081177,
      "rewards/rejected": -0.6391235589981079,
      "step": 570
    },
    {
      "epoch": 0.050929686299475335,
      "grad_norm": 7.543278217315674,
      "learning_rate": 6.433333333333333e-06,
      "logits/chosen": 3.464301347732544,
      "logits/rejected": 3.444230556488037,
      "logps/chosen": -0.7348255515098572,
      "logps/rejected": -1.602164626121521,
      "loss": 0.7316,
      "nll_loss": 0.08772562444210052,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22044768929481506,
      "rewards/margins": 0.26020172238349915,
      "rewards/rejected": -0.4806493818759918,
      "step": 580
    },
    {
      "epoch": 0.05180778433912146,
      "grad_norm": 3.7779767513275146,
      "learning_rate": 6.544444444444445e-06,
      "logits/chosen": 3.633018970489502,
      "logits/rejected": 3.709826946258545,
      "logps/chosen": -0.9298914074897766,
      "logps/rejected": -1.606702446937561,
      "loss": 0.7953,
      "nll_loss": 0.1196284145116806,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.2789674401283264,
      "rewards/margins": 0.20304329693317413,
      "rewards/rejected": -0.48201069235801697,
      "step": 590
    },
    {
      "epoch": 0.05268588237876759,
      "grad_norm": 7.008880615234375,
      "learning_rate": 6.655555555555556e-06,
      "logits/chosen": 3.381080150604248,
      "logits/rejected": 3.444829225540161,
      "logps/chosen": -0.7211336493492126,
      "logps/rejected": -1.423513650894165,
      "loss": 0.7429,
      "nll_loss": 0.06740613281726837,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.216340109705925,
      "rewards/margins": 0.21071402728557587,
      "rewards/rejected": -0.42705410718917847,
      "step": 600
    },
    {
      "epoch": 0.05356398041841372,
      "grad_norm": 5.524548530578613,
      "learning_rate": 6.7666666666666665e-06,
      "logits/chosen": 3.4077486991882324,
      "logits/rejected": 3.377532958984375,
      "logps/chosen": -0.5095429420471191,
      "logps/rejected": -1.0147814750671387,
      "loss": 0.7157,
      "nll_loss": 0.0589555986225605,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.15286290645599365,
      "rewards/margins": 0.1515716016292572,
      "rewards/rejected": -0.30443447828292847,
      "step": 610
    },
    {
      "epoch": 0.054442078458059845,
      "grad_norm": 4.042827606201172,
      "learning_rate": 6.8777777777777785e-06,
      "logits/chosen": 3.716031551361084,
      "logits/rejected": 3.713074207305908,
      "logps/chosen": -0.8152651786804199,
      "logps/rejected": -1.5531432628631592,
      "loss": 0.7318,
      "nll_loss": 0.08608300983905792,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.24457958340644836,
      "rewards/margins": 0.22136345505714417,
      "rewards/rejected": -0.46594300866127014,
      "step": 620
    },
    {
      "epoch": 0.05532017649770597,
      "grad_norm": 4.254418849945068,
      "learning_rate": 6.9888888888888895e-06,
      "logits/chosen": 3.4341864585876465,
      "logits/rejected": 3.458519458770752,
      "logps/chosen": -0.6925168037414551,
      "logps/rejected": -1.5713815689086914,
      "loss": 0.6898,
      "nll_loss": 0.07025544345378876,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.20775504410266876,
      "rewards/margins": 0.26365941762924194,
      "rewards/rejected": -0.4714145064353943,
      "step": 630
    },
    {
      "epoch": 0.05619827453735209,
      "grad_norm": 2.973627805709839,
      "learning_rate": 7.100000000000001e-06,
      "logits/chosen": 3.4850242137908936,
      "logits/rejected": 3.5192267894744873,
      "logps/chosen": -1.1339752674102783,
      "logps/rejected": -1.5586907863616943,
      "loss": 0.8059,
      "nll_loss": 0.10102218389511108,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.34019264578819275,
      "rewards/margins": 0.1274145543575287,
      "rewards/rejected": -0.46760720014572144,
      "step": 640
    },
    {
      "epoch": 0.05707637257699822,
      "grad_norm": 3.3906142711639404,
      "learning_rate": 7.211111111111112e-06,
      "logits/chosen": 3.6287121772766113,
      "logits/rejected": 3.5881965160369873,
      "logps/chosen": -0.7608178853988647,
      "logps/rejected": -1.2269501686096191,
      "loss": 0.7394,
      "nll_loss": 0.0750693827867508,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.228245347738266,
      "rewards/margins": 0.13983972370624542,
      "rewards/rejected": -0.3680850863456726,
      "step": 650
    },
    {
      "epoch": 0.05795447061664435,
      "grad_norm": 0.03068475052714348,
      "learning_rate": 7.322222222222223e-06,
      "logits/chosen": 3.743140697479248,
      "logits/rejected": 3.7635676860809326,
      "logps/chosen": -0.5632290840148926,
      "logps/rejected": -1.5820039510726929,
      "loss": 0.6807,
      "nll_loss": 0.07005371153354645,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.16896870732307434,
      "rewards/margins": 0.30563241243362427,
      "rewards/rejected": -0.47460120916366577,
      "step": 660
    },
    {
      "epoch": 0.058832568656290475,
      "grad_norm": 2.2352776527404785,
      "learning_rate": 7.433333333333334e-06,
      "logits/chosen": 3.5528149604797363,
      "logits/rejected": 3.5446677207946777,
      "logps/chosen": -0.5461568832397461,
      "logps/rejected": -1.2885067462921143,
      "loss": 0.6956,
      "nll_loss": 0.06532245129346848,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16384705901145935,
      "rewards/margins": 0.22270497679710388,
      "rewards/rejected": -0.3865520656108856,
      "step": 670
    },
    {
      "epoch": 0.0597106666959366,
      "grad_norm": 2.6797573566436768,
      "learning_rate": 7.544444444444445e-06,
      "logits/chosen": 3.3902244567871094,
      "logits/rejected": 3.4180169105529785,
      "logps/chosen": -0.6432263255119324,
      "logps/rejected": -1.1323941946029663,
      "loss": 0.7574,
      "nll_loss": 0.07451333105564117,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.19296793639659882,
      "rewards/margins": 0.1467503160238266,
      "rewards/rejected": -0.3397182822227478,
      "step": 680
    },
    {
      "epoch": 0.06058876473558273,
      "grad_norm": 5.877313137054443,
      "learning_rate": 7.655555555555556e-06,
      "logits/chosen": 3.4120171070098877,
      "logits/rejected": 3.4442646503448486,
      "logps/chosen": -0.7911199331283569,
      "logps/rejected": -1.4578664302825928,
      "loss": 0.7751,
      "nll_loss": 0.1019410640001297,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.23733600974082947,
      "rewards/margins": 0.20002400875091553,
      "rewards/rejected": -0.4373599886894226,
      "step": 690
    },
    {
      "epoch": 0.06146686277522886,
      "grad_norm": 3.2915701866149902,
      "learning_rate": 7.766666666666666e-06,
      "logits/chosen": 3.5103302001953125,
      "logits/rejected": 3.4995181560516357,
      "logps/chosen": -0.6160825490951538,
      "logps/rejected": -1.1019346714019775,
      "loss": 0.7363,
      "nll_loss": 0.06433330476284027,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.18482479453086853,
      "rewards/margins": 0.14575564861297607,
      "rewards/rejected": -0.3305804133415222,
      "step": 700
    },
    {
      "epoch": 0.06234496081487498,
      "grad_norm": 3.7106151580810547,
      "learning_rate": 7.877777777777778e-06,
      "logits/chosen": 3.474386692047119,
      "logits/rejected": 3.4553630352020264,
      "logps/chosen": -0.6972242593765259,
      "logps/rejected": -1.5925921201705933,
      "loss": 0.7109,
      "nll_loss": 0.07773466408252716,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.20916728675365448,
      "rewards/margins": 0.2686103284358978,
      "rewards/rejected": -0.4777776300907135,
      "step": 710
    },
    {
      "epoch": 0.0632230588545211,
      "grad_norm": 1.7685184478759766,
      "learning_rate": 7.98888888888889e-06,
      "logits/chosen": 3.6326797008514404,
      "logits/rejected": 3.625549793243408,
      "logps/chosen": -0.5006519556045532,
      "logps/rejected": -1.8575595617294312,
      "loss": 0.6416,
      "nll_loss": 0.05640099197626114,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.15019558370113373,
      "rewards/margins": 0.4070723056793213,
      "rewards/rejected": -0.5572679042816162,
      "step": 720
    },
    {
      "epoch": 0.06410115689416723,
      "grad_norm": 4.595531463623047,
      "learning_rate": 8.1e-06,
      "logits/chosen": 3.6036553382873535,
      "logits/rejected": 3.6813888549804688,
      "logps/chosen": -1.0952359437942505,
      "logps/rejected": -2.0565478801727295,
      "loss": 0.7318,
      "nll_loss": 0.06606093794107437,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.32857078313827515,
      "rewards/margins": 0.28839364647865295,
      "rewards/rejected": -0.6169643998146057,
      "step": 730
    },
    {
      "epoch": 0.06497925493381336,
      "grad_norm": 2.7277488708496094,
      "learning_rate": 8.211111111111112e-06,
      "logits/chosen": 3.4828929901123047,
      "logits/rejected": 3.536961317062378,
      "logps/chosen": -0.6164706945419312,
      "logps/rejected": -2.1122829914093018,
      "loss": 0.6062,
      "nll_loss": 0.040962688624858856,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.18494121730327606,
      "rewards/margins": 0.44874364137649536,
      "rewards/rejected": -0.6336848735809326,
      "step": 740
    },
    {
      "epoch": 0.06585735297345949,
      "grad_norm": 3.6677157878875732,
      "learning_rate": 8.322222222222223e-06,
      "logits/chosen": 3.361325740814209,
      "logits/rejected": 3.362967014312744,
      "logps/chosen": -0.9366201162338257,
      "logps/rejected": -1.7140756845474243,
      "loss": 0.7507,
      "nll_loss": 0.10440067946910858,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.28098607063293457,
      "rewards/margins": 0.23323671519756317,
      "rewards/rejected": -0.5142227411270142,
      "step": 750
    },
    {
      "epoch": 0.06673545101310562,
      "grad_norm": 3.729750633239746,
      "learning_rate": 8.433333333333334e-06,
      "logits/chosen": 3.343749523162842,
      "logits/rejected": 3.401230573654175,
      "logps/chosen": -0.7000004649162292,
      "logps/rejected": -1.9269546270370483,
      "loss": 0.6931,
      "nll_loss": 0.07972760498523712,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.2100001573562622,
      "rewards/margins": 0.3680862486362457,
      "rewards/rejected": -0.5780864357948303,
      "step": 760
    },
    {
      "epoch": 0.06761354905275174,
      "grad_norm": 7.166464328765869,
      "learning_rate": 8.544444444444445e-06,
      "logits/chosen": 3.4859509468078613,
      "logits/rejected": 3.587602138519287,
      "logps/chosen": -0.7638369798660278,
      "logps/rejected": -2.2576065063476562,
      "loss": 0.6832,
      "nll_loss": 0.0738845020532608,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.22915109992027283,
      "rewards/margins": 0.4481307864189148,
      "rewards/rejected": -0.6772819757461548,
      "step": 770
    },
    {
      "epoch": 0.06849164709239787,
      "grad_norm": 2.416330575942993,
      "learning_rate": 8.655555555555557e-06,
      "logits/chosen": 3.408698558807373,
      "logits/rejected": 3.4245800971984863,
      "logps/chosen": -0.8418534398078918,
      "logps/rejected": -1.4894107580184937,
      "loss": 0.757,
      "nll_loss": 0.0903228372335434,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.25255605578422546,
      "rewards/margins": 0.1942671835422516,
      "rewards/rejected": -0.44682326912879944,
      "step": 780
    },
    {
      "epoch": 0.069369745132044,
      "grad_norm": 4.281314373016357,
      "learning_rate": 8.766666666666669e-06,
      "logits/chosen": 3.2489428520202637,
      "logits/rejected": 3.2514851093292236,
      "logps/chosen": -0.8190711736679077,
      "logps/rejected": -1.3487728834152222,
      "loss": 0.7399,
      "nll_loss": 0.07894166558980942,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.24572138488292694,
      "rewards/margins": 0.15891052782535553,
      "rewards/rejected": -0.40463191270828247,
      "step": 790
    },
    {
      "epoch": 0.07024784317169012,
      "grad_norm": 1.3228946924209595,
      "learning_rate": 8.877777777777779e-06,
      "logits/chosen": 3.2964024543762207,
      "logits/rejected": 3.3055152893066406,
      "logps/chosen": -0.8143989443778992,
      "logps/rejected": -1.2753360271453857,
      "loss": 0.75,
      "nll_loss": 0.07704529166221619,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.24431967735290527,
      "rewards/margins": 0.13828110694885254,
      "rewards/rejected": -0.3826007843017578,
      "step": 800
    },
    {
      "epoch": 0.07112594121133625,
      "grad_norm": 3.7781331539154053,
      "learning_rate": 8.988888888888889e-06,
      "logits/chosen": 3.68397855758667,
      "logits/rejected": 3.6694297790527344,
      "logps/chosen": -0.9365280866622925,
      "logps/rejected": -1.7311958074569702,
      "loss": 0.7448,
      "nll_loss": 0.08974708616733551,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.28095847368240356,
      "rewards/margins": 0.23840029537677765,
      "rewards/rejected": -0.51935875415802,
      "step": 810
    },
    {
      "epoch": 0.07200403925098237,
      "grad_norm": 2.7228267192840576,
      "learning_rate": 9.100000000000001e-06,
      "logits/chosen": 3.6748175621032715,
      "logits/rejected": 3.7467494010925293,
      "logps/chosen": -0.5383256077766418,
      "logps/rejected": -2.2832655906677246,
      "loss": 0.6081,
      "nll_loss": 0.06415946036577225,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.16149768233299255,
      "rewards/margins": 0.523482084274292,
      "rewards/rejected": -0.6849797964096069,
      "step": 820
    },
    {
      "epoch": 0.0728821372906285,
      "grad_norm": 1.3846280574798584,
      "learning_rate": 9.211111111111111e-06,
      "logits/chosen": 3.2965283393859863,
      "logits/rejected": 3.329348087310791,
      "logps/chosen": -0.5321189761161804,
      "logps/rejected": -2.2239809036254883,
      "loss": 0.6186,
      "nll_loss": 0.04651743918657303,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15963570773601532,
      "rewards/margins": 0.5075585842132568,
      "rewards/rejected": -0.6671942472457886,
      "step": 830
    },
    {
      "epoch": 0.07376023533027462,
      "grad_norm": 4.5536675453186035,
      "learning_rate": 9.322222222222223e-06,
      "logits/chosen": 3.347224473953247,
      "logits/rejected": 3.3436226844787598,
      "logps/chosen": -0.7508156895637512,
      "logps/rejected": -1.867352843284607,
      "loss": 0.707,
      "nll_loss": 0.0711875781416893,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.2252446860074997,
      "rewards/margins": 0.3349612355232239,
      "rewards/rejected": -0.56020587682724,
      "step": 840
    },
    {
      "epoch": 0.07463833336992075,
      "grad_norm": 4.691596508026123,
      "learning_rate": 9.433333333333335e-06,
      "logits/chosen": 3.253293991088867,
      "logits/rejected": 3.2947421073913574,
      "logps/chosen": -0.9249482154846191,
      "logps/rejected": -1.812909483909607,
      "loss": 0.7568,
      "nll_loss": 0.09252104163169861,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.2774844765663147,
      "rewards/margins": 0.2663884162902832,
      "rewards/rejected": -0.5438728928565979,
      "step": 850
    },
    {
      "epoch": 0.07551643140956688,
      "grad_norm": 4.119868278503418,
      "learning_rate": 9.544444444444445e-06,
      "logits/chosen": 3.3257553577423096,
      "logits/rejected": 3.3328521251678467,
      "logps/chosen": -0.8130607604980469,
      "logps/rejected": -2.2179336547851562,
      "loss": 0.7061,
      "nll_loss": 0.09296734631061554,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.2439182549715042,
      "rewards/margins": 0.42146188020706177,
      "rewards/rejected": -0.6653801202774048,
      "step": 860
    },
    {
      "epoch": 0.076394529449213,
      "grad_norm": 4.941491603851318,
      "learning_rate": 9.655555555555556e-06,
      "logits/chosen": 3.2392685413360596,
      "logits/rejected": 3.2623963356018066,
      "logps/chosen": -0.7361981272697449,
      "logps/rejected": -1.6700445413589478,
      "loss": 0.7312,
      "nll_loss": 0.08274148404598236,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.22085945308208466,
      "rewards/margins": 0.28015393018722534,
      "rewards/rejected": -0.5010133385658264,
      "step": 870
    },
    {
      "epoch": 0.07727262748885913,
      "grad_norm": 1.4032851457595825,
      "learning_rate": 9.766666666666667e-06,
      "logits/chosen": 3.2956814765930176,
      "logits/rejected": 3.318169355392456,
      "logps/chosen": -0.7056166529655457,
      "logps/rejected": -1.8036190271377563,
      "loss": 0.7136,
      "nll_loss": 0.09712977707386017,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.21168498694896698,
      "rewards/margins": 0.3294007182121277,
      "rewards/rejected": -0.5410857200622559,
      "step": 880
    },
    {
      "epoch": 0.07815072552850526,
      "grad_norm": 3.9460878372192383,
      "learning_rate": 9.877777777777778e-06,
      "logits/chosen": 3.4804458618164062,
      "logits/rejected": 3.5131962299346924,
      "logps/chosen": -0.9113373756408691,
      "logps/rejected": -1.8960120677947998,
      "loss": 0.7272,
      "nll_loss": 0.08563139289617538,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.2734012007713318,
      "rewards/margins": 0.295402467250824,
      "rewards/rejected": -0.5688036680221558,
      "step": 890
    },
    {
      "epoch": 0.07902882356815139,
      "grad_norm": 5.001852989196777,
      "learning_rate": 9.98888888888889e-06,
      "logits/chosen": 3.5693771839141846,
      "logits/rejected": 3.613219738006592,
      "logps/chosen": -0.7482819557189941,
      "logps/rejected": -1.611090898513794,
      "loss": 0.7488,
      "nll_loss": 0.10075131803750992,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.2244846075773239,
      "rewards/margins": 0.25884273648262024,
      "rewards/rejected": -0.48332732915878296,
      "step": 900
    },
    {
      "epoch": 0.07990692160779751,
      "grad_norm": 3.9788780212402344,
      "learning_rate": 9.98888888888889e-06,
      "logits/chosen": 3.418684720993042,
      "logits/rejected": 3.4473800659179688,
      "logps/chosen": -0.5167075991630554,
      "logps/rejected": -1.0832570791244507,
      "loss": 0.7187,
      "nll_loss": 0.053985703736543655,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.155012309551239,
      "rewards/margins": 0.16996484994888306,
      "rewards/rejected": -0.3249771296977997,
      "step": 910
    },
    {
      "epoch": 0.08078501964744364,
      "grad_norm": 1.5054136514663696,
      "learning_rate": 9.976543209876544e-06,
      "logits/chosen": 3.389498233795166,
      "logits/rejected": 3.4252688884735107,
      "logps/chosen": -0.6502631902694702,
      "logps/rejected": -1.8272225856781006,
      "loss": 0.6756,
      "nll_loss": 0.07306591421365738,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1950789839029312,
      "rewards/margins": 0.35308781266212463,
      "rewards/rejected": -0.5481668710708618,
      "step": 920
    },
    {
      "epoch": 0.08166311768708977,
      "grad_norm": 2.810540199279785,
      "learning_rate": 9.964197530864198e-06,
      "logits/chosen": 3.4912326335906982,
      "logits/rejected": 3.503628969192505,
      "logps/chosen": -0.4884684681892395,
      "logps/rejected": -1.4588502645492554,
      "loss": 0.6613,
      "nll_loss": 0.04402286559343338,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.14654052257537842,
      "rewards/margins": 0.29111456871032715,
      "rewards/rejected": -0.4376550614833832,
      "step": 930
    },
    {
      "epoch": 0.0825412157267359,
      "grad_norm": 5.386466979980469,
      "learning_rate": 9.951851851851853e-06,
      "logits/chosen": 3.386685848236084,
      "logits/rejected": 3.4002914428710938,
      "logps/chosen": -0.5482162237167358,
      "logps/rejected": -1.572486162185669,
      "loss": 0.6616,
      "nll_loss": 0.053943734616041183,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.16446486115455627,
      "rewards/margins": 0.307280957698822,
      "rewards/rejected": -0.4717458188533783,
      "step": 940
    },
    {
      "epoch": 0.08341931376638202,
      "grad_norm": 1.676483154296875,
      "learning_rate": 9.939506172839507e-06,
      "logits/chosen": 3.3065590858459473,
      "logits/rejected": 3.3416500091552734,
      "logps/chosen": -0.5273550748825073,
      "logps/rejected": -1.707932472229004,
      "loss": 0.6729,
      "nll_loss": 0.06074246019124985,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.1582065373659134,
      "rewards/margins": 0.3541732430458069,
      "rewards/rejected": -0.5123798251152039,
      "step": 950
    },
    {
      "epoch": 0.08429741180602815,
      "grad_norm": 2.689913272857666,
      "learning_rate": 9.927160493827162e-06,
      "logits/chosen": 3.2740864753723145,
      "logits/rejected": 3.335360050201416,
      "logps/chosen": -0.7466616630554199,
      "logps/rejected": -2.0391454696655273,
      "loss": 0.7055,
      "nll_loss": 0.09172563254833221,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2239985167980194,
      "rewards/margins": 0.3877451419830322,
      "rewards/rejected": -0.6117436289787292,
      "step": 960
    },
    {
      "epoch": 0.08517550984567426,
      "grad_norm": 3.589853286743164,
      "learning_rate": 9.914814814814816e-06,
      "logits/chosen": 3.0097994804382324,
      "logits/rejected": 3.0853917598724365,
      "logps/chosen": -0.5722948312759399,
      "logps/rejected": -1.9204362630844116,
      "loss": 0.6501,
      "nll_loss": 0.051983099430799484,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.17168846726417542,
      "rewards/margins": 0.4044424593448639,
      "rewards/rejected": -0.5761309266090393,
      "step": 970
    },
    {
      "epoch": 0.08605360788532039,
      "grad_norm": 1.7398462295532227,
      "learning_rate": 9.90246913580247e-06,
      "logits/chosen": 3.3110098838806152,
      "logits/rejected": 3.4121768474578857,
      "logps/chosen": -0.6699460744857788,
      "logps/rejected": -2.0870394706726074,
      "loss": 0.6724,
      "nll_loss": 0.053984154015779495,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20098385214805603,
      "rewards/margins": 0.4251279830932617,
      "rewards/rejected": -0.6261118054389954,
      "step": 980
    },
    {
      "epoch": 0.08693170592496652,
      "grad_norm": 5.3601861000061035,
      "learning_rate": 9.890123456790123e-06,
      "logits/chosen": 3.1401820182800293,
      "logits/rejected": 3.127436399459839,
      "logps/chosen": -0.7694223523139954,
      "logps/rejected": -1.6481168270111084,
      "loss": 0.737,
      "nll_loss": 0.09365083277225494,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.230826735496521,
      "rewards/margins": 0.2636083662509918,
      "rewards/rejected": -0.49443507194519043,
      "step": 990
    },
    {
      "epoch": 0.08780980396461265,
      "grad_norm": 2.6405630111694336,
      "learning_rate": 9.877777777777778e-06,
      "logits/chosen": 3.1894805431365967,
      "logits/rejected": 3.2360007762908936,
      "logps/chosen": -0.5739088654518127,
      "logps/rejected": -2.0212159156799316,
      "loss": 0.6616,
      "nll_loss": 0.05945644527673721,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1721726506948471,
      "rewards/margins": 0.4341921806335449,
      "rewards/rejected": -0.606364905834198,
      "step": 1000
    },
    {
      "epoch": 0.08868790200425877,
      "grad_norm": 2.2846076488494873,
      "learning_rate": 9.865432098765432e-06,
      "logits/chosen": 3.2782859802246094,
      "logits/rejected": 3.2814033031463623,
      "logps/chosen": -0.5302027463912964,
      "logps/rejected": -1.8145701885223389,
      "loss": 0.6543,
      "nll_loss": 0.05663750320672989,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.15906082093715668,
      "rewards/margins": 0.3853102922439575,
      "rewards/rejected": -0.5443711280822754,
      "step": 1010
    },
    {
      "epoch": 0.0895660000439049,
      "grad_norm": 13.04986572265625,
      "learning_rate": 9.853086419753087e-06,
      "logits/chosen": 2.8904287815093994,
      "logits/rejected": 2.907032012939453,
      "logps/chosen": -1.1254570484161377,
      "logps/rejected": -2.3068795204162598,
      "loss": 0.7832,
      "nll_loss": 0.1419171392917633,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.33763712644577026,
      "rewards/margins": 0.354426771402359,
      "rewards/rejected": -0.6920639276504517,
      "step": 1020
    },
    {
      "epoch": 0.09044409808355103,
      "grad_norm": 2.556560754776001,
      "learning_rate": 9.840740740740743e-06,
      "logits/chosen": 3.151669502258301,
      "logits/rejected": 3.1690382957458496,
      "logps/chosen": -0.7756383419036865,
      "logps/rejected": -1.6224708557128906,
      "loss": 0.7551,
      "nll_loss": 0.09495635330677032,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.23269149661064148,
      "rewards/margins": 0.25404977798461914,
      "rewards/rejected": -0.486741304397583,
      "step": 1030
    },
    {
      "epoch": 0.09132219612319716,
      "grad_norm": 1.3183997869491577,
      "learning_rate": 9.828395061728397e-06,
      "logits/chosen": 3.195861339569092,
      "logits/rejected": 3.2810165882110596,
      "logps/chosen": -0.5329464077949524,
      "logps/rejected": -1.2915513515472412,
      "loss": 0.7152,
      "nll_loss": 0.054315369576215744,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.15988394618034363,
      "rewards/margins": 0.22758150100708008,
      "rewards/rejected": -0.3874654471874237,
      "step": 1040
    },
    {
      "epoch": 0.09220029416284328,
      "grad_norm": 1.674479603767395,
      "learning_rate": 9.81604938271605e-06,
      "logits/chosen": 3.139688491821289,
      "logits/rejected": 3.2257683277130127,
      "logps/chosen": -0.7113819122314453,
      "logps/rejected": -2.1473631858825684,
      "loss": 0.679,
      "nll_loss": 0.04762103408575058,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21341457962989807,
      "rewards/margins": 0.4307943284511566,
      "rewards/rejected": -0.6442088484764099,
      "step": 1050
    },
    {
      "epoch": 0.09307839220248941,
      "grad_norm": 1.9272900819778442,
      "learning_rate": 9.803703703703704e-06,
      "logits/chosen": 3.403465986251831,
      "logits/rejected": 3.385577440261841,
      "logps/chosen": -0.9791383743286133,
      "logps/rejected": -2.0091757774353027,
      "loss": 0.7499,
      "nll_loss": 0.11029829829931259,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.29374146461486816,
      "rewards/margins": 0.309011310338974,
      "rewards/rejected": -0.6027528047561646,
      "step": 1060
    },
    {
      "epoch": 0.09395649024213554,
      "grad_norm": 3.530672073364258,
      "learning_rate": 9.791358024691359e-06,
      "logits/chosen": 2.975001573562622,
      "logits/rejected": 3.063398838043213,
      "logps/chosen": -0.9404687881469727,
      "logps/rejected": -1.803180456161499,
      "loss": 0.7547,
      "nll_loss": 0.079728864133358,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.28214067220687866,
      "rewards/margins": 0.2588135600090027,
      "rewards/rejected": -0.5409542322158813,
      "step": 1070
    },
    {
      "epoch": 0.09483458828178166,
      "grad_norm": 7.528195381164551,
      "learning_rate": 9.779012345679013e-06,
      "logits/chosen": 3.1206297874450684,
      "logits/rejected": 3.173870086669922,
      "logps/chosen": -0.6394155621528625,
      "logps/rejected": -1.1264150142669678,
      "loss": 0.7336,
      "nll_loss": 0.06591827422380447,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.19182467460632324,
      "rewards/margins": 0.14609983563423157,
      "rewards/rejected": -0.3379245400428772,
      "step": 1080
    },
    {
      "epoch": 0.09571268632142779,
      "grad_norm": 2.3711354732513428,
      "learning_rate": 9.766666666666667e-06,
      "logits/chosen": 3.2346444129943848,
      "logits/rejected": 3.3024227619171143,
      "logps/chosen": -0.7532138824462891,
      "logps/rejected": -1.9977819919586182,
      "loss": 0.6926,
      "nll_loss": 0.0733107179403305,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.22596418857574463,
      "rewards/margins": 0.3733704090118408,
      "rewards/rejected": -0.5993345975875854,
      "step": 1090
    },
    {
      "epoch": 0.09659078436107392,
      "grad_norm": 2.3100759983062744,
      "learning_rate": 9.754320987654322e-06,
      "logits/chosen": 3.0819344520568848,
      "logits/rejected": 3.081664562225342,
      "logps/chosen": -0.39054492115974426,
      "logps/rejected": -1.3339643478393555,
      "loss": 0.6571,
      "nll_loss": 0.03647618740797043,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.11716349422931671,
      "rewards/margins": 0.2830258309841156,
      "rewards/rejected": -0.4001893401145935,
      "step": 1100
    },
    {
      "epoch": 0.09746888240072005,
      "grad_norm": 4.387854099273682,
      "learning_rate": 9.741975308641976e-06,
      "logits/chosen": 3.0971732139587402,
      "logits/rejected": 3.105783224105835,
      "logps/chosen": -0.6461865305900574,
      "logps/rejected": -1.5687153339385986,
      "loss": 0.7209,
      "nll_loss": 0.08219017088413239,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.19385597109794617,
      "rewards/margins": 0.2767586410045624,
      "rewards/rejected": -0.47061461210250854,
      "step": 1110
    },
    {
      "epoch": 0.09834698044036616,
      "grad_norm": 1.8574669361114502,
      "learning_rate": 9.72962962962963e-06,
      "logits/chosen": 3.2028121948242188,
      "logits/rejected": 3.188230037689209,
      "logps/chosen": -0.6974012851715088,
      "logps/rejected": -2.043318033218384,
      "loss": 0.6628,
      "nll_loss": 0.06341539323329926,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.20922040939331055,
      "rewards/margins": 0.40377503633499146,
      "rewards/rejected": -0.612995445728302,
      "step": 1120
    },
    {
      "epoch": 0.09922507848001229,
      "grad_norm": 1.4223850965499878,
      "learning_rate": 9.717283950617285e-06,
      "logits/chosen": 3.1057040691375732,
      "logits/rejected": 3.1665711402893066,
      "logps/chosen": -0.4518910348415375,
      "logps/rejected": -1.782041311264038,
      "loss": 0.6529,
      "nll_loss": 0.04685738682746887,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1355672925710678,
      "rewards/margins": 0.39904507994651794,
      "rewards/rejected": -0.5346124172210693,
      "step": 1130
    },
    {
      "epoch": 0.10010317651965842,
      "grad_norm": 0.6512376666069031,
      "learning_rate": 9.70493827160494e-06,
      "logits/chosen": 3.062418222427368,
      "logits/rejected": 3.1085588932037354,
      "logps/chosen": -0.7759238481521606,
      "logps/rejected": -2.237229585647583,
      "loss": 0.6829,
      "nll_loss": 0.08105801045894623,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.2327771931886673,
      "rewards/margins": 0.43839168548583984,
      "rewards/rejected": -0.6711689233779907,
      "step": 1140
    },
    {
      "epoch": 0.10098127455930454,
      "grad_norm": 2.3270082473754883,
      "learning_rate": 9.692592592592594e-06,
      "logits/chosen": 2.8819382190704346,
      "logits/rejected": 2.9401650428771973,
      "logps/chosen": -0.5062090754508972,
      "logps/rejected": -1.868971824645996,
      "loss": 0.6794,
      "nll_loss": 0.05651511624455452,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1518627107143402,
      "rewards/margins": 0.40882882475852966,
      "rewards/rejected": -0.5606915354728699,
      "step": 1150
    },
    {
      "epoch": 0.10185937259895067,
      "grad_norm": 1.2098430395126343,
      "learning_rate": 9.680246913580248e-06,
      "logits/chosen": 2.7454497814178467,
      "logits/rejected": 2.780897617340088,
      "logps/chosen": -0.9837905168533325,
      "logps/rejected": -2.0501391887664795,
      "loss": 0.7538,
      "nll_loss": 0.10150198638439178,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.2951371669769287,
      "rewards/margins": 0.319904625415802,
      "rewards/rejected": -0.6150418519973755,
      "step": 1160
    },
    {
      "epoch": 0.1027374706385968,
      "grad_norm": 2.469829559326172,
      "learning_rate": 9.667901234567903e-06,
      "logits/chosen": 2.8060965538024902,
      "logits/rejected": 2.8710038661956787,
      "logps/chosen": -0.6813799738883972,
      "logps/rejected": -2.1527724266052246,
      "loss": 0.6474,
      "nll_loss": 0.06949031352996826,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2044139802455902,
      "rewards/margins": 0.44141775369644165,
      "rewards/rejected": -0.6458317637443542,
      "step": 1170
    },
    {
      "epoch": 0.10361556867824293,
      "grad_norm": 3.5877394676208496,
      "learning_rate": 9.655555555555556e-06,
      "logits/chosen": 3.1048672199249268,
      "logits/rejected": 3.1411147117614746,
      "logps/chosen": -0.4382111132144928,
      "logps/rejected": -2.12353253364563,
      "loss": 0.624,
      "nll_loss": 0.048050910234451294,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.13146333396434784,
      "rewards/margins": 0.5055964589118958,
      "rewards/rejected": -0.6370598077774048,
      "step": 1180
    },
    {
      "epoch": 0.10449366671788905,
      "grad_norm": 13.380763053894043,
      "learning_rate": 9.64320987654321e-06,
      "logits/chosen": 2.6727805137634277,
      "logits/rejected": 2.7236101627349854,
      "logps/chosen": -0.7069253325462341,
      "logps/rejected": -2.2335355281829834,
      "loss": 0.6386,
      "nll_loss": 0.05090578272938728,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.21207761764526367,
      "rewards/margins": 0.45798301696777344,
      "rewards/rejected": -0.6700606346130371,
      "step": 1190
    },
    {
      "epoch": 0.10537176475753518,
      "grad_norm": 7.120882034301758,
      "learning_rate": 9.630864197530864e-06,
      "logits/chosen": 2.8762526512145996,
      "logits/rejected": 2.901745319366455,
      "logps/chosen": -0.7335812449455261,
      "logps/rejected": -2.8476223945617676,
      "loss": 0.6497,
      "nll_loss": 0.07983629405498505,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2200743705034256,
      "rewards/margins": 0.6342123746871948,
      "rewards/rejected": -0.854286789894104,
      "step": 1200
    },
    {
      "epoch": 0.10624986279718131,
      "grad_norm": 5.2290263175964355,
      "learning_rate": 9.618518518518519e-06,
      "logits/chosen": 3.0229249000549316,
      "logits/rejected": 2.959900379180908,
      "logps/chosen": -1.3016353845596313,
      "logps/rejected": -1.7729085683822632,
      "loss": 0.8886,
      "nll_loss": 0.14201593399047852,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.3904905915260315,
      "rewards/margins": 0.14138197898864746,
      "rewards/rejected": -0.5318726301193237,
      "step": 1210
    },
    {
      "epoch": 0.10712796083682743,
      "grad_norm": 1.350420355796814,
      "learning_rate": 9.606172839506173e-06,
      "logits/chosen": 2.695782423019409,
      "logits/rejected": 2.6982076168060303,
      "logps/chosen": -0.5528481602668762,
      "logps/rejected": -1.7437477111816406,
      "loss": 0.7039,
      "nll_loss": 0.06110968068242073,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.16585442423820496,
      "rewards/margins": 0.3572699725627899,
      "rewards/rejected": -0.5231243371963501,
      "step": 1220
    },
    {
      "epoch": 0.10800605887647356,
      "grad_norm": 4.9116291999816895,
      "learning_rate": 9.593827160493828e-06,
      "logits/chosen": 2.8747756481170654,
      "logits/rejected": 2.797008514404297,
      "logps/chosen": -0.7696909308433533,
      "logps/rejected": -1.8576171398162842,
      "loss": 0.7164,
      "nll_loss": 0.08876083791255951,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.23090729117393494,
      "rewards/margins": 0.3263779282569885,
      "rewards/rejected": -0.5572851896286011,
      "step": 1230
    },
    {
      "epoch": 0.10888415691611969,
      "grad_norm": 3.6960697174072266,
      "learning_rate": 9.581481481481482e-06,
      "logits/chosen": 3.121314287185669,
      "logits/rejected": 3.1768624782562256,
      "logps/chosen": -0.597256064414978,
      "logps/rejected": -2.1361727714538574,
      "loss": 0.6718,
      "nll_loss": 0.06965653598308563,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.17917683720588684,
      "rewards/margins": 0.4616750180721283,
      "rewards/rejected": -0.6408518552780151,
      "step": 1240
    },
    {
      "epoch": 0.10976225495576582,
      "grad_norm": 1.3148126602172852,
      "learning_rate": 9.569135802469136e-06,
      "logits/chosen": 2.9931716918945312,
      "logits/rejected": 3.0196568965911865,
      "logps/chosen": -0.8401254415512085,
      "logps/rejected": -2.2920265197753906,
      "loss": 0.7173,
      "nll_loss": 0.08757736533880234,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.2520376443862915,
      "rewards/margins": 0.4355703294277191,
      "rewards/rejected": -0.6876079440116882,
      "step": 1250
    },
    {
      "epoch": 0.11064035299541194,
      "grad_norm": 3.798100471496582,
      "learning_rate": 9.556790123456791e-06,
      "logits/chosen": 2.9449126720428467,
      "logits/rejected": 2.9711549282073975,
      "logps/chosen": -0.6954627633094788,
      "logps/rejected": -1.5479028224945068,
      "loss": 0.7229,
      "nll_loss": 0.06693422794342041,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.20863886177539825,
      "rewards/margins": 0.2557320296764374,
      "rewards/rejected": -0.4643709063529968,
      "step": 1260
    },
    {
      "epoch": 0.11151845103505806,
      "grad_norm": 2.8658065795898438,
      "learning_rate": 9.544444444444445e-06,
      "logits/chosen": 3.015810966491699,
      "logits/rejected": 3.0757055282592773,
      "logps/chosen": -1.2240221500396729,
      "logps/rejected": -1.8749526739120483,
      "loss": 0.8117,
      "nll_loss": 0.10025894641876221,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.3672066628932953,
      "rewards/margins": 0.1952790915966034,
      "rewards/rejected": -0.5624858140945435,
      "step": 1270
    },
    {
      "epoch": 0.11239654907470419,
      "grad_norm": 2.584207534790039,
      "learning_rate": 9.5320987654321e-06,
      "logits/chosen": 3.0405101776123047,
      "logits/rejected": 3.0836472511291504,
      "logps/chosen": -0.5894891619682312,
      "logps/rejected": -1.7727954387664795,
      "loss": 0.6706,
      "nll_loss": 0.057862233370542526,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.17684674263000488,
      "rewards/margins": 0.35499197244644165,
      "rewards/rejected": -0.5318387150764465,
      "step": 1280
    },
    {
      "epoch": 0.11327464711435031,
      "grad_norm": 0.8380700945854187,
      "learning_rate": 9.519753086419754e-06,
      "logits/chosen": 3.015899896621704,
      "logits/rejected": 2.9938347339630127,
      "logps/chosen": -0.48675793409347534,
      "logps/rejected": -1.6840702295303345,
      "loss": 0.6618,
      "nll_loss": 0.06300728023052216,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.14602738618850708,
      "rewards/margins": 0.35919371247291565,
      "rewards/rejected": -0.5052211880683899,
      "step": 1290
    },
    {
      "epoch": 0.11415274515399644,
      "grad_norm": 1.4230750799179077,
      "learning_rate": 9.507407407407409e-06,
      "logits/chosen": 3.2599899768829346,
      "logits/rejected": 3.2479500770568848,
      "logps/chosen": -0.5513351559638977,
      "logps/rejected": -1.5504658222198486,
      "loss": 0.7042,
      "nll_loss": 0.05995137244462967,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.16540054976940155,
      "rewards/margins": 0.299739271402359,
      "rewards/rejected": -0.46513980627059937,
      "step": 1300
    },
    {
      "epoch": 0.11503084319364257,
      "grad_norm": 1.0455182790756226,
      "learning_rate": 9.495061728395063e-06,
      "logits/chosen": 3.0312843322753906,
      "logits/rejected": 3.068418025970459,
      "logps/chosen": -0.5078593492507935,
      "logps/rejected": -2.3526813983917236,
      "loss": 0.6117,
      "nll_loss": 0.04707217961549759,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.152357816696167,
      "rewards/margins": 0.5534465909004211,
      "rewards/rejected": -0.7058044672012329,
      "step": 1310
    },
    {
      "epoch": 0.1159089412332887,
      "grad_norm": 3.6651833057403564,
      "learning_rate": 9.482716049382716e-06,
      "logits/chosen": 3.071345806121826,
      "logits/rejected": 3.0848429203033447,
      "logps/chosen": -0.6071802973747253,
      "logps/rejected": -1.6902376413345337,
      "loss": 0.6877,
      "nll_loss": 0.06448554247617722,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.1821540892124176,
      "rewards/margins": 0.324917197227478,
      "rewards/rejected": -0.507071316242218,
      "step": 1320
    },
    {
      "epoch": 0.11678703927293482,
      "grad_norm": 5.458474159240723,
      "learning_rate": 9.47037037037037e-06,
      "logits/chosen": 2.9955544471740723,
      "logits/rejected": 2.9541006088256836,
      "logps/chosen": -1.1582845449447632,
      "logps/rejected": -2.687746047973633,
      "loss": 0.8277,
      "nll_loss": 0.14388300478458405,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.34748542308807373,
      "rewards/margins": 0.45883846282958984,
      "rewards/rejected": -0.8063238859176636,
      "step": 1330
    },
    {
      "epoch": 0.11766513731258095,
      "grad_norm": 1.8496476411819458,
      "learning_rate": 9.458024691358025e-06,
      "logits/chosen": 2.8806967735290527,
      "logits/rejected": 2.922480344772339,
      "logps/chosen": -0.3559941351413727,
      "logps/rejected": -2.068563938140869,
      "loss": 0.5969,
      "nll_loss": 0.03910910710692406,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.10679824650287628,
      "rewards/margins": 0.5137708783149719,
      "rewards/rejected": -0.6205691695213318,
      "step": 1340
    },
    {
      "epoch": 0.11854323535222708,
      "grad_norm": 8.205083847045898,
      "learning_rate": 9.44567901234568e-06,
      "logits/chosen": 2.9301178455352783,
      "logits/rejected": 2.972548246383667,
      "logps/chosen": -0.6576313972473145,
      "logps/rejected": -1.3083826303482056,
      "loss": 0.7648,
      "nll_loss": 0.0889785960316658,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.19728945195674896,
      "rewards/margins": 0.19522538781166077,
      "rewards/rejected": -0.39251479506492615,
      "step": 1350
    },
    {
      "epoch": 0.1194213333918732,
      "grad_norm": 3.6549758911132812,
      "learning_rate": 9.433333333333335e-06,
      "logits/chosen": 3.0893311500549316,
      "logits/rejected": 3.068948745727539,
      "logps/chosen": -0.6783910989761353,
      "logps/rejected": -2.4969258308410645,
      "loss": 0.6481,
      "nll_loss": 0.06912653148174286,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.203517347574234,
      "rewards/margins": 0.545560359954834,
      "rewards/rejected": -0.7490777969360352,
      "step": 1360
    },
    {
      "epoch": 0.12029943143151933,
      "grad_norm": 1.5456334352493286,
      "learning_rate": 9.42098765432099e-06,
      "logits/chosen": 2.9339780807495117,
      "logits/rejected": 2.9962122440338135,
      "logps/chosen": -0.46310439705848694,
      "logps/rejected": -2.436547040939331,
      "loss": 0.6248,
      "nll_loss": 0.06015176698565483,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.13893131911754608,
      "rewards/margins": 0.5920329093933105,
      "rewards/rejected": -0.730964183807373,
      "step": 1370
    },
    {
      "epoch": 0.12117752947116546,
      "grad_norm": 2.95572829246521,
      "learning_rate": 9.408641975308642e-06,
      "logits/chosen": 2.7811484336853027,
      "logits/rejected": 2.875540018081665,
      "logps/chosen": -0.6040056347846985,
      "logps/rejected": -2.7113006114959717,
      "loss": 0.6342,
      "nll_loss": 0.042366378009319305,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.18120168149471283,
      "rewards/margins": 0.6321884393692017,
      "rewards/rejected": -0.8133901357650757,
      "step": 1380
    },
    {
      "epoch": 0.12205562751081159,
      "grad_norm": 5.596220970153809,
      "learning_rate": 9.396296296296297e-06,
      "logits/chosen": 2.9219090938568115,
      "logits/rejected": 2.9703197479248047,
      "logps/chosen": -1.1313722133636475,
      "logps/rejected": -3.376211166381836,
      "loss": 0.6689,
      "nll_loss": 0.11231324821710587,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.3394116759300232,
      "rewards/margins": 0.6734517216682434,
      "rewards/rejected": -1.0128633975982666,
      "step": 1390
    },
    {
      "epoch": 0.12293372555045771,
      "grad_norm": 4.154329776763916,
      "learning_rate": 9.383950617283951e-06,
      "logits/chosen": 2.875544548034668,
      "logits/rejected": 2.909510374069214,
      "logps/chosen": -0.7795349955558777,
      "logps/rejected": -3.0624213218688965,
      "loss": 0.6413,
      "nll_loss": 0.09788934886455536,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.23386052250862122,
      "rewards/margins": 0.6848658919334412,
      "rewards/rejected": -0.9187263250350952,
      "step": 1400
    },
    {
      "epoch": 0.12381182359010383,
      "grad_norm": 0.9190550446510315,
      "learning_rate": 9.371604938271605e-06,
      "logits/chosen": 2.800968885421753,
      "logits/rejected": 2.7952873706817627,
      "logps/chosen": -0.5842548608779907,
      "logps/rejected": -2.0285487174987793,
      "loss": 0.6672,
      "nll_loss": 0.06469441950321198,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.17527645826339722,
      "rewards/margins": 0.43328824639320374,
      "rewards/rejected": -0.6085646748542786,
      "step": 1410
    },
    {
      "epoch": 0.12468992162974996,
      "grad_norm": 0.4985824525356293,
      "learning_rate": 9.35925925925926e-06,
      "logits/chosen": 2.738670825958252,
      "logits/rejected": 2.7031972408294678,
      "logps/chosen": -0.714606761932373,
      "logps/rejected": -1.1444988250732422,
      "loss": 0.7813,
      "nll_loss": 0.07031063735485077,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.21438205242156982,
      "rewards/margins": 0.12896756827831268,
      "rewards/rejected": -0.3433496356010437,
      "step": 1420
    },
    {
      "epoch": 0.1255680196693961,
      "grad_norm": 3.7305266857147217,
      "learning_rate": 9.346913580246914e-06,
      "logits/chosen": 2.893129825592041,
      "logits/rejected": 2.9149386882781982,
      "logps/chosen": -0.46482163667678833,
      "logps/rejected": -2.2275373935699463,
      "loss": 0.6239,
      "nll_loss": 0.051440030336380005,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.13944648206233978,
      "rewards/margins": 0.5288147926330566,
      "rewards/rejected": -0.6682612299919128,
      "step": 1430
    },
    {
      "epoch": 0.1264461177090422,
      "grad_norm": 1.8036631345748901,
      "learning_rate": 9.334567901234569e-06,
      "logits/chosen": 2.547828197479248,
      "logits/rejected": 2.5787394046783447,
      "logps/chosen": -0.6464110612869263,
      "logps/rejected": -2.2243905067443848,
      "loss": 0.6899,
      "nll_loss": 0.06981117278337479,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.19392332434654236,
      "rewards/margins": 0.47339382767677307,
      "rewards/rejected": -0.6673170924186707,
      "step": 1440
    },
    {
      "epoch": 0.12732421574868835,
      "grad_norm": 1.5003271102905273,
      "learning_rate": 9.322222222222223e-06,
      "logits/chosen": 2.7942347526550293,
      "logits/rejected": 2.790160655975342,
      "logps/chosen": -0.45981842279434204,
      "logps/rejected": -1.9567676782608032,
      "loss": 0.6536,
      "nll_loss": 0.0419035442173481,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1379455327987671,
      "rewards/margins": 0.4490847587585449,
      "rewards/rejected": -0.587030291557312,
      "step": 1450
    },
    {
      "epoch": 0.12820231378833447,
      "grad_norm": 3.1813344955444336,
      "learning_rate": 9.309876543209878e-06,
      "logits/chosen": 2.6398301124572754,
      "logits/rejected": 2.6550662517547607,
      "logps/chosen": -0.6792389154434204,
      "logps/rejected": -2.9893813133239746,
      "loss": 0.6317,
      "nll_loss": 0.08011049032211304,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2037716805934906,
      "rewards/margins": 0.6930428147315979,
      "rewards/rejected": -0.8968144655227661,
      "step": 1460
    },
    {
      "epoch": 0.1290804118279806,
      "grad_norm": 6.025292873382568,
      "learning_rate": 9.297530864197532e-06,
      "logits/chosen": 2.5487821102142334,
      "logits/rejected": 2.6483190059661865,
      "logps/chosen": -0.3606962561607361,
      "logps/rejected": -2.5816264152526855,
      "loss": 0.6086,
      "nll_loss": 0.04057370498776436,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.10820887982845306,
      "rewards/margins": 0.6662790179252625,
      "rewards/rejected": -0.7744879126548767,
      "step": 1470
    },
    {
      "epoch": 0.12995850986762672,
      "grad_norm": 31.56429100036621,
      "learning_rate": 9.285185185185186e-06,
      "logits/chosen": 2.4623022079467773,
      "logits/rejected": 2.409700393676758,
      "logps/chosen": -0.9133744239807129,
      "logps/rejected": -3.7820403575897217,
      "loss": 0.8332,
      "nll_loss": 0.23980839550495148,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.27401235699653625,
      "rewards/margins": 0.8605998158454895,
      "rewards/rejected": -1.1346122026443481,
      "step": 1480
    },
    {
      "epoch": 0.13083660790727283,
      "grad_norm": 0.035230621695518494,
      "learning_rate": 9.27283950617284e-06,
      "logits/chosen": 2.52907133102417,
      "logits/rejected": 2.530890464782715,
      "logps/chosen": -1.2194797992706299,
      "logps/rejected": -2.8537631034851074,
      "loss": 0.7781,
      "nll_loss": 0.14254291355609894,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.36584392189979553,
      "rewards/margins": 0.49028509855270386,
      "rewards/rejected": -0.856128990650177,
      "step": 1490
    },
    {
      "epoch": 0.13171470594691898,
      "grad_norm": 6.077812671661377,
      "learning_rate": 9.260493827160495e-06,
      "logits/chosen": 2.5622482299804688,
      "logits/rejected": 2.607653856277466,
      "logps/chosen": -0.8888137936592102,
      "logps/rejected": -2.8897366523742676,
      "loss": 0.6866,
      "nll_loss": 0.07926015555858612,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.2666441798210144,
      "rewards/margins": 0.6002769470214844,
      "rewards/rejected": -0.8669211268424988,
      "step": 1500
    },
    {
      "epoch": 0.1325928039865651,
      "grad_norm": 3.651186466217041,
      "learning_rate": 9.24814814814815e-06,
      "logits/chosen": 2.693009853363037,
      "logits/rejected": 2.6718087196350098,
      "logps/chosen": -0.8477522134780884,
      "logps/rejected": -1.7480132579803467,
      "loss": 0.7543,
      "nll_loss": 0.09234277904033661,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.25432562828063965,
      "rewards/margins": 0.2700783610343933,
      "rewards/rejected": -0.5244040489196777,
      "step": 1510
    },
    {
      "epoch": 0.13347090202621123,
      "grad_norm": 2.8306994438171387,
      "learning_rate": 9.235802469135802e-06,
      "logits/chosen": 2.8524880409240723,
      "logits/rejected": 2.8818418979644775,
      "logps/chosen": -0.49177321791648865,
      "logps/rejected": -1.9612071514129639,
      "loss": 0.6686,
      "nll_loss": 0.08659791201353073,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14753195643424988,
      "rewards/margins": 0.4408302307128906,
      "rewards/rejected": -0.5883622169494629,
      "step": 1520
    },
    {
      "epoch": 0.13434900006585734,
      "grad_norm": 3.763047456741333,
      "learning_rate": 9.223456790123457e-06,
      "logits/chosen": 2.698072671890259,
      "logits/rejected": 2.754232883453369,
      "logps/chosen": -0.798735499382019,
      "logps/rejected": -2.2809195518493652,
      "loss": 0.6913,
      "nll_loss": 0.07482419162988663,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2396206557750702,
      "rewards/margins": 0.44465526938438416,
      "rewards/rejected": -0.6842759251594543,
      "step": 1530
    },
    {
      "epoch": 0.13522709810550348,
      "grad_norm": 3.579263925552368,
      "learning_rate": 9.211111111111111e-06,
      "logits/chosen": 3.0055601596832275,
      "logits/rejected": 3.015162706375122,
      "logps/chosen": -0.6990305185317993,
      "logps/rejected": -1.879294991493225,
      "loss": 0.6897,
      "nll_loss": 0.06048674136400223,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.20970916748046875,
      "rewards/margins": 0.35407939553260803,
      "rewards/rejected": -0.5637885332107544,
      "step": 1540
    },
    {
      "epoch": 0.1361051961451496,
      "grad_norm": 1.4635862112045288,
      "learning_rate": 9.198765432098766e-06,
      "logits/chosen": 2.665912628173828,
      "logits/rejected": 2.7371814250946045,
      "logps/chosen": -0.644806981086731,
      "logps/rejected": -2.0770316123962402,
      "loss": 0.6949,
      "nll_loss": 0.07454844564199448,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.19344215095043182,
      "rewards/margins": 0.4296673834323883,
      "rewards/rejected": -0.6231095194816589,
      "step": 1550
    },
    {
      "epoch": 0.13698329418479574,
      "grad_norm": 3.593688726425171,
      "learning_rate": 9.18641975308642e-06,
      "logits/chosen": 3.0721404552459717,
      "logits/rejected": 3.028421401977539,
      "logps/chosen": -0.8886274099349976,
      "logps/rejected": -2.1320443153381348,
      "loss": 0.7047,
      "nll_loss": 0.07659469544887543,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.2665882408618927,
      "rewards/margins": 0.3730250298976898,
      "rewards/rejected": -0.6396132707595825,
      "step": 1560
    },
    {
      "epoch": 0.13786139222444185,
      "grad_norm": 6.7611470222473145,
      "learning_rate": 9.174074074074074e-06,
      "logits/chosen": 2.952538013458252,
      "logits/rejected": 2.9351909160614014,
      "logps/chosen": -0.5501828193664551,
      "logps/rejected": -2.4539520740509033,
      "loss": 0.6445,
      "nll_loss": 0.07492227852344513,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.16505484282970428,
      "rewards/margins": 0.5711307525634766,
      "rewards/rejected": -0.7361854910850525,
      "step": 1570
    },
    {
      "epoch": 0.138739490264088,
      "grad_norm": 0.00917895883321762,
      "learning_rate": 9.161728395061729e-06,
      "logits/chosen": 2.942192792892456,
      "logits/rejected": 2.9979898929595947,
      "logps/chosen": -0.4612821638584137,
      "logps/rejected": -1.4370297193527222,
      "loss": 0.6585,
      "nll_loss": 0.04147753119468689,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.13838467001914978,
      "rewards/margins": 0.2927243113517761,
      "rewards/rejected": -0.43110889196395874,
      "step": 1580
    },
    {
      "epoch": 0.1396175883037341,
      "grad_norm": 1.7900971174240112,
      "learning_rate": 9.149382716049383e-06,
      "logits/chosen": 2.7242748737335205,
      "logits/rejected": 2.7570395469665527,
      "logps/chosen": -0.5272113084793091,
      "logps/rejected": -1.8239033222198486,
      "loss": 0.6752,
      "nll_loss": 0.07194850593805313,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1581634134054184,
      "rewards/margins": 0.3890075981616974,
      "rewards/rejected": -0.5471709966659546,
      "step": 1590
    },
    {
      "epoch": 0.14049568634338025,
      "grad_norm": 7.255580902099609,
      "learning_rate": 9.137037037037038e-06,
      "logits/chosen": 2.852332592010498,
      "logits/rejected": 2.8654465675354004,
      "logps/chosen": -0.8652445077896118,
      "logps/rejected": -2.7340779304504395,
      "loss": 0.7026,
      "nll_loss": 0.08957532793283463,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.25957340002059937,
      "rewards/margins": 0.5606500506401062,
      "rewards/rejected": -0.8202234506607056,
      "step": 1600
    },
    {
      "epoch": 0.14137378438302636,
      "grad_norm": 0.27785807847976685,
      "learning_rate": 9.124691358024692e-06,
      "logits/chosen": 2.8517799377441406,
      "logits/rejected": 2.871009111404419,
      "logps/chosen": -0.45925140380859375,
      "logps/rejected": -1.38749098777771,
      "loss": 0.6784,
      "nll_loss": 0.046599697321653366,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.13777543604373932,
      "rewards/margins": 0.2784718871116638,
      "rewards/rejected": -0.41624727845191956,
      "step": 1610
    },
    {
      "epoch": 0.1422518824226725,
      "grad_norm": 2.3490424156188965,
      "learning_rate": 9.112345679012347e-06,
      "logits/chosen": 3.001574754714966,
      "logits/rejected": 2.9703142642974854,
      "logps/chosen": -0.4932584762573242,
      "logps/rejected": -2.1252918243408203,
      "loss": 0.6813,
      "nll_loss": 0.08429791033267975,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1479775607585907,
      "rewards/margins": 0.48961010575294495,
      "rewards/rejected": -0.6375876665115356,
      "step": 1620
    },
    {
      "epoch": 0.14312998046231862,
      "grad_norm": 3.299020767211914,
      "learning_rate": 9.100000000000001e-06,
      "logits/chosen": 2.8275907039642334,
      "logits/rejected": 2.9383082389831543,
      "logps/chosen": -0.6793197989463806,
      "logps/rejected": -2.526850700378418,
      "loss": 0.6513,
      "nll_loss": 0.05579754710197449,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.20379595458507538,
      "rewards/margins": 0.5542593002319336,
      "rewards/rejected": -0.7580552101135254,
      "step": 1630
    },
    {
      "epoch": 0.14400807850196473,
      "grad_norm": 0.914357602596283,
      "learning_rate": 9.087654320987655e-06,
      "logits/chosen": 2.987334966659546,
      "logits/rejected": 2.9897654056549072,
      "logps/chosen": -0.665000319480896,
      "logps/rejected": -2.4969944953918457,
      "loss": 0.6926,
      "nll_loss": 0.07977604120969772,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.19950011372566223,
      "rewards/margins": 0.5495983362197876,
      "rewards/rejected": -0.7490984797477722,
      "step": 1640
    },
    {
      "epoch": 0.14488617654161087,
      "grad_norm": 2.353787422180176,
      "learning_rate": 9.075308641975308e-06,
      "logits/chosen": 2.7523000240325928,
      "logits/rejected": 2.805290937423706,
      "logps/chosen": -0.34468549489974976,
      "logps/rejected": -2.1994361877441406,
      "loss": 0.6206,
      "nll_loss": 0.04046661779284477,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1034056693315506,
      "rewards/margins": 0.5564252138137817,
      "rewards/rejected": -0.6598309278488159,
      "step": 1650
    },
    {
      "epoch": 0.145764274581257,
      "grad_norm": 9.683286666870117,
      "learning_rate": 9.062962962962964e-06,
      "logits/chosen": 2.8048713207244873,
      "logits/rejected": 2.8911221027374268,
      "logps/chosen": -0.8389989137649536,
      "logps/rejected": -1.6681379079818726,
      "loss": 0.7894,
      "nll_loss": 0.09216944873332977,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.25169968605041504,
      "rewards/margins": 0.24874171614646912,
      "rewards/rejected": -0.5004413723945618,
      "step": 1660
    },
    {
      "epoch": 0.14664237262090313,
      "grad_norm": 9.369816780090332,
      "learning_rate": 9.050617283950619e-06,
      "logits/chosen": 2.8822600841522217,
      "logits/rejected": 2.904628038406372,
      "logps/chosen": -0.8616389036178589,
      "logps/rejected": -2.4774065017700195,
      "loss": 0.6733,
      "nll_loss": 0.052507419139146805,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.2584916949272156,
      "rewards/margins": 0.48473024368286133,
      "rewards/rejected": -0.7432219386100769,
      "step": 1670
    },
    {
      "epoch": 0.14752047066054924,
      "grad_norm": 1.7723337411880493,
      "learning_rate": 9.038271604938273e-06,
      "logits/chosen": 3.0452260971069336,
      "logits/rejected": 3.092968225479126,
      "logps/chosen": -0.5382004380226135,
      "logps/rejected": -1.853811264038086,
      "loss": 0.6863,
      "nll_loss": 0.07374037802219391,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.16146014630794525,
      "rewards/margins": 0.39468324184417725,
      "rewards/rejected": -0.5561434030532837,
      "step": 1680
    },
    {
      "epoch": 0.14839856870019538,
      "grad_norm": 4.466056823730469,
      "learning_rate": 9.025925925925927e-06,
      "logits/chosen": 2.8592844009399414,
      "logits/rejected": 2.896420955657959,
      "logps/chosen": -0.5004099011421204,
      "logps/rejected": -1.6472933292388916,
      "loss": 0.6771,
      "nll_loss": 0.05312333256006241,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1501229852437973,
      "rewards/margins": 0.34406501054763794,
      "rewards/rejected": -0.49418801069259644,
      "step": 1690
    },
    {
      "epoch": 0.1492766667398415,
      "grad_norm": 2.437678098678589,
      "learning_rate": 9.013580246913582e-06,
      "logits/chosen": 2.8448596000671387,
      "logits/rejected": 2.8915913105010986,
      "logps/chosen": -0.6149319410324097,
      "logps/rejected": -1.89218008518219,
      "loss": 0.673,
      "nll_loss": 0.05624104663729668,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.18447960913181305,
      "rewards/margins": 0.3831743597984314,
      "rewards/rejected": -0.567654013633728,
      "step": 1700
    },
    {
      "epoch": 0.15015476477948764,
      "grad_norm": 3.493530750274658,
      "learning_rate": 9.001234567901236e-06,
      "logits/chosen": 2.781919479370117,
      "logits/rejected": 2.7699952125549316,
      "logps/chosen": -0.6766859889030457,
      "logps/rejected": -2.136669635772705,
      "loss": 0.6503,
      "nll_loss": 0.06224127486348152,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2030058205127716,
      "rewards/margins": 0.43799519538879395,
      "rewards/rejected": -0.6410009264945984,
      "step": 1710
    },
    {
      "epoch": 0.15103286281913375,
      "grad_norm": 3.9590651988983154,
      "learning_rate": 8.988888888888889e-06,
      "logits/chosen": 2.562682628631592,
      "logits/rejected": 2.5419199466705322,
      "logps/chosen": -0.965559184551239,
      "logps/rejected": -2.73313570022583,
      "loss": 0.7286,
      "nll_loss": 0.10021784156560898,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.2896677553653717,
      "rewards/margins": 0.5302730798721313,
      "rewards/rejected": -0.8199408650398254,
      "step": 1720
    },
    {
      "epoch": 0.1519109608587799,
      "grad_norm": 6.649216175079346,
      "learning_rate": 8.976543209876543e-06,
      "logits/chosen": 2.754970073699951,
      "logits/rejected": 2.800556182861328,
      "logps/chosen": -0.7695094347000122,
      "logps/rejected": -2.4826478958129883,
      "loss": 0.7055,
      "nll_loss": 0.05943988636136055,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.23085281252861023,
      "rewards/margins": 0.5139415264129639,
      "rewards/rejected": -0.7447944283485413,
      "step": 1730
    },
    {
      "epoch": 0.152789058898426,
      "grad_norm": 3.0816867351531982,
      "learning_rate": 8.964197530864198e-06,
      "logits/chosen": 2.7005701065063477,
      "logits/rejected": 2.8037772178649902,
      "logps/chosen": -0.7151850461959839,
      "logps/rejected": -2.9252617359161377,
      "loss": 0.6407,
      "nll_loss": 0.06549613177776337,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2145555019378662,
      "rewards/margins": 0.6630231142044067,
      "rewards/rejected": -0.8775785565376282,
      "step": 1740
    },
    {
      "epoch": 0.15366715693807215,
      "grad_norm": 3.4558446407318115,
      "learning_rate": 8.951851851851852e-06,
      "logits/chosen": 2.783221483230591,
      "logits/rejected": 2.7478537559509277,
      "logps/chosen": -0.3045424818992615,
      "logps/rejected": -1.9958875179290771,
      "loss": 0.5977,
      "nll_loss": 0.030114714056253433,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09136275202035904,
      "rewards/margins": 0.507403552532196,
      "rewards/rejected": -0.5987662672996521,
      "step": 1750
    },
    {
      "epoch": 0.15454525497771826,
      "grad_norm": 2.812927722930908,
      "learning_rate": 8.939506172839507e-06,
      "logits/chosen": 2.567533016204834,
      "logits/rejected": 2.5942509174346924,
      "logps/chosen": -0.7922319173812866,
      "logps/rejected": -1.9200432300567627,
      "loss": 0.7427,
      "nll_loss": 0.07532784342765808,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.23766958713531494,
      "rewards/margins": 0.33834341168403625,
      "rewards/rejected": -0.5760129690170288,
      "step": 1760
    },
    {
      "epoch": 0.1554233530173644,
      "grad_norm": 4.2586750984191895,
      "learning_rate": 8.927160493827161e-06,
      "logits/chosen": 2.8497743606567383,
      "logits/rejected": 2.857257604598999,
      "logps/chosen": -1.0099961757659912,
      "logps/rejected": -3.4478023052215576,
      "loss": 0.6624,
      "nll_loss": 0.1071515902876854,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.302998811006546,
      "rewards/margins": 0.7313419580459595,
      "rewards/rejected": -1.034340739250183,
      "step": 1770
    },
    {
      "epoch": 0.15630145105701052,
      "grad_norm": 1.7801121473312378,
      "learning_rate": 8.914814814814816e-06,
      "logits/chosen": 2.7116096019744873,
      "logits/rejected": 2.7383835315704346,
      "logps/chosen": -0.6959985494613647,
      "logps/rejected": -2.842719554901123,
      "loss": 0.626,
      "nll_loss": 0.07402163743972778,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2087995707988739,
      "rewards/margins": 0.6440162658691406,
      "rewards/rejected": -0.8528158068656921,
      "step": 1780
    },
    {
      "epoch": 0.15717954909665663,
      "grad_norm": 2.949233055114746,
      "learning_rate": 8.90246913580247e-06,
      "logits/chosen": 2.5691332817077637,
      "logits/rejected": 2.6201071739196777,
      "logps/chosen": -0.7052744626998901,
      "logps/rejected": -2.180938720703125,
      "loss": 0.6607,
      "nll_loss": 0.058830149471759796,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21158234775066376,
      "rewards/margins": 0.4426993429660797,
      "rewards/rejected": -0.6542816758155823,
      "step": 1790
    },
    {
      "epoch": 0.15805764713630277,
      "grad_norm": 0.5553939938545227,
      "learning_rate": 8.890123456790124e-06,
      "logits/chosen": 2.677717685699463,
      "logits/rejected": 2.782273292541504,
      "logps/chosen": -0.4674352705478668,
      "logps/rejected": -3.086191177368164,
      "loss": 0.5701,
      "nll_loss": 0.04577519744634628,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.14023058116436005,
      "rewards/margins": 0.7856268286705017,
      "rewards/rejected": -0.925857424736023,
      "step": 1800
    },
    {
      "epoch": 0.15893574517594888,
      "grad_norm": 2.8835935592651367,
      "learning_rate": 8.877777777777779e-06,
      "logits/chosen": 2.745272636413574,
      "logits/rejected": 2.806513547897339,
      "logps/chosen": -0.5373865962028503,
      "logps/rejected": -3.408268451690674,
      "loss": 0.6131,
      "nll_loss": 0.053614210337400436,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.16121599078178406,
      "rewards/margins": 0.8612645864486694,
      "rewards/rejected": -1.0224807262420654,
      "step": 1810
    },
    {
      "epoch": 0.15981384321559503,
      "grad_norm": 6.393036365509033,
      "learning_rate": 8.865432098765433e-06,
      "logits/chosen": 2.6081528663635254,
      "logits/rejected": 2.663269281387329,
      "logps/chosen": -0.6596813201904297,
      "logps/rejected": -3.0106234550476074,
      "loss": 0.6523,
      "nll_loss": 0.07575313746929169,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.19790442287921906,
      "rewards/margins": 0.7052826881408691,
      "rewards/rejected": -0.903187096118927,
      "step": 1820
    },
    {
      "epoch": 0.16069194125524114,
      "grad_norm": 8.32457160949707,
      "learning_rate": 8.853086419753088e-06,
      "logits/chosen": 2.1919102668762207,
      "logits/rejected": 2.2420222759246826,
      "logps/chosen": -0.6085657477378845,
      "logps/rejected": -3.387340545654297,
      "loss": 0.6396,
      "nll_loss": 0.05589609593153,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1825697273015976,
      "rewards/margins": 0.8336323499679565,
      "rewards/rejected": -1.0162022113800049,
      "step": 1830
    },
    {
      "epoch": 0.16157003929488728,
      "grad_norm": 8.0199556350708,
      "learning_rate": 8.840740740740742e-06,
      "logits/chosen": 2.2459845542907715,
      "logits/rejected": 2.2368826866149902,
      "logps/chosen": -0.904313862323761,
      "logps/rejected": -3.4034416675567627,
      "loss": 0.7623,
      "nll_loss": 0.13848955929279327,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.27129411697387695,
      "rewards/margins": 0.7497383952140808,
      "rewards/rejected": -1.0210325717926025,
      "step": 1840
    },
    {
      "epoch": 0.1624481373345334,
      "grad_norm": 3.155104398727417,
      "learning_rate": 8.828395061728395e-06,
      "logits/chosen": 2.594691514968872,
      "logits/rejected": 2.571620464324951,
      "logps/chosen": -1.037233591079712,
      "logps/rejected": -2.723869800567627,
      "loss": 0.7723,
      "nll_loss": 0.12755393981933594,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.3111700415611267,
      "rewards/margins": 0.5059908628463745,
      "rewards/rejected": -0.817160964012146,
      "step": 1850
    },
    {
      "epoch": 0.16332623537417953,
      "grad_norm": 3.548657178878784,
      "learning_rate": 8.81604938271605e-06,
      "logits/chosen": 2.7853002548217773,
      "logits/rejected": 2.796757221221924,
      "logps/chosen": -0.5054196119308472,
      "logps/rejected": -1.9281543493270874,
      "loss": 0.6821,
      "nll_loss": 0.06374648213386536,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.15162590146064758,
      "rewards/margins": 0.42682045698165894,
      "rewards/rejected": -0.5784463286399841,
      "step": 1860
    },
    {
      "epoch": 0.16420433341382565,
      "grad_norm": 5.429587364196777,
      "learning_rate": 8.803703703703704e-06,
      "logits/chosen": 2.6975064277648926,
      "logits/rejected": 2.7702746391296387,
      "logps/chosen": -0.7951546907424927,
      "logps/rejected": -2.666987895965576,
      "loss": 0.6656,
      "nll_loss": 0.06336641311645508,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.2385464459657669,
      "rewards/margins": 0.5615500211715698,
      "rewards/rejected": -0.8000965118408203,
      "step": 1870
    },
    {
      "epoch": 0.1650824314534718,
      "grad_norm": 1.150895357131958,
      "learning_rate": 8.791358024691358e-06,
      "logits/chosen": 2.687932252883911,
      "logits/rejected": 2.754683017730713,
      "logps/chosen": -0.7847083806991577,
      "logps/rejected": -2.204124927520752,
      "loss": 0.7186,
      "nll_loss": 0.08406446129083633,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.23541252315044403,
      "rewards/margins": 0.42582497000694275,
      "rewards/rejected": -0.6612375378608704,
      "step": 1880
    },
    {
      "epoch": 0.1659605294931179,
      "grad_norm": 2.4268455505371094,
      "learning_rate": 8.779012345679012e-06,
      "logits/chosen": 2.5794925689697266,
      "logits/rejected": 2.6048686504364014,
      "logps/chosen": -0.6307061314582825,
      "logps/rejected": -2.8818671703338623,
      "loss": 0.619,
      "nll_loss": 0.07149704545736313,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.1892118602991104,
      "rewards/margins": 0.6753484010696411,
      "rewards/rejected": -0.8645601272583008,
      "step": 1890
    },
    {
      "epoch": 0.16683862753276404,
      "grad_norm": 5.715259075164795,
      "learning_rate": 8.766666666666669e-06,
      "logits/chosen": 2.707252025604248,
      "logits/rejected": 2.6988537311553955,
      "logps/chosen": -0.6423169374465942,
      "logps/rejected": -1.875507116317749,
      "loss": 0.6782,
      "nll_loss": 0.04863595962524414,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.19269508123397827,
      "rewards/margins": 0.3699570894241333,
      "rewards/rejected": -0.5626521706581116,
      "step": 1900
    },
    {
      "epoch": 0.16771672557241016,
      "grad_norm": 1.6416656970977783,
      "learning_rate": 8.754320987654323e-06,
      "logits/chosen": 2.792642593383789,
      "logits/rejected": 2.8566908836364746,
      "logps/chosen": -0.6888980269432068,
      "logps/rejected": -2.5319087505340576,
      "loss": 0.653,
      "nll_loss": 0.06277020275592804,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.206669420003891,
      "rewards/margins": 0.5529031753540039,
      "rewards/rejected": -0.7595726251602173,
      "step": 1910
    },
    {
      "epoch": 0.1685948236120563,
      "grad_norm": 1.5920031070709229,
      "learning_rate": 8.741975308641976e-06,
      "logits/chosen": 2.6872434616088867,
      "logits/rejected": 2.6933059692382812,
      "logps/chosen": -0.5184003710746765,
      "logps/rejected": -1.9327194690704346,
      "loss": 0.6515,
      "nll_loss": 0.06715475022792816,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.15552011132240295,
      "rewards/margins": 0.42429572343826294,
      "rewards/rejected": -0.5798158049583435,
      "step": 1920
    },
    {
      "epoch": 0.1694729216517024,
      "grad_norm": 4.420103073120117,
      "learning_rate": 8.72962962962963e-06,
      "logits/chosen": 2.835470676422119,
      "logits/rejected": 2.8618292808532715,
      "logps/chosen": -0.7457289695739746,
      "logps/rejected": -2.064795970916748,
      "loss": 0.7263,
      "nll_loss": 0.10260520875453949,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.22371868789196014,
      "rewards/margins": 0.39572006464004517,
      "rewards/rejected": -0.6194387674331665,
      "step": 1930
    },
    {
      "epoch": 0.17035101969134853,
      "grad_norm": 2.6691431999206543,
      "learning_rate": 8.717283950617285e-06,
      "logits/chosen": 2.6595866680145264,
      "logits/rejected": 2.683384656906128,
      "logps/chosen": -0.5285651087760925,
      "logps/rejected": -1.9386451244354248,
      "loss": 0.6966,
      "nll_loss": 0.07593102753162384,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1585695445537567,
      "rewards/margins": 0.4230240285396576,
      "rewards/rejected": -0.5815936326980591,
      "step": 1940
    },
    {
      "epoch": 0.17122911773099467,
      "grad_norm": 1.895266056060791,
      "learning_rate": 8.704938271604939e-06,
      "logits/chosen": 2.801657199859619,
      "logits/rejected": 2.762845516204834,
      "logps/chosen": -0.6032005548477173,
      "logps/rejected": -1.6262376308441162,
      "loss": 0.6888,
      "nll_loss": 0.060188956558704376,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.18096016347408295,
      "rewards/margins": 0.30691108107566833,
      "rewards/rejected": -0.4878712594509125,
      "step": 1950
    },
    {
      "epoch": 0.17210721577064078,
      "grad_norm": 1.7787566184997559,
      "learning_rate": 8.692592592592593e-06,
      "logits/chosen": 2.8991875648498535,
      "logits/rejected": 2.936009645462036,
      "logps/chosen": -0.5252435803413391,
      "logps/rejected": -1.9678659439086914,
      "loss": 0.6254,
      "nll_loss": 0.047948211431503296,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.15757307410240173,
      "rewards/margins": 0.432786762714386,
      "rewards/rejected": -0.5903598070144653,
      "step": 1960
    },
    {
      "epoch": 0.17298531381028692,
      "grad_norm": 1.326920747756958,
      "learning_rate": 8.680246913580248e-06,
      "logits/chosen": 2.5659279823303223,
      "logits/rejected": 2.590271472930908,
      "logps/chosen": -0.5297742486000061,
      "logps/rejected": -2.028113842010498,
      "loss": 0.6545,
      "nll_loss": 0.05374212935566902,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.15893225371837616,
      "rewards/margins": 0.4495018422603607,
      "rewards/rejected": -0.6084341406822205,
      "step": 1970
    },
    {
      "epoch": 0.17386341184993304,
      "grad_norm": 4.611727714538574,
      "learning_rate": 8.667901234567902e-06,
      "logits/chosen": 2.6264684200286865,
      "logits/rejected": 2.632913589477539,
      "logps/chosen": -0.693698525428772,
      "logps/rejected": -3.1472320556640625,
      "loss": 0.6069,
      "nll_loss": 0.05521649122238159,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.20810957252979279,
      "rewards/margins": 0.7360601425170898,
      "rewards/rejected": -0.9441697001457214,
      "step": 1980
    },
    {
      "epoch": 0.17474150988957918,
      "grad_norm": 2.2870845794677734,
      "learning_rate": 8.655555555555557e-06,
      "logits/chosen": 2.3779568672180176,
      "logits/rejected": 2.4282214641571045,
      "logps/chosen": -0.24996769428253174,
      "logps/rejected": -2.317931890487671,
      "loss": 0.5852,
      "nll_loss": 0.045966412872076035,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07499031722545624,
      "rewards/margins": 0.6203892230987549,
      "rewards/rejected": -0.6953796148300171,
      "step": 1990
    },
    {
      "epoch": 0.1756196079292253,
      "grad_norm": 1.1437376737594604,
      "learning_rate": 8.643209876543211e-06,
      "logits/chosen": 2.380004644393921,
      "logits/rejected": 2.3685264587402344,
      "logps/chosen": -0.5816354751586914,
      "logps/rejected": -2.3618433475494385,
      "loss": 0.6945,
      "nll_loss": 0.07123078405857086,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.17449061572551727,
      "rewards/margins": 0.5340624451637268,
      "rewards/rejected": -0.70855313539505,
      "step": 2000
    },
    {
      "epoch": 0.17649770596887143,
      "grad_norm": 9.37255573272705,
      "learning_rate": 8.630864197530865e-06,
      "logits/chosen": 2.211916446685791,
      "logits/rejected": 2.220418930053711,
      "logps/chosen": -0.5349146723747253,
      "logps/rejected": -2.8715481758117676,
      "loss": 0.6366,
      "nll_loss": 0.06667280942201614,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.16047440469264984,
      "rewards/margins": 0.7009900808334351,
      "rewards/rejected": -0.8614645004272461,
      "step": 2010
    },
    {
      "epoch": 0.17737580400851755,
      "grad_norm": 2.2170488834381104,
      "learning_rate": 8.61851851851852e-06,
      "logits/chosen": 1.94232177734375,
      "logits/rejected": 1.9853718280792236,
      "logps/chosen": -0.2896527945995331,
      "logps/rejected": -3.063877820968628,
      "loss": 0.6189,
      "nll_loss": 0.046251922845840454,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.08689583837985992,
      "rewards/margins": 0.8322674632072449,
      "rewards/rejected": -0.9191633462905884,
      "step": 2020
    },
    {
      "epoch": 0.1782539020481637,
      "grad_norm": 8.343827247619629,
      "learning_rate": 8.606172839506174e-06,
      "logits/chosen": 1.9204469919204712,
      "logits/rejected": 1.948312759399414,
      "logps/chosen": -1.1184568405151367,
      "logps/rejected": -3.0377535820007324,
      "loss": 0.8225,
      "nll_loss": 0.12297489494085312,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3355370759963989,
      "rewards/margins": 0.5757889747619629,
      "rewards/rejected": -0.9113261103630066,
      "step": 2030
    },
    {
      "epoch": 0.1791320000878098,
      "grad_norm": 13.199078559875488,
      "learning_rate": 8.593827160493829e-06,
      "logits/chosen": 2.0205349922180176,
      "logits/rejected": 2.0275635719299316,
      "logps/chosen": -1.428043007850647,
      "logps/rejected": -2.872222661972046,
      "loss": 0.9056,
      "nll_loss": 0.1807326227426529,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.4284129738807678,
      "rewards/margins": 0.4332540035247803,
      "rewards/rejected": -0.8616668581962585,
      "step": 2040
    },
    {
      "epoch": 0.18001009812745594,
      "grad_norm": 2.260270357131958,
      "learning_rate": 8.581481481481481e-06,
      "logits/chosen": 2.2045745849609375,
      "logits/rejected": 2.187514543533325,
      "logps/chosen": -0.4845626950263977,
      "logps/rejected": -2.2464981079101562,
      "loss": 0.6956,
      "nll_loss": 0.06010964512825012,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1453687846660614,
      "rewards/margins": 0.5285806059837341,
      "rewards/rejected": -0.6739493608474731,
      "step": 2050
    },
    {
      "epoch": 0.18088819616710206,
      "grad_norm": 2.032146453857422,
      "learning_rate": 8.569135802469136e-06,
      "logits/chosen": 2.2264397144317627,
      "logits/rejected": 2.265310764312744,
      "logps/chosen": -1.0286977291107178,
      "logps/rejected": -3.1123714447021484,
      "loss": 0.7175,
      "nll_loss": 0.11090108007192612,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.3086093068122864,
      "rewards/margins": 0.6251022815704346,
      "rewards/rejected": -0.9337115287780762,
      "step": 2060
    },
    {
      "epoch": 0.1817662942067482,
      "grad_norm": 0.9217659831047058,
      "learning_rate": 8.55679012345679e-06,
      "logits/chosen": 2.2858176231384277,
      "logits/rejected": 2.315831422805786,
      "logps/chosen": -0.6487756967544556,
      "logps/rejected": -2.1730990409851074,
      "loss": 0.6862,
      "nll_loss": 0.07243213802576065,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.19463272392749786,
      "rewards/margins": 0.4572969377040863,
      "rewards/rejected": -0.6519297361373901,
      "step": 2070
    },
    {
      "epoch": 0.1826443922463943,
      "grad_norm": 4.933953285217285,
      "learning_rate": 8.544444444444445e-06,
      "logits/chosen": 2.4036874771118164,
      "logits/rejected": 2.3770546913146973,
      "logps/chosen": -0.9166983366012573,
      "logps/rejected": -2.5144124031066895,
      "loss": 0.7355,
      "nll_loss": 0.12400822341442108,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.27500951290130615,
      "rewards/margins": 0.47931423783302307,
      "rewards/rejected": -0.7543236613273621,
      "step": 2080
    },
    {
      "epoch": 0.18352249028604042,
      "grad_norm": 5.920201778411865,
      "learning_rate": 8.532098765432099e-06,
      "logits/chosen": 2.504974603652954,
      "logits/rejected": 2.541961193084717,
      "logps/chosen": -0.8963934779167175,
      "logps/rejected": -2.740062713623047,
      "loss": 0.7162,
      "nll_loss": 0.1004381999373436,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.2689180374145508,
      "rewards/margins": 0.5531007051467896,
      "rewards/rejected": -0.8220188021659851,
      "step": 2090
    },
    {
      "epoch": 0.18440058832568657,
      "grad_norm": 0.10835571587085724,
      "learning_rate": 8.519753086419754e-06,
      "logits/chosen": 2.54685640335083,
      "logits/rejected": 2.627911329269409,
      "logps/chosen": -0.33035722374916077,
      "logps/rejected": -2.806840419769287,
      "loss": 0.561,
      "nll_loss": 0.03572739288210869,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09910716861486435,
      "rewards/margins": 0.7429450154304504,
      "rewards/rejected": -0.8420522809028625,
      "step": 2100
    },
    {
      "epoch": 0.18527868636533268,
      "grad_norm": 5.237948417663574,
      "learning_rate": 8.507407407407408e-06,
      "logits/chosen": 2.4657981395721436,
      "logits/rejected": 2.541999101638794,
      "logps/chosen": -0.5284551382064819,
      "logps/rejected": -2.2460741996765137,
      "loss": 0.6753,
      "nll_loss": 0.055809833109378815,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.15853655338287354,
      "rewards/margins": 0.515285849571228,
      "rewards/rejected": -0.6738223433494568,
      "step": 2110
    },
    {
      "epoch": 0.18615678440497882,
      "grad_norm": 6.183863162994385,
      "learning_rate": 8.495061728395062e-06,
      "logits/chosen": 2.370246171951294,
      "logits/rejected": 2.388896942138672,
      "logps/chosen": -0.5437101125717163,
      "logps/rejected": -2.8997700214385986,
      "loss": 0.6217,
      "nll_loss": 0.07455585151910782,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1631130427122116,
      "rewards/margins": 0.7068179249763489,
      "rewards/rejected": -0.8699310421943665,
      "step": 2120
    },
    {
      "epoch": 0.18703488244462493,
      "grad_norm": 3.2471072673797607,
      "learning_rate": 8.482716049382717e-06,
      "logits/chosen": 2.301064968109131,
      "logits/rejected": 2.3306994438171387,
      "logps/chosen": -1.0358989238739014,
      "logps/rejected": -3.3612143993377686,
      "loss": 0.6543,
      "nll_loss": 0.08921568840742111,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.3107697069644928,
      "rewards/margins": 0.6975947618484497,
      "rewards/rejected": -1.0083644390106201,
      "step": 2130
    },
    {
      "epoch": 0.18791298048427107,
      "grad_norm": 2.2561800479888916,
      "learning_rate": 8.470370370370371e-06,
      "logits/chosen": 2.35896635055542,
      "logits/rejected": 2.424318790435791,
      "logps/chosen": -0.42497625946998596,
      "logps/rejected": -3.6937179565429688,
      "loss": 0.5684,
      "nll_loss": 0.049736388027668,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12749287486076355,
      "rewards/margins": 0.9806225895881653,
      "rewards/rejected": -1.1081154346466064,
      "step": 2140
    },
    {
      "epoch": 0.1887910785239172,
      "grad_norm": 3.721003293991089,
      "learning_rate": 8.458024691358026e-06,
      "logits/chosen": 2.294174909591675,
      "logits/rejected": 2.332521915435791,
      "logps/chosen": -0.3734773099422455,
      "logps/rejected": -2.520681858062744,
      "loss": 0.6118,
      "nll_loss": 0.03998088836669922,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11204320192337036,
      "rewards/margins": 0.6441613435745239,
      "rewards/rejected": -0.7562046051025391,
      "step": 2150
    },
    {
      "epoch": 0.18966917656356333,
      "grad_norm": 0.09796835482120514,
      "learning_rate": 8.44567901234568e-06,
      "logits/chosen": 2.144991397857666,
      "logits/rejected": 2.220353364944458,
      "logps/chosen": -0.5873435139656067,
      "logps/rejected": -3.0226333141326904,
      "loss": 0.6662,
      "nll_loss": 0.05847520753741264,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.17620307207107544,
      "rewards/margins": 0.7305869460105896,
      "rewards/rejected": -0.9067900776863098,
      "step": 2160
    },
    {
      "epoch": 0.19054727460320944,
      "grad_norm": 5.563838005065918,
      "learning_rate": 8.433333333333334e-06,
      "logits/chosen": 2.4212334156036377,
      "logits/rejected": 2.4930455684661865,
      "logps/chosen": -1.0709176063537598,
      "logps/rejected": -3.6808440685272217,
      "loss": 0.705,
      "nll_loss": 0.1328948587179184,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3212752938270569,
      "rewards/margins": 0.7829779386520386,
      "rewards/rejected": -1.1042531728744507,
      "step": 2170
    },
    {
      "epoch": 0.19142537264285558,
      "grad_norm": 2.7498562335968018,
      "learning_rate": 8.420987654320987e-06,
      "logits/chosen": 2.3793070316314697,
      "logits/rejected": 2.4112510681152344,
      "logps/chosen": -0.8520647883415222,
      "logps/rejected": -3.673933506011963,
      "loss": 0.6735,
      "nll_loss": 0.11274605989456177,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.25561946630477905,
      "rewards/margins": 0.8465606570243835,
      "rewards/rejected": -1.102180004119873,
      "step": 2180
    },
    {
      "epoch": 0.1923034706825017,
      "grad_norm": 2.2553093433380127,
      "learning_rate": 8.408641975308642e-06,
      "logits/chosen": 2.5548908710479736,
      "logits/rejected": 2.6078009605407715,
      "logps/chosen": -0.457774817943573,
      "logps/rejected": -3.5358822345733643,
      "loss": 0.6123,
      "nll_loss": 0.05915825441479683,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13733243942260742,
      "rewards/margins": 0.9234321713447571,
      "rewards/rejected": -1.0607647895812988,
      "step": 2190
    },
    {
      "epoch": 0.19318156872214784,
      "grad_norm": 2.100526809692383,
      "learning_rate": 8.396296296296296e-06,
      "logits/chosen": 2.438559055328369,
      "logits/rejected": 2.4799530506134033,
      "logps/chosen": -0.5425572395324707,
      "logps/rejected": -2.516019582748413,
      "loss": 0.6859,
      "nll_loss": 0.06712070107460022,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1627671718597412,
      "rewards/margins": 0.5920388698577881,
      "rewards/rejected": -0.7548059821128845,
      "step": 2200
    },
    {
      "epoch": 0.19405966676179395,
      "grad_norm": 0.8824114203453064,
      "learning_rate": 8.383950617283952e-06,
      "logits/chosen": 2.6323628425598145,
      "logits/rejected": 2.694988489151001,
      "logps/chosen": -0.5362733006477356,
      "logps/rejected": -3.8703293800354004,
      "loss": 0.5774,
      "nll_loss": 0.0370684489607811,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.16088199615478516,
      "rewards/margins": 1.0002167224884033,
      "rewards/rejected": -1.1610987186431885,
      "step": 2210
    },
    {
      "epoch": 0.1949377648014401,
      "grad_norm": 7.0314717292785645,
      "learning_rate": 8.371604938271607e-06,
      "logits/chosen": 2.701634407043457,
      "logits/rejected": 2.749321222305298,
      "logps/chosen": -0.914169430732727,
      "logps/rejected": -2.8599061965942383,
      "loss": 0.7608,
      "nll_loss": 0.10523217916488647,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2742508053779602,
      "rewards/margins": 0.5837210416793823,
      "rewards/rejected": -0.8579719662666321,
      "step": 2220
    },
    {
      "epoch": 0.1958158628410862,
      "grad_norm": 0.8926441669464111,
      "learning_rate": 8.359259259259261e-06,
      "logits/chosen": 2.576303720474243,
      "logits/rejected": 2.606104850769043,
      "logps/chosen": -0.8748048543930054,
      "logps/rejected": -2.003169536590576,
      "loss": 0.7253,
      "nll_loss": 0.07492824643850327,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.2624414563179016,
      "rewards/margins": 0.33850938081741333,
      "rewards/rejected": -0.6009508371353149,
      "step": 2230
    },
    {
      "epoch": 0.19669396088073232,
      "grad_norm": 1.0316241979599,
      "learning_rate": 8.346913580246915e-06,
      "logits/chosen": 2.5414836406707764,
      "logits/rejected": 2.6227262020111084,
      "logps/chosen": -0.6043969988822937,
      "logps/rejected": -3.0870282649993896,
      "loss": 0.5888,
      "nll_loss": 0.04015268385410309,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.18131910264492035,
      "rewards/margins": 0.7447894215583801,
      "rewards/rejected": -0.9261085391044617,
      "step": 2240
    },
    {
      "epoch": 0.19757205892037846,
      "grad_norm": 7.171932220458984,
      "learning_rate": 8.334567901234568e-06,
      "logits/chosen": 2.5014796257019043,
      "logits/rejected": 2.4662632942199707,
      "logps/chosen": -0.5610159039497375,
      "logps/rejected": -2.8177947998046875,
      "loss": 0.6317,
      "nll_loss": 0.06384368985891342,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.16830478608608246,
      "rewards/margins": 0.6770337224006653,
      "rewards/rejected": -0.8453385233879089,
      "step": 2250
    },
    {
      "epoch": 0.19845015696002458,
      "grad_norm": 3.9543685913085938,
      "learning_rate": 8.322222222222223e-06,
      "logits/chosen": 2.628187656402588,
      "logits/rejected": 2.585869312286377,
      "logps/chosen": -0.62028568983078,
      "logps/rejected": -2.7624683380126953,
      "loss": 0.6441,
      "nll_loss": 0.06617014110088348,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.18608573079109192,
      "rewards/margins": 0.6426547765731812,
      "rewards/rejected": -0.8287404775619507,
      "step": 2260
    },
    {
      "epoch": 0.19932825499967072,
      "grad_norm": 4.6166815757751465,
      "learning_rate": 8.309876543209877e-06,
      "logits/chosen": 2.4966917037963867,
      "logits/rejected": 2.537562847137451,
      "logps/chosen": -0.963221549987793,
      "logps/rejected": -3.5489554405212402,
      "loss": 0.663,
      "nll_loss": 0.06778384000062943,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.28896647691726685,
      "rewards/margins": 0.7757201790809631,
      "rewards/rejected": -1.06468665599823,
      "step": 2270
    },
    {
      "epoch": 0.20020635303931683,
      "grad_norm": 11.345588684082031,
      "learning_rate": 8.297530864197531e-06,
      "logits/chosen": 2.5486679077148438,
      "logits/rejected": 2.4989380836486816,
      "logps/chosen": -0.6900479197502136,
      "logps/rejected": -2.254317045211792,
      "loss": 0.7252,
      "nll_loss": 0.07977007329463959,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.20701436698436737,
      "rewards/margins": 0.4692806601524353,
      "rewards/rejected": -0.6762951016426086,
      "step": 2280
    },
    {
      "epoch": 0.20108445107896297,
      "grad_norm": 0.1733788251876831,
      "learning_rate": 8.285185185185186e-06,
      "logits/chosen": 2.6525139808654785,
      "logits/rejected": 2.702258586883545,
      "logps/chosen": -0.6712150573730469,
      "logps/rejected": -3.3791470527648926,
      "loss": 0.6285,
      "nll_loss": 0.06629864871501923,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.20136454701423645,
      "rewards/margins": 0.8123796582221985,
      "rewards/rejected": -1.0137441158294678,
      "step": 2290
    },
    {
      "epoch": 0.20196254911860909,
      "grad_norm": 2.2842109203338623,
      "learning_rate": 8.27283950617284e-06,
      "logits/chosen": 2.430759906768799,
      "logits/rejected": 2.509899616241455,
      "logps/chosen": -0.612421989440918,
      "logps/rejected": -2.978832960128784,
      "loss": 0.6656,
      "nll_loss": 0.0659157857298851,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18372659385204315,
      "rewards/margins": 0.7099233865737915,
      "rewards/rejected": -0.8936498761177063,
      "step": 2300
    },
    {
      "epoch": 0.20284064715825523,
      "grad_norm": 6.801368713378906,
      "learning_rate": 8.260493827160495e-06,
      "logits/chosen": 2.424044609069824,
      "logits/rejected": 2.4345576763153076,
      "logps/chosen": -0.4499019682407379,
      "logps/rejected": -3.497096300125122,
      "loss": 0.5669,
      "nll_loss": 0.03968087583780289,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.13497060537338257,
      "rewards/margins": 0.9141584634780884,
      "rewards/rejected": -1.0491290092468262,
      "step": 2310
    },
    {
      "epoch": 0.20371874519790134,
      "grad_norm": 4.946938514709473,
      "learning_rate": 8.248148148148149e-06,
      "logits/chosen": 2.310943841934204,
      "logits/rejected": 2.327831983566284,
      "logps/chosen": -0.6226638555526733,
      "logps/rejected": -5.598433971405029,
      "loss": 0.4657,
      "nll_loss": 0.053391944617033005,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.18679918348789215,
      "rewards/margins": 1.4927312135696411,
      "rewards/rejected": -1.679530382156372,
      "step": 2320
    },
    {
      "epoch": 0.20459684323754748,
      "grad_norm": 24.650245666503906,
      "learning_rate": 8.235802469135803e-06,
      "logits/chosen": 2.0664007663726807,
      "logits/rejected": 2.158742904663086,
      "logps/chosen": -2.7977170944213867,
      "logps/rejected": -6.907550811767578,
      "loss": 1.5409,
      "nll_loss": 0.6891128420829773,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8393152356147766,
      "rewards/margins": 1.2329500913619995,
      "rewards/rejected": -2.072265148162842,
      "step": 2330
    },
    {
      "epoch": 0.2054749412771936,
      "grad_norm": 2.8847906589508057,
      "learning_rate": 8.223456790123458e-06,
      "logits/chosen": 2.0425117015838623,
      "logits/rejected": 2.046208381652832,
      "logps/chosen": -0.8856005668640137,
      "logps/rejected": -3.2776896953582764,
      "loss": 0.7465,
      "nll_loss": 0.10851933062076569,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.2656802237033844,
      "rewards/margins": 0.717626690864563,
      "rewards/rejected": -0.983306884765625,
      "step": 2340
    },
    {
      "epoch": 0.20635303931683974,
      "grad_norm": 6.361351013183594,
      "learning_rate": 8.211111111111112e-06,
      "logits/chosen": 2.2434327602386475,
      "logits/rejected": 2.3048999309539795,
      "logps/chosen": -0.753734290599823,
      "logps/rejected": -3.09961199760437,
      "loss": 0.6477,
      "nll_loss": 0.05653975531458855,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.22612027823925018,
      "rewards/margins": 0.7037633657455444,
      "rewards/rejected": -0.929883599281311,
      "step": 2350
    },
    {
      "epoch": 0.20723113735648585,
      "grad_norm": 4.796877384185791,
      "learning_rate": 8.198765432098767e-06,
      "logits/chosen": 2.420621156692505,
      "logits/rejected": 2.454742670059204,
      "logps/chosen": -0.5336098670959473,
      "logps/rejected": -2.5250916481018066,
      "loss": 0.6861,
      "nll_loss": 0.07172581553459167,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.16008298099040985,
      "rewards/margins": 0.5974445343017578,
      "rewards/rejected": -0.7575275897979736,
      "step": 2360
    },
    {
      "epoch": 0.208109235396132,
      "grad_norm": 6.713689804077148,
      "learning_rate": 8.186419753086421e-06,
      "logits/chosen": 2.700634241104126,
      "logits/rejected": 2.761862277984619,
      "logps/chosen": -0.9955413937568665,
      "logps/rejected": -3.3551056385040283,
      "loss": 0.7475,
      "nll_loss": 0.11970362812280655,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2986624538898468,
      "rewards/margins": 0.7078693509101868,
      "rewards/rejected": -1.0065317153930664,
      "step": 2370
    },
    {
      "epoch": 0.2089873334357781,
      "grad_norm": 5.509604454040527,
      "learning_rate": 8.174074074074074e-06,
      "logits/chosen": 2.601839780807495,
      "logits/rejected": 2.6365180015563965,
      "logps/chosen": -0.7382031679153442,
      "logps/rejected": -2.511854648590088,
      "loss": 0.6877,
      "nll_loss": 0.058238618075847626,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2214609682559967,
      "rewards/margins": 0.5320954918861389,
      "rewards/rejected": -0.7535563707351685,
      "step": 2380
    },
    {
      "epoch": 0.20986543147542422,
      "grad_norm": 0.9899409413337708,
      "learning_rate": 8.161728395061728e-06,
      "logits/chosen": 2.59370493888855,
      "logits/rejected": 2.653318166732788,
      "logps/chosen": -0.43372973799705505,
      "logps/rejected": -2.1165356636047363,
      "loss": 0.6498,
      "nll_loss": 0.055299948900938034,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.1301189363002777,
      "rewards/margins": 0.5048418045043945,
      "rewards/rejected": -0.6349607706069946,
      "step": 2390
    },
    {
      "epoch": 0.21074352951507036,
      "grad_norm": 2.4443204402923584,
      "learning_rate": 8.149382716049383e-06,
      "logits/chosen": 2.621575355529785,
      "logits/rejected": 2.6399495601654053,
      "logps/chosen": -0.6197006702423096,
      "logps/rejected": -2.7705483436584473,
      "loss": 0.6775,
      "nll_loss": 0.09681596606969833,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.1859102100133896,
      "rewards/margins": 0.6452543139457703,
      "rewards/rejected": -0.831164538860321,
      "step": 2400
    },
    {
      "epoch": 0.21162162755471647,
      "grad_norm": 3.302237033843994,
      "learning_rate": 8.137037037037037e-06,
      "logits/chosen": 2.6685478687286377,
      "logits/rejected": 2.645535469055176,
      "logps/chosen": -0.5314685106277466,
      "logps/rejected": -2.784475803375244,
      "loss": 0.6021,
      "nll_loss": 0.05336238071322441,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.1594405472278595,
      "rewards/margins": 0.675902247428894,
      "rewards/rejected": -0.8353427052497864,
      "step": 2410
    },
    {
      "epoch": 0.21249972559436262,
      "grad_norm": 1.3908727169036865,
      "learning_rate": 8.124691358024692e-06,
      "logits/chosen": 2.372706413269043,
      "logits/rejected": 2.4324469566345215,
      "logps/chosen": -0.5971062779426575,
      "logps/rejected": -2.159764051437378,
      "loss": 0.6943,
      "nll_loss": 0.05865710228681564,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.179131880402565,
      "rewards/margins": 0.4687972664833069,
      "rewards/rejected": -0.6479291915893555,
      "step": 2420
    },
    {
      "epoch": 0.21337782363400873,
      "grad_norm": 2.2388875484466553,
      "learning_rate": 8.112345679012346e-06,
      "logits/chosen": 2.378962993621826,
      "logits/rejected": 2.4710259437561035,
      "logps/chosen": -0.6657778024673462,
      "logps/rejected": -2.7295007705688477,
      "loss": 0.6557,
      "nll_loss": 0.07417738437652588,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.19973333179950714,
      "rewards/margins": 0.6191169619560242,
      "rewards/rejected": -0.8188502192497253,
      "step": 2430
    },
    {
      "epoch": 0.21425592167365487,
      "grad_norm": 11.22681713104248,
      "learning_rate": 8.1e-06,
      "logits/chosen": 2.5654773712158203,
      "logits/rejected": 2.5966382026672363,
      "logps/chosen": -0.5373214483261108,
      "logps/rejected": -2.4600396156311035,
      "loss": 0.6436,
      "nll_loss": 0.07008077204227448,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.16119642555713654,
      "rewards/margins": 0.5768154263496399,
      "rewards/rejected": -0.7380119562149048,
      "step": 2440
    },
    {
      "epoch": 0.21513401971330098,
      "grad_norm": 3.76454496383667,
      "learning_rate": 8.087654320987655e-06,
      "logits/chosen": 2.541652202606201,
      "logits/rejected": 2.5319390296936035,
      "logps/chosen": -0.7174406051635742,
      "logps/rejected": -2.493823528289795,
      "loss": 0.694,
      "nll_loss": 0.08293718844652176,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.21523217856884003,
      "rewards/margins": 0.5329148173332214,
      "rewards/rejected": -0.7481471300125122,
      "step": 2450
    },
    {
      "epoch": 0.21601211775294712,
      "grad_norm": 6.552890777587891,
      "learning_rate": 8.07530864197531e-06,
      "logits/chosen": 2.4907002449035645,
      "logits/rejected": 2.483581066131592,
      "logps/chosen": -0.5098174214363098,
      "logps/rejected": -2.033862829208374,
      "loss": 0.6884,
      "nll_loss": 0.06638985127210617,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.15294523537158966,
      "rewards/margins": 0.4572136402130127,
      "rewards/rejected": -0.6101589202880859,
      "step": 2460
    },
    {
      "epoch": 0.21689021579259324,
      "grad_norm": 1.994023323059082,
      "learning_rate": 8.062962962962964e-06,
      "logits/chosen": 2.6207573413848877,
      "logits/rejected": 2.5812675952911377,
      "logps/chosen": -0.7166529297828674,
      "logps/rejected": -2.042680263519287,
      "loss": 0.673,
      "nll_loss": 0.04922042042016983,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.21499589085578918,
      "rewards/margins": 0.3978081941604614,
      "rewards/rejected": -0.6128040552139282,
      "step": 2470
    },
    {
      "epoch": 0.21776831383223938,
      "grad_norm": 1.6430639028549194,
      "learning_rate": 8.050617283950618e-06,
      "logits/chosen": 2.2188925743103027,
      "logits/rejected": 2.256579875946045,
      "logps/chosen": -0.4708133637905121,
      "logps/rejected": -1.9442718029022217,
      "loss": 0.643,
      "nll_loss": 0.043398790061473846,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.14124402403831482,
      "rewards/margins": 0.44203758239746094,
      "rewards/rejected": -0.5832816362380981,
      "step": 2480
    },
    {
      "epoch": 0.2186464118718855,
      "grad_norm": 2.5793867111206055,
      "learning_rate": 8.038271604938272e-06,
      "logits/chosen": 2.5893726348876953,
      "logits/rejected": 2.5955262184143066,
      "logps/chosen": -0.9521909952163696,
      "logps/rejected": -2.2524361610412598,
      "loss": 0.7949,
      "nll_loss": 0.09755026549100876,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.28565728664398193,
      "rewards/margins": 0.3900734782218933,
      "rewards/rejected": -0.6757307648658752,
      "step": 2490
    },
    {
      "epoch": 0.21952450991153163,
      "grad_norm": 3.0991997718811035,
      "learning_rate": 8.025925925925927e-06,
      "logits/chosen": 2.4973714351654053,
      "logits/rejected": 2.5837717056274414,
      "logps/chosen": -0.7727764248847961,
      "logps/rejected": -1.6810334920883179,
      "loss": 0.7202,
      "nll_loss": 0.08514519035816193,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.23183290660381317,
      "rewards/margins": 0.2724771797657013,
      "rewards/rejected": -0.5043100714683533,
      "step": 2500
    },
    {
      "epoch": 0.22040260795117775,
      "grad_norm": 0.889065682888031,
      "learning_rate": 8.01358024691358e-06,
      "logits/chosen": 2.623380422592163,
      "logits/rejected": 2.6181862354278564,
      "logps/chosen": -0.6605237126350403,
      "logps/rejected": -2.3106017112731934,
      "loss": 0.6626,
      "nll_loss": 0.056938063353300095,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.19815710186958313,
      "rewards/margins": 0.4950234889984131,
      "rewards/rejected": -0.6931806206703186,
      "step": 2510
    },
    {
      "epoch": 0.2212807059908239,
      "grad_norm": 1.9629164934158325,
      "learning_rate": 8.001234567901234e-06,
      "logits/chosen": 2.725886583328247,
      "logits/rejected": 2.8111538887023926,
      "logps/chosen": -0.9530594944953918,
      "logps/rejected": -2.9670989513397217,
      "loss": 0.6788,
      "nll_loss": 0.08854852616786957,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.28591784834861755,
      "rewards/margins": 0.6042118072509766,
      "rewards/rejected": -0.8901296854019165,
      "step": 2520
    },
    {
      "epoch": 0.22215880403047,
      "grad_norm": 2.8490424156188965,
      "learning_rate": 7.98888888888889e-06,
      "logits/chosen": 2.6199440956115723,
      "logits/rejected": 2.5510783195495605,
      "logps/chosen": -0.7600888013839722,
      "logps/rejected": -2.3557190895080566,
      "loss": 0.6933,
      "nll_loss": 0.07890000194311142,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.22802665829658508,
      "rewards/margins": 0.47868919372558594,
      "rewards/rejected": -0.7067158222198486,
      "step": 2530
    },
    {
      "epoch": 0.22303690207011612,
      "grad_norm": 2.2429914474487305,
      "learning_rate": 7.976543209876545e-06,
      "logits/chosen": 2.657155752182007,
      "logits/rejected": 2.730384349822998,
      "logps/chosen": -0.6944109201431274,
      "logps/rejected": -2.3378891944885254,
      "loss": 0.6597,
      "nll_loss": 0.07231010496616364,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20832328498363495,
      "rewards/margins": 0.4930434226989746,
      "rewards/rejected": -0.7013667821884155,
      "step": 2540
    },
    {
      "epoch": 0.22391500010976226,
      "grad_norm": 8.76103401184082,
      "learning_rate": 7.964197530864199e-06,
      "logits/chosen": 2.425436019897461,
      "logits/rejected": 2.432227849960327,
      "logps/chosen": -0.8007850646972656,
      "logps/rejected": -1.2312095165252686,
      "loss": 0.8001,
      "nll_loss": 0.08954410254955292,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.24023552238941193,
      "rewards/margins": 0.1291273534297943,
      "rewards/rejected": -0.36936289072036743,
      "step": 2550
    },
    {
      "epoch": 0.22479309814940837,
      "grad_norm": 0.9743072986602783,
      "learning_rate": 7.951851851851853e-06,
      "logits/chosen": 2.5851752758026123,
      "logits/rejected": 2.6295394897460938,
      "logps/chosen": -0.5336810946464539,
      "logps/rejected": -1.8357422351837158,
      "loss": 0.6645,
      "nll_loss": 0.0451740063726902,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.16010431945323944,
      "rewards/margins": 0.39061832427978516,
      "rewards/rejected": -0.5507226586341858,
      "step": 2560
    },
    {
      "epoch": 0.2256711961890545,
      "grad_norm": 16.18039321899414,
      "learning_rate": 7.939506172839508e-06,
      "logits/chosen": 2.548182487487793,
      "logits/rejected": 2.511976718902588,
      "logps/chosen": -0.34628647565841675,
      "logps/rejected": -1.9476335048675537,
      "loss": 0.6362,
      "nll_loss": 0.043379928916692734,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1038859486579895,
      "rewards/margins": 0.4804041385650635,
      "rewards/rejected": -0.584290087223053,
      "step": 2570
    },
    {
      "epoch": 0.22654929422870063,
      "grad_norm": 0.8062020540237427,
      "learning_rate": 7.92716049382716e-06,
      "logits/chosen": 2.4778008460998535,
      "logits/rejected": 2.540804147720337,
      "logps/chosen": -0.5255548357963562,
      "logps/rejected": -2.09405779838562,
      "loss": 0.6816,
      "nll_loss": 0.07233087718486786,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.15766644477844238,
      "rewards/margins": 0.4705510139465332,
      "rewards/rejected": -0.6282175183296204,
      "step": 2580
    },
    {
      "epoch": 0.22742739226834677,
      "grad_norm": 0.9400202631950378,
      "learning_rate": 7.914814814814815e-06,
      "logits/chosen": 2.3705923557281494,
      "logits/rejected": 2.3822696208953857,
      "logps/chosen": -0.29244324564933777,
      "logps/rejected": -1.8687235116958618,
      "loss": 0.643,
      "nll_loss": 0.0392463319003582,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08773298561573029,
      "rewards/margins": 0.47288402915000916,
      "rewards/rejected": -0.5606169700622559,
      "step": 2590
    },
    {
      "epoch": 0.22830549030799288,
      "grad_norm": 2.667971134185791,
      "learning_rate": 7.90246913580247e-06,
      "logits/chosen": 2.3539376258850098,
      "logits/rejected": 2.370870351791382,
      "logps/chosen": -0.8493935465812683,
      "logps/rejected": -2.638115882873535,
      "loss": 0.7404,
      "nll_loss": 0.10912100225687027,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2548181116580963,
      "rewards/margins": 0.5366166830062866,
      "rewards/rejected": -0.7914347648620605,
      "step": 2600
    },
    {
      "epoch": 0.22918358834763902,
      "grad_norm": 2.8281736373901367,
      "learning_rate": 7.890123456790124e-06,
      "logits/chosen": 2.146486759185791,
      "logits/rejected": 2.068389415740967,
      "logps/chosen": -0.5109766125679016,
      "logps/rejected": -1.839223861694336,
      "loss": 0.6687,
      "nll_loss": 0.056281328201293945,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15329298377037048,
      "rewards/margins": 0.39847415685653687,
      "rewards/rejected": -0.5517671704292297,
      "step": 2610
    },
    {
      "epoch": 0.23006168638728514,
      "grad_norm": 0.8495587706565857,
      "learning_rate": 7.877777777777778e-06,
      "logits/chosen": 2.6396515369415283,
      "logits/rejected": 2.7024216651916504,
      "logps/chosen": -0.6976840496063232,
      "logps/rejected": -2.1378865242004395,
      "loss": 0.6814,
      "nll_loss": 0.056956302374601364,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.20930524170398712,
      "rewards/margins": 0.43206077814102173,
      "rewards/rejected": -0.6413660049438477,
      "step": 2620
    },
    {
      "epoch": 0.23093978442693128,
      "grad_norm": 3.5757088661193848,
      "learning_rate": 7.865432098765433e-06,
      "logits/chosen": 2.367114305496216,
      "logits/rejected": 2.4596433639526367,
      "logps/chosen": -0.8233789205551147,
      "logps/rejected": -3.531553268432617,
      "loss": 0.6048,
      "nll_loss": 0.07218165695667267,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.24701371788978577,
      "rewards/margins": 0.8124523162841797,
      "rewards/rejected": -1.059466004371643,
      "step": 2630
    },
    {
      "epoch": 0.2318178824665774,
      "grad_norm": 1.7588326930999756,
      "learning_rate": 7.853086419753087e-06,
      "logits/chosen": 2.2890639305114746,
      "logits/rejected": 2.2860817909240723,
      "logps/chosen": -0.31422901153564453,
      "logps/rejected": -2.4267849922180176,
      "loss": 0.584,
      "nll_loss": 0.03038870170712471,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.09426870942115784,
      "rewards/margins": 0.6337667107582092,
      "rewards/rejected": -0.7280355095863342,
      "step": 2640
    },
    {
      "epoch": 0.23269598050622353,
      "grad_norm": 4.534848213195801,
      "learning_rate": 7.840740740740741e-06,
      "logits/chosen": 2.2893667221069336,
      "logits/rejected": 2.3961310386657715,
      "logps/chosen": -1.1099700927734375,
      "logps/rejected": -3.46467924118042,
      "loss": 0.7172,
      "nll_loss": 0.1096486896276474,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3329910337924957,
      "rewards/margins": 0.7064129114151001,
      "rewards/rejected": -1.0394039154052734,
      "step": 2650
    },
    {
      "epoch": 0.23357407854586965,
      "grad_norm": 4.673385143280029,
      "learning_rate": 7.828395061728396e-06,
      "logits/chosen": 2.296675205230713,
      "logits/rejected": 2.2739694118499756,
      "logps/chosen": -0.5930169820785522,
      "logps/rejected": -2.1158299446105957,
      "loss": 0.6747,
      "nll_loss": 0.06488887220621109,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1779050976037979,
      "rewards/margins": 0.4568440020084381,
      "rewards/rejected": -0.6347490549087524,
      "step": 2660
    },
    {
      "epoch": 0.23445217658551576,
      "grad_norm": 4.619322776794434,
      "learning_rate": 7.81604938271605e-06,
      "logits/chosen": 2.316483974456787,
      "logits/rejected": 2.2958438396453857,
      "logps/chosen": -0.8736156225204468,
      "logps/rejected": -2.8213837146759033,
      "loss": 0.7145,
      "nll_loss": 0.0816427692770958,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2620847225189209,
      "rewards/margins": 0.5843304395675659,
      "rewards/rejected": -0.8464152216911316,
      "step": 2670
    },
    {
      "epoch": 0.2353302746251619,
      "grad_norm": 2.9810938835144043,
      "learning_rate": 7.803703703703705e-06,
      "logits/chosen": 2.459575891494751,
      "logits/rejected": 2.5659358501434326,
      "logps/chosen": -0.7047882080078125,
      "logps/rejected": -2.6978650093078613,
      "loss": 0.6257,
      "nll_loss": 0.048231981694698334,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21143648028373718,
      "rewards/margins": 0.5979229807853699,
      "rewards/rejected": -0.8093594312667847,
      "step": 2680
    },
    {
      "epoch": 0.23620837266480801,
      "grad_norm": 1.5284343957901,
      "learning_rate": 7.791358024691359e-06,
      "logits/chosen": 2.327259063720703,
      "logits/rejected": 2.342414379119873,
      "logps/chosen": -0.630928635597229,
      "logps/rejected": -1.5021655559539795,
      "loss": 0.749,
      "nll_loss": 0.08067157119512558,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.18927858769893646,
      "rewards/margins": 0.26137107610702515,
      "rewards/rejected": -0.4506497383117676,
      "step": 2690
    },
    {
      "epoch": 0.23708647070445416,
      "grad_norm": 2.211660385131836,
      "learning_rate": 7.779012345679014e-06,
      "logits/chosen": 2.6317856311798096,
      "logits/rejected": 2.615809679031372,
      "logps/chosen": -0.5954752564430237,
      "logps/rejected": -2.3459115028381348,
      "loss": 0.6778,
      "nll_loss": 0.08433017879724503,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.17864257097244263,
      "rewards/margins": 0.5251308679580688,
      "rewards/rejected": -0.7037734389305115,
      "step": 2700
    },
    {
      "epoch": 0.23796456874410027,
      "grad_norm": 1.088346004486084,
      "learning_rate": 7.766666666666666e-06,
      "logits/chosen": 2.3896658420562744,
      "logits/rejected": 2.4483752250671387,
      "logps/chosen": -0.9959812164306641,
      "logps/rejected": -2.1024794578552246,
      "loss": 0.7702,
      "nll_loss": 0.07456602156162262,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.29879438877105713,
      "rewards/margins": 0.33194953203201294,
      "rewards/rejected": -0.6307438611984253,
      "step": 2710
    },
    {
      "epoch": 0.2388426667837464,
      "grad_norm": 0.024926647543907166,
      "learning_rate": 7.75432098765432e-06,
      "logits/chosen": 2.4478306770324707,
      "logits/rejected": 2.464536190032959,
      "logps/chosen": -0.5027719736099243,
      "logps/rejected": -2.434457540512085,
      "loss": 0.6162,
      "nll_loss": 0.037077441811561584,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.15083159506320953,
      "rewards/margins": 0.5795056819915771,
      "rewards/rejected": -0.730337381362915,
      "step": 2720
    },
    {
      "epoch": 0.23972076482339252,
      "grad_norm": 2.2997021675109863,
      "learning_rate": 7.741975308641975e-06,
      "logits/chosen": 2.381772518157959,
      "logits/rejected": 2.4063820838928223,
      "logps/chosen": -0.2956869602203369,
      "logps/rejected": -3.0764455795288086,
      "loss": 0.5462,
      "nll_loss": 0.0379708856344223,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.08870609104633331,
      "rewards/margins": 0.834227442741394,
      "rewards/rejected": -0.9229336977005005,
      "step": 2730
    },
    {
      "epoch": 0.24059886286303866,
      "grad_norm": 0.1084788367152214,
      "learning_rate": 7.72962962962963e-06,
      "logits/chosen": 2.582411766052246,
      "logits/rejected": 2.6022956371307373,
      "logps/chosen": -0.8210613131523132,
      "logps/rejected": -1.958987832069397,
      "loss": 0.7403,
      "nll_loss": 0.07586108148097992,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.24631838500499725,
      "rewards/margins": 0.34137797355651855,
      "rewards/rejected": -0.5876964330673218,
      "step": 2740
    },
    {
      "epoch": 0.24147696090268478,
      "grad_norm": 3.7977356910705566,
      "learning_rate": 7.717283950617284e-06,
      "logits/chosen": 2.4403786659240723,
      "logits/rejected": 2.398824453353882,
      "logps/chosen": -0.5754778981208801,
      "logps/rejected": -2.9741971492767334,
      "loss": 0.631,
      "nll_loss": 0.06641169637441635,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.17264336347579956,
      "rewards/margins": 0.7196158170700073,
      "rewards/rejected": -0.8922592401504517,
      "step": 2750
    },
    {
      "epoch": 0.24235505894233092,
      "grad_norm": 1.7483222484588623,
      "learning_rate": 7.70493827160494e-06,
      "logits/chosen": 2.11322021484375,
      "logits/rejected": 2.1725521087646484,
      "logps/chosen": -0.5592783689498901,
      "logps/rejected": -1.9913737773895264,
      "loss": 0.671,
      "nll_loss": 0.07330699265003204,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.16778354346752167,
      "rewards/margins": 0.4296286106109619,
      "rewards/rejected": -0.5974121689796448,
      "step": 2760
    },
    {
      "epoch": 0.24323315698197703,
      "grad_norm": 1.0311236381530762,
      "learning_rate": 7.692592592592594e-06,
      "logits/chosen": 2.309854507446289,
      "logits/rejected": 2.313572406768799,
      "logps/chosen": -0.6930335760116577,
      "logps/rejected": -2.353086233139038,
      "loss": 0.7228,
      "nll_loss": 0.0806727483868599,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20791009068489075,
      "rewards/margins": 0.498015820980072,
      "rewards/rejected": -0.7059258818626404,
      "step": 2770
    },
    {
      "epoch": 0.24411125502162317,
      "grad_norm": 4.491925239562988,
      "learning_rate": 7.680246913580247e-06,
      "logits/chosen": 2.534341335296631,
      "logits/rejected": 2.560044050216675,
      "logps/chosen": -0.536239743232727,
      "logps/rejected": -2.622220993041992,
      "loss": 0.6372,
      "nll_loss": 0.05687220022082329,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.1608719378709793,
      "rewards/margins": 0.625794529914856,
      "rewards/rejected": -0.7866664528846741,
      "step": 2780
    },
    {
      "epoch": 0.2449893530612693,
      "grad_norm": 0.7264915108680725,
      "learning_rate": 7.667901234567902e-06,
      "logits/chosen": 2.2637343406677246,
      "logits/rejected": 2.312181234359741,
      "logps/chosen": -0.7516659498214722,
      "logps/rejected": -1.9516077041625977,
      "loss": 0.7264,
      "nll_loss": 0.07927460223436356,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.22549979388713837,
      "rewards/margins": 0.3599824607372284,
      "rewards/rejected": -0.5854822993278503,
      "step": 2790
    },
    {
      "epoch": 0.24586745110091543,
      "grad_norm": 3.0847902297973633,
      "learning_rate": 7.655555555555556e-06,
      "logits/chosen": 2.367601156234741,
      "logits/rejected": 2.403787612915039,
      "logps/chosen": -0.41557034850120544,
      "logps/rejected": -2.6111361980438232,
      "loss": 0.6177,
      "nll_loss": 0.04188116267323494,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12467111647129059,
      "rewards/margins": 0.6586698889732361,
      "rewards/rejected": -0.7833409905433655,
      "step": 2800
    },
    {
      "epoch": 0.24674554914056154,
      "grad_norm": 4.381664752960205,
      "learning_rate": 7.64320987654321e-06,
      "logits/chosen": 2.159615993499756,
      "logits/rejected": 2.1967501640319824,
      "logps/chosen": -0.8560575246810913,
      "logps/rejected": -2.5341227054595947,
      "loss": 0.6988,
      "nll_loss": 0.0791650265455246,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2568172812461853,
      "rewards/margins": 0.5034195780754089,
      "rewards/rejected": -0.7602368593215942,
      "step": 2810
    },
    {
      "epoch": 0.24762364718020766,
      "grad_norm": 1.5694470405578613,
      "learning_rate": 7.630864197530865e-06,
      "logits/chosen": 2.573787212371826,
      "logits/rejected": 2.6199076175689697,
      "logps/chosen": -0.3929263949394226,
      "logps/rejected": -2.310509443283081,
      "loss": 0.5938,
      "nll_loss": 0.04064936563372612,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.11787792295217514,
      "rewards/margins": 0.5752750039100647,
      "rewards/rejected": -0.6931529641151428,
      "step": 2820
    },
    {
      "epoch": 0.2485017452198538,
      "grad_norm": 0.7673569321632385,
      "learning_rate": 7.618518518518519e-06,
      "logits/chosen": 2.1453781127929688,
      "logits/rejected": 2.2463371753692627,
      "logps/chosen": -0.40726566314697266,
      "logps/rejected": -2.0054023265838623,
      "loss": 0.658,
      "nll_loss": 0.048675037920475006,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.12217970192432404,
      "rewards/margins": 0.4794410765171051,
      "rewards/rejected": -0.6016206741333008,
      "step": 2830
    },
    {
      "epoch": 0.2493798432594999,
      "grad_norm": 0.3016711473464966,
      "learning_rate": 7.606172839506173e-06,
      "logits/chosen": 2.3802428245544434,
      "logits/rejected": 2.4284090995788574,
      "logps/chosen": -0.2703506350517273,
      "logps/rejected": -2.1822762489318848,
      "loss": 0.6137,
      "nll_loss": 0.03810378909111023,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.08110519498586655,
      "rewards/margins": 0.5735777020454407,
      "rewards/rejected": -0.6546828746795654,
      "step": 2840
    },
    {
      "epoch": 0.250257941299146,
      "grad_norm": 7.489548683166504,
      "learning_rate": 7.593827160493827e-06,
      "logits/chosen": 2.248429298400879,
      "logits/rejected": 2.308821678161621,
      "logps/chosen": -0.8789108991622925,
      "logps/rejected": -3.1246941089630127,
      "loss": 0.664,
      "nll_loss": 0.07486443221569061,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2636732757091522,
      "rewards/margins": 0.6737349033355713,
      "rewards/rejected": -0.9374082684516907,
      "step": 2850
    },
    {
      "epoch": 0.2511360393387922,
      "grad_norm": 1.4026418924331665,
      "learning_rate": 7.581481481481482e-06,
      "logits/chosen": 2.4665749073028564,
      "logits/rejected": 2.4932830333709717,
      "logps/chosen": -0.6312376260757446,
      "logps/rejected": -2.414559841156006,
      "loss": 0.6769,
      "nll_loss": 0.043715715408325195,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.18937130272388458,
      "rewards/margins": 0.5349966883659363,
      "rewards/rejected": -0.7243679761886597,
      "step": 2860
    },
    {
      "epoch": 0.2520141373784383,
      "grad_norm": 6.41206169128418,
      "learning_rate": 7.569135802469136e-06,
      "logits/chosen": 1.9791500568389893,
      "logits/rejected": 2.065732479095459,
      "logps/chosen": -0.3990306854248047,
      "logps/rejected": -2.8924102783203125,
      "loss": 0.5879,
      "nll_loss": 0.03257184475660324,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11970920860767365,
      "rewards/margins": 0.7480138540267944,
      "rewards/rejected": -0.8677231073379517,
      "step": 2870
    },
    {
      "epoch": 0.2528922354180844,
      "grad_norm": 0.007237335667014122,
      "learning_rate": 7.5567901234567905e-06,
      "logits/chosen": 2.199582099914551,
      "logits/rejected": 2.2399208545684814,
      "logps/chosen": -0.5396376848220825,
      "logps/rejected": -2.956026554107666,
      "loss": 0.6215,
      "nll_loss": 0.05001888796687126,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16189131140708923,
      "rewards/margins": 0.7249167561531067,
      "rewards/rejected": -0.8868080377578735,
      "step": 2880
    },
    {
      "epoch": 0.25377033345773053,
      "grad_norm": 1.490009069442749,
      "learning_rate": 7.544444444444445e-06,
      "logits/chosen": 2.2380213737487793,
      "logits/rejected": 2.329550266265869,
      "logps/chosen": -0.5914583206176758,
      "logps/rejected": -3.31396484375,
      "loss": 0.6415,
      "nll_loss": 0.07680721580982208,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17743751406669617,
      "rewards/margins": 0.8167519569396973,
      "rewards/rejected": -0.9941895604133606,
      "step": 2890
    },
    {
      "epoch": 0.2546484314973767,
      "grad_norm": 4.014424800872803,
      "learning_rate": 7.5320987654321e-06,
      "logits/chosen": 1.9832671880722046,
      "logits/rejected": 2.1117234230041504,
      "logps/chosen": -0.7208765745162964,
      "logps/rejected": -2.1712794303894043,
      "loss": 0.7748,
      "nll_loss": 0.10638797283172607,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.21626298129558563,
      "rewards/margins": 0.43512091040611267,
      "rewards/rejected": -0.6513839364051819,
      "step": 2900
    },
    {
      "epoch": 0.2555265295370228,
      "grad_norm": 0.9837947487831116,
      "learning_rate": 7.519753086419753e-06,
      "logits/chosen": 2.266629695892334,
      "logits/rejected": 2.3368468284606934,
      "logps/chosen": -0.5821598172187805,
      "logps/rejected": -3.631063938140869,
      "loss": 0.622,
      "nll_loss": 0.04266344755887985,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1746479570865631,
      "rewards/margins": 0.9146712422370911,
      "rewards/rejected": -1.0893189907073975,
      "step": 2910
    },
    {
      "epoch": 0.25640462757666893,
      "grad_norm": 7.946182727813721,
      "learning_rate": 7.507407407407407e-06,
      "logits/chosen": 2.081207275390625,
      "logits/rejected": 2.123415946960449,
      "logps/chosen": -0.6722933053970337,
      "logps/rejected": -3.213026762008667,
      "loss": 0.6752,
      "nll_loss": 0.09485231339931488,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.2016880065202713,
      "rewards/margins": 0.762220025062561,
      "rewards/rejected": -0.9639080762863159,
      "step": 2920
    },
    {
      "epoch": 0.25728272561631504,
      "grad_norm": 3.1857895851135254,
      "learning_rate": 7.495061728395062e-06,
      "logits/chosen": 2.0671546459198,
      "logits/rejected": 2.1429953575134277,
      "logps/chosen": -1.1341451406478882,
      "logps/rejected": -3.0532522201538086,
      "loss": 0.7432,
      "nll_loss": 0.07530729472637177,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.34024354815483093,
      "rewards/margins": 0.5757321119308472,
      "rewards/rejected": -0.9159756898880005,
      "step": 2930
    },
    {
      "epoch": 0.2581608236559612,
      "grad_norm": 5.178243637084961,
      "learning_rate": 7.482716049382717e-06,
      "logits/chosen": 2.082383394241333,
      "logits/rejected": 2.076977252960205,
      "logps/chosen": -0.3990221321582794,
      "logps/rejected": -1.7076250314712524,
      "loss": 0.6855,
      "nll_loss": 0.06153715401887894,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1197066530585289,
      "rewards/margins": 0.39258089661598206,
      "rewards/rejected": -0.5122874975204468,
      "step": 2940
    },
    {
      "epoch": 0.2590389216956073,
      "grad_norm": 1.9402194023132324,
      "learning_rate": 7.4703703703703715e-06,
      "logits/chosen": 2.0955326557159424,
      "logits/rejected": 2.119300603866577,
      "logps/chosen": -0.48326557874679565,
      "logps/rejected": -2.141892194747925,
      "loss": 0.6575,
      "nll_loss": 0.05360071733593941,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.14497968554496765,
      "rewards/margins": 0.49758806824684143,
      "rewards/rejected": -0.6425677537918091,
      "step": 2950
    },
    {
      "epoch": 0.25991701973525344,
      "grad_norm": 2.156696081161499,
      "learning_rate": 7.458024691358026e-06,
      "logits/chosen": 2.2110159397125244,
      "logits/rejected": 2.210599422454834,
      "logps/chosen": -0.7016893625259399,
      "logps/rejected": -1.9251207113265991,
      "loss": 0.7172,
      "nll_loss": 0.06567586958408356,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.21050682663917542,
      "rewards/margins": 0.36702945828437805,
      "rewards/rejected": -0.5775362253189087,
      "step": 2960
    },
    {
      "epoch": 0.26079511777489955,
      "grad_norm": 4.643383026123047,
      "learning_rate": 7.44567901234568e-06,
      "logits/chosen": 2.3343098163604736,
      "logits/rejected": 2.3111183643341064,
      "logps/chosen": -0.546606183052063,
      "logps/rejected": -1.9335724115371704,
      "loss": 0.6684,
      "nll_loss": 0.04829854518175125,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.1639818698167801,
      "rewards/margins": 0.4160899519920349,
      "rewards/rejected": -0.5800718069076538,
      "step": 2970
    },
    {
      "epoch": 0.26167321581454567,
      "grad_norm": 3.739795684814453,
      "learning_rate": 7.433333333333334e-06,
      "logits/chosen": 2.259247303009033,
      "logits/rejected": 2.3139965534210205,
      "logps/chosen": -0.6394690871238708,
      "logps/rejected": -3.2223987579345703,
      "loss": 0.6667,
      "nll_loss": 0.07620217651128769,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1918407380580902,
      "rewards/margins": 0.774878978729248,
      "rewards/rejected": -0.9667198061943054,
      "step": 2980
    },
    {
      "epoch": 0.26255131385419184,
      "grad_norm": 3.2304673194885254,
      "learning_rate": 7.420987654320988e-06,
      "logits/chosen": 2.221369504928589,
      "logits/rejected": 2.2490549087524414,
      "logps/chosen": -0.3039132356643677,
      "logps/rejected": -3.022956371307373,
      "loss": 0.5563,
      "nll_loss": 0.030767951160669327,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.09117396920919418,
      "rewards/margins": 0.8157129287719727,
      "rewards/rejected": -0.9068870544433594,
      "step": 2990
    },
    {
      "epoch": 0.26342941189383795,
      "grad_norm": 1.1860368251800537,
      "learning_rate": 7.408641975308643e-06,
      "logits/chosen": 2.0464189052581787,
      "logits/rejected": 2.0965323448181152,
      "logps/chosen": -0.32864516973495483,
      "logps/rejected": -2.536147356033325,
      "loss": 0.617,
      "nll_loss": 0.044617362320423126,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.09859354794025421,
      "rewards/margins": 0.6622506380081177,
      "rewards/rejected": -0.7608442306518555,
      "step": 3000
    },
    {
      "epoch": 0.26430750993348406,
      "grad_norm": 1.1873282194137573,
      "learning_rate": 7.396296296296297e-06,
      "logits/chosen": 2.152050256729126,
      "logits/rejected": 2.2446908950805664,
      "logps/chosen": -0.4739529490470886,
      "logps/rejected": -3.3722312450408936,
      "loss": 0.6149,
      "nll_loss": 0.06702496111392975,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.14218589663505554,
      "rewards/margins": 0.8694835901260376,
      "rewards/rejected": -1.011669397354126,
      "step": 3010
    },
    {
      "epoch": 0.2651856079731302,
      "grad_norm": 4.482595920562744,
      "learning_rate": 7.3839506172839516e-06,
      "logits/chosen": 2.094207286834717,
      "logits/rejected": 2.200845241546631,
      "logps/chosen": -1.2236783504486084,
      "logps/rejected": -4.37764835357666,
      "loss": 0.6561,
      "nll_loss": 0.09786146134138107,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.3671035170555115,
      "rewards/margins": 0.9461910128593445,
      "rewards/rejected": -1.3132946491241455,
      "step": 3020
    },
    {
      "epoch": 0.26606370601277635,
      "grad_norm": 0.8290062546730042,
      "learning_rate": 7.371604938271606e-06,
      "logits/chosen": 1.9782556295394897,
      "logits/rejected": 2.009742021560669,
      "logps/chosen": -0.425149142742157,
      "logps/rejected": -3.8111705780029297,
      "loss": 0.5745,
      "nll_loss": 0.042985234409570694,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.12754476070404053,
      "rewards/margins": 1.0158064365386963,
      "rewards/rejected": -1.1433511972427368,
      "step": 3030
    },
    {
      "epoch": 0.26694180405242246,
      "grad_norm": 7.664605140686035,
      "learning_rate": 7.3592592592592595e-06,
      "logits/chosen": 2.083406448364258,
      "logits/rejected": 2.1995325088500977,
      "logps/chosen": -1.0140199661254883,
      "logps/rejected": -3.976250410079956,
      "loss": 0.759,
      "nll_loss": 0.07868107408285141,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3042060136795044,
      "rewards/margins": 0.8886691927909851,
      "rewards/rejected": -1.1928752660751343,
      "step": 3040
    },
    {
      "epoch": 0.2678199020920686,
      "grad_norm": 5.215522766113281,
      "learning_rate": 7.346913580246914e-06,
      "logits/chosen": 2.006986141204834,
      "logits/rejected": 2.1079816818237305,
      "logps/chosen": -0.6254408955574036,
      "logps/rejected": -2.870253801345825,
      "loss": 0.6843,
      "nll_loss": 0.07333754748106003,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1876322478055954,
      "rewards/margins": 0.6734437346458435,
      "rewards/rejected": -0.8610760569572449,
      "step": 3050
    },
    {
      "epoch": 0.2686980001317147,
      "grad_norm": 1.4749705791473389,
      "learning_rate": 7.334567901234568e-06,
      "logits/chosen": 2.1230270862579346,
      "logits/rejected": 2.2760751247406006,
      "logps/chosen": -0.9829801321029663,
      "logps/rejected": -2.8155903816223145,
      "loss": 0.7342,
      "nll_loss": 0.10188277065753937,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2948940396308899,
      "rewards/margins": 0.5497831702232361,
      "rewards/rejected": -0.8446771502494812,
      "step": 3060
    },
    {
      "epoch": 0.26957609817136086,
      "grad_norm": 9.571310043334961,
      "learning_rate": 7.322222222222223e-06,
      "logits/chosen": 2.225337028503418,
      "logits/rejected": 2.2470784187316895,
      "logps/chosen": -0.7389670014381409,
      "logps/rejected": -2.8324790000915527,
      "loss": 0.6649,
      "nll_loss": 0.0714489072561264,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.2216901034116745,
      "rewards/margins": 0.628053605556488,
      "rewards/rejected": -0.8497437238693237,
      "step": 3070
    },
    {
      "epoch": 0.27045419621100697,
      "grad_norm": 1.4797980785369873,
      "learning_rate": 7.309876543209877e-06,
      "logits/chosen": 2.0399069786071777,
      "logits/rejected": 2.1479554176330566,
      "logps/chosen": -0.7341340780258179,
      "logps/rejected": -2.218003749847412,
      "loss": 0.7062,
      "nll_loss": 0.057169754058122635,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.22024023532867432,
      "rewards/margins": 0.4451608657836914,
      "rewards/rejected": -0.6654011011123657,
      "step": 3080
    },
    {
      "epoch": 0.2713322942506531,
      "grad_norm": 11.532062530517578,
      "learning_rate": 7.297530864197532e-06,
      "logits/chosen": 2.1539671421051025,
      "logits/rejected": 2.135166645050049,
      "logps/chosen": -0.9680719375610352,
      "logps/rejected": -2.85876202583313,
      "loss": 0.7423,
      "nll_loss": 0.0898386538028717,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2904215455055237,
      "rewards/margins": 0.5672070980072021,
      "rewards/rejected": -0.8576286435127258,
      "step": 3090
    },
    {
      "epoch": 0.2722103922902992,
      "grad_norm": 0.5979923605918884,
      "learning_rate": 7.285185185185186e-06,
      "logits/chosen": 2.2366061210632324,
      "logits/rejected": 2.297550916671753,
      "logps/chosen": -0.9880453944206238,
      "logps/rejected": -3.887558698654175,
      "loss": 0.6517,
      "nll_loss": 0.08710993081331253,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2964136004447937,
      "rewards/margins": 0.8698541522026062,
      "rewards/rejected": -1.1662677526474,
      "step": 3100
    },
    {
      "epoch": 0.27308849032994537,
      "grad_norm": 6.401426792144775,
      "learning_rate": 7.27283950617284e-06,
      "logits/chosen": 2.1483216285705566,
      "logits/rejected": 2.219287395477295,
      "logps/chosen": -0.8323481678962708,
      "logps/rejected": -1.9748185873031616,
      "loss": 0.7784,
      "nll_loss": 0.11523783206939697,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2497044801712036,
      "rewards/margins": 0.34274110198020935,
      "rewards/rejected": -0.5924455523490906,
      "step": 3110
    },
    {
      "epoch": 0.2739665883695915,
      "grad_norm": 2.1834819316864014,
      "learning_rate": 7.260493827160494e-06,
      "logits/chosen": 2.21921968460083,
      "logits/rejected": 2.247816801071167,
      "logps/chosen": -0.8979324102401733,
      "logps/rejected": -2.4821817874908447,
      "loss": 0.753,
      "nll_loss": 0.09049418568611145,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.26937970519065857,
      "rewards/margins": 0.47527486085891724,
      "rewards/rejected": -0.7446545362472534,
      "step": 3120
    },
    {
      "epoch": 0.2748446864092376,
      "grad_norm": 2.105536937713623,
      "learning_rate": 7.2481481481481485e-06,
      "logits/chosen": 2.3172378540039062,
      "logits/rejected": 2.3811116218566895,
      "logps/chosen": -0.33308374881744385,
      "logps/rejected": -1.8823559284210205,
      "loss": 0.6691,
      "nll_loss": 0.04073493555188179,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09992513060569763,
      "rewards/margins": 0.4647817015647888,
      "rewards/rejected": -0.5647068023681641,
      "step": 3130
    },
    {
      "epoch": 0.2757227844488837,
      "grad_norm": 5.6695556640625,
      "learning_rate": 7.235802469135803e-06,
      "logits/chosen": 2.484741687774658,
      "logits/rejected": 2.524019718170166,
      "logps/chosen": -0.6708775758743286,
      "logps/rejected": -2.733649730682373,
      "loss": 0.675,
      "nll_loss": 0.06934002041816711,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.20126327872276306,
      "rewards/margins": 0.6188317537307739,
      "rewards/rejected": -0.8200949430465698,
      "step": 3140
    },
    {
      "epoch": 0.2766008824885298,
      "grad_norm": 0.11841005086898804,
      "learning_rate": 7.223456790123457e-06,
      "logits/chosen": 2.3487932682037354,
      "logits/rejected": 2.4092326164245605,
      "logps/chosen": -0.5150425434112549,
      "logps/rejected": -1.9013131856918335,
      "loss": 0.7062,
      "nll_loss": 0.05403406545519829,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.15451276302337646,
      "rewards/margins": 0.4158812463283539,
      "rewards/rejected": -0.570393979549408,
      "step": 3150
    },
    {
      "epoch": 0.277478980528176,
      "grad_norm": 2.9380528926849365,
      "learning_rate": 7.211111111111112e-06,
      "logits/chosen": 2.3656086921691895,
      "logits/rejected": 2.4282584190368652,
      "logps/chosen": -0.6937441825866699,
      "logps/rejected": -2.480012893676758,
      "loss": 0.701,
      "nll_loss": 0.07696821540594101,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.20812325179576874,
      "rewards/margins": 0.5358806848526001,
      "rewards/rejected": -0.74400395154953,
      "step": 3160
    },
    {
      "epoch": 0.2783570785678221,
      "grad_norm": 5.344911575317383,
      "learning_rate": 7.198765432098766e-06,
      "logits/chosen": 2.2785983085632324,
      "logits/rejected": 2.3243610858917236,
      "logps/chosen": -0.7472286224365234,
      "logps/rejected": -1.1668407917022705,
      "loss": 0.7779,
      "nll_loss": 0.07855083793401718,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.22416862845420837,
      "rewards/margins": 0.12588365375995636,
      "rewards/rejected": -0.35005226731300354,
      "step": 3170
    },
    {
      "epoch": 0.2792351766074682,
      "grad_norm": 0.625324547290802,
      "learning_rate": 7.18641975308642e-06,
      "logits/chosen": 2.2220301628112793,
      "logits/rejected": 2.2504611015319824,
      "logps/chosen": -0.7789251208305359,
      "logps/rejected": -2.3314409255981445,
      "loss": 0.7352,
      "nll_loss": 0.06989389657974243,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.23367755115032196,
      "rewards/margins": 0.4657546877861023,
      "rewards/rejected": -0.6994322538375854,
      "step": 3180
    },
    {
      "epoch": 0.28011327464711433,
      "grad_norm": 2.6372480392456055,
      "learning_rate": 7.174074074074074e-06,
      "logits/chosen": 2.337601900100708,
      "logits/rejected": 2.3855247497558594,
      "logps/chosen": -0.4849260747432709,
      "logps/rejected": -2.1613705158233643,
      "loss": 0.6273,
      "nll_loss": 0.052817367017269135,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.14547783136367798,
      "rewards/margins": 0.5029333829879761,
      "rewards/rejected": -0.6484112739562988,
      "step": 3190
    },
    {
      "epoch": 0.2809913726867605,
      "grad_norm": 2.161203145980835,
      "learning_rate": 7.1617283950617285e-06,
      "logits/chosen": 2.1061933040618896,
      "logits/rejected": 2.1634392738342285,
      "logps/chosen": -0.5152336955070496,
      "logps/rejected": -2.2968502044677734,
      "loss": 0.6311,
      "nll_loss": 0.04893555864691734,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.15457013249397278,
      "rewards/margins": 0.5344849824905396,
      "rewards/rejected": -0.6890550851821899,
      "step": 3200
    }
  ],
  "logging_steps": 10,
  "max_steps": 9000,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}