{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 363,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.002761477390403866,
      "grad_norm": 0.5197265148162842,
      "learning_rate": 0.0,
      "log_odds_chosen": 0.4317269027233124,
      "log_odds_ratio": -0.5042418837547302,
      "logits/chosen": -0.5456271767616272,
      "logits/rejected": -0.10779725015163422,
      "logps/chosen": -1.998489260673523,
      "logps/rejected": -2.3799774646759033,
      "loss": 2.1834,
      "nll_loss": 2.1329703330993652,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.199848935008049,
      "rewards/margins": 0.03814878687262535,
      "rewards/rejected": -0.23799772560596466,
      "step": 1
    },
    {
      "epoch": 0.005522954780807732,
      "grad_norm": 0.7380698323249817,
      "learning_rate": 2.7027027027027028e-08,
      "log_odds_chosen": 0.4411306381225586,
      "log_odds_ratio": -0.4988963305950165,
      "logits/chosen": -0.6594648361206055,
      "logits/rejected": -0.07251911610364914,
      "logps/chosen": -2.0794589519500732,
      "logps/rejected": -2.473759889602661,
      "loss": 2.2463,
      "nll_loss": 2.196385383605957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20794589817523956,
      "rewards/margins": 0.039430104196071625,
      "rewards/rejected": -0.24737600982189178,
      "step": 2
    },
    {
      "epoch": 0.008284432171211598,
      "grad_norm": 0.6366102695465088,
      "learning_rate": 5.4054054054054056e-08,
      "log_odds_chosen": 0.5236693620681763,
      "log_odds_ratio": -0.46964502334594727,
      "logits/chosen": -0.7439606785774231,
      "logits/rejected": -0.04276692494750023,
      "logps/chosen": -2.012474775314331,
      "logps/rejected": -2.480654716491699,
      "loss": 2.1674,
      "nll_loss": 2.1203911304473877,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20124749839305878,
      "rewards/margins": 0.046817973256111145,
      "rewards/rejected": -0.24806547164916992,
      "step": 3
    },
    {
      "epoch": 0.011045909561615464,
      "grad_norm": 0.5186454653739929,
      "learning_rate": 8.108108108108108e-08,
      "log_odds_chosen": 0.4754515290260315,
      "log_odds_ratio": -0.4913516938686371,
      "logits/chosen": -0.5739610195159912,
      "logits/rejected": -0.0752713680267334,
      "logps/chosen": -2.0060904026031494,
      "logps/rejected": -2.4304208755493164,
      "loss": 2.1537,
      "nll_loss": 2.1045989990234375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20060904324054718,
      "rewards/margins": 0.042433060705661774,
      "rewards/rejected": -0.24304211139678955,
      "step": 4
    },
    {
      "epoch": 0.013807386952019331,
      "grad_norm": 0.5336912274360657,
      "learning_rate": 1.0810810810810811e-07,
      "log_odds_chosen": 0.4733606278896332,
      "log_odds_ratio": -0.49613577127456665,
      "logits/chosen": -0.5785455703735352,
      "logits/rejected": -0.12771189212799072,
      "logps/chosen": -1.9649040699005127,
      "logps/rejected": -2.382744789123535,
      "loss": 2.1184,
      "nll_loss": 2.0687689781188965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19649039208889008,
      "rewards/margins": 0.04178408533334732,
      "rewards/rejected": -0.23827451467514038,
      "step": 5
    },
    {
      "epoch": 0.016568864342423197,
      "grad_norm": 0.5385575890541077,
      "learning_rate": 1.3513513513513515e-07,
      "log_odds_chosen": 0.3951405882835388,
      "log_odds_ratio": -0.5181410312652588,
      "logits/chosen": -0.5337327718734741,
      "logits/rejected": -0.22749024629592896,
      "logps/chosen": -2.0121028423309326,
      "logps/rejected": -2.3618290424346924,
      "loss": 2.1777,
      "nll_loss": 2.125922203063965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20121026039123535,
      "rewards/margins": 0.034972622990608215,
      "rewards/rejected": -0.23618288338184357,
      "step": 6
    },
    {
      "epoch": 0.019330341732827064,
      "grad_norm": 0.5466561913490295,
      "learning_rate": 1.6216216216216215e-07,
      "log_odds_chosen": 0.2610325217247009,
      "log_odds_ratio": -0.5772624015808105,
      "logits/chosen": -0.5413729548454285,
      "logits/rejected": -0.23400373756885529,
      "logps/chosen": -2.0744194984436035,
      "logps/rejected": -2.308685779571533,
      "loss": 2.237,
      "nll_loss": 2.1792986392974854,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20744195580482483,
      "rewards/margins": 0.023426644504070282,
      "rewards/rejected": -0.23086859285831451,
      "step": 7
    },
    {
      "epoch": 0.022091819123230928,
      "grad_norm": 0.5931817889213562,
      "learning_rate": 1.891891891891892e-07,
      "log_odds_chosen": 0.41551724076271057,
      "log_odds_ratio": -0.5085166692733765,
      "logits/chosen": -0.4540097415447235,
      "logits/rejected": -0.042557284235954285,
      "logps/chosen": -2.0818376541137695,
      "logps/rejected": -2.4534404277801514,
      "loss": 2.247,
      "nll_loss": 2.1960983276367188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20818378031253815,
      "rewards/margins": 0.03716025501489639,
      "rewards/rejected": -0.24534404277801514,
      "step": 8
    },
    {
      "epoch": 0.024853296513634795,
      "grad_norm": 0.6041191816329956,
      "learning_rate": 2.1621621621621622e-07,
      "log_odds_chosen": 0.6630181670188904,
      "log_odds_ratio": -0.42459195852279663,
      "logits/chosen": -0.7538988590240479,
      "logits/rejected": -0.16427314281463623,
      "logps/chosen": -1.9715290069580078,
      "logps/rejected": -2.5645241737365723,
      "loss": 2.1468,
      "nll_loss": 2.104363441467285,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19715292751789093,
      "rewards/margins": 0.05929948389530182,
      "rewards/rejected": -0.25645241141319275,
      "step": 9
    },
    {
      "epoch": 0.027614773904038662,
      "grad_norm": 0.738123893737793,
      "learning_rate": 2.4324324324324326e-07,
      "log_odds_chosen": 0.44313472509384155,
      "log_odds_ratio": -0.5019431710243225,
      "logits/chosen": -0.7369757890701294,
      "logits/rejected": -0.23782339692115784,
      "logps/chosen": -2.0911521911621094,
      "logps/rejected": -2.489513874053955,
      "loss": 2.2565,
      "nll_loss": 2.2063329219818115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20911525189876556,
      "rewards/margins": 0.03983616083860397,
      "rewards/rejected": -0.24895140528678894,
      "step": 10
    },
    {
      "epoch": 0.030376251294442526,
      "grad_norm": 0.66302490234375,
      "learning_rate": 2.702702702702703e-07,
      "log_odds_chosen": 0.5352292060852051,
      "log_odds_ratio": -0.4679567217826843,
      "logits/chosen": -0.6730350255966187,
      "logits/rejected": -0.14047637581825256,
      "logps/chosen": -2.055230140686035,
      "logps/rejected": -2.537259340286255,
      "loss": 2.2245,
      "nll_loss": 2.1777424812316895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20552299916744232,
      "rewards/margins": 0.04820293188095093,
      "rewards/rejected": -0.25372591614723206,
      "step": 11
    },
    {
      "epoch": 0.03313772868484639,
      "grad_norm": 0.5557176470756531,
      "learning_rate": 2.972972972972973e-07,
      "log_odds_chosen": 0.4027867913246155,
      "log_odds_ratio": -0.5220299959182739,
      "logits/chosen": -0.5875533223152161,
      "logits/rejected": -0.006882909685373306,
      "logps/chosen": -1.9575071334838867,
      "logps/rejected": -2.3148741722106934,
      "loss": 2.1445,
      "nll_loss": 2.092282772064209,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19575071334838867,
      "rewards/margins": 0.03573668748140335,
      "rewards/rejected": -0.23148740828037262,
      "step": 12
    },
    {
      "epoch": 0.03589920607525026,
      "grad_norm": 0.46403586864471436,
      "learning_rate": 3.243243243243243e-07,
      "log_odds_chosen": 0.4915584325790405,
      "log_odds_ratio": -0.479642391204834,
      "logits/chosen": -0.4917553961277008,
      "logits/rejected": -0.08394598215818405,
      "logps/chosen": -1.88984215259552,
      "logps/rejected": -2.3196663856506348,
      "loss": 2.0677,
      "nll_loss": 2.019725799560547,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1889842301607132,
      "rewards/margins": 0.04298242926597595,
      "rewards/rejected": -0.23196667432785034,
      "step": 13
    },
    {
      "epoch": 0.03866068346565413,
      "grad_norm": 0.6981692314147949,
      "learning_rate": 3.5135135135135134e-07,
      "log_odds_chosen": 0.4690040051937103,
      "log_odds_ratio": -0.4980979561805725,
      "logits/chosen": -0.687742292881012,
      "logits/rejected": -0.013176449574530125,
      "logps/chosen": -2.086107015609741,
      "logps/rejected": -2.506309747695923,
      "loss": 2.233,
      "nll_loss": 2.183150053024292,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2086106836795807,
      "rewards/margins": 0.0420202761888504,
      "rewards/rejected": -0.2506310045719147,
      "step": 14
    },
    {
      "epoch": 0.04142216085605799,
      "grad_norm": 0.7050800323486328,
      "learning_rate": 3.783783783783784e-07,
      "log_odds_chosen": 0.5713032484054565,
      "log_odds_ratio": -0.448469340801239,
      "logits/chosen": -0.7780197262763977,
      "logits/rejected": 0.1294655054807663,
      "logps/chosen": -2.0061349868774414,
      "logps/rejected": -2.514050006866455,
      "loss": 2.1671,
      "nll_loss": 2.1222198009490967,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20061349868774414,
      "rewards/margins": 0.05079151317477226,
      "rewards/rejected": -0.2514050006866455,
      "step": 15
    },
    {
      "epoch": 0.044183638246461855,
      "grad_norm": 0.49071159958839417,
      "learning_rate": 4.054054054054054e-07,
      "log_odds_chosen": 0.4512079954147339,
      "log_odds_ratio": -0.5018619894981384,
      "logits/chosen": -0.6061773300170898,
      "logits/rejected": 0.05348123610019684,
      "logps/chosen": -1.8963017463684082,
      "logps/rejected": -2.288022994995117,
      "loss": 2.0746,
      "nll_loss": 2.0244240760803223,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1896301507949829,
      "rewards/margins": 0.03917212039232254,
      "rewards/rejected": -0.22880229353904724,
      "step": 16
    },
    {
      "epoch": 0.04694511563686572,
      "grad_norm": 0.5817921161651611,
      "learning_rate": 4.3243243243243244e-07,
      "log_odds_chosen": 0.5201853513717651,
      "log_odds_ratio": -0.47411778569221497,
      "logits/chosen": -0.5998549461364746,
      "logits/rejected": 0.055297549813985825,
      "logps/chosen": -1.987619161605835,
      "logps/rejected": -2.451605796813965,
      "loss": 2.1345,
      "nll_loss": 2.0870776176452637,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19876189529895782,
      "rewards/margins": 0.04639868065714836,
      "rewards/rejected": -0.24516057968139648,
      "step": 17
    },
    {
      "epoch": 0.04970659302726959,
      "grad_norm": 0.6028745174407959,
      "learning_rate": 4.594594594594595e-07,
      "log_odds_chosen": 0.6340630054473877,
      "log_odds_ratio": -0.4299185276031494,
      "logits/chosen": -0.6498495936393738,
      "logits/rejected": -0.0934261828660965,
      "logps/chosen": -2.0205729007720947,
      "logps/rejected": -2.59153151512146,
      "loss": 2.1659,
      "nll_loss": 2.122877359390259,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20205730199813843,
      "rewards/margins": 0.05709584802389145,
      "rewards/rejected": -0.2591531574726105,
      "step": 18
    },
    {
      "epoch": 0.05246807041767346,
      "grad_norm": 0.6153688430786133,
      "learning_rate": 4.864864864864865e-07,
      "log_odds_chosen": 0.23762521147727966,
      "log_odds_ratio": -0.5856046676635742,
      "logits/chosen": -0.6359980702400208,
      "logits/rejected": -0.058345817029476166,
      "logps/chosen": -2.055912494659424,
      "logps/rejected": -2.265796661376953,
      "loss": 2.2221,
      "nll_loss": 2.163560152053833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20559124648571014,
      "rewards/margins": 0.02098841220140457,
      "rewards/rejected": -0.22657965123653412,
      "step": 19
    },
    {
      "epoch": 0.055229547808077324,
      "grad_norm": 0.5243191123008728,
      "learning_rate": 5.135135135135134e-07,
      "log_odds_chosen": 0.43903401494026184,
      "log_odds_ratio": -0.5044845342636108,
      "logits/chosen": -0.5481252670288086,
      "logits/rejected": 0.012299822643399239,
      "logps/chosen": -1.9716014862060547,
      "logps/rejected": -2.3587422370910645,
      "loss": 2.1563,
      "nll_loss": 2.1058452129364014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19716015458106995,
      "rewards/margins": 0.03871406614780426,
      "rewards/rejected": -0.2358742207288742,
      "step": 20
    },
    {
      "epoch": 0.057991025198481184,
      "grad_norm": 0.6418594717979431,
      "learning_rate": 5.405405405405406e-07,
      "log_odds_chosen": 0.3529980778694153,
      "log_odds_ratio": -0.5401824712753296,
      "logits/chosen": -0.45990100502967834,
      "logits/rejected": -0.03464129567146301,
      "logps/chosen": -2.1360199451446533,
      "logps/rejected": -2.4547152519226074,
      "loss": 2.2806,
      "nll_loss": 2.2266147136688232,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2136019915342331,
      "rewards/margins": 0.03186953812837601,
      "rewards/rejected": -0.2454715520143509,
      "step": 21
    },
    {
      "epoch": 0.06075250258888505,
      "grad_norm": 0.573546290397644,
      "learning_rate": 5.675675675675675e-07,
      "log_odds_chosen": 0.5019404888153076,
      "log_odds_ratio": -0.48850372433662415,
      "logits/chosen": -0.6077824831008911,
      "logits/rejected": -0.18484465777873993,
      "logps/chosen": -1.9800050258636475,
      "logps/rejected": -2.425489664077759,
      "loss": 2.1345,
      "nll_loss": 2.0856926441192627,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19800050556659698,
      "rewards/margins": 0.04454846680164337,
      "rewards/rejected": -0.24254897236824036,
      "step": 22
    },
    {
      "epoch": 0.06351397997928893,
      "grad_norm": 0.5493528246879578,
      "learning_rate": 5.945945945945947e-07,
      "log_odds_chosen": 0.39711329340934753,
      "log_odds_ratio": -0.523137629032135,
      "logits/chosen": -0.5691142082214355,
      "logits/rejected": -0.3585430979728699,
      "logps/chosen": -1.9641809463500977,
      "logps/rejected": -2.3188183307647705,
      "loss": 2.1462,
      "nll_loss": 2.093902587890625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19641812145709991,
      "rewards/margins": 0.035463735461235046,
      "rewards/rejected": -0.23188185691833496,
      "step": 23
    },
    {
      "epoch": 0.06627545736969279,
      "grad_norm": 0.5700852870941162,
      "learning_rate": 6.216216216216216e-07,
      "log_odds_chosen": 0.5202399492263794,
      "log_odds_ratio": -0.4676600396633148,
      "logits/chosen": -0.5802191495895386,
      "logits/rejected": 0.039508990943431854,
      "logps/chosen": -1.967350959777832,
      "logps/rejected": -2.429182767868042,
      "loss": 2.1365,
      "nll_loss": 2.0896944999694824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19673511385917664,
      "rewards/margins": 0.04618315398693085,
      "rewards/rejected": -0.24291826784610748,
      "step": 24
    },
    {
      "epoch": 0.06903693476009665,
      "grad_norm": 0.7011380791664124,
      "learning_rate": 6.486486486486486e-07,
      "log_odds_chosen": 0.6565573215484619,
      "log_odds_ratio": -0.4259224534034729,
      "logits/chosen": -0.649218738079071,
      "logits/rejected": -0.06181307137012482,
      "logps/chosen": -1.9984382390975952,
      "logps/rejected": -2.588566780090332,
      "loss": 2.1586,
      "nll_loss": 2.1160356998443604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19984383881092072,
      "rewards/margins": 0.059012845158576965,
      "rewards/rejected": -0.2588566839694977,
      "step": 25
    },
    {
      "epoch": 0.07179841215050052,
      "grad_norm": 0.6201441884040833,
      "learning_rate": 6.756756756756756e-07,
      "log_odds_chosen": 0.3787028193473816,
      "log_odds_ratio": -0.5289822816848755,
      "logits/chosen": -0.5880062580108643,
      "logits/rejected": -0.11418405175209045,
      "logps/chosen": -2.0192718505859375,
      "logps/rejected": -2.356654644012451,
      "loss": 2.1955,
      "nll_loss": 2.142634153366089,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20192717015743256,
      "rewards/margins": 0.03373831510543823,
      "rewards/rejected": -0.23566550016403198,
      "step": 26
    },
    {
      "epoch": 0.07455988954090438,
      "grad_norm": 0.44776439666748047,
      "learning_rate": 7.027027027027027e-07,
      "log_odds_chosen": 0.4692346453666687,
      "log_odds_ratio": -0.49579283595085144,
      "logits/chosen": -0.43663290143013,
      "logits/rejected": -0.17443624138832092,
      "logps/chosen": -1.8659433126449585,
      "logps/rejected": -2.274401903152466,
      "loss": 2.0463,
      "nll_loss": 1.9967215061187744,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18659432232379913,
      "rewards/margins": 0.040845878422260284,
      "rewards/rejected": -0.22744019329547882,
      "step": 27
    },
    {
      "epoch": 0.07732136693130826,
      "grad_norm": 0.5262994170188904,
      "learning_rate": 7.297297297297297e-07,
      "log_odds_chosen": 0.5275386571884155,
      "log_odds_ratio": -0.4684979319572449,
      "logits/chosen": -0.5315351486206055,
      "logits/rejected": -0.21029916405677795,
      "logps/chosen": -1.9571702480316162,
      "logps/rejected": -2.424804449081421,
      "loss": 2.1246,
      "nll_loss": 2.077768325805664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19571705162525177,
      "rewards/margins": 0.04676342010498047,
      "rewards/rejected": -0.24248044192790985,
      "step": 28
    },
    {
      "epoch": 0.08008284432171212,
      "grad_norm": 0.5570821762084961,
      "learning_rate": 7.567567567567568e-07,
      "log_odds_chosen": 0.4756891429424286,
      "log_odds_ratio": -0.49199897050857544,
      "logits/chosen": -0.5677704811096191,
      "logits/rejected": -0.034404173493385315,
      "logps/chosen": -2.0466325283050537,
      "logps/rejected": -2.471601724624634,
      "loss": 2.2042,
      "nll_loss": 2.1550400257110596,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2046632468700409,
      "rewards/margins": 0.04249695688486099,
      "rewards/rejected": -0.24716021120548248,
      "step": 29
    },
    {
      "epoch": 0.08284432171211598,
      "grad_norm": 0.6109534502029419,
      "learning_rate": 7.837837837837838e-07,
      "log_odds_chosen": 0.645262598991394,
      "log_odds_ratio": -0.42898693680763245,
      "logits/chosen": -0.5536071062088013,
      "logits/rejected": -0.004539853893220425,
      "logps/chosen": -2.027257204055786,
      "logps/rejected": -2.6078548431396484,
      "loss": 2.1758,
      "nll_loss": 2.1329245567321777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20272573828697205,
      "rewards/margins": 0.05805976688861847,
      "rewards/rejected": -0.2607854902744293,
      "step": 30
    },
    {
      "epoch": 0.08560579910251985,
      "grad_norm": 0.6070998907089233,
      "learning_rate": 8.108108108108108e-07,
      "log_odds_chosen": 0.5565178990364075,
      "log_odds_ratio": -0.45878711342811584,
      "logits/chosen": -0.5867283344268799,
      "logits/rejected": -0.06864497065544128,
      "logps/chosen": -2.0168516635894775,
      "logps/rejected": -2.5162007808685303,
      "loss": 2.1855,
      "nll_loss": 2.1396615505218506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20168517529964447,
      "rewards/margins": 0.04993489012122154,
      "rewards/rejected": -0.2516200542449951,
      "step": 31
    },
    {
      "epoch": 0.08836727649292371,
      "grad_norm": 0.4846738278865814,
      "learning_rate": 8.378378378378377e-07,
      "log_odds_chosen": 0.43661972880363464,
      "log_odds_ratio": -0.5039945840835571,
      "logits/chosen": -0.4310506582260132,
      "logits/rejected": 0.003593362867832184,
      "logps/chosen": -2.0135138034820557,
      "logps/rejected": -2.402101516723633,
      "loss": 2.1852,
      "nll_loss": 2.134814500808716,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20135138928890228,
      "rewards/margins": 0.03885875269770622,
      "rewards/rejected": -0.24021016061306,
      "step": 32
    },
    {
      "epoch": 0.09112875388332758,
      "grad_norm": 0.5441243648529053,
      "learning_rate": 8.648648648648649e-07,
      "log_odds_chosen": 0.5726721286773682,
      "log_odds_ratio": -0.45256343483924866,
      "logits/chosen": -0.4675900936126709,
      "logits/rejected": -0.058118633925914764,
      "logps/chosen": -1.9284062385559082,
      "logps/rejected": -2.4360082149505615,
      "loss": 2.0956,
      "nll_loss": 2.0503551959991455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19284063577651978,
      "rewards/margins": 0.05076019838452339,
      "rewards/rejected": -0.24360083043575287,
      "step": 33
    },
    {
      "epoch": 0.09389023127373144,
      "grad_norm": 0.4893564283847809,
      "learning_rate": 8.918918918918918e-07,
      "log_odds_chosen": 0.6038653254508972,
      "log_odds_ratio": -0.45129159092903137,
      "logits/chosen": -0.41903308033943176,
      "logits/rejected": -0.08722685277462006,
      "logps/chosen": -1.9256908893585205,
      "logps/rejected": -2.4649672508239746,
      "loss": 2.0767,
      "nll_loss": 2.0315608978271484,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1925690770149231,
      "rewards/margins": 0.053927645087242126,
      "rewards/rejected": -0.24649673700332642,
      "step": 34
    },
    {
      "epoch": 0.09665170866413532,
      "grad_norm": 0.6620936393737793,
      "learning_rate": 9.18918918918919e-07,
      "log_odds_chosen": 0.41170692443847656,
      "log_odds_ratio": -0.512043833732605,
      "logits/chosen": -0.8187747001647949,
      "logits/rejected": -0.20862913131713867,
      "logps/chosen": -1.982971429824829,
      "logps/rejected": -2.3442938327789307,
      "loss": 2.1551,
      "nll_loss": 2.1038753986358643,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1982971578836441,
      "rewards/margins": 0.03613222762942314,
      "rewards/rejected": -0.23442938923835754,
      "step": 35
    },
    {
      "epoch": 0.09941318605453918,
      "grad_norm": 0.6175864338874817,
      "learning_rate": 9.459459459459459e-07,
      "log_odds_chosen": 0.6069755554199219,
      "log_odds_ratio": -0.4442310333251953,
      "logits/chosen": -0.6989039182662964,
      "logits/rejected": 0.013947159051895142,
      "logps/chosen": -1.9784715175628662,
      "logps/rejected": -2.521758794784546,
      "loss": 2.1381,
      "nll_loss": 2.093674659729004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1978471428155899,
      "rewards/margins": 0.054328739643096924,
      "rewards/rejected": -0.252175897359848,
      "step": 36
    },
    {
      "epoch": 0.10217466344494304,
      "grad_norm": 0.6760265827178955,
      "learning_rate": 9.72972972972973e-07,
      "log_odds_chosen": 0.5962284803390503,
      "log_odds_ratio": -0.445417582988739,
      "logits/chosen": -0.5475363731384277,
      "logits/rejected": -0.18748612701892853,
      "logps/chosen": -2.120981216430664,
      "logps/rejected": -2.6638612747192383,
      "loss": 2.2946,
      "nll_loss": 2.2500131130218506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2120981216430664,
      "rewards/margins": 0.054287999868392944,
      "rewards/rejected": -0.26638612151145935,
      "step": 37
    },
    {
      "epoch": 0.10493614083534691,
      "grad_norm": 0.5492844581604004,
      "learning_rate": 1e-06,
      "log_odds_chosen": 0.429279625415802,
      "log_odds_ratio": -0.5045007467269897,
      "logits/chosen": -0.564774751663208,
      "logits/rejected": -0.1553649604320526,
      "logps/chosen": -1.9513108730316162,
      "logps/rejected": -2.3313405513763428,
      "loss": 2.1348,
      "nll_loss": 2.0843911170959473,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1951310932636261,
      "rewards/margins": 0.03800297528505325,
      "rewards/rejected": -0.23313407599925995,
      "step": 38
    },
    {
      "epoch": 0.10769761822575077,
      "grad_norm": 0.6983578205108643,
      "learning_rate": 9.999767832624e-07,
      "log_odds_chosen": 0.49402672052383423,
      "log_odds_ratio": -0.49974578619003296,
      "logits/chosen": -0.49002426862716675,
      "logits/rejected": 0.011857733130455017,
      "logps/chosen": -2.0831823348999023,
      "logps/rejected": -2.533005475997925,
      "loss": 2.2348,
      "nll_loss": 2.1848011016845703,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20831823348999023,
      "rewards/margins": 0.04498231038451195,
      "rewards/rejected": -0.25330057740211487,
      "step": 39
    },
    {
      "epoch": 0.11045909561615465,
      "grad_norm": 0.7787635922431946,
      "learning_rate": 9.999071352056673e-07,
      "log_odds_chosen": 0.2771000862121582,
      "log_odds_ratio": -0.5739508271217346,
      "logits/chosen": -0.6959272623062134,
      "logits/rejected": -0.16685396432876587,
      "logps/chosen": -2.1053991317749023,
      "logps/rejected": -2.3543496131896973,
      "loss": 2.2778,
      "nll_loss": 2.2204456329345703,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21053989231586456,
      "rewards/margins": 0.02489505149424076,
      "rewards/rejected": -0.23543494939804077,
      "step": 40
    },
    {
      "epoch": 0.11322057300655851,
      "grad_norm": 0.4911040961742401,
      "learning_rate": 9.99791062297805e-07,
      "log_odds_chosen": 0.48859286308288574,
      "log_odds_ratio": -0.48782065510749817,
      "logits/chosen": -0.5469129681587219,
      "logits/rejected": -0.11792521923780441,
      "logps/chosen": -1.9237489700317383,
      "logps/rejected": -2.355954647064209,
      "loss": 2.0893,
      "nll_loss": 2.0404934883117676,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19237489998340607,
      "rewards/margins": 0.043220579624176025,
      "rewards/rejected": -0.2355954945087433,
      "step": 41
    },
    {
      "epoch": 0.11598205039696237,
      "grad_norm": 0.6618607640266418,
      "learning_rate": 9.996285753181497e-07,
      "log_odds_chosen": 0.3562512695789337,
      "log_odds_ratio": -0.5404328107833862,
      "logits/chosen": -0.6230701804161072,
      "logits/rejected": -0.18673820793628693,
      "logps/chosen": -2.1178598403930664,
      "logps/rejected": -2.437302589416504,
      "loss": 2.2876,
      "nll_loss": 2.2335362434387207,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21178597211837769,
      "rewards/margins": 0.031944289803504944,
      "rewards/rejected": -0.24373027682304382,
      "step": 42
    },
    {
      "epoch": 0.11874352778736624,
      "grad_norm": 0.5659067034721375,
      "learning_rate": 9.99419689356372e-07,
      "log_odds_chosen": 0.4233945906162262,
      "log_odds_ratio": -0.5130196809768677,
      "logits/chosen": -0.5873526930809021,
      "logits/rejected": -0.1797933280467987,
      "logps/chosen": -1.9770363569259644,
      "logps/rejected": -2.3484270572662354,
      "loss": 2.1586,
      "nll_loss": 2.107292652130127,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19770362973213196,
      "rewards/margins": 0.037139080464839935,
      "rewards/rejected": -0.2348427027463913,
      "step": 43
    },
    {
      "epoch": 0.1215050051777701,
      "grad_norm": 0.5917359590530396,
      "learning_rate": 9.99164423811074e-07,
      "log_odds_chosen": 0.3507523834705353,
      "log_odds_ratio": -0.5445480942726135,
      "logits/chosen": -0.5411754846572876,
      "logits/rejected": -0.20721937716007233,
      "logps/chosen": -2.068720817565918,
      "logps/rejected": -2.381938934326172,
      "loss": 2.2314,
      "nll_loss": 2.1769025325775146,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2068721055984497,
      "rewards/margins": 0.031321801245212555,
      "rewards/rejected": -0.23819389939308167,
      "step": 44
    },
    {
      "epoch": 0.12426648256817398,
      "grad_norm": 0.5997647643089294,
      "learning_rate": 9.988628023879881e-07,
      "log_odds_chosen": 0.6433253884315491,
      "log_odds_ratio": -0.42948102951049805,
      "logits/chosen": -0.6800805926322937,
      "logits/rejected": -0.3857523500919342,
      "logps/chosen": -1.972766399383545,
      "logps/rejected": -2.5487842559814453,
      "loss": 2.1234,
      "nll_loss": 2.080449342727661,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19727663695812225,
      "rewards/margins": 0.05760182440280914,
      "rewards/rejected": -0.254878431558609,
      "step": 45
    },
    {
      "epoch": 0.12702795995857785,
      "grad_norm": 0.5174111127853394,
      "learning_rate": 9.985148530977764e-07,
      "log_odds_chosen": 0.5635263919830322,
      "log_odds_ratio": -0.4585229754447937,
      "logits/chosen": -0.5617026090621948,
      "logits/rejected": -0.14804458618164062,
      "logps/chosen": -1.8502693176269531,
      "logps/rejected": -2.3388235569000244,
      "loss": 2.0084,
      "nll_loss": 1.9625658988952637,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18502694368362427,
      "rewards/margins": 0.04885541647672653,
      "rewards/rejected": -0.2338823527097702,
      "step": 46
    },
    {
      "epoch": 0.1297894373489817,
      "grad_norm": 0.5050185322761536,
      "learning_rate": 9.981206082534287e-07,
      "log_odds_chosen": 0.4026612937450409,
      "log_odds_ratio": -0.525580883026123,
      "logits/chosen": -0.4673263728618622,
      "logits/rejected": -0.16735760867595673,
      "logps/chosen": -2.0518088340759277,
      "logps/rejected": -2.41474986076355,
      "loss": 2.2222,
      "nll_loss": 2.1696157455444336,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20518088340759277,
      "rewards/margins": 0.036294080317020416,
      "rewards/rejected": -0.24147498607635498,
      "step": 47
    },
    {
      "epoch": 0.13255091473938557,
      "grad_norm": 0.6147828102111816,
      "learning_rate": 9.976801044672607e-07,
      "log_odds_chosen": 0.18149018287658691,
      "log_odds_ratio": -0.6318742632865906,
      "logits/chosen": -0.4836263060569763,
      "logits/rejected": -0.11841636896133423,
      "logps/chosen": -2.127021074295044,
      "logps/rejected": -2.3001065254211426,
      "loss": 2.29,
      "nll_loss": 2.2268035411834717,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21270211040973663,
      "rewards/margins": 0.0173085518181324,
      "rewards/rejected": -0.23001064360141754,
      "step": 48
    },
    {
      "epoch": 0.13531239212978943,
      "grad_norm": 0.6266311407089233,
      "learning_rate": 9.97193382647516e-07,
      "log_odds_chosen": 0.6347547769546509,
      "log_odds_ratio": -0.43759316205978394,
      "logits/chosen": -0.5783393383026123,
      "logits/rejected": -0.030999187380075455,
      "logps/chosen": -2.064052104949951,
      "logps/rejected": -2.6410865783691406,
      "loss": 2.2133,
      "nll_loss": 2.169518232345581,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20640523731708527,
      "rewards/margins": 0.0577034056186676,
      "rewards/rejected": -0.26410865783691406,
      "step": 49
    },
    {
      "epoch": 0.1380738695201933,
      "grad_norm": 0.6744863390922546,
      "learning_rate": 9.966604879945656e-07,
      "log_odds_chosen": 0.5657740831375122,
      "log_odds_ratio": -0.47156062722206116,
      "logits/chosen": -0.6663930416107178,
      "logits/rejected": -0.027224615216255188,
      "logps/chosen": -2.0237436294555664,
      "logps/rejected": -2.534412384033203,
      "loss": 2.1689,
      "nll_loss": 2.1217703819274902,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20237436890602112,
      "rewards/margins": 0.05106687173247337,
      "rewards/rejected": -0.2534412443637848,
      "step": 50
    },
    {
      "epoch": 0.14083534691059718,
      "grad_norm": 0.6870261430740356,
      "learning_rate": 9.96081469996711e-07,
      "log_odds_chosen": 0.4837898910045624,
      "log_odds_ratio": -0.48495402932167053,
      "logits/chosen": -0.5758293271064758,
      "logits/rejected": -0.31325966119766235,
      "logps/chosen": -2.0443923473358154,
      "logps/rejected": -2.476912498474121,
      "loss": 2.2087,
      "nll_loss": 2.1601643562316895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20443923771381378,
      "rewards/margins": 0.043251991271972656,
      "rewards/rejected": -0.24769122898578644,
      "step": 51
    },
    {
      "epoch": 0.14359682430100104,
      "grad_norm": 0.5728748440742493,
      "learning_rate": 9.954563824255877e-07,
      "log_odds_chosen": 0.43097832798957825,
      "log_odds_ratio": -0.5058196783065796,
      "logits/chosen": -0.5012402534484863,
      "logits/rejected": -0.1452914923429489,
      "logps/chosen": -1.9914681911468506,
      "logps/rejected": -2.3732481002807617,
      "loss": 2.1486,
      "nll_loss": 2.098066806793213,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1991468220949173,
      "rewards/margins": 0.03817800432443619,
      "rewards/rejected": -0.23732483386993408,
      "step": 52
    },
    {
      "epoch": 0.1463583016914049,
      "grad_norm": 0.6349831819534302,
      "learning_rate": 9.947852833311724e-07,
      "log_odds_chosen": 0.3815571665763855,
      "log_odds_ratio": -0.526985228061676,
      "logits/chosen": -0.643978476524353,
      "logits/rejected": -0.23492318391799927,
      "logps/chosen": -1.957843542098999,
      "logps/rejected": -2.2938966751098633,
      "loss": 2.136,
      "nll_loss": 2.083264112472534,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19578438997268677,
      "rewards/margins": 0.033605292439460754,
      "rewards/rejected": -0.22938966751098633,
      "step": 53
    },
    {
      "epoch": 0.14911977908180876,
      "grad_norm": 0.5989128351211548,
      "learning_rate": 9.94068235036391e-07,
      "log_odds_chosen": 0.39456459879875183,
      "log_odds_ratio": -0.5237154364585876,
      "logits/chosen": -0.6117141842842102,
      "logits/rejected": -0.17834247648715973,
      "logps/chosen": -1.9467459917068481,
      "logps/rejected": -2.2964022159576416,
      "loss": 2.1273,
      "nll_loss": 2.0749144554138184,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19467462599277496,
      "rewards/margins": 0.034965626895427704,
      "rewards/rejected": -0.22964024543762207,
      "step": 54
    },
    {
      "epoch": 0.15188125647221262,
      "grad_norm": 0.5899688005447388,
      "learning_rate": 9.933053041313323e-07,
      "log_odds_chosen": 0.3930458724498749,
      "log_odds_ratio": -0.5195820927619934,
      "logits/chosen": -0.6809389591217041,
      "logits/rejected": -0.11192372441291809,
      "logps/chosen": -2.004241704940796,
      "logps/rejected": -2.3520522117614746,
      "loss": 2.1823,
      "nll_loss": 2.130378246307373,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2004241794347763,
      "rewards/margins": 0.03478105366230011,
      "rewards/rejected": -0.2352052479982376,
      "step": 55
    },
    {
      "epoch": 0.1546427338626165,
      "grad_norm": 0.7014657258987427,
      "learning_rate": 9.924965614670628e-07,
      "log_odds_chosen": 0.6613426804542542,
      "log_odds_ratio": -0.42668840289115906,
      "logits/chosen": -0.6904681324958801,
      "logits/rejected": -0.11418459564447403,
      "logps/chosen": -2.0051043033599854,
      "logps/rejected": -2.600146770477295,
      "loss": 2.1764,
      "nll_loss": 2.133692502975464,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2005104273557663,
      "rewards/margins": 0.05950424075126648,
      "rewards/rejected": -0.26001468300819397,
      "step": 56
    },
    {
      "epoch": 0.15740421125302037,
      "grad_norm": 0.5079272389411926,
      "learning_rate": 9.916420821490472e-07,
      "log_odds_chosen": 0.4238623380661011,
      "log_odds_ratio": -0.5145986080169678,
      "logits/chosen": -0.5487760901451111,
      "logits/rejected": -0.05774221196770668,
      "logps/chosen": -1.8575114011764526,
      "logps/rejected": -2.229008674621582,
      "loss": 2.0278,
      "nll_loss": 1.9763325452804565,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18575115501880646,
      "rewards/margins": 0.03714972361922264,
      "rewards/rejected": -0.2229008823633194,
      "step": 57
    },
    {
      "epoch": 0.16016568864342423,
      "grad_norm": 0.6976705193519592,
      "learning_rate": 9.90741945530174e-07,
      "log_odds_chosen": 0.5120624303817749,
      "log_odds_ratio": -0.4859912693500519,
      "logits/chosen": -0.5908156037330627,
      "logits/rejected": -0.12501533329486847,
      "logps/chosen": -2.0053253173828125,
      "logps/rejected": -2.459195852279663,
      "loss": 2.172,
      "nll_loss": 2.123382091522217,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20053252577781677,
      "rewards/margins": 0.04538704454898834,
      "rewards/rejected": -0.2459195852279663,
      "step": 58
    },
    {
      "epoch": 0.1629271660338281,
      "grad_norm": 0.6146124601364136,
      "learning_rate": 9.89796235203386e-07,
      "log_odds_chosen": 0.29176169633865356,
      "log_odds_ratio": -0.5638565421104431,
      "logits/chosen": -0.5347757339477539,
      "logits/rejected": -0.20498931407928467,
      "logps/chosen": -1.9904438257217407,
      "logps/rejected": -2.247553825378418,
      "loss": 2.1691,
      "nll_loss": 2.1127238273620605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1990443915128708,
      "rewards/margins": 0.025710972025990486,
      "rewards/rejected": -0.22475537657737732,
      "step": 59
    },
    {
      "epoch": 0.16568864342423195,
      "grad_norm": 0.5901596546173096,
      "learning_rate": 9.888050389939172e-07,
      "log_odds_chosen": 0.37348029017448425,
      "log_odds_ratio": -0.5292580723762512,
      "logits/chosen": -0.5264938473701477,
      "logits/rejected": 0.09010139852762222,
      "logps/chosen": -2.0238590240478516,
      "logps/rejected": -2.355509042739868,
      "loss": 2.1964,
      "nll_loss": 2.143430233001709,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20238588750362396,
      "rewards/margins": 0.03316497802734375,
      "rewards/rejected": -0.2355508804321289,
      "step": 60
    },
    {
      "epoch": 0.16845012081463584,
      "grad_norm": 0.5306107997894287,
      "learning_rate": 9.877684489511365e-07,
      "log_odds_chosen": 0.6772121787071228,
      "log_odds_ratio": -0.41606950759887695,
      "logits/chosen": -0.5884420871734619,
      "logits/rejected": -0.17208705842494965,
      "logps/chosen": -1.9372531175613403,
      "logps/rejected": -2.540231943130493,
      "loss": 2.0812,
      "nll_loss": 2.0395634174346924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19372530281543732,
      "rewards/margins": 0.060297898948192596,
      "rewards/rejected": -0.2540232241153717,
      "step": 61
    },
    {
      "epoch": 0.1712115982050397,
      "grad_norm": 0.708717942237854,
      "learning_rate": 9.866865613400006e-07,
      "log_odds_chosen": 0.40706658363342285,
      "log_odds_ratio": -0.51793372631073,
      "logits/chosen": -0.5416155457496643,
      "logits/rejected": 0.13652700185775757,
      "logps/chosen": -2.111114740371704,
      "logps/rejected": -2.476184129714966,
      "loss": 2.2753,
      "nll_loss": 2.2235543727874756,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21111147105693817,
      "rewards/margins": 0.03650694340467453,
      "rewards/rejected": -0.2476184070110321,
      "step": 62
    },
    {
      "epoch": 0.17397307559544356,
      "grad_norm": 0.5680522918701172,
      "learning_rate": 9.855594766321122e-07,
      "log_odds_chosen": 0.5157344341278076,
      "log_odds_ratio": -0.4737292230129242,
      "logits/chosen": -0.5442768335342407,
      "logits/rejected": -0.11401443928480148,
      "logps/chosen": -2.0177366733551025,
      "logps/rejected": -2.4775257110595703,
      "loss": 2.1671,
      "nll_loss": 2.1197335720062256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20177365839481354,
      "rewards/margins": 0.04597889631986618,
      "rewards/rejected": -0.24775256216526031,
      "step": 63
    },
    {
      "epoch": 0.17673455298584742,
      "grad_norm": 0.5254662036895752,
      "learning_rate": 9.843872994963912e-07,
      "log_odds_chosen": 0.581967830657959,
      "log_odds_ratio": -0.4526694416999817,
      "logits/chosen": -0.6353936791419983,
      "logits/rejected": -0.27343085408210754,
      "logps/chosen": -1.987318515777588,
      "logps/rejected": -2.509692668914795,
      "loss": 2.1343,
      "nll_loss": 2.089053153991699,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19873185455799103,
      "rewards/margins": 0.052237413823604584,
      "rewards/rejected": -0.250969260931015,
      "step": 64
    },
    {
      "epoch": 0.17949603037625128,
      "grad_norm": 0.5754149556159973,
      "learning_rate": 9.831701387893532e-07,
      "log_odds_chosen": 0.3633577227592468,
      "log_odds_ratio": -0.5337969064712524,
      "logits/chosen": -0.5239946842193604,
      "logits/rejected": -0.029723022133111954,
      "logps/chosen": -1.9162522554397583,
      "logps/rejected": -2.2321159839630127,
      "loss": 2.0987,
      "nll_loss": 2.0453133583068848,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19162523746490479,
      "rewards/margins": 0.031586356461048126,
      "rewards/rejected": -0.22321158647537231,
      "step": 65
    },
    {
      "epoch": 0.18225750776665517,
      "grad_norm": 0.5621442198753357,
      "learning_rate": 9.819081075450013e-07,
      "log_odds_chosen": 0.5473266839981079,
      "log_odds_ratio": -0.4831118583679199,
      "logits/chosen": -0.7125363945960999,
      "logits/rejected": -0.14726145565509796,
      "logps/chosen": -1.882205605506897,
      "logps/rejected": -2.363412380218506,
      "loss": 2.0493,
      "nll_loss": 2.000981092453003,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1882205605506897,
      "rewards/margins": 0.048120684921741486,
      "rewards/rejected": -0.23634123802185059,
      "step": 66
    },
    {
      "epoch": 0.18501898515705903,
      "grad_norm": 0.5734560489654541,
      "learning_rate": 9.80601322964329e-07,
      "log_odds_chosen": 0.33386579155921936,
      "log_odds_ratio": -0.5444394946098328,
      "logits/chosen": -0.5026360750198364,
      "logits/rejected": 0.03345927596092224,
      "logps/chosen": -2.0293185710906982,
      "logps/rejected": -2.324599504470825,
      "loss": 2.173,
      "nll_loss": 2.1185953617095947,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20293188095092773,
      "rewards/margins": 0.029528062790632248,
      "rewards/rejected": -0.2324599325656891,
      "step": 67
    },
    {
      "epoch": 0.1877804625474629,
      "grad_norm": 0.6155676245689392,
      "learning_rate": 9.792499064044342e-07,
      "log_odds_chosen": 0.4463171362876892,
      "log_odds_ratio": -0.5067868232727051,
      "logits/chosen": -0.373102605342865,
      "logits/rejected": 0.08332100510597229,
      "logps/chosen": -2.0481481552124023,
      "logps/rejected": -2.4466750621795654,
      "loss": 2.239,
      "nll_loss": 2.188358783721924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2048148363828659,
      "rewards/margins": 0.03985270857810974,
      "rewards/rejected": -0.24466755986213684,
      "step": 68
    },
    {
      "epoch": 0.19054193993786675,
      "grad_norm": 0.7794233560562134,
      "learning_rate": 9.778539833672523e-07,
      "log_odds_chosen": 0.5371280908584595,
      "log_odds_ratio": -0.4709131121635437,
      "logits/chosen": -0.6216379404067993,
      "logits/rejected": 0.006563026458024979,
      "logps/chosen": -2.126814365386963,
      "logps/rejected": -2.6152100563049316,
      "loss": 2.2866,
      "nll_loss": 2.2394604682922363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21268144249916077,
      "rewards/margins": 0.04883957654237747,
      "rewards/rejected": -0.26152101159095764,
      "step": 69
    },
    {
      "epoch": 0.19330341732827064,
      "grad_norm": 0.55165696144104,
      "learning_rate": 9.764136834878985e-07,
      "log_odds_chosen": 0.6038047075271606,
      "log_odds_ratio": -0.4510645270347595,
      "logits/chosen": -0.4978086054325104,
      "logits/rejected": -0.26690828800201416,
      "logps/chosen": -2.0604536533355713,
      "logps/rejected": -2.6057753562927246,
      "loss": 2.2046,
      "nll_loss": 2.159515619277954,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20604537427425385,
      "rewards/margins": 0.05453217402100563,
      "rewards/rejected": -0.260577529668808,
      "step": 70
    },
    {
      "epoch": 0.1960648947186745,
      "grad_norm": 0.7500314116477966,
      "learning_rate": 9.749291405226304e-07,
      "log_odds_chosen": 0.44653454422950745,
      "log_odds_ratio": -0.5012515187263489,
      "logits/chosen": -0.5977927446365356,
      "logits/rejected": -0.13603514432907104,
      "logps/chosen": -2.0395045280456543,
      "logps/rejected": -2.440225124359131,
      "loss": 2.2189,
      "nll_loss": 2.1687891483306885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.203950434923172,
      "rewards/margins": 0.040072064846754074,
      "rewards/rejected": -0.24402251839637756,
      "step": 71
    },
    {
      "epoch": 0.19882637210907836,
      "grad_norm": 0.7076852917671204,
      "learning_rate": 9.734004923364256e-07,
      "log_odds_chosen": 0.5996865630149841,
      "log_odds_ratio": -0.4596526026725769,
      "logits/chosen": -0.5302853584289551,
      "logits/rejected": -0.14227402210235596,
      "logps/chosen": -1.9468610286712646,
      "logps/rejected": -2.459900140762329,
      "loss": 2.1054,
      "nll_loss": 2.0594077110290527,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19468611478805542,
      "rewards/margins": 0.05130389332771301,
      "rewards/rejected": -0.24599000811576843,
      "step": 72
    },
    {
      "epoch": 0.20158784949948222,
      "grad_norm": 0.6143236756324768,
      "learning_rate": 9.718278808901796e-07,
      "log_odds_chosen": 0.396225243806839,
      "log_odds_ratio": -0.5162422060966492,
      "logits/chosen": -0.43135014176368713,
      "logits/rejected": -0.058280639350414276,
      "logps/chosen": -2.100261688232422,
      "logps/rejected": -2.455887794494629,
      "loss": 2.2568,
      "nll_loss": 2.205214738845825,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21002618968486786,
      "rewards/margins": 0.03556258603930473,
      "rewards/rejected": -0.2455887794494629,
      "step": 73
    },
    {
      "epoch": 0.20434932688988608,
      "grad_norm": 0.519507110118866,
      "learning_rate": 9.702114522275216e-07,
      "log_odds_chosen": 0.4036938548088074,
      "log_odds_ratio": -0.5151816010475159,
      "logits/chosen": -0.4134013056755066,
      "logits/rejected": -0.1517806351184845,
      "logps/chosen": -1.9940909147262573,
      "logps/rejected": -2.3503451347351074,
      "loss": 2.1793,
      "nll_loss": 2.127819538116455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19940906763076782,
      "rewards/margins": 0.03562544658780098,
      "rewards/rejected": -0.2350345253944397,
      "step": 74
    },
    {
      "epoch": 0.20711080428028997,
      "grad_norm": 0.782429575920105,
      "learning_rate": 9.685513564612519e-07,
      "log_odds_chosen": 0.49272727966308594,
      "log_odds_ratio": -0.48521819710731506,
      "logits/chosen": -0.6792095899581909,
      "logits/rejected": -0.06243826448917389,
      "logps/chosen": -2.024515151977539,
      "logps/rejected": -2.4666824340820312,
      "loss": 2.2057,
      "nll_loss": 2.1572229862213135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20245149731636047,
      "rewards/margins": 0.04421672224998474,
      "rewards/rejected": -0.2466682344675064,
      "step": 75
    },
    {
      "epoch": 0.20987228167069383,
      "grad_norm": 0.6543245911598206,
      "learning_rate": 9.66847747759402e-07,
      "log_odds_chosen": 0.47571614384651184,
      "log_odds_ratio": -0.48933419585227966,
      "logits/chosen": -0.4802244007587433,
      "logits/rejected": -0.22477136552333832,
      "logps/chosen": -2.1057422161102295,
      "logps/rejected": -2.5357770919799805,
      "loss": 2.2481,
      "nll_loss": 2.1991918087005615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2105741947889328,
      "rewards/margins": 0.04300351440906525,
      "rewards/rejected": -0.25357770919799805,
      "step": 76
    },
    {
      "epoch": 0.2126337590610977,
      "grad_norm": 0.6939597725868225,
      "learning_rate": 9.651007843309163e-07,
      "log_odds_chosen": 0.41576477885246277,
      "log_odds_ratio": -0.5240318775177002,
      "logits/chosen": -0.6152012944221497,
      "logits/rejected": -0.16751326620578766,
      "logps/chosen": -2.0883710384368896,
      "logps/rejected": -2.464268922805786,
      "loss": 2.2666,
      "nll_loss": 2.2142295837402344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2088371068239212,
      "rewards/margins": 0.037589773535728455,
      "rewards/rejected": -0.24642688035964966,
      "step": 77
    },
    {
      "epoch": 0.21539523645150155,
      "grad_norm": 0.6390596628189087,
      "learning_rate": 9.63310628410961e-07,
      "log_odds_chosen": 0.3899291455745697,
      "log_odds_ratio": -0.5355044007301331,
      "logits/chosen": -0.4170103073120117,
      "logits/rejected": 0.048835985362529755,
      "logps/chosen": -2.106487274169922,
      "logps/rejected": -2.456912040710449,
      "loss": 2.266,
      "nll_loss": 2.212411880493164,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21064873039722443,
      "rewards/margins": 0.03504248708486557,
      "rewards/rejected": -0.2456912249326706,
      "step": 78
    },
    {
      "epoch": 0.2181567138419054,
      "grad_norm": 0.5294399857521057,
      "learning_rate": 9.614774462458572e-07,
      "log_odds_chosen": 0.466531366109848,
      "log_odds_ratio": -0.4890137314796448,
      "logits/chosen": -0.43251708149909973,
      "logits/rejected": -0.08842866122722626,
      "logps/chosen": -1.989919662475586,
      "logps/rejected": -2.4037232398986816,
      "loss": 2.1759,
      "nll_loss": 2.1269915103912354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19899196922779083,
      "rewards/margins": 0.04138034209609032,
      "rewards/rejected": -0.24037230014801025,
      "step": 79
    },
    {
      "epoch": 0.2209181912323093,
      "grad_norm": 0.6490361094474792,
      "learning_rate": 9.596014080776421e-07,
      "log_odds_chosen": 0.39948511123657227,
      "log_odds_ratio": -0.524588406085968,
      "logits/chosen": -0.5656427145004272,
      "logits/rejected": -0.10723835974931717,
      "logps/chosen": -1.9778105020523071,
      "logps/rejected": -2.329385280609131,
      "loss": 2.1511,
      "nll_loss": 2.098690986633301,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.197781041264534,
      "rewards/margins": 0.035157497972249985,
      "rewards/rejected": -0.23293854296207428,
      "step": 80
    },
    {
      "epoch": 0.22367966862271316,
      "grad_norm": 0.5923272967338562,
      "learning_rate": 9.576826881282595e-07,
      "log_odds_chosen": 0.38708534836769104,
      "log_odds_ratio": -0.521623432636261,
      "logits/chosen": -0.4615509808063507,
      "logits/rejected": -0.004090195521712303,
      "logps/chosen": -1.9872735738754272,
      "logps/rejected": -2.3287017345428467,
      "loss": 2.1555,
      "nll_loss": 2.1033740043640137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19872736930847168,
      "rewards/margins": 0.03414280712604523,
      "rewards/rejected": -0.2328701764345169,
      "step": 81
    },
    {
      "epoch": 0.22644114601311702,
      "grad_norm": 0.6121114492416382,
      "learning_rate": 9.55721464583379e-07,
      "log_odds_chosen": 0.3890568017959595,
      "log_odds_ratio": -0.5230833888053894,
      "logits/chosen": -0.5602201819419861,
      "logits/rejected": -0.051301341503858566,
      "logps/chosen": -2.073209762573242,
      "logps/rejected": -2.421408176422119,
      "loss": 2.2536,
      "nll_loss": 2.2012758255004883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20732098817825317,
      "rewards/margins": 0.03481980785727501,
      "rewards/rejected": -0.24214079976081848,
      "step": 82
    },
    {
      "epoch": 0.22920262340352088,
      "grad_norm": 0.6667964458465576,
      "learning_rate": 9.53717919575851e-07,
      "log_odds_chosen": 0.6576822400093079,
      "log_odds_ratio": -0.42426633834838867,
      "logits/chosen": -0.5727198123931885,
      "logits/rejected": -0.1270349621772766,
      "logps/chosen": -2.023730993270874,
      "logps/rejected": -2.616180181503296,
      "loss": 2.1898,
      "nll_loss": 2.1473617553710938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20237311720848083,
      "rewards/margins": 0.059244900941848755,
      "rewards/rejected": -0.2616180181503296,
      "step": 83
    },
    {
      "epoch": 0.23196410079392474,
      "grad_norm": 0.693790853023529,
      "learning_rate": 9.516722391687902e-07,
      "log_odds_chosen": 0.44280722737312317,
      "log_odds_ratio": -0.500329315662384,
      "logits/chosen": -0.5797449350357056,
      "logits/rejected": -0.06375281512737274,
      "logps/chosen": -2.0299980640411377,
      "logps/rejected": -2.422809362411499,
      "loss": 2.2156,
      "nll_loss": 2.165597915649414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2029997855424881,
      "rewards/margins": 0.039281152188777924,
      "rewards/rejected": -0.2422809600830078,
      "step": 84
    },
    {
      "epoch": 0.23472557818432863,
      "grad_norm": 0.5140150189399719,
      "learning_rate": 9.495846133382972e-07,
      "log_odds_chosen": 0.5956349968910217,
      "log_odds_ratio": -0.45164743065834045,
      "logits/chosen": -0.6015070080757141,
      "logits/rejected": -0.1540786623954773,
      "logps/chosen": -1.9280236959457397,
      "logps/rejected": -2.460214138031006,
      "loss": 2.0991,
      "nll_loss": 2.0539848804473877,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19280238449573517,
      "rewards/margins": 0.05321904271841049,
      "rewards/rejected": -0.24602143466472626,
      "step": 85
    },
    {
      "epoch": 0.2374870555747325,
      "grad_norm": 0.6148126721382141,
      "learning_rate": 9.474552359558165e-07,
      "log_odds_chosen": 0.5965796709060669,
      "log_odds_ratio": -0.4580463767051697,
      "logits/chosen": -0.42590850591659546,
      "logits/rejected": -0.22703289985656738,
      "logps/chosen": -2.051675796508789,
      "logps/rejected": -2.593811511993408,
      "loss": 2.22,
      "nll_loss": 2.1741926670074463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20516757667064667,
      "rewards/margins": 0.05421356484293938,
      "rewards/rejected": -0.25938117504119873,
      "step": 86
    },
    {
      "epoch": 0.24024853296513635,
      "grad_norm": 0.5130142569541931,
      "learning_rate": 9.452843047701322e-07,
      "log_odds_chosen": 0.5396848917007446,
      "log_odds_ratio": -0.46959275007247925,
      "logits/chosen": -0.4120473861694336,
      "logits/rejected": -0.19751112163066864,
      "logps/chosen": -1.9355332851409912,
      "logps/rejected": -2.4127590656280518,
      "loss": 2.1023,
      "nll_loss": 2.0553040504455566,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19355332851409912,
      "rewards/margins": 0.04772257059812546,
      "rewards/rejected": -0.24127590656280518,
      "step": 87
    },
    {
      "epoch": 0.2430100103555402,
      "grad_norm": 0.5595256686210632,
      "learning_rate": 9.430720213890029e-07,
      "log_odds_chosen": 0.613072395324707,
      "log_odds_ratio": -0.4729953408241272,
      "logits/chosen": -0.6118651032447815,
      "logits/rejected": -0.12474207580089569,
      "logps/chosen": -1.8794748783111572,
      "logps/rejected": -2.42482590675354,
      "loss": 2.0608,
      "nll_loss": 2.0134880542755127,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18794748187065125,
      "rewards/margins": 0.05453511327505112,
      "rewards/rejected": -0.24248260259628296,
      "step": 88
    },
    {
      "epoch": 0.24577148774594407,
      "grad_norm": 0.4620777368545532,
      "learning_rate": 9.408185912604394e-07,
      "log_odds_chosen": 0.6550320982933044,
      "log_odds_ratio": -0.4278123378753662,
      "logits/chosen": -0.4233023524284363,
      "logits/rejected": -0.2336420714855194,
      "logps/chosen": -1.9594345092773438,
      "logps/rejected": -2.544253349304199,
      "loss": 2.1261,
      "nll_loss": 2.083322286605835,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1959434449672699,
      "rewards/margins": 0.058481886982917786,
      "rewards/rejected": -0.2544253170490265,
      "step": 89
    },
    {
      "epoch": 0.24853296513634796,
      "grad_norm": 0.5894656181335449,
      "learning_rate": 9.385242236536259e-07,
      "log_odds_chosen": 0.6060933470726013,
      "log_odds_ratio": -0.43741244077682495,
      "logits/chosen": -0.588950514793396,
      "logits/rejected": 0.0337841734290123,
      "logps/chosen": -1.9820064306259155,
      "logps/rejected": -2.522369146347046,
      "loss": 2.147,
      "nll_loss": 2.1032943725585938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19820062816143036,
      "rewards/margins": 0.054036274552345276,
      "rewards/rejected": -0.25223690271377563,
      "step": 90
    },
    {
      "epoch": 0.2512944425267518,
      "grad_norm": 0.5386000275611877,
      "learning_rate": 9.36189131639485e-07,
      "log_odds_chosen": 0.4552825093269348,
      "log_odds_ratio": -0.5021163821220398,
      "logits/chosen": -0.4553697109222412,
      "logits/rejected": -0.03825013339519501,
      "logps/chosen": -1.960836410522461,
      "logps/rejected": -2.3634746074676514,
      "loss": 2.1238,
      "nll_loss": 2.0735952854156494,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1960836499929428,
      "rewards/margins": 0.040263839066028595,
      "rewards/rejected": -0.2363474816083908,
      "step": 91
    },
    {
      "epoch": 0.2540559199171557,
      "grad_norm": 0.5585065484046936,
      "learning_rate": 9.338135320708911e-07,
      "log_odds_chosen": 0.4675421714782715,
      "log_odds_ratio": -0.4915333390235901,
      "logits/chosen": -0.4344261884689331,
      "logits/rejected": -0.3970990478992462,
      "logps/chosen": -2.0036003589630127,
      "logps/rejected": -2.4209625720977783,
      "loss": 2.1638,
      "nll_loss": 2.1146421432495117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20036004483699799,
      "rewards/margins": 0.04173620790243149,
      "rewards/rejected": -0.24209626019001007,
      "step": 92
    },
    {
      "epoch": 0.25681739730755954,
      "grad_norm": 0.6691899299621582,
      "learning_rate": 9.313976455625314e-07,
      "log_odds_chosen": 0.5349908471107483,
      "log_odds_ratio": -0.4698356091976166,
      "logits/chosen": -0.5215790867805481,
      "logits/rejected": -0.08899018168449402,
      "logps/chosen": -2.011092185974121,
      "logps/rejected": -2.4887406826019287,
      "loss": 2.1803,
      "nll_loss": 2.133345603942871,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20110921561717987,
      "rewards/margins": 0.047764845192432404,
      "rewards/rejected": -0.24887408316135406,
      "step": 93
    },
    {
      "epoch": 0.2595788746979634,
      "grad_norm": 0.5345549583435059,
      "learning_rate": 9.289416964704185e-07,
      "log_odds_chosen": 0.47327086329460144,
      "log_odds_ratio": -0.4930804371833801,
      "logits/chosen": -0.646896481513977,
      "logits/rejected": -0.12612032890319824,
      "logps/chosen": -1.8881816864013672,
      "logps/rejected": -2.3036322593688965,
      "loss": 2.0468,
      "nll_loss": 1.9974749088287354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18881818652153015,
      "rewards/margins": 0.0415450744330883,
      "rewards/rejected": -0.23036324977874756,
      "step": 94
    },
    {
      "epoch": 0.26234035208836726,
      "grad_norm": 0.4902014434337616,
      "learning_rate": 9.264459128710549e-07,
      "log_odds_chosen": 0.37959468364715576,
      "log_odds_ratio": -0.5257529616355896,
      "logits/chosen": -0.4130203127861023,
      "logits/rejected": -0.09607210755348206,
      "logps/chosen": -1.908294439315796,
      "logps/rejected": -2.240814447402954,
      "loss": 2.0786,
      "nll_loss": 2.02600359916687,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19082944095134735,
      "rewards/margins": 0.033252015709877014,
      "rewards/rejected": -0.22408147156238556,
      "step": 95
    },
    {
      "epoch": 0.26510182947877114,
      "grad_norm": 0.5475373268127441,
      "learning_rate": 9.239105265402525e-07,
      "log_odds_chosen": 0.30384886264801025,
      "log_odds_ratio": -0.5576225519180298,
      "logits/chosen": -0.44069942831993103,
      "logits/rejected": -0.046746112406253815,
      "logps/chosen": -2.078669548034668,
      "logps/rejected": -2.350083112716675,
      "loss": 2.2571,
      "nll_loss": 2.2013750076293945,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20786696672439575,
      "rewards/margins": 0.027141369879245758,
      "rewards/rejected": -0.2350083291530609,
      "step": 96
    },
    {
      "epoch": 0.26786330686917503,
      "grad_norm": 0.547146737575531,
      "learning_rate": 9.213357729316076e-07,
      "log_odds_chosen": 0.5971243381500244,
      "log_odds_ratio": -0.455092191696167,
      "logits/chosen": -0.3572068214416504,
      "logits/rejected": -0.017504265531897545,
      "logps/chosen": -1.9446320533752441,
      "logps/rejected": -2.483046770095825,
      "loss": 2.1207,
      "nll_loss": 2.075225353240967,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19446319341659546,
      "rewards/margins": 0.05384145677089691,
      "rewards/rejected": -0.24830466508865356,
      "step": 97
    },
    {
      "epoch": 0.27062478425957887,
      "grad_norm": 0.6391997337341309,
      "learning_rate": 9.187218911546361e-07,
      "log_odds_chosen": 0.42829424142837524,
      "log_odds_ratio": -0.5071159601211548,
      "logits/chosen": -0.4863595962524414,
      "logits/rejected": -0.14603319764137268,
      "logps/chosen": -2.0261991024017334,
      "logps/rejected": -2.406280517578125,
      "loss": 2.1936,
      "nll_loss": 2.142874002456665,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20261992514133453,
      "rewards/margins": 0.038008153438568115,
      "rewards/rejected": -0.24062804877758026,
      "step": 98
    },
    {
      "epoch": 0.27338626164998275,
      "grad_norm": 0.6325942277908325,
      "learning_rate": 9.160691239525673e-07,
      "log_odds_chosen": 0.5877907276153564,
      "log_odds_ratio": -0.44585251808166504,
      "logits/chosen": -0.5181315541267395,
      "logits/rejected": -0.2302314043045044,
      "logps/chosen": -2.076399087905884,
      "logps/rejected": -2.6072998046875,
      "loss": 2.2289,
      "nll_loss": 2.1842758655548096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2076399028301239,
      "rewards/margins": 0.05309012532234192,
      "rewards/rejected": -0.2607300281524658,
      "step": 99
    },
    {
      "epoch": 0.2761477390403866,
      "grad_norm": 0.5501412749290466,
      "learning_rate": 9.133777176798012e-07,
      "log_odds_chosen": 0.3657677173614502,
      "log_odds_ratio": -0.531266987323761,
      "logits/chosen": -0.586296021938324,
      "logits/rejected": -0.014902745373547077,
      "logps/chosen": -1.947542428970337,
      "logps/rejected": -2.2680420875549316,
      "loss": 2.1386,
      "nll_loss": 2.0854897499084473,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1947542428970337,
      "rewards/margins": 0.03204997628927231,
      "rewards/rejected": -0.2268042117357254,
      "step": 100
    },
    {
      "epoch": 0.2789092164307905,
      "grad_norm": 0.5406647324562073,
      "learning_rate": 9.10647922279031e-07,
      "log_odds_chosen": 0.3731037974357605,
      "log_odds_ratio": -0.5326068997383118,
      "logits/chosen": -0.3845860958099365,
      "logits/rejected": -0.06091095507144928,
      "logps/chosen": -2.0511059761047363,
      "logps/rejected": -2.387570858001709,
      "loss": 2.2256,
      "nll_loss": 2.172375440597534,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2051105797290802,
      "rewards/margins": 0.033646512776613235,
      "rewards/rejected": -0.23875710368156433,
      "step": 101
    },
    {
      "epoch": 0.28167069382119436,
      "grad_norm": 0.5214187502861023,
      "learning_rate": 9.078799912580303e-07,
      "log_odds_chosen": 0.6036267280578613,
      "log_odds_ratio": -0.443120539188385,
      "logits/chosen": -0.5061873197555542,
      "logits/rejected": -0.11495180428028107,
      "logps/chosen": -1.8962457180023193,
      "logps/rejected": -2.426725387573242,
      "loss": 2.0868,
      "nll_loss": 2.0424931049346924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18962456285953522,
      "rewards/margins": 0.05304800719022751,
      "rewards/rejected": -0.24267259240150452,
      "step": 102
    },
    {
      "epoch": 0.2844321712115982,
      "grad_norm": 0.575564980506897,
      "learning_rate": 9.050741816661126e-07,
      "log_odds_chosen": 0.372819721698761,
      "log_odds_ratio": -0.5326889157295227,
      "logits/chosen": -0.5737372040748596,
      "logits/rejected": -0.16316528618335724,
      "logps/chosen": -1.9487695693969727,
      "logps/rejected": -2.277163505554199,
      "loss": 2.1286,
      "nll_loss": 2.075375556945801,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19487696886062622,
      "rewards/margins": 0.03283938020467758,
      "rewards/rejected": -0.2277163714170456,
      "step": 103
    },
    {
      "epoch": 0.2871936486020021,
      "grad_norm": 0.4972224533557892,
      "learning_rate": 9.022307540702576e-07,
      "log_odds_chosen": 0.41442954540252686,
      "log_odds_ratio": -0.512852668762207,
      "logits/chosen": -0.5097107291221619,
      "logits/rejected": -0.024050669744610786,
      "logps/chosen": -1.859932780265808,
      "logps/rejected": -2.2202517986297607,
      "loss": 2.0482,
      "nll_loss": 1.9969029426574707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1859932690858841,
      "rewards/margins": 0.036031924188137054,
      "rewards/rejected": -0.22202520072460175,
      "step": 104
    },
    {
      "epoch": 0.2899551259924059,
      "grad_norm": 0.4960035979747772,
      "learning_rate": 8.993499725309148e-07,
      "log_odds_chosen": 0.5264571309089661,
      "log_odds_ratio": -0.4678184688091278,
      "logits/chosen": -0.42354345321655273,
      "logits/rejected": -0.06721704453229904,
      "logps/chosen": -1.8945815563201904,
      "logps/rejected": -2.3582100868225098,
      "loss": 2.0585,
      "nll_loss": 2.011746644973755,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18945816159248352,
      "rewards/margins": 0.04636286199092865,
      "rewards/rejected": -0.23582102358341217,
      "step": 105
    },
    {
      "epoch": 0.2927166033828098,
      "grad_norm": 0.5499945282936096,
      "learning_rate": 8.964321045774806e-07,
      "log_odds_chosen": 0.5790612101554871,
      "log_odds_ratio": -0.46018439531326294,
      "logits/chosen": -0.48419511318206787,
      "logits/rejected": -0.043670542538166046,
      "logps/chosen": -1.9003938436508179,
      "logps/rejected": -2.4120233058929443,
      "loss": 2.0632,
      "nll_loss": 2.017177104949951,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19003938138484955,
      "rewards/margins": 0.05116293579339981,
      "rewards/rejected": -0.24120232462882996,
      "step": 106
    },
    {
      "epoch": 0.2954780807732137,
      "grad_norm": 0.5568175911903381,
      "learning_rate": 8.934774211834537e-07,
      "log_odds_chosen": 0.45170602202415466,
      "log_odds_ratio": -0.4972607493400574,
      "logits/chosen": -0.4519391357898712,
      "logits/rejected": -0.08241380751132965,
      "logps/chosen": -1.9787864685058594,
      "logps/rejected": -2.3785712718963623,
      "loss": 2.1426,
      "nll_loss": 2.0928850173950195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1978786587715149,
      "rewards/margins": 0.03997847065329552,
      "rewards/rejected": -0.23785711824893951,
      "step": 107
    },
    {
      "epoch": 0.2982395581636175,
      "grad_norm": 0.5856341123580933,
      "learning_rate": 8.904861967412701e-07,
      "log_odds_chosen": 0.5225868821144104,
      "log_odds_ratio": -0.47116899490356445,
      "logits/chosen": -0.4344129264354706,
      "logits/rejected": -0.13722169399261475,
      "logps/chosen": -2.0127432346343994,
      "logps/rejected": -2.480672836303711,
      "loss": 2.1647,
      "nll_loss": 2.1175570487976074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2012743055820465,
      "rewards/margins": 0.04679298400878906,
      "rewards/rejected": -0.24806730449199677,
      "step": 108
    },
    {
      "epoch": 0.3010010355540214,
      "grad_norm": 0.6680328845977783,
      "learning_rate": 8.87458709036822e-07,
      "log_odds_chosen": 0.6864393949508667,
      "log_odds_ratio": -0.4224277436733246,
      "logits/chosen": -0.5091022849082947,
      "logits/rejected": -0.2857436239719391,
      "logps/chosen": -2.0367543697357178,
      "logps/rejected": -2.6599154472351074,
      "loss": 2.1829,
      "nll_loss": 2.1406257152557373,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20367543399333954,
      "rewards/margins": 0.06231612712144852,
      "rewards/rejected": -0.26599156856536865,
      "step": 109
    },
    {
      "epoch": 0.30376251294442524,
      "grad_norm": 0.5433963537216187,
      "learning_rate": 8.843952392236593e-07,
      "log_odds_chosen": 0.33481231331825256,
      "log_odds_ratio": -0.5607271790504456,
      "logits/chosen": -0.36713624000549316,
      "logits/rejected": -0.14656810462474823,
      "logps/chosen": -1.9623396396636963,
      "logps/rejected": -2.2557480335235596,
      "loss": 2.1606,
      "nll_loss": 2.1045403480529785,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19623397290706635,
      "rewards/margins": 0.029340829700231552,
      "rewards/rejected": -0.225574791431427,
      "step": 110
    },
    {
      "epoch": 0.30652399033482913,
      "grad_norm": 0.5330284833908081,
      "learning_rate": 8.812960717968818e-07,
      "log_odds_chosen": 0.46152007579803467,
      "log_odds_ratio": -0.4966501295566559,
      "logits/chosen": -0.4940492808818817,
      "logits/rejected": -0.11987907439470291,
      "logps/chosen": -1.8650836944580078,
      "logps/rejected": -2.264942169189453,
      "loss": 2.043,
      "nll_loss": 1.9932986497879028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18650837242603302,
      "rewards/margins": 0.03998585417866707,
      "rewards/rejected": -0.2264942228794098,
      "step": 111
    },
    {
      "epoch": 0.309285467725233,
      "grad_norm": 0.47368451952934265,
      "learning_rate": 8.781614945667168e-07,
      "log_odds_chosen": 0.4548957347869873,
      "log_odds_ratio": -0.5047125220298767,
      "logits/chosen": -0.3089265525341034,
      "logits/rejected": -0.09748263657093048,
      "logps/chosen": -1.9140433073043823,
      "logps/rejected": -2.316842794418335,
      "loss": 2.0869,
      "nll_loss": 2.0364115238189697,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1914043426513672,
      "rewards/margins": 0.04027996212244034,
      "rewards/rejected": -0.23168428242206573,
      "step": 112
    },
    {
      "epoch": 0.31204694511563685,
      "grad_norm": 0.6315337419509888,
      "learning_rate": 8.749917986317928e-07,
      "log_odds_chosen": 0.45678791403770447,
      "log_odds_ratio": -0.49500221014022827,
      "logits/chosen": -0.5219292640686035,
      "logits/rejected": 0.00903545506298542,
      "logps/chosen": -1.9943602085113525,
      "logps/rejected": -2.399494171142578,
      "loss": 2.1749,
      "nll_loss": 2.125378131866455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1994360387325287,
      "rewards/margins": 0.04051339626312256,
      "rewards/rejected": -0.23994943499565125,
      "step": 113
    },
    {
      "epoch": 0.31480842250604074,
      "grad_norm": 0.5757876038551331,
      "learning_rate": 8.717872783521047e-07,
      "log_odds_chosen": 0.583101749420166,
      "log_odds_ratio": -0.45457175374031067,
      "logits/chosen": -0.38971248269081116,
      "logits/rejected": -0.1346980631351471,
      "logps/chosen": -1.9794937372207642,
      "logps/rejected": -2.500247001647949,
      "loss": 2.1542,
      "nll_loss": 2.1087803840637207,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1979493647813797,
      "rewards/margins": 0.05207531899213791,
      "rewards/rejected": -0.2500247061252594,
      "step": 114
    },
    {
      "epoch": 0.3175698998964446,
      "grad_norm": 0.4948897957801819,
      "learning_rate": 8.685482313216783e-07,
      "log_odds_chosen": 0.43383458256721497,
      "log_odds_ratio": -0.5028841495513916,
      "logits/chosen": -0.2275230586528778,
      "logits/rejected": -0.16718950867652893,
      "logps/chosen": -1.9936423301696777,
      "logps/rejected": -2.376922607421875,
      "loss": 2.1491,
      "nll_loss": 2.0988006591796875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19936421513557434,
      "rewards/margins": 0.038328029215335846,
      "rewards/rejected": -0.23769225180149078,
      "step": 115
    },
    {
      "epoch": 0.32033137728684846,
      "grad_norm": 0.5214484930038452,
      "learning_rate": 8.652749583409339e-07,
      "log_odds_chosen": 0.40033382177352905,
      "log_odds_ratio": -0.522826075553894,
      "logits/chosen": -0.3641853928565979,
      "logits/rejected": -0.07613816857337952,
      "logps/chosen": -1.979244589805603,
      "logps/rejected": -2.330749273300171,
      "loss": 2.1521,
      "nll_loss": 2.0998451709747314,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19792446494102478,
      "rewards/margins": 0.03515046834945679,
      "rewards/rejected": -0.23307493329048157,
      "step": 116
    },
    {
      "epoch": 0.32309285467725235,
      "grad_norm": 0.5448740124702454,
      "learning_rate": 8.619677633887508e-07,
      "log_odds_chosen": 0.37107452750205994,
      "log_odds_ratio": -0.5365285277366638,
      "logits/chosen": -0.3629532754421234,
      "logits/rejected": -0.17828039824962616,
      "logps/chosen": -2.061443328857422,
      "logps/rejected": -2.393622636795044,
      "loss": 2.233,
      "nll_loss": 2.1793932914733887,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2061443328857422,
      "rewards/margins": 0.03321794420480728,
      "rewards/rejected": -0.23936225473880768,
      "step": 117
    },
    {
      "epoch": 0.3258543320676562,
      "grad_norm": 0.5429673194885254,
      "learning_rate": 8.586269535942385e-07,
      "log_odds_chosen": 0.36125749349594116,
      "log_odds_ratio": -0.535487949848175,
      "logits/chosen": -0.5306198596954346,
      "logits/rejected": -0.4250182509422302,
      "logps/chosen": -1.9498947858810425,
      "logps/rejected": -2.2681477069854736,
      "loss": 2.1266,
      "nll_loss": 2.0730719566345215,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19498948752880096,
      "rewards/margins": 0.03182529658079147,
      "rewards/rejected": -0.22681477665901184,
      "step": 118
    },
    {
      "epoch": 0.32861580945806007,
      "grad_norm": 0.6338194012641907,
      "learning_rate": 8.552528392082146e-07,
      "log_odds_chosen": 0.45459601283073425,
      "log_odds_ratio": -0.4954701364040375,
      "logits/chosen": -0.2504444420337677,
      "logits/rejected": -0.0979476124048233,
      "logps/chosen": -2.117143154144287,
      "logps/rejected": -2.522320508956909,
      "loss": 2.2909,
      "nll_loss": 2.2413687705993652,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21171432733535767,
      "rewards/margins": 0.04051773250102997,
      "rewards/rejected": -0.25223207473754883,
      "step": 119
    },
    {
      "epoch": 0.3313772868484639,
      "grad_norm": 0.48553842306137085,
      "learning_rate": 8.518457335743924e-07,
      "log_odds_chosen": 0.5631441473960876,
      "log_odds_ratio": -0.4539083242416382,
      "logits/chosen": -0.4194352924823761,
      "logits/rejected": -0.02024952881038189,
      "logps/chosen": -1.8635163307189941,
      "logps/rejected": -2.3571970462799072,
      "loss": 2.0269,
      "nll_loss": 1.981484293937683,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18635164201259613,
      "rewards/margins": 0.04936806112527847,
      "rewards/rejected": -0.235719695687294,
      "step": 120
    },
    {
      "epoch": 0.3341387642388678,
      "grad_norm": 0.5075821280479431,
      "learning_rate": 8.48405953100282e-07,
      "log_odds_chosen": 0.5342307686805725,
      "log_odds_ratio": -0.46748292446136475,
      "logits/chosen": -0.4761042892932892,
      "logits/rejected": -0.09555709362030029,
      "logps/chosen": -1.9163572788238525,
      "logps/rejected": -2.3868184089660645,
      "loss": 2.0688,
      "nll_loss": 2.022080898284912,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19163572788238525,
      "rewards/margins": 0.04704609513282776,
      "rewards/rejected": -0.23868180811405182,
      "step": 121
    },
    {
      "epoch": 0.3369002416292717,
      "grad_norm": 0.4722713232040405,
      "learning_rate": 8.449338172278058e-07,
      "log_odds_chosen": 0.42079102993011475,
      "log_odds_ratio": -0.5151209235191345,
      "logits/chosen": -0.3123060464859009,
      "logits/rejected": -0.14001092314720154,
      "logps/chosen": -1.95204496383667,
      "logps/rejected": -2.3235373497009277,
      "loss": 2.1379,
      "nll_loss": 2.0863630771636963,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.195204496383667,
      "rewards/margins": 0.03714926540851593,
      "rewards/rejected": -0.23235377669334412,
      "step": 122
    },
    {
      "epoch": 0.3396617190196755,
      "grad_norm": 0.563771665096283,
      "learning_rate": 8.414296484036338e-07,
      "log_odds_chosen": 0.27497193217277527,
      "log_odds_ratio": -0.5697895884513855,
      "logits/chosen": -0.3733835816383362,
      "logits/rejected": -0.04408363625407219,
      "logps/chosen": -2.1177093982696533,
      "logps/rejected": -2.364516496658325,
      "loss": 2.2738,
      "nll_loss": 2.2168140411376953,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2117709368467331,
      "rewards/margins": 0.02468070387840271,
      "rewards/rejected": -0.2364516258239746,
      "step": 123
    },
    {
      "epoch": 0.3424231964100794,
      "grad_norm": 0.716310977935791,
      "learning_rate": 8.378937720492383e-07,
      "log_odds_chosen": 0.3946109414100647,
      "log_odds_ratio": -0.5247021317481995,
      "logits/chosen": -0.6494585275650024,
      "logits/rejected": 0.044096484780311584,
      "logps/chosen": -2.059776544570923,
      "logps/rejected": -2.4121546745300293,
      "loss": 2.2261,
      "nll_loss": 2.173640727996826,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2059776484966278,
      "rewards/margins": 0.03523779660463333,
      "rewards/rejected": -0.24121545255184174,
      "step": 124
    },
    {
      "epoch": 0.34518467380048323,
      "grad_norm": 0.5385156273841858,
      "learning_rate": 8.343265165306734e-07,
      "log_odds_chosen": 0.6251612901687622,
      "log_odds_ratio": -0.44978296756744385,
      "logits/chosen": -0.4387221932411194,
      "logits/rejected": -0.38548457622528076,
      "logps/chosen": -1.9126592874526978,
      "logps/rejected": -2.4681596755981445,
      "loss": 2.0951,
      "nll_loss": 2.0500893592834473,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19126594066619873,
      "rewards/margins": 0.055550046265125275,
      "rewards/rejected": -0.2468159794807434,
      "step": 125
    },
    {
      "epoch": 0.3479461511908871,
      "grad_norm": 0.545859158039093,
      "learning_rate": 8.307282131280804e-07,
      "log_odds_chosen": 0.5865803956985474,
      "log_odds_ratio": -0.44584840536117554,
      "logits/chosen": -0.44923073053359985,
      "logits/rejected": -0.2851545214653015,
      "logps/chosen": -1.9255836009979248,
      "logps/rejected": -2.4458699226379395,
      "loss": 2.1001,
      "nll_loss": 2.0555202960968018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19255836308002472,
      "rewards/margins": 0.05202864110469818,
      "rewards/rejected": -0.2445870041847229,
      "step": 126
    },
    {
      "epoch": 0.350707628581291,
      "grad_norm": 0.5951908826828003,
      "learning_rate": 8.27099196004923e-07,
      "log_odds_chosen": 0.3327331244945526,
      "log_odds_ratio": -0.5520588159561157,
      "logits/chosen": -0.31916385889053345,
      "logits/rejected": 0.10464876145124435,
      "logps/chosen": -2.09009051322937,
      "logps/rejected": -2.3857364654541016,
      "loss": 2.2761,
      "nll_loss": 2.220856189727783,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2090090662240982,
      "rewards/margins": 0.02956460230052471,
      "rewards/rejected": -0.23857367038726807,
      "step": 127
    },
    {
      "epoch": 0.35346910597169484,
      "grad_norm": 0.5234466195106506,
      "learning_rate": 8.23439802176954e-07,
      "log_odds_chosen": 0.5177618861198425,
      "log_odds_ratio": -0.48196426033973694,
      "logits/chosen": -0.3624882400035858,
      "logits/rejected": -0.07198012620210648,
      "logps/chosen": -1.9356945753097534,
      "logps/rejected": -2.396468162536621,
      "loss": 2.1149,
      "nll_loss": 2.0666897296905518,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19356943666934967,
      "rewards/margins": 0.04607734829187393,
      "rewards/rejected": -0.2396468222141266,
      "step": 128
    },
    {
      "epoch": 0.35623058336209873,
      "grad_norm": 0.5753765106201172,
      "learning_rate": 8.19750371480919e-07,
      "log_odds_chosen": 0.35112565755844116,
      "log_odds_ratio": -0.5461571216583252,
      "logits/chosen": -0.4609984755516052,
      "logits/rejected": -0.013083010911941528,
      "logps/chosen": -2.0177810192108154,
      "logps/rejected": -2.329488754272461,
      "loss": 2.1926,
      "nll_loss": 2.1379547119140625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2017781138420105,
      "rewards/margins": 0.031170757487416267,
      "rewards/rejected": -0.23294886946678162,
      "step": 129
    },
    {
      "epoch": 0.35899206075250256,
      "grad_norm": 0.5582403540611267,
      "learning_rate": 8.160312465429952e-07,
      "log_odds_chosen": 0.6578480005264282,
      "log_odds_ratio": -0.4275791049003601,
      "logits/chosen": -0.46288540959358215,
      "logits/rejected": -0.20930448174476624,
      "logps/chosen": -1.945103645324707,
      "logps/rejected": -2.5350096225738525,
      "loss": 2.1233,
      "nll_loss": 2.0805039405822754,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.194510355591774,
      "rewards/margins": 0.05899059772491455,
      "rewards/rejected": -0.25350093841552734,
      "step": 130
    },
    {
      "epoch": 0.36175353814290645,
      "grad_norm": 0.589810848236084,
      "learning_rate": 8.122827727469737e-07,
      "log_odds_chosen": 0.5904497504234314,
      "log_odds_ratio": -0.44798266887664795,
      "logits/chosen": -0.32802432775497437,
      "logits/rejected": -0.0018861368298530579,
      "logps/chosen": -2.0113017559051514,
      "logps/rejected": -2.541584014892578,
      "loss": 2.1695,
      "nll_loss": 2.1246747970581055,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2011301964521408,
      "rewards/margins": 0.05302818864583969,
      "rewards/rejected": -0.2541583776473999,
      "step": 131
    },
    {
      "epoch": 0.36451501553331034,
      "grad_norm": 0.6079692840576172,
      "learning_rate": 8.085052982021847e-07,
      "log_odds_chosen": 0.5775200128555298,
      "log_odds_ratio": -0.4538944363594055,
      "logits/chosen": -0.46243995428085327,
      "logits/rejected": -0.2528253197669983,
      "logps/chosen": -1.9396477937698364,
      "logps/rejected": -2.4543814659118652,
      "loss": 2.1143,
      "nll_loss": 2.068951368331909,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19396479427814484,
      "rewards/margins": 0.05147337168455124,
      "rewards/rejected": -0.24543817341327667,
      "step": 132
    },
    {
      "epoch": 0.36727649292371417,
      "grad_norm": 0.5014004111289978,
      "learning_rate": 8.046991737111695e-07,
      "log_odds_chosen": 0.48234912753105164,
      "log_odds_ratio": -0.48756104707717896,
      "logits/chosen": -0.42011186480522156,
      "logits/rejected": -0.011914369650185108,
      "logps/chosen": -1.9627089500427246,
      "logps/rejected": -2.392961263656616,
      "loss": 2.138,
      "nll_loss": 2.089262008666992,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1962708979845047,
      "rewards/margins": 0.04302523285150528,
      "rewards/rejected": -0.2392961084842682,
      "step": 133
    },
    {
      "epoch": 0.37003797031411806,
      "grad_norm": 0.5740203261375427,
      "learning_rate": 8.008647527371022e-07,
      "log_odds_chosen": 0.5004888772964478,
      "log_odds_ratio": -0.4775448441505432,
      "logits/chosen": -0.5055060982704163,
      "logits/rejected": -0.4352371096611023,
      "logps/chosen": -1.966503620147705,
      "logps/rejected": -2.409017324447632,
      "loss": 2.1428,
      "nll_loss": 2.0950794219970703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19665035605430603,
      "rewards/margins": 0.044251397252082825,
      "rewards/rejected": -0.24090173840522766,
      "step": 134
    },
    {
      "epoch": 0.37279944770452195,
      "grad_norm": 0.44002586603164673,
      "learning_rate": 7.970023913709651e-07,
      "log_odds_chosen": 0.5827584862709045,
      "log_odds_ratio": -0.45719826221466064,
      "logits/chosen": -0.29188770055770874,
      "logits/rejected": -0.261316180229187,
      "logps/chosen": -1.857527256011963,
      "logps/rejected": -2.367795944213867,
      "loss": 2.0367,
      "nll_loss": 1.99098539352417,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1857527196407318,
      "rewards/margins": 0.051026877015829086,
      "rewards/rejected": -0.2367796003818512,
      "step": 135
    },
    {
      "epoch": 0.3755609250949258,
      "grad_norm": 0.5160555243492126,
      "learning_rate": 7.931124482984801e-07,
      "log_odds_chosen": 0.49602457880973816,
      "log_odds_ratio": -0.4820690155029297,
      "logits/chosen": -0.422038733959198,
      "logits/rejected": 0.061051055788993835,
      "logps/chosen": -2.003971576690674,
      "logps/rejected": -2.44795560836792,
      "loss": 2.1561,
      "nll_loss": 2.1079189777374268,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20039717853069305,
      "rewards/margins": 0.04439838230609894,
      "rewards/rejected": -0.2447955459356308,
      "step": 136
    },
    {
      "epoch": 0.37832240248532967,
      "grad_norm": 0.4655568599700928,
      "learning_rate": 7.891952847667972e-07,
      "log_odds_chosen": 0.5271226763725281,
      "log_odds_ratio": -0.4714434742927551,
      "logits/chosen": -0.3394045829772949,
      "logits/rejected": -0.14115437865257263,
      "logps/chosen": -1.862820029258728,
      "logps/rejected": -2.3243649005889893,
      "loss": 2.037,
      "nll_loss": 1.9898964166641235,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18628200888633728,
      "rewards/margins": 0.04615447670221329,
      "rewards/rejected": -0.23243649303913116,
      "step": 137
    },
    {
      "epoch": 0.3810838798757335,
      "grad_norm": 0.4939066469669342,
      "learning_rate": 7.85251264550948e-07,
      "log_odds_chosen": 0.4060474634170532,
      "log_odds_ratio": -0.5139645934104919,
      "logits/chosen": -0.3042515516281128,
      "logits/rejected": -0.1826004981994629,
      "logps/chosen": -1.9575718641281128,
      "logps/rejected": -2.3145861625671387,
      "loss": 2.1244,
      "nll_loss": 2.0729963779449463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.195757195353508,
      "rewards/margins": 0.03570141643285751,
      "rewards/rejected": -0.2314586043357849,
      "step": 138
    },
    {
      "epoch": 0.3838453572661374,
      "grad_norm": 0.5114497542381287,
      "learning_rate": 7.812807539200621e-07,
      "log_odds_chosen": 0.42092645168304443,
      "log_odds_ratio": -0.5130881667137146,
      "logits/chosen": -0.24258123338222504,
      "logits/rejected": -0.031729258596897125,
      "logps/chosen": -2.042588710784912,
      "logps/rejected": -2.4170773029327393,
      "loss": 2.206,
      "nll_loss": 2.1546831130981445,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20425887405872345,
      "rewards/margins": 0.03744884580373764,
      "rewards/rejected": -0.2417077273130417,
      "step": 139
    },
    {
      "epoch": 0.3866068346565413,
      "grad_norm": 0.45012837648391724,
      "learning_rate": 7.772841216033532e-07,
      "log_odds_chosen": 0.417616069316864,
      "log_odds_ratio": -0.5143554210662842,
      "logits/chosen": -0.43615201115608215,
      "logits/rejected": -0.06629861146211624,
      "logps/chosen": -1.8509984016418457,
      "logps/rejected": -2.215993881225586,
      "loss": 2.0275,
      "nll_loss": 1.9760750532150269,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18509985506534576,
      "rewards/margins": 0.03649955242872238,
      "rewards/rejected": -0.22159941494464874,
      "step": 140
    },
    {
      "epoch": 0.3893683120469451,
      "grad_norm": 0.4387165904045105,
      "learning_rate": 7.732617387558768e-07,
      "log_odds_chosen": 0.5448954701423645,
      "log_odds_ratio": -0.469321608543396,
      "logits/chosen": -0.30067747831344604,
      "logits/rejected": -0.32880979776382446,
      "logps/chosen": -1.8875839710235596,
      "logps/rejected": -2.372188091278076,
      "loss": 2.0445,
      "nll_loss": 1.997605800628662,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18875840306282043,
      "rewards/margins": 0.04846040904521942,
      "rewards/rejected": -0.23721879720687866,
      "step": 141
    },
    {
      "epoch": 0.392129789437349,
      "grad_norm": 0.5102280378341675,
      "learning_rate": 7.69213978924061e-07,
      "log_odds_chosen": 0.6055910587310791,
      "log_odds_ratio": -0.4432864785194397,
      "logits/chosen": -0.5158487558364868,
      "logits/rejected": -0.1547757089138031,
      "logps/chosen": -1.8841460943222046,
      "logps/rejected": -2.4185075759887695,
      "loss": 2.0458,
      "nll_loss": 2.0015206336975098,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18841463327407837,
      "rewards/margins": 0.053436145186424255,
      "rewards/rejected": -0.24185076355934143,
      "step": 142
    },
    {
      "epoch": 0.39489126682775283,
      "grad_norm": 0.5141972899436951,
      "learning_rate": 7.651412180110175e-07,
      "log_odds_chosen": 0.3155019283294678,
      "log_odds_ratio": -0.554594099521637,
      "logits/chosen": -0.4066213369369507,
      "logits/rejected": -0.22623799741268158,
      "logps/chosen": -1.8567359447479248,
      "logps/rejected": -2.1304125785827637,
      "loss": 2.0609,
      "nll_loss": 2.00547456741333,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1856735795736313,
      "rewards/margins": 0.027367670089006424,
      "rewards/rejected": -0.2130412757396698,
      "step": 143
    },
    {
      "epoch": 0.3976527442181567,
      "grad_norm": 0.5515993237495422,
      "learning_rate": 7.610438342416319e-07,
      "log_odds_chosen": 0.5125759840011597,
      "log_odds_ratio": -0.47270989418029785,
      "logits/chosen": -0.5050479769706726,
      "logits/rejected": -0.2101307213306427,
      "logps/chosen": -1.9838966131210327,
      "logps/rejected": -2.439676284790039,
      "loss": 2.1363,
      "nll_loss": 2.088998556137085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1983896642923355,
      "rewards/margins": 0.045577969402074814,
      "rewards/rejected": -0.24396765232086182,
      "step": 144
    },
    {
      "epoch": 0.4004142216085606,
      "grad_norm": 0.6079866886138916,
      "learning_rate": 7.569222081274395e-07,
      "log_odds_chosen": 0.5498825311660767,
      "log_odds_ratio": -0.4596479535102844,
      "logits/chosen": -0.6153779029846191,
      "logits/rejected": -0.051476310938596725,
      "logps/chosen": -1.9400660991668701,
      "logps/rejected": -2.424722194671631,
      "loss": 2.1104,
      "nll_loss": 2.0644783973693848,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19400662183761597,
      "rewards/margins": 0.04846561700105667,
      "rewards/rejected": -0.24247224628925323,
      "step": 145
    },
    {
      "epoch": 0.40317569899896444,
      "grad_norm": 0.4408237040042877,
      "learning_rate": 7.527767224312882e-07,
      "log_odds_chosen": 0.5145049095153809,
      "log_odds_ratio": -0.4705893397331238,
      "logits/chosen": -0.42787209153175354,
      "logits/rejected": -0.06426948308944702,
      "logps/chosen": -1.9053359031677246,
      "logps/rejected": -2.3589413166046143,
      "loss": 2.0674,
      "nll_loss": 2.0203795433044434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1905336081981659,
      "rewards/margins": 0.045360542833805084,
      "rewards/rejected": -0.23589414358139038,
      "step": 146
    },
    {
      "epoch": 0.4059371763893683,
      "grad_norm": 0.4828256070613861,
      "learning_rate": 7.486077621317925e-07,
      "log_odds_chosen": 0.49437594413757324,
      "log_odds_ratio": -0.488892138004303,
      "logits/chosen": -0.2916801869869232,
      "logits/rejected": -0.15828007459640503,
      "logps/chosen": -2.0035698413848877,
      "logps/rejected": -2.446871280670166,
      "loss": 2.16,
      "nll_loss": 2.1111345291137695,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20035697519779205,
      "rewards/margins": 0.044330157339572906,
      "rewards/rejected": -0.24468713998794556,
      "step": 147
    },
    {
      "epoch": 0.40869865377977216,
      "grad_norm": 0.46756893396377563,
      "learning_rate": 7.444157143875819e-07,
      "log_odds_chosen": 0.7566857933998108,
      "log_odds_ratio": -0.3968402147293091,
      "logits/chosen": -0.6170058250427246,
      "logits/rejected": -0.38097241520881653,
      "logps/chosen": -1.7716487646102905,
      "logps/rejected": -2.4373488426208496,
      "loss": 1.9515,
      "nll_loss": 1.9118363857269287,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17716486752033234,
      "rewards/margins": 0.06657001376152039,
      "rewards/rejected": -0.24373489618301392,
      "step": 148
    },
    {
      "epoch": 0.41146013117017605,
      "grad_norm": 0.5957549810409546,
      "learning_rate": 7.402009685013462e-07,
      "log_odds_chosen": 0.5112757682800293,
      "log_odds_ratio": -0.4749317467212677,
      "logits/chosen": -0.46232783794403076,
      "logits/rejected": -0.10341174900531769,
      "logps/chosen": -1.9221307039260864,
      "logps/rejected": -2.370673418045044,
      "loss": 2.0902,
      "nll_loss": 2.0427207946777344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19221307337284088,
      "rewards/margins": 0.04485424980521202,
      "rewards/rejected": -0.2370673269033432,
      "step": 149
    },
    {
      "epoch": 0.41422160856057993,
      "grad_norm": 0.5625369548797607,
      "learning_rate": 7.359639158836827e-07,
      "log_odds_chosen": 0.5237774848937988,
      "log_odds_ratio": -0.4707946479320526,
      "logits/chosen": -0.2808252274990082,
      "logits/rejected": -0.08281365036964417,
      "logps/chosen": -2.0850653648376465,
      "logps/rejected": -2.557412624359131,
      "loss": 2.2278,
      "nll_loss": 2.180760383605957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2085065394639969,
      "rewards/margins": 0.04723470285534859,
      "rewards/rejected": -0.2557412385940552,
      "step": 150
    },
    {
      "epoch": 0.41698308595098377,
      "grad_norm": 0.5004662871360779,
      "learning_rate": 7.317049500167465e-07,
      "log_odds_chosen": 0.4554722011089325,
      "log_odds_ratio": -0.4992949068546295,
      "logits/chosen": -0.29819634556770325,
      "logits/rejected": -0.08140935748815536,
      "logps/chosen": -1.935859203338623,
      "logps/rejected": -2.3361001014709473,
      "loss": 2.1062,
      "nll_loss": 2.056227922439575,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19358593225479126,
      "rewards/margins": 0.040024079382419586,
      "rewards/rejected": -0.23361001908779144,
      "step": 151
    },
    {
      "epoch": 0.41974456334138766,
      "grad_norm": 0.5074776411056519,
      "learning_rate": 7.274244664177097e-07,
      "log_odds_chosen": 0.5130349397659302,
      "log_odds_ratio": -0.4809737801551819,
      "logits/chosen": -0.31135305762290955,
      "logits/rejected": -0.24845655262470245,
      "logps/chosen": -1.9829801321029663,
      "logps/rejected": -2.4422216415405273,
      "loss": 2.137,
      "nll_loss": 2.0889265537261963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19829802215099335,
      "rewards/margins": 0.04592413455247879,
      "rewards/rejected": -0.24422214925289154,
      "step": 152
    },
    {
      "epoch": 0.4225060407317915,
      "grad_norm": 0.5020032525062561,
      "learning_rate": 7.231228626020303e-07,
      "log_odds_chosen": 0.5971647500991821,
      "log_odds_ratio": -0.4452877938747406,
      "logits/chosen": -0.360279381275177,
      "logits/rejected": -0.07742983847856522,
      "logps/chosen": -1.9082156419754028,
      "logps/rejected": -2.4285898208618164,
      "loss": 2.0708,
      "nll_loss": 2.026240348815918,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1908215582370758,
      "rewards/margins": 0.052037447690963745,
      "rewards/rejected": -0.24285900592803955,
      "step": 153
    },
    {
      "epoch": 0.4252675181221954,
      "grad_norm": 0.4719047248363495,
      "learning_rate": 7.188005380465364e-07,
      "log_odds_chosen": 0.3675041198730469,
      "log_odds_ratio": -0.5354775190353394,
      "logits/chosen": -0.36417311429977417,
      "logits/rejected": -0.17365439236164093,
      "logps/chosen": -1.8679873943328857,
      "logps/rejected": -2.1895949840545654,
      "loss": 2.0629,
      "nll_loss": 2.009333372116089,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18679876625537872,
      "rewards/margins": 0.03216075152158737,
      "rewards/rejected": -0.2189595103263855,
      "step": 154
    },
    {
      "epoch": 0.42802899551259926,
      "grad_norm": 0.5571091771125793,
      "learning_rate": 7.144578941523282e-07,
      "log_odds_chosen": 0.5827471017837524,
      "log_odds_ratio": -0.45410656929016113,
      "logits/chosen": -0.34637928009033203,
      "logits/rejected": -0.028011344373226166,
      "logps/chosen": -1.9241366386413574,
      "logps/rejected": -2.4405038356781006,
      "loss": 2.1127,
      "nll_loss": 2.0673089027404785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19241367280483246,
      "rewards/margins": 0.0516367070376873,
      "rewards/rejected": -0.24405038356781006,
      "step": 155
    },
    {
      "epoch": 0.4307904729030031,
      "grad_norm": 0.507463812828064,
      "learning_rate": 7.100953342075009e-07,
      "log_odds_chosen": 0.31627729535102844,
      "log_odds_ratio": -0.5548453330993652,
      "logits/chosen": -0.2541901171207428,
      "logits/rejected": -0.1983020305633545,
      "logps/chosen": -2.0070505142211914,
      "logps/rejected": -2.2861552238464355,
      "loss": 2.1742,
      "nll_loss": 2.118666410446167,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20070505142211914,
      "rewards/margins": 0.027910443022847176,
      "rewards/rejected": -0.22861552238464355,
      "step": 156
    },
    {
      "epoch": 0.433551950293407,
      "grad_norm": 0.47030338644981384,
      "learning_rate": 7.057132633496923e-07,
      "log_odds_chosen": 0.44779300689697266,
      "log_odds_ratio": -0.49846184253692627,
      "logits/chosen": -0.3302999436855316,
      "logits/rejected": -0.07695013284683228,
      "logps/chosen": -1.912306308746338,
      "logps/rejected": -2.3054590225219727,
      "loss": 2.0937,
      "nll_loss": 2.0438175201416016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1912306398153305,
      "rewards/margins": 0.03931526839733124,
      "rewards/rejected": -0.23054589331150055,
      "step": 157
    },
    {
      "epoch": 0.4363134276838108,
      "grad_norm": 0.4665156900882721,
      "learning_rate": 7.013120885284598e-07,
      "log_odds_chosen": 0.41706544160842896,
      "log_odds_ratio": -0.525128960609436,
      "logits/chosen": -0.4476760923862457,
      "logits/rejected": -0.12284443527460098,
      "logps/chosen": -1.8804253339767456,
      "logps/rejected": -2.24664306640625,
      "loss": 2.0497,
      "nll_loss": 1.9972330331802368,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1880425363779068,
      "rewards/margins": 0.03662176430225372,
      "rewards/rejected": -0.22466430068016052,
      "step": 158
    },
    {
      "epoch": 0.4390749050742147,
      "grad_norm": 0.5610730648040771,
      "learning_rate": 6.968922184674867e-07,
      "log_odds_chosen": 0.32004255056381226,
      "log_odds_ratio": -0.5479089021682739,
      "logits/chosen": -0.277768611907959,
      "logits/rejected": -0.021672092378139496,
      "logps/chosen": -2.057568073272705,
      "logps/rejected": -2.340733289718628,
      "loss": 2.238,
      "nll_loss": 2.183201313018799,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20575681328773499,
      "rewards/margins": 0.028316490352153778,
      "rewards/rejected": -0.23407329618930817,
      "step": 159
    },
    {
      "epoch": 0.4418363824646186,
      "grad_norm": 0.4623549282550812,
      "learning_rate": 6.924540636266272e-07,
      "log_odds_chosen": 0.44459542632102966,
      "log_odds_ratio": -0.5021770596504211,
      "logits/chosen": -0.2480657398700714,
      "logits/rejected": -0.10325874388217926,
      "logps/chosen": -1.8401743173599243,
      "logps/rejected": -2.2281768321990967,
      "loss": 2.0379,
      "nll_loss": 1.987722635269165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18401743471622467,
      "rewards/margins": 0.03880026936531067,
      "rewards/rejected": -0.22281770408153534,
      "step": 160
    },
    {
      "epoch": 0.4445978598550224,
      "grad_norm": 0.4706402122974396,
      "learning_rate": 6.879980361637865e-07,
      "log_odds_chosen": 0.4396761953830719,
      "log_odds_ratio": -0.5012038350105286,
      "logits/chosen": -0.2594497501850128,
      "logits/rejected": -0.14531859755516052,
      "logps/chosen": -1.9227181673049927,
      "logps/rejected": -2.310340166091919,
      "loss": 2.085,
      "nll_loss": 2.034846544265747,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19227181375026703,
      "rewards/margins": 0.03876221179962158,
      "rewards/rejected": -0.2310340255498886,
      "step": 161
    },
    {
      "epoch": 0.4473593372454263,
      "grad_norm": 0.5076239109039307,
      "learning_rate": 6.83524549896646e-07,
      "log_odds_chosen": 0.5196196436882019,
      "log_odds_ratio": -0.4721822738647461,
      "logits/chosen": -0.4366263449192047,
      "logits/rejected": -0.18334615230560303,
      "logps/chosen": -1.980088233947754,
      "logps/rejected": -2.4419891834259033,
      "loss": 2.1421,
      "nll_loss": 2.0949079990386963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19800883531570435,
      "rewards/margins": 0.046190083026885986,
      "rewards/rejected": -0.24419891834259033,
      "step": 162
    },
    {
      "epoch": 0.45012081463583015,
      "grad_norm": 0.5030075907707214,
      "learning_rate": 6.790340202642331e-07,
      "log_odds_chosen": 0.5108202695846558,
      "log_odds_ratio": -0.4829539954662323,
      "logits/chosen": -0.37305155396461487,
      "logits/rejected": -0.20123453438282013,
      "logps/chosen": -1.914867639541626,
      "logps/rejected": -2.361052989959717,
      "loss": 2.0855,
      "nll_loss": 2.0372049808502197,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19148677587509155,
      "rewards/margins": 0.04461853206157684,
      "rewards/rejected": -0.2361053079366684,
      "step": 163
    },
    {
      "epoch": 0.45288229202623403,
      "grad_norm": 0.4399136006832123,
      "learning_rate": 6.745268642883404e-07,
      "log_odds_chosen": 0.5083030462265015,
      "log_odds_ratio": -0.47626441717147827,
      "logits/chosen": -0.31777992844581604,
      "logits/rejected": -0.25371578335762024,
      "logps/chosen": -1.8217023611068726,
      "logps/rejected": -2.261169672012329,
      "loss": 1.9987,
      "nll_loss": 1.9510999917984009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18217025697231293,
      "rewards/margins": 0.04394673556089401,
      "rewards/rejected": -0.22611698508262634,
      "step": 164
    },
    {
      "epoch": 0.4556437694166379,
      "grad_norm": 0.45623862743377686,
      "learning_rate": 6.700035005347982e-07,
      "log_odds_chosen": 0.5798279047012329,
      "log_odds_ratio": -0.4491708278656006,
      "logits/chosen": -0.25216957926750183,
      "logits/rejected": -0.1132553368806839,
      "logps/chosen": -2.0085461139678955,
      "logps/rejected": -2.5288069248199463,
      "loss": 2.1813,
      "nll_loss": 2.1363625526428223,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2008546143770218,
      "rewards/margins": 0.05202607810497284,
      "rewards/rejected": -0.25288069248199463,
      "step": 165
    },
    {
      "epoch": 0.45840524680704176,
      "grad_norm": 0.5783258676528931,
      "learning_rate": 6.654643490746041e-07,
      "log_odds_chosen": 0.4070507884025574,
      "log_odds_ratio": -0.5265771746635437,
      "logits/chosen": -0.46417152881622314,
      "logits/rejected": -0.16523872315883636,
      "logps/chosen": -1.994320273399353,
      "logps/rejected": -2.3601064682006836,
      "loss": 2.1739,
      "nll_loss": 2.1212868690490723,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19943203032016754,
      "rewards/margins": 0.03657860308885574,
      "rewards/rejected": -0.2360106259584427,
      "step": 166
    },
    {
      "epoch": 0.46116672419744564,
      "grad_norm": 0.48164114356040955,
      "learning_rate": 6.609098314449115e-07,
      "log_odds_chosen": 0.5174700021743774,
      "log_odds_ratio": -0.4722740054130554,
      "logits/chosen": -0.19338321685791016,
      "logits/rejected": 0.05244123563170433,
      "logps/chosen": -1.9246100187301636,
      "logps/rejected": -2.383852005004883,
      "loss": 2.1152,
      "nll_loss": 2.0679879188537598,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1924610137939453,
      "rewards/margins": 0.04592423141002655,
      "rewards/rejected": -0.23838523030281067,
      "step": 167
    },
    {
      "epoch": 0.4639282015878495,
      "grad_norm": 0.49352937936782837,
      "learning_rate": 6.563403706098832e-07,
      "log_odds_chosen": 0.6114981174468994,
      "log_odds_ratio": -0.45163875818252563,
      "logits/chosen": -0.33679214119911194,
      "logits/rejected": -0.13914185762405396,
      "logps/chosen": -1.9058302640914917,
      "logps/rejected": -2.4507896900177,
      "loss": 2.0875,
      "nll_loss": 2.0423457622528076,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1905830204486847,
      "rewards/margins": 0.054495953023433685,
      "rewards/rejected": -0.24507898092269897,
      "step": 168
    },
    {
      "epoch": 0.46668967897825336,
      "grad_norm": 0.4441838562488556,
      "learning_rate": 6.517563909214119e-07,
      "log_odds_chosen": 0.5005499124526978,
      "log_odds_ratio": -0.47856655716896057,
      "logits/chosen": -0.27010205388069153,
      "logits/rejected": -0.1331631988286972,
      "logps/chosen": -1.912389874458313,
      "logps/rejected": -2.352985143661499,
      "loss": 2.0798,
      "nll_loss": 2.0319559574127197,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19123899936676025,
      "rewards/margins": 0.04405952990055084,
      "rewards/rejected": -0.2352985292673111,
      "step": 169
    },
    {
      "epoch": 0.46945115636865725,
      "grad_norm": 0.499590128660202,
      "learning_rate": 6.47158318079712e-07,
      "log_odds_chosen": 0.5453026294708252,
      "log_odds_ratio": -0.4767185151576996,
      "logits/chosen": -0.39645957946777344,
      "logits/rejected": -0.2863028347492218,
      "logps/chosen": -1.8735414743423462,
      "logps/rejected": -2.361140012741089,
      "loss": 2.0483,
      "nll_loss": 2.000643730163574,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18735414743423462,
      "rewards/margins": 0.04875984787940979,
      "rewards/rejected": -0.2361140102148056,
      "step": 170
    },
    {
      "epoch": 0.4722126337590611,
      "grad_norm": 0.5450389981269836,
      "learning_rate": 6.425465790937861e-07,
      "log_odds_chosen": 0.44236356019973755,
      "log_odds_ratio": -0.5045577883720398,
      "logits/chosen": -0.3227909505367279,
      "logits/rejected": 0.0022758357226848602,
      "logps/chosen": -1.9548108577728271,
      "logps/rejected": -2.3471572399139404,
      "loss": 2.143,
      "nll_loss": 2.0925097465515137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.195481076836586,
      "rewards/margins": 0.039234623312950134,
      "rewards/rejected": -0.23471572995185852,
      "step": 171
    },
    {
      "epoch": 0.474974111149465,
      "grad_norm": 0.5155457854270935,
      "learning_rate": 6.379216022417695e-07,
      "log_odds_chosen": 0.5473636984825134,
      "log_odds_ratio": -0.4568098783493042,
      "logits/chosen": -0.4189146161079407,
      "logits/rejected": -0.024255044758319855,
      "logps/chosen": -1.8866467475891113,
      "logps/rejected": -2.366701126098633,
      "loss": 2.0709,
      "nll_loss": 2.025205612182617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18866465985774994,
      "rewards/margins": 0.04800545424222946,
      "rewards/rejected": -0.23667012155056,
      "step": 172
    },
    {
      "epoch": 0.4777355885398688,
      "grad_norm": 0.5155901908874512,
      "learning_rate": 6.332838170311585e-07,
      "log_odds_chosen": 0.45225203037261963,
      "log_odds_ratio": -0.5040324330329895,
      "logits/chosen": -0.3719104826450348,
      "logits/rejected": -0.13081157207489014,
      "logps/chosen": -1.9039151668548584,
      "logps/rejected": -2.3025238513946533,
      "loss": 2.081,
      "nll_loss": 2.0305662155151367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19039152562618256,
      "rewards/margins": 0.03986084833741188,
      "rewards/rejected": -0.23025237023830414,
      "step": 173
    },
    {
      "epoch": 0.4804970659302727,
      "grad_norm": 0.5719819664955139,
      "learning_rate": 6.286336541589223e-07,
      "log_odds_chosen": 0.23404161632061005,
      "log_odds_ratio": -0.5945478677749634,
      "logits/chosen": -0.37860414385795593,
      "logits/rejected": -0.1406262069940567,
      "logps/chosen": -2.040283441543579,
      "logps/rejected": -2.2495615482330322,
      "loss": 2.2208,
      "nll_loss": 2.161356210708618,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20402835309505463,
      "rewards/margins": 0.020927807316184044,
      "rewards/rejected": -0.2249561846256256,
      "step": 174
    },
    {
      "epoch": 0.4832585433206766,
      "grad_norm": 0.4385439157485962,
      "learning_rate": 6.239715454715053e-07,
      "log_odds_chosen": 0.47704774141311646,
      "log_odds_ratio": -0.48626744747161865,
      "logits/chosen": -0.2991381883621216,
      "logits/rejected": -0.25715842843055725,
      "logps/chosen": -1.9338128566741943,
      "logps/rejected": -2.3564324378967285,
      "loss": 2.0894,
      "nll_loss": 2.0407304763793945,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19338127970695496,
      "rewards/margins": 0.04226197302341461,
      "rewards/rejected": -0.23564325273036957,
      "step": 175
    },
    {
      "epoch": 0.4860200207110804,
      "grad_norm": 0.5189127922058105,
      "learning_rate": 6.192979239247242e-07,
      "log_odds_chosen": 0.41903528571128845,
      "log_odds_ratio": -0.513340950012207,
      "logits/chosen": -0.45005002617836,
      "logits/rejected": -0.3438834547996521,
      "logps/chosen": -1.9110206365585327,
      "logps/rejected": -2.2788233757019043,
      "loss": 2.0786,
      "nll_loss": 2.0272610187530518,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19110207259655,
      "rewards/margins": 0.036780260503292084,
      "rewards/rejected": -0.22788234055042267,
      "step": 176
    },
    {
      "epoch": 0.4887814981014843,
      "grad_norm": 0.5391824841499329,
      "learning_rate": 6.146132235435591e-07,
      "log_odds_chosen": 0.4565315842628479,
      "log_odds_ratio": -0.5045695304870605,
      "logits/chosen": -0.3154350221157074,
      "logits/rejected": -0.13199269771575928,
      "logps/chosen": -1.9622068405151367,
      "logps/rejected": -2.364424467086792,
      "loss": 2.145,
      "nll_loss": 2.094515562057495,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19622071087360382,
      "rewards/margins": 0.040221765637397766,
      "rewards/rejected": -0.2364424616098404,
      "step": 177
    },
    {
      "epoch": 0.49154297549188813,
      "grad_norm": 0.5079851150512695,
      "learning_rate": 6.099178793818478e-07,
      "log_odds_chosen": 0.7125115990638733,
      "log_odds_ratio": -0.4052090644836426,
      "logits/chosen": -0.4896236062049866,
      "logits/rejected": -0.18183393776416779,
      "logps/chosen": -1.8969542980194092,
      "logps/rejected": -2.526519298553467,
      "loss": 2.0695,
      "nll_loss": 2.0289804935455322,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18969543278217316,
      "rewards/margins": 0.06295648217201233,
      "rewards/rejected": -0.2526519298553467,
      "step": 178
    },
    {
      "epoch": 0.494304452882292,
      "grad_norm": 0.42423373460769653,
      "learning_rate": 6.052123274818841e-07,
      "log_odds_chosen": 0.518886923789978,
      "log_odds_ratio": -0.4682368338108063,
      "logits/chosen": -0.22320327162742615,
      "logits/rejected": -0.1859840303659439,
      "logps/chosen": -1.9779009819030762,
      "logps/rejected": -2.437370777130127,
      "loss": 2.1241,
      "nll_loss": 2.0772347450256348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19779011607170105,
      "rewards/margins": 0.045946985483169556,
      "rewards/rejected": -0.2437371015548706,
      "step": 179
    },
    {
      "epoch": 0.4970659302726959,
      "grad_norm": 0.5107156038284302,
      "learning_rate": 6.004970048339225e-07,
      "log_odds_chosen": 0.3471386730670929,
      "log_odds_ratio": -0.538905918598175,
      "logits/chosen": -0.34241271018981934,
      "logits/rejected": -0.13514721393585205,
      "logps/chosen": -1.964866042137146,
      "logps/rejected": -2.2716946601867676,
      "loss": 2.1479,
      "nll_loss": 2.0939929485321045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19648662209510803,
      "rewards/margins": 0.03068285994231701,
      "rewards/rejected": -0.2271694540977478,
      "step": 180
    },
    {
      "epoch": 0.49982740766309974,
      "grad_norm": 0.4654123783111572,
      "learning_rate": 5.957723493355976e-07,
      "log_odds_chosen": 0.4619404077529907,
      "log_odds_ratio": -0.49577149748802185,
      "logits/chosen": -0.21615736186504364,
      "logits/rejected": -0.024188008159399033,
      "logps/chosen": -1.9399676322937012,
      "logps/rejected": -2.3466637134552,
      "loss": 2.1414,
      "nll_loss": 2.0918264389038086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19399675726890564,
      "rewards/margins": 0.04066961258649826,
      "rewards/rejected": -0.2346663475036621,
      "step": 181
    },
    {
      "epoch": 0.5025888850535036,
      "grad_norm": 0.4790768027305603,
      "learning_rate": 5.910387997512573e-07,
      "log_odds_chosen": 0.519871175289154,
      "log_odds_ratio": -0.47471052408218384,
      "logits/chosen": -0.271729052066803,
      "logits/rejected": -0.45565640926361084,
      "logps/chosen": -1.9683092832565308,
      "logps/rejected": -2.43229603767395,
      "loss": 2.1393,
      "nll_loss": 2.091820001602173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19683094322681427,
      "rewards/margins": 0.04639867693185806,
      "rewards/rejected": -0.24322962760925293,
      "step": 182
    },
    {
      "epoch": 0.5053503624439075,
      "grad_norm": 0.48759087920188904,
      "learning_rate": 5.862967956712159e-07,
      "log_odds_chosen": 0.4985675811767578,
      "log_odds_ratio": -0.48592039942741394,
      "logits/chosen": -0.3652348518371582,
      "logits/rejected": 0.008702604100108147,
      "logps/chosen": -1.855039119720459,
      "logps/rejected": -2.292175769805908,
      "loss": 2.0347,
      "nll_loss": 1.9861171245574951,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18550390005111694,
      "rewards/margins": 0.04371367394924164,
      "rewards/rejected": -0.22921758890151978,
      "step": 183
    },
    {
      "epoch": 0.5081118398343114,
      "grad_norm": 0.4419066607952118,
      "learning_rate": 5.815467774709313e-07,
      "log_odds_chosen": 0.5235296487808228,
      "log_odds_ratio": -0.46782034635543823,
      "logits/chosen": -0.32331582903862,
      "logits/rejected": -0.20873025059700012,
      "logps/chosen": -1.8986095190048218,
      "logps/rejected": -2.3583438396453857,
      "loss": 2.0622,
      "nll_loss": 2.0154149532318115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18986095488071442,
      "rewards/margins": 0.045973438769578934,
      "rewards/rejected": -0.23583440482616425,
      "step": 184
    },
    {
      "epoch": 0.5108733172247152,
      "grad_norm": 0.40547627210617065,
      "learning_rate": 5.767891862701081e-07,
      "log_odds_chosen": 0.5548373460769653,
      "log_odds_ratio": -0.46065008640289307,
      "logits/chosen": -0.3630341589450836,
      "logits/rejected": -0.1256704479455948,
      "logps/chosen": -1.796975016593933,
      "logps/rejected": -2.2778377532958984,
      "loss": 1.9619,
      "nll_loss": 1.915809988975525,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17969751358032227,
      "rewards/margins": 0.048086270689964294,
      "rewards/rejected": -0.22778378427028656,
      "step": 185
    },
    {
      "epoch": 0.5136347946151191,
      "grad_norm": 0.5163060426712036,
      "learning_rate": 5.720244638917323e-07,
      "log_odds_chosen": 0.37705615162849426,
      "log_odds_ratio": -0.5264884233474731,
      "logits/chosen": -0.22042424976825714,
      "logits/rejected": -0.07070097327232361,
      "logps/chosen": -1.9638121128082275,
      "logps/rejected": -2.2940421104431152,
      "loss": 2.1194,
      "nll_loss": 2.066725254058838,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19638122618198395,
      "rewards/margins": 0.03302296996116638,
      "rewards/rejected": -0.22940419614315033,
      "step": 186
    },
    {
      "epoch": 0.516396272005523,
      "grad_norm": 0.47353076934814453,
      "learning_rate": 5.672530528210404e-07,
      "log_odds_chosen": 0.265331506729126,
      "log_odds_ratio": -0.5742963552474976,
      "logits/chosen": -0.2964926064014435,
      "logits/rejected": -0.006437892094254494,
      "logps/chosen": -1.9184975624084473,
      "logps/rejected": -2.148369550704956,
      "loss": 2.1004,
      "nll_loss": 2.042937994003296,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19184976816177368,
      "rewards/margins": 0.022987185046076775,
      "rewards/rejected": -0.2148369401693344,
      "step": 187
    },
    {
      "epoch": 0.5191577493959268,
      "grad_norm": 0.5041413903236389,
      "learning_rate": 5.624753961644281e-07,
      "log_odds_chosen": 0.4017038941383362,
      "log_odds_ratio": -0.5178303718566895,
      "logits/chosen": -0.19358986616134644,
      "logits/rejected": -0.14603643119335175,
      "logps/chosen": -2.017169237136841,
      "logps/rejected": -2.3748221397399902,
      "loss": 2.184,
      "nll_loss": 2.1322598457336426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20171691477298737,
      "rewards/margins": 0.035765327513217926,
      "rewards/rejected": -0.2374822348356247,
      "step": 188
    },
    {
      "epoch": 0.5219192267863307,
      "grad_norm": 0.46041780710220337,
      "learning_rate": 5.576919376082989e-07,
      "log_odds_chosen": 0.4141581654548645,
      "log_odds_ratio": -0.5125820636749268,
      "logits/chosen": -0.2839770019054413,
      "logits/rejected": -0.13594995439052582,
      "logps/chosen": -1.9662584066390991,
      "logps/rejected": -2.332793712615967,
      "loss": 2.1302,
      "nll_loss": 2.0789811611175537,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19662585854530334,
      "rewards/margins": 0.036653533577919006,
      "rewards/rejected": -0.23327937722206116,
      "step": 189
    },
    {
      "epoch": 0.5246807041767345,
      "grad_norm": 0.47012194991111755,
      "learning_rate": 5.529031213778614e-07,
      "log_odds_chosen": 0.4835112690925598,
      "log_odds_ratio": -0.4840419888496399,
      "logits/chosen": -0.389356791973114,
      "logits/rejected": -0.23288729786872864,
      "logps/chosen": -1.896279215812683,
      "logps/rejected": -2.3211445808410645,
      "loss": 2.0948,
      "nll_loss": 2.0464413166046143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18962793052196503,
      "rewards/margins": 0.042486537247896194,
      "rewards/rejected": -0.23211446404457092,
      "step": 190
    },
    {
      "epoch": 0.5274421815671384,
      "grad_norm": 0.5491785407066345,
      "learning_rate": 5.481093921958749e-07,
      "log_odds_chosen": 0.4995926320552826,
      "log_odds_ratio": -0.47897040843963623,
      "logits/chosen": -0.38450419902801514,
      "logits/rejected": -0.15095578134059906,
      "logps/chosen": -2.0216710567474365,
      "logps/rejected": -2.4666261672973633,
      "loss": 2.1912,
      "nll_loss": 2.1433069705963135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20216712355613708,
      "rewards/margins": 0.04449552297592163,
      "rewards/rejected": -0.24666264653205872,
      "step": 191
    },
    {
      "epoch": 0.5302036589575423,
      "grad_norm": 0.48054239153862,
      "learning_rate": 5.433111952413494e-07,
      "log_odds_chosen": 0.4785778820514679,
      "log_odds_ratio": -0.48475128412246704,
      "logits/chosen": -0.26365604996681213,
      "logits/rejected": -0.18638935685157776,
      "logps/chosen": -1.9027060270309448,
      "logps/rejected": -2.322059392929077,
      "loss": 2.0587,
      "nll_loss": 2.0102057456970215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19027060270309448,
      "rewards/margins": 0.041935332119464874,
      "rewards/rejected": -0.23220594227313995,
      "step": 192
    },
    {
      "epoch": 0.5329651363479462,
      "grad_norm": 0.41837820410728455,
      "learning_rate": 5.385089761082039e-07,
      "log_odds_chosen": 0.4972219169139862,
      "log_odds_ratio": -0.47896653413772583,
      "logits/chosen": -0.26745283603668213,
      "logits/rejected": -0.1336251199245453,
      "logps/chosen": -1.8878390789031982,
      "logps/rejected": -2.3233118057250977,
      "loss": 2.056,
      "nll_loss": 2.008148670196533,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1887839138507843,
      "rewards/margins": 0.04354727268218994,
      "rewards/rejected": -0.23233118653297424,
      "step": 193
    },
    {
      "epoch": 0.5357266137383501,
      "grad_norm": 0.45828357338905334,
      "learning_rate": 5.33703180763884e-07,
      "log_odds_chosen": 0.46731892228126526,
      "log_odds_ratio": -0.4991610050201416,
      "logits/chosen": -0.3924446702003479,
      "logits/rejected": -0.07468175888061523,
      "logps/chosen": -1.887953281402588,
      "logps/rejected": -2.2966556549072266,
      "loss": 2.0681,
      "nll_loss": 2.018134593963623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1887953132390976,
      "rewards/margins": 0.04087024927139282,
      "rewards/rejected": -0.2296655774116516,
      "step": 194
    },
    {
      "epoch": 0.5384880911287538,
      "grad_norm": 0.4831075072288513,
      "learning_rate": 5.288942555079478e-07,
      "log_odds_chosen": 0.6895397901535034,
      "log_odds_ratio": -0.4520753026008606,
      "logits/chosen": -0.3334307074546814,
      "logits/rejected": -0.0966208279132843,
      "logps/chosen": -1.911513090133667,
      "logps/rejected": -2.509132146835327,
      "loss": 2.1089,
      "nll_loss": 2.063706159591675,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19115132093429565,
      "rewards/margins": 0.059761881828308105,
      "rewards/rejected": -0.25091320276260376,
      "step": 195
    },
    {
      "epoch": 0.5412495685191577,
      "grad_norm": 0.42661550641059875,
      "learning_rate": 5.240826469306186e-07,
      "log_odds_chosen": 0.39442360401153564,
      "log_odds_ratio": -0.5208821892738342,
      "logits/chosen": -0.3010050058364868,
      "logits/rejected": -0.5069546103477478,
      "logps/chosen": -1.8927909135818481,
      "logps/rejected": -2.239405632019043,
      "loss": 2.0735,
      "nll_loss": 2.021390199661255,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18927910923957825,
      "rewards/margins": 0.034661464393138885,
      "rewards/rejected": -0.22394056618213654,
      "step": 196
    },
    {
      "epoch": 0.5440110459095616,
      "grad_norm": 0.511133074760437,
      "learning_rate": 5.192688018713113e-07,
      "log_odds_chosen": 0.43289878964424133,
      "log_odds_ratio": -0.5032349228858948,
      "logits/chosen": -0.29611679911613464,
      "logits/rejected": -0.14694276452064514,
      "logps/chosen": -1.93662428855896,
      "logps/rejected": -2.316537618637085,
      "loss": 2.1202,
      "nll_loss": 2.0698323249816895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19366243481636047,
      "rewards/margins": 0.03799131512641907,
      "rewards/rejected": -0.23165374994277954,
      "step": 197
    },
    {
      "epoch": 0.5467725232999655,
      "grad_norm": 0.45973795652389526,
      "learning_rate": 5.144531673771363e-07,
      "log_odds_chosen": 0.4756642282009125,
      "log_odds_ratio": -0.4891282320022583,
      "logits/chosen": -0.17253297567367554,
      "logits/rejected": -0.18779116868972778,
      "logps/chosen": -1.9230740070343018,
      "logps/rejected": -2.343052387237549,
      "loss": 2.0922,
      "nll_loss": 2.043295383453369,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19230739772319794,
      "rewards/margins": 0.04199780896306038,
      "rewards/rejected": -0.2343052327632904,
      "step": 198
    },
    {
      "epoch": 0.5495340006903694,
      "grad_norm": 0.4573759138584137,
      "learning_rate": 5.096361906613835e-07,
      "log_odds_chosen": 0.3954029977321625,
      "log_odds_ratio": -0.5324212312698364,
      "logits/chosen": -0.3204929530620575,
      "logits/rejected": -0.29722410440444946,
      "logps/chosen": -1.8643978834152222,
      "logps/rejected": -2.213501453399658,
      "loss": 2.0449,
      "nll_loss": 1.9916300773620605,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18643978238105774,
      "rewards/margins": 0.03491034358739853,
      "rewards/rejected": -0.22135013341903687,
      "step": 199
    },
    {
      "epoch": 0.5522954780807732,
      "grad_norm": 0.478037029504776,
      "learning_rate": 5.048183190619903e-07,
      "log_odds_chosen": 0.5303144454956055,
      "log_odds_ratio": -0.4676993787288666,
      "logits/chosen": -0.29242661595344543,
      "logits/rejected": -0.18273036181926727,
      "logps/chosen": -1.899263620376587,
      "logps/rejected": -2.36576247215271,
      "loss": 2.0796,
      "nll_loss": 2.032811164855957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1899263709783554,
      "rewards/margins": 0.046649858355522156,
      "rewards/rejected": -0.23657621443271637,
      "step": 200
    },
    {
      "epoch": 0.5550569554711771,
      "grad_norm": 0.4479018747806549,
      "learning_rate": 5e-07,
      "log_odds_chosen": 0.40334001183509827,
      "log_odds_ratio": -0.5163565874099731,
      "logits/chosen": -0.20333202183246613,
      "logits/rejected": -0.3748874068260193,
      "logps/chosen": -1.9522382020950317,
      "logps/rejected": -2.308758497238159,
      "loss": 2.1284,
      "nll_loss": 2.076796531677246,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1952238380908966,
      "rewards/margins": 0.0356520339846611,
      "rewards/rejected": -0.2308758646249771,
      "step": 201
    },
    {
      "epoch": 0.557818432861581,
      "grad_norm": 0.4593195617198944,
      "learning_rate": 4.951816809380097e-07,
      "log_odds_chosen": 0.4540878236293793,
      "log_odds_ratio": -0.4991394877433777,
      "logits/chosen": -0.374197393655777,
      "logits/rejected": -0.28211909532546997,
      "logps/chosen": -1.8836652040481567,
      "logps/rejected": -2.2832422256469727,
      "loss": 2.0662,
      "nll_loss": 2.0162971019744873,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18836651742458344,
      "rewards/margins": 0.03995770215988159,
      "rewards/rejected": -0.22832421958446503,
      "step": 202
    },
    {
      "epoch": 0.5605799102519848,
      "grad_norm": 0.4635930359363556,
      "learning_rate": 4.903638093386167e-07,
      "log_odds_chosen": 0.509470522403717,
      "log_odds_ratio": -0.4782922863960266,
      "logits/chosen": -0.26355335116386414,
      "logits/rejected": -0.31064000725746155,
      "logps/chosen": -1.9495270252227783,
      "logps/rejected": -2.3999316692352295,
      "loss": 2.111,
      "nll_loss": 2.063145875930786,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19495271146297455,
      "rewards/margins": 0.04504048079252243,
      "rewards/rejected": -0.23999318480491638,
      "step": 203
    },
    {
      "epoch": 0.5633413876423887,
      "grad_norm": 0.44470760226249695,
      "learning_rate": 4.855468326228638e-07,
      "log_odds_chosen": 0.620209813117981,
      "log_odds_ratio": -0.4331510663032532,
      "logits/chosen": -0.3340487480163574,
      "logits/rejected": -0.2821475565433502,
      "logps/chosen": -1.9127272367477417,
      "logps/rejected": -2.4635629653930664,
      "loss": 2.0928,
      "nll_loss": 2.049443244934082,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19127273559570312,
      "rewards/margins": 0.05508355423808098,
      "rewards/rejected": -0.24635626375675201,
      "step": 204
    },
    {
      "epoch": 0.5661028650327925,
      "grad_norm": 0.4645419716835022,
      "learning_rate": 4.807311981286888e-07,
      "log_odds_chosen": 0.3717229962348938,
      "log_odds_ratio": -0.5261833667755127,
      "logits/chosen": -0.2799780070781708,
      "logits/rejected": -0.0387892983853817,
      "logps/chosen": -1.9042205810546875,
      "logps/rejected": -2.2294371128082275,
      "loss": 2.0765,
      "nll_loss": 2.0238375663757324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19042205810546875,
      "rewards/margins": 0.03252164646983147,
      "rewards/rejected": -0.22294369339942932,
      "step": 205
    },
    {
      "epoch": 0.5688643424231964,
      "grad_norm": 0.44044730067253113,
      "learning_rate": 4.7591735306938134e-07,
      "log_odds_chosen": 0.5784755349159241,
      "log_odds_ratio": -0.46954020857810974,
      "logits/chosen": -0.2254553735256195,
      "logits/rejected": -0.3041171431541443,
      "logps/chosen": -1.8773598670959473,
      "logps/rejected": -2.385300636291504,
      "loss": 2.0672,
      "nll_loss": 2.020278215408325,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18773600459098816,
      "rewards/margins": 0.0507940798997879,
      "rewards/rejected": -0.23853006958961487,
      "step": 206
    },
    {
      "epoch": 0.5716258198136003,
      "grad_norm": 0.4391086995601654,
      "learning_rate": 4.7110574449205214e-07,
      "log_odds_chosen": 0.5138649344444275,
      "log_odds_ratio": -0.4766671359539032,
      "logits/chosen": -0.15283432602882385,
      "logits/rejected": -0.22331276535987854,
      "logps/chosen": -2.0644686222076416,
      "logps/rejected": -2.5245234966278076,
      "loss": 2.2192,
      "nll_loss": 2.171563148498535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2064468413591385,
      "rewards/margins": 0.046005476266145706,
      "rewards/rejected": -0.2524523437023163,
      "step": 207
    },
    {
      "epoch": 0.5743872972040042,
      "grad_norm": 0.46477314829826355,
      "learning_rate": 4.6629681923611603e-07,
      "log_odds_chosen": 0.41258686780929565,
      "log_odds_ratio": -0.5109040141105652,
      "logits/chosen": -0.10923075675964355,
      "logits/rejected": -0.15614847838878632,
      "logps/chosen": -1.9834563732147217,
      "logps/rejected": -2.348266124725342,
      "loss": 2.15,
      "nll_loss": 2.098912477493286,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19834566116333008,
      "rewards/margins": 0.03648097440600395,
      "rewards/rejected": -0.23482660949230194,
      "step": 208
    },
    {
      "epoch": 0.577148774594408,
      "grad_norm": 0.4983419179916382,
      "learning_rate": 4.614910238917963e-07,
      "log_odds_chosen": 0.440628319978714,
      "log_odds_ratio": -0.5007336735725403,
      "logits/chosen": -0.37217968702316284,
      "logits/rejected": -0.14922872185707092,
      "logps/chosen": -1.8617953062057495,
      "logps/rejected": -2.244419574737549,
      "loss": 2.0429,
      "nll_loss": 1.9927852153778076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.186179518699646,
      "rewards/margins": 0.03826242685317993,
      "rewards/rejected": -0.22444197535514832,
      "step": 209
    },
    {
      "epoch": 0.5799102519848118,
      "grad_norm": 0.4707476794719696,
      "learning_rate": 4.5668880475865067e-07,
      "log_odds_chosen": 0.46539944410324097,
      "log_odds_ratio": -0.49586039781570435,
      "logits/chosen": -0.2454988956451416,
      "logits/rejected": 0.020261432975530624,
      "logps/chosen": -1.9095300436019897,
      "logps/rejected": -2.321131706237793,
      "loss": 2.0876,
      "nll_loss": 2.038007974624634,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19095300137996674,
      "rewards/margins": 0.04116016626358032,
      "rewards/rejected": -0.23211316764354706,
      "step": 210
    },
    {
      "epoch": 0.5826717293752157,
      "grad_norm": 0.369704008102417,
      "learning_rate": 4.5189060780412515e-07,
      "log_odds_chosen": 0.423761248588562,
      "log_odds_ratio": -0.5302607417106628,
      "logits/chosen": -0.1828247755765915,
      "logits/rejected": -0.021833106875419617,
      "logps/chosen": -1.8902254104614258,
      "logps/rejected": -2.2649874687194824,
      "loss": 2.0762,
      "nll_loss": 2.0231916904449463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18902254104614258,
      "rewards/margins": 0.03747619688510895,
      "rewards/rejected": -0.22649873793125153,
      "step": 211
    },
    {
      "epoch": 0.5854332067656196,
      "grad_norm": 0.39925074577331543,
      "learning_rate": 4.4709687862213864e-07,
      "log_odds_chosen": 0.5902334451675415,
      "log_odds_ratio": -0.4505179226398468,
      "logits/chosen": -0.2861301302909851,
      "logits/rejected": -0.41044506430625916,
      "logps/chosen": -1.8154157400131226,
      "logps/rejected": -2.3315727710723877,
      "loss": 1.9792,
      "nll_loss": 1.9341650009155273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1815415918827057,
      "rewards/margins": 0.05161570757627487,
      "rewards/rejected": -0.23315729200839996,
      "step": 212
    },
    {
      "epoch": 0.5881946841560235,
      "grad_norm": 0.38932037353515625,
      "learning_rate": 4.4230806239170113e-07,
      "log_odds_chosen": 0.49800512194633484,
      "log_odds_ratio": -0.48540055751800537,
      "logits/chosen": -0.14587503671646118,
      "logits/rejected": -0.3000808656215668,
      "logps/chosen": -1.8713995218276978,
      "logps/rejected": -2.304245948791504,
      "loss": 2.0255,
      "nll_loss": 1.9769891500473022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18713994324207306,
      "rewards/margins": 0.04328465461730957,
      "rewards/rejected": -0.23042459785938263,
      "step": 213
    },
    {
      "epoch": 0.5909561615464274,
      "grad_norm": 0.4413486421108246,
      "learning_rate": 4.3752460383557194e-07,
      "log_odds_chosen": 0.5172650218009949,
      "log_odds_ratio": -0.4785919189453125,
      "logits/chosen": -0.2637350261211395,
      "logits/rejected": -0.11889218538999557,
      "logps/chosen": -1.9498810768127441,
      "logps/rejected": -2.4114248752593994,
      "loss": 2.1077,
      "nll_loss": 2.059854507446289,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19498810172080994,
      "rewards/margins": 0.046154364943504333,
      "rewards/rejected": -0.24114246666431427,
      "step": 214
    },
    {
      "epoch": 0.5937176389368312,
      "grad_norm": 0.4996843636035919,
      "learning_rate": 4.3274694717895964e-07,
      "log_odds_chosen": 0.41755494475364685,
      "log_odds_ratio": -0.5150564908981323,
      "logits/chosen": -0.18713760375976562,
      "logits/rejected": -0.066367506980896,
      "logps/chosen": -1.9931713342666626,
      "logps/rejected": -2.36261248588562,
      "loss": 2.1841,
      "nll_loss": 2.132643222808838,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19931714236736298,
      "rewards/margins": 0.03694409132003784,
      "rewards/rejected": -0.23626123368740082,
      "step": 215
    },
    {
      "epoch": 0.596479116327235,
      "grad_norm": 0.4241478145122528,
      "learning_rate": 4.2797553610826797e-07,
      "log_odds_chosen": 0.48448869585990906,
      "log_odds_ratio": -0.48890987038612366,
      "logits/chosen": -0.31950169801712036,
      "logits/rejected": -0.01856943964958191,
      "logps/chosen": -1.8249506950378418,
      "logps/rejected": -2.24398136138916,
      "loss": 2.0009,
      "nll_loss": 1.9520447254180908,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1824950873851776,
      "rewards/margins": 0.041903071105480194,
      "rewards/rejected": -0.22439813613891602,
      "step": 216
    },
    {
      "epoch": 0.5992405937176389,
      "grad_norm": 0.4482674300670624,
      "learning_rate": 4.2321081372989186e-07,
      "log_odds_chosen": 0.34753182530403137,
      "log_odds_ratio": -0.5502303242683411,
      "logits/chosen": -0.2806357741355896,
      "logits/rejected": -0.11674074083566666,
      "logps/chosen": -1.9155105352401733,
      "logps/rejected": -2.2155590057373047,
      "loss": 2.1157,
      "nll_loss": 2.0606534481048584,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1915510594844818,
      "rewards/margins": 0.030004845932126045,
      "rewards/rejected": -0.2215559035539627,
      "step": 217
    },
    {
      "epoch": 0.6020020711080428,
      "grad_norm": 0.5122972130775452,
      "learning_rate": 4.184532225290686e-07,
      "log_odds_chosen": 0.4445059299468994,
      "log_odds_ratio": -0.4998216927051544,
      "logits/chosen": -0.29193422198295593,
      "logits/rejected": -0.1805831342935562,
      "logps/chosen": -1.9241386651992798,
      "logps/rejected": -2.317504644393921,
      "loss": 2.092,
      "nll_loss": 2.0420260429382324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19241386651992798,
      "rewards/margins": 0.03933658450841904,
      "rewards/rejected": -0.2317504584789276,
      "step": 218
    },
    {
      "epoch": 0.6047635484984467,
      "grad_norm": 0.4178394079208374,
      "learning_rate": 4.13703204328784e-07,
      "log_odds_chosen": 0.5128558874130249,
      "log_odds_ratio": -0.4747515618801117,
      "logits/chosen": -0.3372417986392975,
      "logits/rejected": -0.2523457705974579,
      "logps/chosen": -1.8720359802246094,
      "logps/rejected": -2.3234829902648926,
      "loss": 2.043,
      "nll_loss": 1.9955538511276245,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18720358610153198,
      "rewards/margins": 0.0451447069644928,
      "rewards/rejected": -0.23234832286834717,
      "step": 219
    },
    {
      "epoch": 0.6075250258888505,
      "grad_norm": 0.5029446482658386,
      "learning_rate": 4.089612002487428e-07,
      "log_odds_chosen": 0.3836578130722046,
      "log_odds_ratio": -0.5439907312393188,
      "logits/chosen": -0.19269907474517822,
      "logits/rejected": -0.2113722860813141,
      "logps/chosen": -1.9919250011444092,
      "logps/rejected": -2.3357558250427246,
      "loss": 2.1533,
      "nll_loss": 2.0988759994506836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19919249415397644,
      "rewards/margins": 0.034383051097393036,
      "rewards/rejected": -0.23357556760311127,
      "step": 220
    },
    {
      "epoch": 0.6102865032792544,
      "grad_norm": 0.4052739441394806,
      "learning_rate": 4.042276506644024e-07,
      "log_odds_chosen": 0.5609914660453796,
      "log_odds_ratio": -0.4547596871852875,
      "logits/chosen": -0.1459711492061615,
      "logits/rejected": -0.10660164058208466,
      "logps/chosen": -1.8867138624191284,
      "logps/rejected": -2.378152847290039,
      "loss": 2.0517,
      "nll_loss": 2.0061984062194824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18867139518260956,
      "rewards/margins": 0.049143895506858826,
      "rewards/rejected": -0.2378152757883072,
      "step": 221
    },
    {
      "epoch": 0.6130479806696583,
      "grad_norm": 0.40826407074928284,
      "learning_rate": 3.995029951660776e-07,
      "log_odds_chosen": 0.6102659106254578,
      "log_odds_ratio": -0.44388866424560547,
      "logits/chosen": -0.23413187265396118,
      "logits/rejected": -0.14933869242668152,
      "logps/chosen": -1.9388023614883423,
      "logps/rejected": -2.4753732681274414,
      "loss": 2.1057,
      "nll_loss": 2.0613319873809814,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19388024508953094,
      "rewards/margins": 0.05365708842873573,
      "rewards/rejected": -0.24753732979297638,
      "step": 222
    },
    {
      "epoch": 0.6158094580600622,
      "grad_norm": 0.40428611636161804,
      "learning_rate": 3.9478767251811595e-07,
      "log_odds_chosen": 0.38443854451179504,
      "log_odds_ratio": -0.5237889289855957,
      "logits/chosen": -0.2519416809082031,
      "logits/rejected": -0.4330897927284241,
      "logps/chosen": -1.9040776491165161,
      "logps/rejected": -2.2454843521118164,
      "loss": 2.074,
      "nll_loss": 2.0216562747955322,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19040776789188385,
      "rewards/margins": 0.03414066135883331,
      "rewards/rejected": -0.22454842925071716,
      "step": 223
    },
    {
      "epoch": 0.618570935450466,
      "grad_norm": 0.41897815465927124,
      "learning_rate": 3.9008212061815207e-07,
      "log_odds_chosen": 0.48172813653945923,
      "log_odds_ratio": -0.48397549986839294,
      "logits/chosen": -0.24099406599998474,
      "logits/rejected": -0.16496577858924866,
      "logps/chosen": -1.8474040031433105,
      "logps/rejected": -2.265615224838257,
      "loss": 2.0232,
      "nll_loss": 1.9748239517211914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18474040925502777,
      "rewards/margins": 0.04182111844420433,
      "rewards/rejected": -0.2265615463256836,
      "step": 224
    },
    {
      "epoch": 0.6213324128408698,
      "grad_norm": 0.3934837281703949,
      "learning_rate": 3.853867764564409e-07,
      "log_odds_chosen": 0.5578025579452515,
      "log_odds_ratio": -0.4562433063983917,
      "logits/chosen": -0.34043368697166443,
      "logits/rejected": -0.3277437686920166,
      "logps/chosen": -1.803032636642456,
      "logps/rejected": -2.2886483669281006,
      "loss": 1.9669,
      "nll_loss": 1.9212396144866943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18030327558517456,
      "rewards/margins": 0.04856157302856445,
      "rewards/rejected": -0.228864848613739,
      "step": 225
    },
    {
      "epoch": 0.6240938902312737,
      "grad_norm": 0.4615204334259033,
      "learning_rate": 3.8070207607527585e-07,
      "log_odds_chosen": 0.3832344710826874,
      "log_odds_ratio": -0.5226789712905884,
      "logits/chosen": -0.09884040057659149,
      "logits/rejected": -0.15861357748508453,
      "logps/chosen": -2.0035581588745117,
      "logps/rejected": -2.344210147857666,
      "loss": 2.1831,
      "nll_loss": 2.130826950073242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20035582780838013,
      "rewards/margins": 0.034065183252096176,
      "rewards/rejected": -0.2344210296869278,
      "step": 226
    },
    {
      "epoch": 0.6268553676216776,
      "grad_norm": 0.5194337368011475,
      "learning_rate": 3.7602845452849463e-07,
      "log_odds_chosen": 0.547633171081543,
      "log_odds_ratio": -0.466709703207016,
      "logits/chosen": -0.19283336400985718,
      "logits/rejected": -0.17510032653808594,
      "logps/chosen": -2.0914061069488525,
      "logps/rejected": -2.586927890777588,
      "loss": 2.2513,
      "nll_loss": 2.2046549320220947,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2091406285762787,
      "rewards/margins": 0.049552179872989655,
      "rewards/rejected": -0.25869280099868774,
      "step": 227
    },
    {
      "epoch": 0.6296168450120815,
      "grad_norm": 0.4143467843532562,
      "learning_rate": 3.7136634584107783e-07,
      "log_odds_chosen": 0.29817482829093933,
      "log_odds_ratio": -0.5645455121994019,
      "logits/chosen": -0.08032770454883575,
      "logits/rejected": -0.28935497999191284,
      "logps/chosen": -2.012125253677368,
      "logps/rejected": -2.2754557132720947,
      "loss": 2.1851,
      "nll_loss": 2.128629684448242,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.201212540268898,
      "rewards/margins": 0.026333047077059746,
      "rewards/rejected": -0.22754555940628052,
      "step": 228
    },
    {
      "epoch": 0.6323783224024854,
      "grad_norm": 0.4432436525821686,
      "learning_rate": 3.6671618296884143e-07,
      "log_odds_chosen": 0.4121524393558502,
      "log_odds_ratio": -0.5227192640304565,
      "logits/chosen": -0.2652166783809662,
      "logits/rejected": -0.21302977204322815,
      "logps/chosen": -1.8965659141540527,
      "logps/rejected": -2.2580127716064453,
      "loss": 2.079,
      "nll_loss": 2.0267221927642822,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.189656600356102,
      "rewards/margins": 0.036144666373729706,
      "rewards/rejected": -0.2258012592792511,
      "step": 229
    },
    {
      "epoch": 0.6351397997928891,
      "grad_norm": 0.4152255058288574,
      "learning_rate": 3.6207839775823047e-07,
      "log_odds_chosen": 0.5571652054786682,
      "log_odds_ratio": -0.46520212292671204,
      "logits/chosen": -0.4695712625980377,
      "logits/rejected": -0.3200719356536865,
      "logps/chosen": -1.744187593460083,
      "logps/rejected": -2.221540927886963,
      "loss": 1.938,
      "nll_loss": 1.8914943933486938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17441876232624054,
      "rewards/margins": 0.04773533344268799,
      "rewards/rejected": -0.22215408086776733,
      "step": 230
    },
    {
      "epoch": 0.637901277183293,
      "grad_norm": 0.4536137282848358,
      "learning_rate": 3.57453420906214e-07,
      "log_odds_chosen": 0.25317129492759705,
      "log_odds_ratio": -0.5782453417778015,
      "logits/chosen": -0.3376019597053528,
      "logits/rejected": -0.2664189040660858,
      "logps/chosen": -1.913726568222046,
      "logps/rejected": -2.133842706680298,
      "loss": 2.0915,
      "nll_loss": 2.033724069595337,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19137263298034668,
      "rewards/margins": 0.022011613473296165,
      "rewards/rejected": -0.2133842557668686,
      "step": 231
    },
    {
      "epoch": 0.6406627545736969,
      "grad_norm": 0.5167147517204285,
      "learning_rate": 3.5284168192028805e-07,
      "log_odds_chosen": 0.5250340700149536,
      "log_odds_ratio": -0.4787348508834839,
      "logits/chosen": -0.24234242737293243,
      "logits/rejected": -0.3100574314594269,
      "logps/chosen": -1.9809167385101318,
      "logps/rejected": -2.453472852706909,
      "loss": 2.153,
      "nll_loss": 2.10510516166687,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19809167087078094,
      "rewards/margins": 0.047255612909793854,
      "rewards/rejected": -0.2453472763299942,
      "step": 232
    },
    {
      "epoch": 0.6434242319641008,
      "grad_norm": 0.41366466879844666,
      "learning_rate": 3.482436090785882e-07,
      "log_odds_chosen": 0.5098401308059692,
      "log_odds_ratio": -0.48293790221214294,
      "logits/chosen": -0.26800814270973206,
      "logits/rejected": -0.3802124261856079,
      "logps/chosen": -1.917423129081726,
      "logps/rejected": -2.3675589561462402,
      "loss": 2.0912,
      "nll_loss": 2.042955160140991,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19174233078956604,
      "rewards/margins": 0.04501357674598694,
      "rewards/rejected": -0.23675590753555298,
      "step": 233
    },
    {
      "epoch": 0.6461857093545047,
      "grad_norm": 0.4223003387451172,
      "learning_rate": 3.4365962939011693e-07,
      "log_odds_chosen": 0.4395076036453247,
      "log_odds_ratio": -0.5000528693199158,
      "logits/chosen": -0.1607169806957245,
      "logits/rejected": -0.3638402819633484,
      "logps/chosen": -1.9584696292877197,
      "logps/rejected": -2.34755539894104,
      "loss": 2.1174,
      "nll_loss": 2.067427158355713,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19584694504737854,
      "rewards/margins": 0.03890858590602875,
      "rewards/rejected": -0.23475554585456848,
      "step": 234
    },
    {
      "epoch": 0.6489471867449085,
      "grad_norm": 0.44036707282066345,
      "learning_rate": 3.3909016855508867e-07,
      "log_odds_chosen": 0.3687521815299988,
      "log_odds_ratio": -0.5295863747596741,
      "logits/chosen": -0.2543387711048126,
      "logits/rejected": -0.21828660368919373,
      "logps/chosen": -1.9281657934188843,
      "logps/rejected": -2.2500669956207275,
      "loss": 2.1205,
      "nll_loss": 2.0675907135009766,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1928165853023529,
      "rewards/margins": 0.03219012916088104,
      "rewards/rejected": -0.22500671446323395,
      "step": 235
    },
    {
      "epoch": 0.6517086641353124,
      "grad_norm": 0.42869260907173157,
      "learning_rate": 3.345356509253958e-07,
      "log_odds_chosen": 0.4268803298473358,
      "log_odds_ratio": -0.513598620891571,
      "logits/chosen": -0.28990602493286133,
      "logits/rejected": -0.32511040568351746,
      "logps/chosen": -1.867914080619812,
      "logps/rejected": -2.24053955078125,
      "loss": 2.0196,
      "nll_loss": 1.968214511871338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18679141998291016,
      "rewards/margins": 0.03726254403591156,
      "rewards/rejected": -0.22405396401882172,
      "step": 236
    },
    {
      "epoch": 0.6544701415257163,
      "grad_norm": 0.43179935216903687,
      "learning_rate": 3.299964994652017e-07,
      "log_odds_chosen": 0.40741828083992004,
      "log_odds_ratio": -0.5262683033943176,
      "logits/chosen": -0.29947933554649353,
      "logits/rejected": -0.21216286718845367,
      "logps/chosen": -1.8077688217163086,
      "logps/rejected": -2.1621196269989014,
      "loss": 1.9985,
      "nll_loss": 1.945885181427002,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18077686429023743,
      "rewards/margins": 0.03543510288000107,
      "rewards/rejected": -0.21621198952198029,
      "step": 237
    },
    {
      "epoch": 0.6572316189161201,
      "grad_norm": 0.5112394690513611,
      "learning_rate": 3.2547313571165967e-07,
      "log_odds_chosen": 0.46831125020980835,
      "log_odds_ratio": -0.5017648935317993,
      "logits/chosen": -0.39401593804359436,
      "logits/rejected": -0.1256256252527237,
      "logps/chosen": -1.954957127571106,
      "logps/rejected": -2.3675594329833984,
      "loss": 2.1357,
      "nll_loss": 2.0854828357696533,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19549569487571716,
      "rewards/margins": 0.04126022756099701,
      "rewards/rejected": -0.23675595223903656,
      "step": 238
    },
    {
      "epoch": 0.659993096306524,
      "grad_norm": 0.44042789936065674,
      "learning_rate": 3.209659797357669e-07,
      "log_odds_chosen": 0.5182204246520996,
      "log_odds_ratio": -0.4714072644710541,
      "logits/chosen": -0.18000362813472748,
      "logits/rejected": -0.04683329910039902,
      "logps/chosen": -1.928417444229126,
      "logps/rejected": -2.384457588195801,
      "loss": 2.0979,
      "nll_loss": 2.05072021484375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19284175336360931,
      "rewards/margins": 0.045604005455970764,
      "rewards/rejected": -0.23844575881958008,
      "step": 239
    },
    {
      "epoch": 0.6627545736969278,
      "grad_norm": 0.43828555941581726,
      "learning_rate": 3.1647545010335395e-07,
      "log_odds_chosen": 0.6641589403152466,
      "log_odds_ratio": -0.4267301559448242,
      "logits/chosen": -0.3079659342765808,
      "logits/rejected": -0.1823115348815918,
      "logps/chosen": -1.8825758695602417,
      "logps/rejected": -2.47467303276062,
      "loss": 2.0502,
      "nll_loss": 2.007534980773926,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1882575899362564,
      "rewards/margins": 0.059209711849689484,
      "rewards/rejected": -0.2474672943353653,
      "step": 240
    },
    {
      "epoch": 0.6655160510873317,
      "grad_norm": 0.3798598349094391,
      "learning_rate": 3.120019638362136e-07,
      "log_odds_chosen": 0.4808882772922516,
      "log_odds_ratio": -0.48572006821632385,
      "logits/chosen": -0.11574582755565643,
      "logits/rejected": -0.10241278260946274,
      "logps/chosen": -1.864730715751648,
      "logps/rejected": -2.2846121788024902,
      "loss": 2.0244,
      "nll_loss": 1.9758269786834717,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1864730715751648,
      "rewards/margins": 0.04198816418647766,
      "rewards/rejected": -0.22846123576164246,
      "step": 241
    },
    {
      "epoch": 0.6682775284777356,
      "grad_norm": 0.4033946990966797,
      "learning_rate": 3.075459363733727e-07,
      "log_odds_chosen": 0.5772659182548523,
      "log_odds_ratio": -0.4489781856536865,
      "logits/chosen": -0.21031953394412994,
      "logits/rejected": -0.16377201676368713,
      "logps/chosen": -1.8277872800827026,
      "logps/rejected": -2.3323185443878174,
      "loss": 1.9902,
      "nll_loss": 1.945261836051941,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1827787160873413,
      "rewards/margins": 0.05045313015580177,
      "rewards/rejected": -0.23323185741901398,
      "step": 242
    },
    {
      "epoch": 0.6710390058681395,
      "grad_norm": 0.4857328534126282,
      "learning_rate": 3.031077815325132e-07,
      "log_odds_chosen": 0.5548877716064453,
      "log_odds_ratio": -0.4721669554710388,
      "logits/chosen": -0.16459237039089203,
      "logits/rejected": -0.19354675710201263,
      "logps/chosen": -1.9921672344207764,
      "logps/rejected": -2.489896297454834,
      "loss": 2.1594,
      "nll_loss": 2.1121487617492676,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19921672344207764,
      "rewards/margins": 0.04977291822433472,
      "rewards/rejected": -0.24898964166641235,
      "step": 243
    },
    {
      "epoch": 0.6738004832585434,
      "grad_norm": 0.3858998417854309,
      "learning_rate": 2.9868791147154025e-07,
      "log_odds_chosen": 0.39727193117141724,
      "log_odds_ratio": -0.5191612839698792,
      "logits/chosen": -0.12661641836166382,
      "logits/rejected": -0.2074287384748459,
      "logps/chosen": -1.843322515487671,
      "logps/rejected": -2.189265012741089,
      "loss": 2.0185,
      "nll_loss": 1.9665818214416504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18433226644992828,
      "rewards/margins": 0.034594256430864334,
      "rewards/rejected": -0.21892650425434113,
      "step": 244
    },
    {
      "epoch": 0.6765619606489471,
      "grad_norm": 0.4435907006263733,
      "learning_rate": 2.942867366503077e-07,
      "log_odds_chosen": 0.539937436580658,
      "log_odds_ratio": -0.4697742462158203,
      "logits/chosen": -0.2335578054189682,
      "logits/rejected": -0.09750137478113174,
      "logps/chosen": -1.9041216373443604,
      "logps/rejected": -2.381821870803833,
      "loss": 2.0762,
      "nll_loss": 2.0292389392852783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19041216373443604,
      "rewards/margins": 0.047770045697689056,
      "rewards/rejected": -0.2381822019815445,
      "step": 245
    },
    {
      "epoch": 0.679323438039351,
      "grad_norm": 0.40660807490348816,
      "learning_rate": 2.8990466579249917e-07,
      "log_odds_chosen": 0.41202157735824585,
      "log_odds_ratio": -0.5189868211746216,
      "logits/chosen": -0.15573078393936157,
      "logits/rejected": -0.20848733186721802,
      "logps/chosen": -1.9583265781402588,
      "logps/rejected": -2.323798179626465,
      "loss": 2.1306,
      "nll_loss": 2.078730583190918,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19583265483379364,
      "rewards/margins": 0.03654715418815613,
      "rewards/rejected": -0.23237979412078857,
      "step": 246
    },
    {
      "epoch": 0.6820849154297549,
      "grad_norm": 0.39858436584472656,
      "learning_rate": 2.8554210584767184e-07,
      "log_odds_chosen": 0.4532400071620941,
      "log_odds_ratio": -0.49805763363838196,
      "logits/chosen": -0.31871944665908813,
      "logits/rejected": -0.33112263679504395,
      "logps/chosen": -1.8041869401931763,
      "logps/rejected": -2.1945743560791016,
      "loss": 1.9834,
      "nll_loss": 1.933552622795105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1804186999797821,
      "rewards/margins": 0.03903874009847641,
      "rewards/rejected": -0.21945743262767792,
      "step": 247
    },
    {
      "epoch": 0.6848463928201588,
      "grad_norm": 0.4925353527069092,
      "learning_rate": 2.811994619534637e-07,
      "log_odds_chosen": 0.4248819351196289,
      "log_odds_ratio": -0.5094651579856873,
      "logits/chosen": -0.24002224206924438,
      "logits/rejected": -0.07596518099308014,
      "logps/chosen": -1.9429184198379517,
      "logps/rejected": -2.3185441493988037,
      "loss": 2.1136,
      "nll_loss": 2.0626585483551025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1942918300628662,
      "rewards/margins": 0.037562571465969086,
      "rewards/rejected": -0.2318544238805771,
      "step": 248
    },
    {
      "epoch": 0.6876078702105627,
      "grad_norm": 0.44815462827682495,
      "learning_rate": 2.768771373979697e-07,
      "log_odds_chosen": 0.6093379855155945,
      "log_odds_ratio": -0.455441415309906,
      "logits/chosen": -0.2777319550514221,
      "logits/rejected": -0.15260514616966248,
      "logps/chosen": -1.785163402557373,
      "logps/rejected": -2.31174635887146,
      "loss": 1.9745,
      "nll_loss": 1.9289604425430298,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17851634323596954,
      "rewards/margins": 0.0526583231985569,
      "rewards/rejected": -0.23117467761039734,
      "step": 249
    },
    {
      "epoch": 0.6903693476009665,
      "grad_norm": 0.4060370624065399,
      "learning_rate": 2.725755335822903e-07,
      "log_odds_chosen": 0.5381920337677002,
      "log_odds_ratio": -0.47121572494506836,
      "logits/chosen": -0.1876389980316162,
      "logits/rejected": -0.3438076674938202,
      "logps/chosen": -1.9416241645812988,
      "logps/rejected": -2.420865058898926,
      "loss": 2.1236,
      "nll_loss": 2.076474189758301,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19416242837905884,
      "rewards/margins": 0.047924067825078964,
      "rewards/rejected": -0.2420864999294281,
      "step": 250
    },
    {
      "epoch": 0.6931308249913704,
      "grad_norm": 0.462128609418869,
      "learning_rate": 2.6829504998325345e-07,
      "log_odds_chosen": 0.5822583436965942,
      "log_odds_ratio": -0.45154690742492676,
      "logits/chosen": -0.27919477224349976,
      "logits/rejected": -0.2352999448776245,
      "logps/chosen": -1.8237282037734985,
      "logps/rejected": -2.3347060680389404,
      "loss": 2.0204,
      "nll_loss": 1.9752583503723145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1823728233575821,
      "rewards/margins": 0.05109778791666031,
      "rewards/rejected": -0.2334706038236618,
      "step": 251
    },
    {
      "epoch": 0.6958923023817742,
      "grad_norm": 0.46492475271224976,
      "learning_rate": 2.640360841163174e-07,
      "log_odds_chosen": 0.4209150969982147,
      "log_odds_ratio": -0.508406937122345,
      "logits/chosen": -0.2801150977611542,
      "logits/rejected": -0.2511332929134369,
      "logps/chosen": -1.9320614337921143,
      "logps/rejected": -2.3003315925598145,
      "loss": 2.0929,
      "nll_loss": 2.042011260986328,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19320616126060486,
      "rewards/margins": 0.03682699054479599,
      "rewards/rejected": -0.23003314435482025,
      "step": 252
    },
    {
      "epoch": 0.6986537797721781,
      "grad_norm": 0.4857609272003174,
      "learning_rate": 2.597990314986538e-07,
      "log_odds_chosen": 0.3778655529022217,
      "log_odds_ratio": -0.531346321105957,
      "logits/chosen": -0.23271705210208893,
      "logits/rejected": -0.19061926007270813,
      "logps/chosen": -1.9238560199737549,
      "logps/rejected": -2.2503604888916016,
      "loss": 2.1045,
      "nll_loss": 2.0513229370117188,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19238559901714325,
      "rewards/margins": 0.032650426030159,
      "rewards/rejected": -0.22503602504730225,
      "step": 253
    },
    {
      "epoch": 0.701415257162582,
      "grad_norm": 0.40434175729751587,
      "learning_rate": 2.5558428561241816e-07,
      "log_odds_chosen": 0.41764160990715027,
      "log_odds_ratio": -0.5145137906074524,
      "logits/chosen": -0.14151108264923096,
      "logits/rejected": -0.293675035238266,
      "logps/chosen": -1.884641408920288,
      "logps/rejected": -2.251406669616699,
      "loss": 2.0596,
      "nll_loss": 2.008186101913452,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18846413493156433,
      "rewards/margins": 0.036676548421382904,
      "rewards/rejected": -0.22514067590236664,
      "step": 254
    },
    {
      "epoch": 0.7041767345529858,
      "grad_norm": 0.4174916446208954,
      "learning_rate": 2.5139223786820744e-07,
      "log_odds_chosen": 0.5133525133132935,
      "log_odds_ratio": -0.4715559780597687,
      "logits/chosen": -0.2591693699359894,
      "logits/rejected": -0.3382716178894043,
      "logps/chosen": -1.9182339906692505,
      "logps/rejected": -2.3702430725097656,
      "loss": 2.0929,
      "nll_loss": 2.045783519744873,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19182339310646057,
      "rewards/margins": 0.04520091414451599,
      "rewards/rejected": -0.23702432215213776,
      "step": 255
    },
    {
      "epoch": 0.7069382119433897,
      "grad_norm": 0.40420272946357727,
      "learning_rate": 2.472232775687119e-07,
      "log_odds_chosen": 0.40430134534835815,
      "log_odds_ratio": -0.5246070623397827,
      "logits/chosen": -0.2077058106660843,
      "logits/rejected": -0.15366941690444946,
      "logps/chosen": -1.9031445980072021,
      "logps/rejected": -2.256007432937622,
      "loss": 2.0681,
      "nll_loss": 2.015625476837158,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19031447172164917,
      "rewards/margins": 0.035286273807287216,
      "rewards/rejected": -0.2256007343530655,
      "step": 256
    },
    {
      "epoch": 0.7096996893337936,
      "grad_norm": 0.44793635606765747,
      "learning_rate": 2.430777918725606e-07,
      "log_odds_chosen": 0.548711895942688,
      "log_odds_ratio": -0.46093201637268066,
      "logits/chosen": -0.2710830867290497,
      "logits/rejected": -0.09219225496053696,
      "logps/chosen": -1.9426264762878418,
      "logps/rejected": -2.4272983074188232,
      "loss": 2.1358,
      "nll_loss": 2.089747905731201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19426265358924866,
      "rewards/margins": 0.048467181622982025,
      "rewards/rejected": -0.24272982776165009,
      "step": 257
    },
    {
      "epoch": 0.7124611667241975,
      "grad_norm": 0.47333893179893494,
      "learning_rate": 2.3895616575836806e-07,
      "log_odds_chosen": 0.4012176990509033,
      "log_odds_ratio": -0.517660915851593,
      "logits/chosen": -0.17184937000274658,
      "logits/rejected": -0.07071331143379211,
      "logps/chosen": -1.9584107398986816,
      "logps/rejected": -2.3111367225646973,
      "loss": 2.1488,
      "nll_loss": 2.096987009048462,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19584107398986816,
      "rewards/margins": 0.03527258336544037,
      "rewards/rejected": -0.23111367225646973,
      "step": 258
    },
    {
      "epoch": 0.7152226441146013,
      "grad_norm": 0.467342346906662,
      "learning_rate": 2.348587819889825e-07,
      "log_odds_chosen": 0.29258960485458374,
      "log_odds_ratio": -0.5616452097892761,
      "logits/chosen": -0.18664903938770294,
      "logits/rejected": -0.1637982726097107,
      "logps/chosen": -1.9444085359573364,
      "logps/rejected": -2.1991188526153564,
      "loss": 2.1166,
      "nll_loss": 2.0604355335235596,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1944408416748047,
      "rewards/margins": 0.025471050292253494,
      "rewards/rejected": -0.21991188824176788,
      "step": 259
    },
    {
      "epoch": 0.7179841215050051,
      "grad_norm": 0.40307050943374634,
      "learning_rate": 2.3078602107593897e-07,
      "log_odds_chosen": 0.5154957175254822,
      "log_odds_ratio": -0.4730759859085083,
      "logits/chosen": -0.34735918045043945,
      "logits/rejected": -0.06134866178035736,
      "logps/chosen": -1.76466965675354,
      "logps/rejected": -2.21045184135437,
      "loss": 1.9385,
      "nll_loss": 1.8912420272827148,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1764669567346573,
      "rewards/margins": 0.044578224420547485,
      "rewards/rejected": -0.22104518115520477,
      "step": 260
    },
    {
      "epoch": 0.720745598895409,
      "grad_norm": 0.40177178382873535,
      "learning_rate": 2.267382612441231e-07,
      "log_odds_chosen": 0.3897167444229126,
      "log_odds_ratio": -0.5248652100563049,
      "logits/chosen": -0.1719050407409668,
      "logits/rejected": 0.005903229117393494,
      "logps/chosen": -1.8464490175247192,
      "logps/rejected": -2.1852574348449707,
      "loss": 2.0366,
      "nll_loss": 1.98414146900177,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1846449077129364,
      "rewards/margins": 0.03388083726167679,
      "rewards/rejected": -0.2185257375240326,
      "step": 261
    },
    {
      "epoch": 0.7235070762858129,
      "grad_norm": 0.41757282614707947,
      "learning_rate": 2.2271587839664668e-07,
      "log_odds_chosen": 0.5213944911956787,
      "log_odds_ratio": -0.4728841483592987,
      "logits/chosen": -0.26425325870513916,
      "logits/rejected": -0.11123887449502945,
      "logps/chosen": -1.8827658891677856,
      "logps/rejected": -2.339275360107422,
      "loss": 2.034,
      "nll_loss": 1.9866759777069092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18827658891677856,
      "rewards/margins": 0.04565096274018288,
      "rewards/rejected": -0.23392753303050995,
      "step": 262
    },
    {
      "epoch": 0.7262685536762168,
      "grad_norm": 0.46502548456192017,
      "learning_rate": 2.1871924607993797e-07,
      "log_odds_chosen": 0.43557122349739075,
      "log_odds_ratio": -0.504368245601654,
      "logits/chosen": -0.15488485991954803,
      "logits/rejected": -0.05642605200409889,
      "logps/chosen": -1.9347072839736938,
      "logps/rejected": -2.3199334144592285,
      "loss": 2.1231,
      "nll_loss": 2.072688341140747,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19347071647644043,
      "rewards/margins": 0.03852261230349541,
      "rewards/rejected": -0.23199334740638733,
      "step": 263
    },
    {
      "epoch": 0.7290300310666207,
      "grad_norm": 0.44662994146347046,
      "learning_rate": 2.1474873544905203e-07,
      "log_odds_chosen": 0.3577326536178589,
      "log_odds_ratio": -0.5319563150405884,
      "logits/chosen": -0.18652108311653137,
      "logits/rejected": -0.0033029038459062576,
      "logps/chosen": -1.8936744928359985,
      "logps/rejected": -2.204425573348999,
      "loss": 2.074,
      "nll_loss": 2.020807981491089,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18936744332313538,
      "rewards/margins": 0.031075095757842064,
      "rewards/rejected": -0.22044254839420319,
      "step": 264
    },
    {
      "epoch": 0.7317915084570245,
      "grad_norm": 0.47074761986732483,
      "learning_rate": 2.1080471523320277e-07,
      "log_odds_chosen": 0.38798317313194275,
      "log_odds_ratio": -0.5263108015060425,
      "logits/chosen": -0.23264719545841217,
      "logits/rejected": -0.19018438458442688,
      "logps/chosen": -1.9208102226257324,
      "logps/rejected": -2.2583324909210205,
      "loss": 2.0875,
      "nll_loss": 2.0348596572875977,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1920810043811798,
      "rewards/margins": 0.033752232789993286,
      "rewards/rejected": -0.2258332371711731,
      "step": 265
    },
    {
      "epoch": 0.7345529858474283,
      "grad_norm": 0.4252146780490875,
      "learning_rate": 2.0688755170151994e-07,
      "log_odds_chosen": 0.38214442133903503,
      "log_odds_ratio": -0.5270596146583557,
      "logits/chosen": -0.19508197903633118,
      "logits/rejected": -0.22396771609783173,
      "logps/chosen": -1.9821109771728516,
      "logps/rejected": -2.3214941024780273,
      "loss": 2.1498,
      "nll_loss": 2.0970942974090576,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19821108877658844,
      "rewards/margins": 0.03393830358982086,
      "rewards/rejected": -0.2321494072675705,
      "step": 266
    },
    {
      "epoch": 0.7373144632378322,
      "grad_norm": 0.4351484477519989,
      "learning_rate": 2.029976086290347e-07,
      "log_odds_chosen": 0.6953536868095398,
      "log_odds_ratio": -0.4082562327384949,
      "logits/chosen": -0.17970114946365356,
      "logits/rejected": -0.4311271905899048,
      "logps/chosen": -1.8674814701080322,
      "logps/rejected": -2.481518507003784,
      "loss": 2.0399,
      "nll_loss": 1.9990986585617065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18674813210964203,
      "rewards/margins": 0.06140372157096863,
      "rewards/rejected": -0.24815186858177185,
      "step": 267
    },
    {
      "epoch": 0.7400759406282361,
      "grad_norm": 0.4341726005077362,
      "learning_rate": 1.991352472628978e-07,
      "log_odds_chosen": 0.48897361755371094,
      "log_odds_ratio": -0.4817885756492615,
      "logits/chosen": -0.13388441503047943,
      "logits/rejected": -0.10476130247116089,
      "logps/chosen": -1.9310623407363892,
      "logps/rejected": -2.3620707988739014,
      "loss": 2.1002,
      "nll_loss": 2.051971197128296,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1931062638759613,
      "rewards/margins": 0.04310084134340286,
      "rewards/rejected": -0.23620708286762238,
      "step": 268
    },
    {
      "epoch": 0.74283741801864,
      "grad_norm": 0.410519003868103,
      "learning_rate": 1.9530082628883055e-07,
      "log_odds_chosen": 0.47287672758102417,
      "log_odds_ratio": -0.4956197142601013,
      "logits/chosen": -0.1638665646314621,
      "logits/rejected": -0.27936267852783203,
      "logps/chosen": -1.8986576795578003,
      "logps/rejected": -2.312922239303589,
      "loss": 2.0796,
      "nll_loss": 2.0300209522247314,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18986576795578003,
      "rewards/margins": 0.041426435112953186,
      "rewards/rejected": -0.23129220306873322,
      "step": 269
    },
    {
      "epoch": 0.7455988954090439,
      "grad_norm": 0.3971397280693054,
      "learning_rate": 1.9149470179781529e-07,
      "log_odds_chosen": 0.3566720187664032,
      "log_odds_ratio": -0.5360889434814453,
      "logits/chosen": -0.2508685290813446,
      "logits/rejected": -0.2334275245666504,
      "logps/chosen": -1.8639934062957764,
      "logps/rejected": -2.1733081340789795,
      "loss": 2.0295,
      "nll_loss": 1.975880742073059,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18639934062957764,
      "rewards/margins": 0.030931470915675163,
      "rewards/rejected": -0.21733081340789795,
      "step": 270
    },
    {
      "epoch": 0.7483603727994477,
      "grad_norm": 0.5120834708213806,
      "learning_rate": 1.877172272530264e-07,
      "log_odds_chosen": 0.5644389390945435,
      "log_odds_ratio": -0.4525200128555298,
      "logits/chosen": -0.26217329502105713,
      "logits/rejected": 0.013759253546595573,
      "logps/chosen": -1.9745893478393555,
      "logps/rejected": -2.4759721755981445,
      "loss": 2.1673,
      "nll_loss": 2.122096061706543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19745893776416779,
      "rewards/margins": 0.050138264894485474,
      "rewards/rejected": -0.24759721755981445,
      "step": 271
    },
    {
      "epoch": 0.7511218501898516,
      "grad_norm": 0.3965185284614563,
      "learning_rate": 1.8396875345700496e-07,
      "log_odds_chosen": 0.4397854804992676,
      "log_odds_ratio": -0.5040175318717957,
      "logits/chosen": -0.2846404016017914,
      "logits/rejected": -0.2431364506483078,
      "logps/chosen": -1.8459501266479492,
      "logps/rejected": -2.2276697158813477,
      "loss": 2.0233,
      "nll_loss": 1.9729148149490356,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1845950186252594,
      "rewards/margins": 0.03817194700241089,
      "rewards/rejected": -0.2227669507265091,
      "step": 272
    },
    {
      "epoch": 0.7538833275802554,
      "grad_norm": 0.40360215306282043,
      "learning_rate": 1.8024962851908105e-07,
      "log_odds_chosen": 0.6198650598526001,
      "log_odds_ratio": -0.4344263970851898,
      "logits/chosen": -0.17764055728912354,
      "logits/rejected": -0.21754667162895203,
      "logps/chosen": -1.898611068725586,
      "logps/rejected": -2.4485225677490234,
      "loss": 2.0506,
      "nll_loss": 2.0071144104003906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18986110389232635,
      "rewards/margins": 0.05499115586280823,
      "rewards/rejected": -0.2448522448539734,
      "step": 273
    },
    {
      "epoch": 0.7566448049706593,
      "grad_norm": 0.3847593069076538,
      "learning_rate": 1.76560197823046e-07,
      "log_odds_chosen": 0.3786264657974243,
      "log_odds_ratio": -0.5261546969413757,
      "logits/chosen": -0.3020249307155609,
      "logits/rejected": -0.26148176193237305,
      "logps/chosen": -1.8321974277496338,
      "logps/rejected": -2.157923460006714,
      "loss": 2.0072,
      "nll_loss": 1.954557180404663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18321974575519562,
      "rewards/margins": 0.03257260471582413,
      "rewards/rejected": -0.21579232811927795,
      "step": 274
    },
    {
      "epoch": 0.7594062823610632,
      "grad_norm": 0.47011545300483704,
      "learning_rate": 1.7290080399507717e-07,
      "log_odds_chosen": 0.3963688015937805,
      "log_odds_ratio": -0.5213245749473572,
      "logits/chosen": -0.2686113715171814,
      "logits/rejected": -0.01877725124359131,
      "logps/chosen": -1.9168490171432495,
      "logps/rejected": -2.2653939723968506,
      "loss": 2.1037,
      "nll_loss": 2.051600217819214,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19168490171432495,
      "rewards/margins": 0.03485449403524399,
      "rewards/rejected": -0.22653940320014954,
      "step": 275
    },
    {
      "epoch": 0.762167759751467,
      "grad_norm": 0.401557594537735,
      "learning_rate": 1.6927178687191952e-07,
      "log_odds_chosen": 0.37216082215309143,
      "log_odds_ratio": -0.5283534526824951,
      "logits/chosen": -0.2824167311191559,
      "logits/rejected": -0.18011616170406342,
      "logps/chosen": -1.7967240810394287,
      "logps/rejected": -2.116696357727051,
      "loss": 1.9662,
      "nll_loss": 1.9133520126342773,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17967242002487183,
      "rewards/margins": 0.031997211277484894,
      "rewards/rejected": -0.21166963875293732,
      "step": 276
    },
    {
      "epoch": 0.7649292371418709,
      "grad_norm": 0.45957428216934204,
      "learning_rate": 1.656734834693266e-07,
      "log_odds_chosen": 0.625089704990387,
      "log_odds_ratio": -0.4314519166946411,
      "logits/chosen": -0.2980007529258728,
      "logits/rejected": -0.1111484244465828,
      "logps/chosen": -1.7964612245559692,
      "logps/rejected": -2.3418169021606445,
      "loss": 1.9706,
      "nll_loss": 1.9274060726165771,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17964611947536469,
      "rewards/margins": 0.05453556776046753,
      "rewards/rejected": -0.23418167233467102,
      "step": 277
    },
    {
      "epoch": 0.7676907145322748,
      "grad_norm": 0.44963356852531433,
      "learning_rate": 1.6210622795076167e-07,
      "log_odds_chosen": 0.45046642422676086,
      "log_odds_ratio": -0.5030001401901245,
      "logits/chosen": -0.1990780085325241,
      "logits/rejected": -0.10728436708450317,
      "logps/chosen": -1.944456934928894,
      "logps/rejected": -2.3431031703948975,
      "loss": 2.1419,
      "nll_loss": 2.091618537902832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1944456845521927,
      "rewards/margins": 0.03986464440822601,
      "rewards/rejected": -0.2343103587627411,
      "step": 278
    },
    {
      "epoch": 0.7704521919226787,
      "grad_norm": 0.4373071491718292,
      "learning_rate": 1.5857035159636622e-07,
      "log_odds_chosen": 0.4049380123615265,
      "log_odds_ratio": -0.5157204866409302,
      "logits/chosen": -0.01834922283887863,
      "logits/rejected": -0.2680577039718628,
      "logps/chosen": -1.9991668462753296,
      "logps/rejected": -2.3591115474700928,
      "loss": 2.1694,
      "nll_loss": 2.117833375930786,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19991669058799744,
      "rewards/margins": 0.03599446266889572,
      "rewards/rejected": -0.23591114580631256,
      "step": 279
    },
    {
      "epoch": 0.7732136693130826,
      "grad_norm": 0.45289215445518494,
      "learning_rate": 1.5506618277219408e-07,
      "log_odds_chosen": 0.571408748626709,
      "log_odds_ratio": -0.4670514762401581,
      "logits/chosen": -0.27342498302459717,
      "logits/rejected": -0.2676662504673004,
      "logps/chosen": -1.9300659894943237,
      "logps/rejected": -2.436277389526367,
      "loss": 2.0928,
      "nll_loss": 2.0461032390594482,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19300661981105804,
      "rewards/margins": 0.05062113702297211,
      "rewards/rejected": -0.24362774193286896,
      "step": 280
    },
    {
      "epoch": 0.7759751467034863,
      "grad_norm": 0.40905460715293884,
      "learning_rate": 1.5159404689971796e-07,
      "log_odds_chosen": 0.3241649270057678,
      "log_odds_ratio": -0.5482557415962219,
      "logits/chosen": -0.2184755951166153,
      "logits/rejected": -0.10185020416975021,
      "logps/chosen": -1.934694766998291,
      "logps/rejected": -2.2194483280181885,
      "loss": 2.1004,
      "nll_loss": 2.045550584793091,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19346947968006134,
      "rewards/margins": 0.028475342318415642,
      "rewards/rejected": -0.22194482386112213,
      "step": 281
    },
    {
      "epoch": 0.7787366240938902,
      "grad_norm": 0.4077555537223816,
      "learning_rate": 1.481542664256075e-07,
      "log_odds_chosen": 0.650532603263855,
      "log_odds_ratio": -0.43051740527153015,
      "logits/chosen": -0.2367272973060608,
      "logits/rejected": -0.507286548614502,
      "logps/chosen": -1.8646379709243774,
      "logps/rejected": -2.4395954608917236,
      "loss": 2.0319,
      "nll_loss": 1.9888720512390137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18646378815174103,
      "rewards/margins": 0.05749577283859253,
      "rewards/rejected": -0.24395956099033356,
      "step": 282
    },
    {
      "epoch": 0.7814981014842941,
      "grad_norm": 0.423104852437973,
      "learning_rate": 1.447471607917854e-07,
      "log_odds_chosen": 0.4105510711669922,
      "log_odds_ratio": -0.5273401737213135,
      "logits/chosen": -0.23630788922309875,
      "logits/rejected": -0.2856886386871338,
      "logps/chosen": -1.8955426216125488,
      "logps/rejected": -2.261371612548828,
      "loss": 2.0688,
      "nll_loss": 2.0160558223724365,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18955425918102264,
      "rewards/margins": 0.03658292070031166,
      "rewards/rejected": -0.2261371612548828,
      "step": 283
    },
    {
      "epoch": 0.784259578874698,
      "grad_norm": 0.45282772183418274,
      "learning_rate": 1.413730464057616e-07,
      "log_odds_chosen": 0.39813441038131714,
      "log_odds_ratio": -0.5250266194343567,
      "logits/chosen": -0.14440934360027313,
      "logits/rejected": -0.14882172644138336,
      "logps/chosen": -1.9959735870361328,
      "logps/rejected": -2.350877046585083,
      "loss": 2.1646,
      "nll_loss": 2.112144708633423,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19959735870361328,
      "rewards/margins": 0.03549034148454666,
      "rewards/rejected": -0.23508770763874054,
      "step": 284
    },
    {
      "epoch": 0.7870210562651019,
      "grad_norm": 0.42741671204566956,
      "learning_rate": 1.3803223661124935e-07,
      "log_odds_chosen": 0.416368305683136,
      "log_odds_ratio": -0.5100723505020142,
      "logits/chosen": -0.20184141397476196,
      "logits/rejected": -0.29025012254714966,
      "logps/chosen": -1.898500680923462,
      "logps/rejected": -2.2631101608276367,
      "loss": 2.0658,
      "nll_loss": 2.0147459506988525,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1898500621318817,
      "rewards/margins": 0.03646095469594002,
      "rewards/rejected": -0.22631105780601501,
      "step": 285
    },
    {
      "epoch": 0.7897825336555057,
      "grad_norm": 0.35584133863449097,
      "learning_rate": 1.3472504165906612e-07,
      "log_odds_chosen": 0.39061859250068665,
      "log_odds_ratio": -0.5203800201416016,
      "logits/chosen": -0.19002895057201385,
      "logits/rejected": -0.28870323300361633,
      "logps/chosen": -1.8227334022521973,
      "logps/rejected": -2.1615891456604004,
      "loss": 1.9882,
      "nll_loss": 1.9361411333084106,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18227334320545197,
      "rewards/margins": 0.03388558700680733,
      "rewards/rejected": -0.216158926486969,
      "step": 286
    },
    {
      "epoch": 0.7925440110459095,
      "grad_norm": 0.39489269256591797,
      "learning_rate": 1.3145176867832165e-07,
      "log_odds_chosen": 0.5395733714103699,
      "log_odds_ratio": -0.4665833115577698,
      "logits/chosen": -0.08378874510526657,
      "logits/rejected": -0.12423109263181686,
      "logps/chosen": -1.8206287622451782,
      "logps/rejected": -2.295069932937622,
      "loss": 1.9996,
      "nll_loss": 1.952966570854187,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18206286430358887,
      "rewards/margins": 0.047444120049476624,
      "rewards/rejected": -0.2295069843530655,
      "step": 287
    },
    {
      "epoch": 0.7953054884363134,
      "grad_norm": 0.4502512216567993,
      "learning_rate": 1.2821272164789543e-07,
      "log_odds_chosen": 0.26743370294570923,
      "log_odds_ratio": -0.5727296471595764,
      "logits/chosen": -0.26246315240859985,
      "logits/rejected": -0.14970630407333374,
      "logps/chosen": -2.0250937938690186,
      "logps/rejected": -2.2614104747772217,
      "loss": 2.1875,
      "nll_loss": 2.1301777362823486,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20250937342643738,
      "rewards/margins": 0.023631669580936432,
      "rewards/rejected": -0.2261410504579544,
      "step": 288
    },
    {
      "epoch": 0.7980669658267173,
      "grad_norm": 0.4383101165294647,
      "learning_rate": 1.2500820136820733e-07,
      "log_odds_chosen": 0.5365539789199829,
      "log_odds_ratio": -0.4690496325492859,
      "logits/chosen": -0.01988196186721325,
      "logits/rejected": -0.18432113528251648,
      "logps/chosen": -1.9984502792358398,
      "logps/rejected": -2.4775490760803223,
      "loss": 2.177,
      "nll_loss": 2.1300911903381348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19984503090381622,
      "rewards/margins": 0.04790988564491272,
      "rewards/rejected": -0.24775493144989014,
      "step": 289
    },
    {
      "epoch": 0.8008284432171212,
      "grad_norm": 0.45397064089775085,
      "learning_rate": 1.2183850543328312e-07,
      "log_odds_chosen": 0.2914395332336426,
      "log_odds_ratio": -0.5629580616950989,
      "logits/chosen": -0.20800651609897614,
      "logits/rejected": -0.1190246194601059,
      "logps/chosen": -1.9052541255950928,
      "logps/rejected": -2.1603102684020996,
      "loss": 2.0907,
      "nll_loss": 2.0344083309173584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19052539765834808,
      "rewards/margins": 0.02550562657415867,
      "rewards/rejected": -0.2160310447216034,
      "step": 290
    },
    {
      "epoch": 0.803589920607525,
      "grad_norm": 0.41133707761764526,
      "learning_rate": 1.187039282031182e-07,
      "log_odds_chosen": 0.42272239923477173,
      "log_odds_ratio": -0.5111529231071472,
      "logits/chosen": -0.13887159526348114,
      "logits/rejected": -0.11638610064983368,
      "logps/chosen": -1.9467461109161377,
      "logps/rejected": -2.319835662841797,
      "loss": 2.1042,
      "nll_loss": 2.0530924797058105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19467462599277496,
      "rewards/margins": 0.0373089462518692,
      "rewards/rejected": -0.23198355734348297,
      "step": 291
    },
    {
      "epoch": 0.8063513979979289,
      "grad_norm": 0.417402982711792,
      "learning_rate": 1.1560476077634069e-07,
      "log_odds_chosen": 0.544741153717041,
      "log_odds_ratio": -0.4601641297340393,
      "logits/chosen": -0.14703384041786194,
      "logits/rejected": 0.04298261180520058,
      "logps/chosen": -1.8048415184020996,
      "logps/rejected": -2.2771944999694824,
      "loss": 1.9973,
      "nll_loss": 1.951300024986267,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18048414587974548,
      "rewards/margins": 0.047235310077667236,
      "rewards/rejected": -0.22771945595741272,
      "step": 292
    },
    {
      "epoch": 0.8091128753883328,
      "grad_norm": 0.45477494597435,
      "learning_rate": 1.1254129096317805e-07,
      "log_odds_chosen": 0.3906061053276062,
      "log_odds_ratio": -0.523013174533844,
      "logits/chosen": -0.13202102482318878,
      "logits/rejected": -0.22473666071891785,
      "logps/chosen": -1.9333416223526,
      "logps/rejected": -2.27662992477417,
      "loss": 2.126,
      "nll_loss": 2.0736703872680664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19333416223526,
      "rewards/margins": 0.03432881087064743,
      "rewards/rejected": -0.22766299545764923,
      "step": 293
    },
    {
      "epoch": 0.8118743527787367,
      "grad_norm": 0.45089539885520935,
      "learning_rate": 1.0951380325872977e-07,
      "log_odds_chosen": 0.5258044600486755,
      "log_odds_ratio": -0.4683087170124054,
      "logits/chosen": -0.227797731757164,
      "logits/rejected": -0.12991823256015778,
      "logps/chosen": -1.835335373878479,
      "logps/rejected": -2.294915199279785,
      "loss": 2.022,
      "nll_loss": 1.9751490354537964,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18353353440761566,
      "rewards/margins": 0.04595799744129181,
      "rewards/rejected": -0.22949153184890747,
      "step": 294
    },
    {
      "epoch": 0.8146358301691405,
      "grad_norm": 0.42870861291885376,
      "learning_rate": 1.0652257881654624e-07,
      "log_odds_chosen": 0.5187486410140991,
      "log_odds_ratio": -0.47181975841522217,
      "logits/chosen": -0.31071117520332336,
      "logits/rejected": -0.15594279766082764,
      "logps/chosen": -1.8465943336486816,
      "logps/rejected": -2.300283670425415,
      "loss": 2.0407,
      "nll_loss": 1.9935269355773926,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1846594363451004,
      "rewards/margins": 0.04536893218755722,
      "rewards/rejected": -0.23002836108207703,
      "step": 295
    },
    {
      "epoch": 0.8173973075595443,
      "grad_norm": 0.41282153129577637,
      "learning_rate": 1.0356789542251936e-07,
      "log_odds_chosen": 0.648618757724762,
      "log_odds_ratio": -0.4256685674190521,
      "logits/chosen": -0.11298441886901855,
      "logits/rejected": -0.3672432601451874,
      "logps/chosen": -1.859349250793457,
      "logps/rejected": -2.432389736175537,
      "loss": 2.0123,
      "nll_loss": 1.9697067737579346,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.185934916138649,
      "rewards/margins": 0.057304054498672485,
      "rewards/rejected": -0.24323897063732147,
      "step": 296
    },
    {
      "epoch": 0.8201587849499482,
      "grad_norm": 0.45917996764183044,
      "learning_rate": 1.006500274690853e-07,
      "log_odds_chosen": 0.5037029385566711,
      "log_odds_ratio": -0.4776912033557892,
      "logits/chosen": -0.16094689071178436,
      "logits/rejected": -0.17968234419822693,
      "logps/chosen": -1.9684001207351685,
      "logps/rejected": -2.416630268096924,
      "loss": 2.1397,
      "nll_loss": 2.091897487640381,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19684001803398132,
      "rewards/margins": 0.04482300952076912,
      "rewards/rejected": -0.24166302382946014,
      "step": 297
    },
    {
      "epoch": 0.8229202623403521,
      "grad_norm": 0.39604610204696655,
      "learning_rate": 9.776924592974256e-08,
      "log_odds_chosen": 0.5864183902740479,
      "log_odds_ratio": -0.45360007882118225,
      "logits/chosen": -0.23677986860275269,
      "logits/rejected": -0.3255308270454407,
      "logps/chosen": -1.8277275562286377,
      "logps/rejected": -2.3369171619415283,
      "loss": 1.9928,
      "nll_loss": 1.9474081993103027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18277277052402496,
      "rewards/margins": 0.05091894418001175,
      "rewards/rejected": -0.2336917221546173,
      "step": 298
    },
    {
      "epoch": 0.825681739730756,
      "grad_norm": 0.4423990249633789,
      "learning_rate": 9.492581833388735e-08,
      "log_odds_chosen": 0.4847714900970459,
      "log_odds_ratio": -0.4888085126876831,
      "logits/chosen": -0.2521671950817108,
      "logits/rejected": -0.30755069851875305,
      "logps/chosen": -1.8848044872283936,
      "logps/rejected": -2.3121137619018555,
      "loss": 2.0551,
      "nll_loss": 2.0062382221221924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1884804368019104,
      "rewards/margins": 0.042730942368507385,
      "rewards/rejected": -0.23121139407157898,
      "step": 299
    },
    {
      "epoch": 0.8284432171211599,
      "grad_norm": 0.46994680166244507,
      "learning_rate": 9.212000874196952e-08,
      "log_odds_chosen": 0.6198639273643494,
      "log_odds_ratio": -0.4352167248725891,
      "logits/chosen": -0.32599297165870667,
      "logits/rejected": -0.30806779861450195,
      "logps/chosen": -1.9247195720672607,
      "logps/rejected": -2.4739463329315186,
      "loss": 2.0882,
      "nll_loss": 2.044715642929077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1924719512462616,
      "rewards/margins": 0.0549226775765419,
      "rewards/rejected": -0.2473946213722229,
      "step": 300
    },
    {
      "epoch": 0.8312046945115636,
      "grad_norm": 0.3998923599720001,
      "learning_rate": 8.935207772096903e-08,
      "log_odds_chosen": 0.5991203784942627,
      "log_odds_ratio": -0.4437808096408844,
      "logits/chosen": -0.2712499499320984,
      "logits/rejected": -0.4473310112953186,
      "logps/chosen": -1.914632797241211,
      "logps/rejected": -2.4458022117614746,
      "loss": 2.0637,
      "nll_loss": 2.0193285942077637,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19146327674388885,
      "rewards/margins": 0.053116947412490845,
      "rewards/rejected": -0.2445802092552185,
      "step": 301
    },
    {
      "epoch": 0.8339661719019675,
      "grad_norm": 0.3971911370754242,
      "learning_rate": 8.662228232019875e-08,
      "log_odds_chosen": 0.6176374554634094,
      "log_odds_ratio": -0.44052228331565857,
      "logits/chosen": -0.3155173063278198,
      "logits/rejected": -0.11423099786043167,
      "logps/chosen": -1.7802956104278564,
      "logps/rejected": -2.3111257553100586,
      "loss": 1.9577,
      "nll_loss": 1.9136923551559448,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17802958190441132,
      "rewards/margins": 0.05308301001787186,
      "rewards/rejected": -0.23111258447170258,
      "step": 302
    },
    {
      "epoch": 0.8367276492923714,
      "grad_norm": 0.4174393117427826,
      "learning_rate": 8.393087604743283e-08,
      "log_odds_chosen": 0.38905197381973267,
      "log_odds_ratio": -0.5197128057479858,
      "logits/chosen": -0.18575572967529297,
      "logits/rejected": -0.0011536306701600552,
      "logps/chosen": -1.935011386871338,
      "logps/rejected": -2.275057315826416,
      "loss": 2.1014,
      "nll_loss": 2.049394369125366,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19350114464759827,
      "rewards/margins": 0.034004583954811096,
      "rewards/rejected": -0.22750572860240936,
      "step": 303
    },
    {
      "epoch": 0.8394891266827753,
      "grad_norm": 0.45562076568603516,
      "learning_rate": 8.127810884536402e-08,
      "log_odds_chosen": 0.5559534430503845,
      "log_odds_ratio": -0.4578395485877991,
      "logits/chosen": -0.13820932805538177,
      "logits/rejected": -0.1494869887828827,
      "logps/chosen": -1.856467843055725,
      "logps/rejected": -2.3439013957977295,
      "loss": 2.0462,
      "nll_loss": 2.0004019737243652,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18564678728580475,
      "rewards/margins": 0.0487433560192585,
      "rewards/rejected": -0.23439012467861176,
      "step": 304
    },
    {
      "epoch": 0.8422506040731792,
      "grad_norm": 0.4482463300228119,
      "learning_rate": 7.866422706839238e-08,
      "log_odds_chosen": 0.348733127117157,
      "log_odds_ratio": -0.5404360890388489,
      "logits/chosen": -0.2583235800266266,
      "logits/rejected": -0.12020088732242584,
      "logps/chosen": -2.0293304920196533,
      "logps/rejected": -2.337681531906128,
      "loss": 2.2194,
      "nll_loss": 2.1653995513916016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20293305814266205,
      "rewards/margins": 0.030835090205073357,
      "rewards/rejected": -0.23376816511154175,
      "step": 305
    },
    {
      "epoch": 0.845012081463583,
      "grad_norm": 0.39394766092300415,
      "learning_rate": 7.608947345974759e-08,
      "log_odds_chosen": 0.6800714135169983,
      "log_odds_ratio": -0.41480177640914917,
      "logits/chosen": -0.17396017909049988,
      "logits/rejected": -0.3051375448703766,
      "logps/chosen": -1.8647377490997314,
      "logps/rejected": -2.4632208347320557,
      "loss": 2.0508,
      "nll_loss": 2.00935959815979,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1864737868309021,
      "rewards/margins": 0.05984831228852272,
      "rewards/rejected": -0.24632208049297333,
      "step": 306
    },
    {
      "epoch": 0.8477735588539869,
      "grad_norm": 0.4351319968700409,
      "learning_rate": 7.355408712894507e-08,
      "log_odds_chosen": 0.4267576038837433,
      "log_odds_ratio": -0.5087683200836182,
      "logits/chosen": -0.1508183628320694,
      "logits/rejected": -0.030818209052085876,
      "logps/chosen": -1.9018332958221436,
      "logps/rejected": -2.275527000427246,
      "loss": 2.0888,
      "nll_loss": 2.037964105606079,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19018332660198212,
      "rewards/margins": 0.037369392812252045,
      "rewards/rejected": -0.22755271196365356,
      "step": 307
    },
    {
      "epoch": 0.8505350362443908,
      "grad_norm": 0.46579980850219727,
      "learning_rate": 7.105830352958142e-08,
      "log_odds_chosen": 0.49559804797172546,
      "log_odds_ratio": -0.48423653841018677,
      "logits/chosen": -0.30391234159469604,
      "logits/rejected": -0.06965890526771545,
      "logps/chosen": -1.8141393661499023,
      "logps/rejected": -2.24297833442688,
      "loss": 1.9927,
      "nll_loss": 1.9442522525787354,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.181413933634758,
      "rewards/margins": 0.04288391023874283,
      "rewards/rejected": -0.22429785132408142,
      "step": 308
    },
    {
      "epoch": 0.8532965136347946,
      "grad_norm": 0.515749454498291,
      "learning_rate": 6.86023544374686e-08,
      "log_odds_chosen": 0.22447392344474792,
      "log_odds_ratio": -0.5952900648117065,
      "logits/chosen": -0.27339303493499756,
      "logits/rejected": -0.06818941235542297,
      "logps/chosen": -1.9380499124526978,
      "logps/rejected": -2.1311028003692627,
      "loss": 2.1235,
      "nll_loss": 2.0640058517456055,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19380497932434082,
      "rewards/margins": 0.01930529810488224,
      "rewards/rejected": -0.2131102830171585,
      "step": 309
    },
    {
      "epoch": 0.8560579910251985,
      "grad_norm": 0.4221779704093933,
      "learning_rate": 6.618646792910893e-08,
      "log_odds_chosen": 0.5058144927024841,
      "log_odds_ratio": -0.480294406414032,
      "logits/chosen": -0.173689603805542,
      "logits/rejected": -0.3814562261104584,
      "logps/chosen": -1.9178377389907837,
      "logps/rejected": -2.3588790893554688,
      "loss": 2.088,
      "nll_loss": 2.0399904251098633,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19178378582000732,
      "rewards/margins": 0.04410414397716522,
      "rewards/rejected": -0.23588792979717255,
      "step": 310
    },
    {
      "epoch": 0.8588194684156023,
      "grad_norm": 0.4498308300971985,
      "learning_rate": 6.381086836051497e-08,
      "log_odds_chosen": 0.2599206268787384,
      "log_odds_ratio": -0.5829892754554749,
      "logits/chosen": -0.13349135220050812,
      "logits/rejected": -0.19249173998832703,
      "logps/chosen": -1.9893162250518799,
      "logps/rejected": -2.2166213989257812,
      "loss": 2.1795,
      "nll_loss": 2.1212141513824463,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19893160462379456,
      "rewards/margins": 0.022730525583028793,
      "rewards/rejected": -0.22166214883327484,
      "step": 311
    },
    {
      "epoch": 0.8615809458060062,
      "grad_norm": 0.4042651057243347,
      "learning_rate": 6.147577634637413e-08,
      "log_odds_chosen": 0.6165303587913513,
      "log_odds_ratio": -0.43272557854652405,
      "logits/chosen": -0.17402523756027222,
      "logits/rejected": -0.2573563754558563,
      "logps/chosen": -1.8469359874725342,
      "logps/rejected": -2.388429880142212,
      "loss": 2.0394,
      "nll_loss": 1.9960887432098389,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1846935898065567,
      "rewards/margins": 0.05414938926696777,
      "rewards/rejected": -0.23884299397468567,
      "step": 312
    },
    {
      "epoch": 0.8643424231964101,
      "grad_norm": 0.38300517201423645,
      "learning_rate": 5.918140873956062e-08,
      "log_odds_chosen": 0.4959363341331482,
      "log_odds_ratio": -0.47913581132888794,
      "logits/chosen": -0.17489324510097504,
      "logits/rejected": -0.23760297894477844,
      "logps/chosen": -1.8430047035217285,
      "logps/rejected": -2.2771506309509277,
      "loss": 1.9984,
      "nll_loss": 1.9504523277282715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1843004673719406,
      "rewards/margins": 0.04341460019350052,
      "rewards/rejected": -0.22771507501602173,
      "step": 313
    },
    {
      "epoch": 0.867103900586814,
      "grad_norm": 0.4202573299407959,
      "learning_rate": 5.692797861099718e-08,
      "log_odds_chosen": 0.5489151477813721,
      "log_odds_ratio": -0.4572814404964447,
      "logits/chosen": -0.08470579981803894,
      "logits/rejected": -0.13348816335201263,
      "logps/chosen": -1.9210916757583618,
      "logps/rejected": -2.406761407852173,
      "loss": 2.0904,
      "nll_loss": 2.0447049140930176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19210916757583618,
      "rewards/margins": 0.048566970974206924,
      "rewards/rejected": -0.2406761199235916,
      "step": 314
    },
    {
      "epoch": 0.8698653779772179,
      "grad_norm": 0.39355114102363586,
      "learning_rate": 5.471569522986774e-08,
      "log_odds_chosen": 0.4892618954181671,
      "log_odds_ratio": -0.49415305256843567,
      "logits/chosen": -0.1918748915195465,
      "logits/rejected": -0.18721967935562134,
      "logps/chosen": -1.8986515998840332,
      "logps/rejected": -2.3301892280578613,
      "loss": 2.0665,
      "nll_loss": 2.0170679092407227,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18986515700817108,
      "rewards/margins": 0.043153755366802216,
      "rewards/rejected": -0.2330189049243927,
      "step": 315
    },
    {
      "epoch": 0.8726268553676216,
      "grad_norm": 0.4295710325241089,
      "learning_rate": 5.25447640441834e-08,
      "log_odds_chosen": 0.42472338676452637,
      "log_odds_ratio": -0.5125213265419006,
      "logits/chosen": -0.17091691493988037,
      "logits/rejected": -0.12590187788009644,
      "logps/chosen": -1.949544906616211,
      "logps/rejected": -2.3239901065826416,
      "loss": 2.1284,
      "nll_loss": 2.077145576477051,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19495448470115662,
      "rewards/margins": 0.037444520741701126,
      "rewards/rejected": -0.23239898681640625,
      "step": 316
    },
    {
      "epoch": 0.8753883327580255,
      "grad_norm": 0.49030354619026184,
      "learning_rate": 5.041538666170281e-08,
      "log_odds_chosen": 0.39470869302749634,
      "log_odds_ratio": -0.5200437307357788,
      "logits/chosen": -0.3289051353931427,
      "logits/rejected": -0.12990185618400574,
      "logps/chosen": -1.9601221084594727,
      "logps/rejected": -2.306885004043579,
      "loss": 2.139,
      "nll_loss": 2.087029218673706,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1960122138261795,
      "rewards/margins": 0.03467627614736557,
      "rewards/rejected": -0.23068848252296448,
      "step": 317
    },
    {
      "epoch": 0.8781498101484294,
      "grad_norm": 0.4893982410430908,
      "learning_rate": 4.832776083120982e-08,
      "log_odds_chosen": 0.4297412931919098,
      "log_odds_ratio": -0.5068095922470093,
      "logits/chosen": -0.1508714258670807,
      "logits/rejected": -0.21032114326953888,
      "logps/chosen": -2.0057082176208496,
      "logps/rejected": -2.38881778717041,
      "loss": 2.1739,
      "nll_loss": 2.1231908798217773,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20057085156440735,
      "rewards/margins": 0.03831096366047859,
      "rewards/rejected": -0.23888179659843445,
      "step": 318
    },
    {
      "epoch": 0.8809112875388333,
      "grad_norm": 0.3878088593482971,
      "learning_rate": 4.6282080424148886e-08,
      "log_odds_chosen": 0.5795797109603882,
      "log_odds_ratio": -0.45386186242103577,
      "logits/chosen": -0.20354528725147247,
      "logits/rejected": -0.38495659828186035,
      "logps/chosen": -1.8539503812789917,
      "logps/rejected": -2.363740921020508,
      "loss": 2.0171,
      "nll_loss": 1.9717295169830322,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1853950470685959,
      "rewards/margins": 0.050979048013687134,
      "rewards/rejected": -0.23637409508228302,
      "step": 319
    },
    {
      "epoch": 0.8836727649292372,
      "grad_norm": 0.4000735580921173,
      "learning_rate": 4.427853541662091e-08,
      "log_odds_chosen": 0.31460365653038025,
      "log_odds_ratio": -0.5543741583824158,
      "logits/chosen": -0.06682717800140381,
      "logits/rejected": -0.2006971538066864,
      "logps/chosen": -1.9457982778549194,
      "logps/rejected": -2.2210662364959717,
      "loss": 2.1125,
      "nll_loss": 2.0570363998413086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1945798248052597,
      "rewards/margins": 0.027526797726750374,
      "rewards/rejected": -0.22210662066936493,
      "step": 320
    },
    {
      "epoch": 0.886434242319641,
      "grad_norm": 0.37986594438552856,
      "learning_rate": 4.2317311871740644e-08,
      "log_odds_chosen": 0.5543714165687561,
      "log_odds_ratio": -0.4679988920688629,
      "logits/chosen": -0.08007542788982391,
      "logits/rejected": -0.38316231966018677,
      "logps/chosen": -1.8993326425552368,
      "logps/rejected": -2.3954031467437744,
      "loss": 2.0726,
      "nll_loss": 2.025761604309082,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18993327021598816,
      "rewards/margins": 0.04960707575082779,
      "rewards/rejected": -0.23954033851623535,
      "step": 321
    },
    {
      "epoch": 0.8891957197100449,
      "grad_norm": 0.4185676574707031,
      "learning_rate": 4.039859192235778e-08,
      "log_odds_chosen": 0.31169041991233826,
      "log_odds_ratio": -0.5566931366920471,
      "logits/chosen": -0.14175063371658325,
      "logits/rejected": -0.11621041595935822,
      "logps/chosen": -1.861968994140625,
      "logps/rejected": -2.1321418285369873,
      "loss": 2.0483,
      "nll_loss": 1.992612600326538,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18619689345359802,
      "rewards/margins": 0.027017315849661827,
      "rewards/rejected": -0.2132142037153244,
      "step": 322
    },
    {
      "epoch": 0.8919571971004487,
      "grad_norm": 0.4270247519016266,
      "learning_rate": 3.852255375414271e-08,
      "log_odds_chosen": 0.3725894093513489,
      "log_odds_ratio": -0.5399094223976135,
      "logits/chosen": -0.2114194631576538,
      "logits/rejected": -0.2174735814332962,
      "logps/chosen": -1.9405810832977295,
      "logps/rejected": -2.270659923553467,
      "loss": 2.1359,
      "nll_loss": 2.081935405731201,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1940581053495407,
      "rewards/margins": 0.03300788253545761,
      "rewards/rejected": -0.22706599533557892,
      "step": 323
    },
    {
      "epoch": 0.8947186744908526,
      "grad_norm": 0.4771701693534851,
      "learning_rate": 3.668937158903901e-08,
      "log_odds_chosen": 0.5787175297737122,
      "log_odds_ratio": -0.45257288217544556,
      "logits/chosen": -0.2463836818933487,
      "logits/rejected": -0.1700681895017624,
      "logps/chosen": -2.015784740447998,
      "logps/rejected": -2.5313968658447266,
      "loss": 2.1772,
      "nll_loss": 2.1319868564605713,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20157849788665771,
      "rewards/margins": 0.05156121402978897,
      "rewards/rejected": -0.2531397044658661,
      "step": 324
    },
    {
      "epoch": 0.8974801518812565,
      "grad_norm": 0.42737582325935364,
      "learning_rate": 3.489921566908371e-08,
      "log_odds_chosen": 0.4688037931919098,
      "log_odds_ratio": -0.4908846318721771,
      "logits/chosen": -0.16491183638572693,
      "logits/rejected": -0.27947261929512024,
      "logps/chosen": -1.8919143676757812,
      "logps/rejected": -2.305262327194214,
      "loss": 2.0642,
      "nll_loss": 2.015066385269165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18919143080711365,
      "rewards/margins": 0.04133479669690132,
      "rewards/rejected": -0.23052623867988586,
      "step": 325
    },
    {
      "epoch": 0.9002416292716603,
      "grad_norm": 0.4799785017967224,
      "learning_rate": 3.3152252240598086e-08,
      "log_odds_chosen": 0.3481528162956238,
      "log_odds_ratio": -0.5378261208534241,
      "logits/chosen": -0.14339762926101685,
      "logits/rejected": 0.09353294223546982,
      "logps/chosen": -2.0005557537078857,
      "logps/rejected": -2.307278633117676,
      "loss": 2.1874,
      "nll_loss": 2.1336636543273926,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2000555843114853,
      "rewards/margins": 0.030672287568449974,
      "rewards/rejected": -0.230727881193161,
      "step": 326
    },
    {
      "epoch": 0.9030031066620642,
      "grad_norm": 0.4857260584831238,
      "learning_rate": 3.144864353874804e-08,
      "log_odds_chosen": 0.41800910234451294,
      "log_odds_ratio": -0.5131241083145142,
      "logits/chosen": -0.24292851984500885,
      "logits/rejected": -0.03285370022058487,
      "logps/chosen": -1.9487199783325195,
      "logps/rejected": -2.3177647590637207,
      "loss": 2.1447,
      "nll_loss": 2.093341827392578,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19487199187278748,
      "rewards/margins": 0.0369044728577137,
      "rewards/rejected": -0.23177646100521088,
      "step": 327
    },
    {
      "epoch": 0.9057645840524681,
      "grad_norm": 0.43344756960868835,
      "learning_rate": 2.978854777247841e-08,
      "log_odds_chosen": 0.5819824934005737,
      "log_odds_ratio": -0.4501224160194397,
      "logits/chosen": -0.26334643363952637,
      "logits/rejected": -0.31945887207984924,
      "logps/chosen": -1.790362000465393,
      "logps/rejected": -2.297755479812622,
      "loss": 1.9433,
      "nll_loss": 1.8982830047607422,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1790362149477005,
      "rewards/margins": 0.05073932558298111,
      "rewards/rejected": -0.2297755479812622,
      "step": 328
    },
    {
      "epoch": 0.908526061442872,
      "grad_norm": 0.42716139554977417,
      "learning_rate": 2.817211910982037e-08,
      "log_odds_chosen": 0.649154543876648,
      "log_odds_ratio": -0.42589282989501953,
      "logits/chosen": -0.202769935131073,
      "logits/rejected": -0.15542913973331451,
      "logps/chosen": -1.8751076459884644,
      "logps/rejected": -2.4470901489257812,
      "loss": 2.0441,
      "nll_loss": 2.0014681816101074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18751075863838196,
      "rewards/margins": 0.057198263704776764,
      "rewards/rejected": -0.24470902979373932,
      "step": 329
    },
    {
      "epoch": 0.9112875388332758,
      "grad_norm": 0.4449755847454071,
      "learning_rate": 2.6599507663574384e-08,
      "log_odds_chosen": 0.43763425946235657,
      "log_odds_ratio": -0.4999912977218628,
      "logits/chosen": -0.14027515053749084,
      "logits/rejected": -0.1446068286895752,
      "logps/chosen": -1.9731489419937134,
      "logps/rejected": -2.3602209091186523,
      "loss": 2.1396,
      "nll_loss": 2.0895612239837646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19731488823890686,
      "rewards/margins": 0.03870721161365509,
      "rewards/rejected": -0.23602209985256195,
      "step": 330
    },
    {
      "epoch": 0.9140490162236796,
      "grad_norm": 0.4197101891040802,
      "learning_rate": 2.5070859477369642e-08,
      "log_odds_chosen": 0.5072610974311829,
      "log_odds_ratio": -0.47924864292144775,
      "logits/chosen": -0.0747680813074112,
      "logits/rejected": -0.3043823838233948,
      "logps/chosen": -1.8657901287078857,
      "logps/rejected": -2.309816360473633,
      "loss": 2.0324,
      "nll_loss": 1.9844887256622314,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18657900393009186,
      "rewards/margins": 0.04440262168645859,
      "rewards/rejected": -0.23098163306713104,
      "step": 331
    },
    {
      "epoch": 0.9168104936140835,
      "grad_norm": 0.3947368562221527,
      "learning_rate": 2.358631651210141e-08,
      "log_odds_chosen": 0.5603442788124084,
      "log_odds_ratio": -0.45878544449806213,
      "logits/chosen": -0.23719505965709686,
      "logits/rejected": -0.3236241340637207,
      "logps/chosen": -1.908208966255188,
      "logps/rejected": -2.4037296772003174,
      "loss": 2.0877,
      "nll_loss": 2.0417838096618652,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19082088768482208,
      "rewards/margins": 0.049552083015441895,
      "rewards/rejected": -0.24037298560142517,
      "step": 332
    },
    {
      "epoch": 0.9195719710044874,
      "grad_norm": 0.4243737459182739,
      "learning_rate": 2.214601663274762e-08,
      "log_odds_chosen": 0.3393900990486145,
      "log_odds_ratio": -0.5531291365623474,
      "logits/chosen": -0.3143972158432007,
      "logits/rejected": -0.14712917804718018,
      "logps/chosen": -1.8259567022323608,
      "logps/rejected": -2.1251866817474365,
      "loss": 2.0087,
      "nll_loss": 1.9534207582473755,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18259567022323608,
      "rewards/margins": 0.02992301806807518,
      "rewards/rejected": -0.21251866221427917,
      "step": 333
    },
    {
      "epoch": 0.9223334483948913,
      "grad_norm": 0.37499305605888367,
      "learning_rate": 2.0750093595565733e-08,
      "log_odds_chosen": 0.49258458614349365,
      "log_odds_ratio": -0.4850914180278778,
      "logits/chosen": -0.14541703462600708,
      "logits/rejected": -0.2299581617116928,
      "logps/chosen": -1.9290242195129395,
      "logps/rejected": -2.363391876220703,
      "loss": 2.0948,
      "nll_loss": 2.0463147163391113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19290241599082947,
      "rewards/margins": 0.04343675822019577,
      "rewards/rejected": -0.23633918166160583,
      "step": 334
    },
    {
      "epoch": 0.9250949257852952,
      "grad_norm": 0.44641590118408203,
      "learning_rate": 1.9398677035671218e-08,
      "log_odds_chosen": 0.42436647415161133,
      "log_odds_ratio": -0.5091593861579895,
      "logits/chosen": -0.1494864821434021,
      "logits/rejected": -0.17076365649700165,
      "logps/chosen": -2.0193638801574707,
      "logps/rejected": -2.395270586013794,
      "loss": 2.1732,
      "nll_loss": 2.122241973876953,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20193640887737274,
      "rewards/margins": 0.03759066015481949,
      "rewards/rejected": -0.23952704668045044,
      "step": 335
    },
    {
      "epoch": 0.927856403175699,
      "grad_norm": 0.4320031702518463,
      "learning_rate": 1.8091892454998593e-08,
      "log_odds_chosen": 0.4674716889858246,
      "log_odds_ratio": -0.4904363751411438,
      "logits/chosen": -0.10691366344690323,
      "logits/rejected": -0.10504031181335449,
      "logps/chosen": -1.911413550376892,
      "logps/rejected": -2.32393741607666,
      "loss": 2.087,
      "nll_loss": 2.037943124771118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19114135205745697,
      "rewards/margins": 0.04125239700078964,
      "rewards/rejected": -0.2323937565088272,
      "step": 336
    },
    {
      "epoch": 0.9306178805661028,
      "grad_norm": 0.396973580121994,
      "learning_rate": 1.682986121064689e-08,
      "log_odds_chosen": 0.2887546718120575,
      "log_odds_ratio": -0.5638278722763062,
      "logits/chosen": -0.2744186520576477,
      "logits/rejected": -0.11924867331981659,
      "logps/chosen": -1.8107192516326904,
      "logps/rejected": -2.056103229522705,
      "loss": 1.9905,
      "nll_loss": 1.934165596961975,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.181071937084198,
      "rewards/margins": 0.024538416415452957,
      "rewards/rejected": -0.20561034977436066,
      "step": 337
    },
    {
      "epoch": 0.9333793579565067,
      "grad_norm": 0.4459056556224823,
      "learning_rate": 1.5612700503608967e-08,
      "log_odds_chosen": 0.45278072357177734,
      "log_odds_ratio": -0.5071058869361877,
      "logits/chosen": -0.36018478870391846,
      "logits/rejected": -0.37708672881126404,
      "logps/chosen": -1.8574512004852295,
      "logps/rejected": -2.2552061080932617,
      "loss": 2.0447,
      "nll_loss": 1.9939401149749756,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18574512004852295,
      "rewards/margins": 0.039775483310222626,
      "rewards/rejected": -0.22552061080932617,
      "step": 338
    },
    {
      "epoch": 0.9361408353469106,
      "grad_norm": 0.40658360719680786,
      "learning_rate": 1.444052336788787e-08,
      "log_odds_chosen": 0.4479530453681946,
      "log_odds_ratio": -0.5013195872306824,
      "logits/chosen": -0.12710537016391754,
      "logits/rejected": -0.2852480709552765,
      "logps/chosen": -1.8056758642196655,
      "logps/rejected": -2.195272445678711,
      "loss": 1.9768,
      "nll_loss": 1.926626205444336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18056757748126984,
      "rewards/margins": 0.03895965963602066,
      "rewards/rejected": -0.2195272445678711,
      "step": 339
    },
    {
      "epoch": 0.9389023127373145,
      "grad_norm": 0.38530224561691284,
      "learning_rate": 1.3313438659999399e-08,
      "log_odds_chosen": 0.5152425765991211,
      "log_odds_ratio": -0.4869002103805542,
      "logits/chosen": -0.14443586766719818,
      "logits/rejected": -0.1850556582212448,
      "logps/chosen": -1.8249857425689697,
      "logps/rejected": -2.2752771377563477,
      "loss": 2.0094,
      "nll_loss": 1.9606778621673584,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18249857425689697,
      "rewards/margins": 0.04502914100885391,
      "rewards/rejected": -0.2275277078151703,
      "step": 340
    },
    {
      "epoch": 0.9416637901277183,
      "grad_norm": 0.45364004373550415,
      "learning_rate": 1.223155104886342e-08,
      "log_odds_chosen": 0.39692723751068115,
      "log_odds_ratio": -0.5225122570991516,
      "logits/chosen": -0.24809373915195465,
      "logits/rejected": -0.06684684753417969,
      "logps/chosen": -1.8658114671707153,
      "logps/rejected": -2.21091365814209,
      "loss": 2.0469,
      "nll_loss": 1.9946162700653076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18658114969730377,
      "rewards/margins": 0.034510232508182526,
      "rewards/rejected": -0.2210913896560669,
      "step": 341
    },
    {
      "epoch": 0.9444252675181222,
      "grad_norm": 0.39424437284469604,
      "learning_rate": 1.119496100608297e-08,
      "log_odds_chosen": 0.3901040554046631,
      "log_odds_ratio": -0.5215116739273071,
      "logits/chosen": -0.2839021384716034,
      "logits/rejected": -0.2984020709991455,
      "logps/chosen": -1.7855983972549438,
      "logps/rejected": -2.112828254699707,
      "loss": 1.9827,
      "nll_loss": 1.930551290512085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17855985462665558,
      "rewards/margins": 0.03272298350930214,
      "rewards/rejected": -0.21128283441066742,
      "step": 342
    },
    {
      "epoch": 0.9471867449085261,
      "grad_norm": 0.42831146717071533,
      "learning_rate": 1.0203764796614056e-08,
      "log_odds_chosen": 0.4961273670196533,
      "log_odds_ratio": -0.481975793838501,
      "logits/chosen": -0.2962571084499359,
      "logits/rejected": -0.22582583129405975,
      "logps/chosen": -1.9064340591430664,
      "logps/rejected": -2.3443710803985596,
      "loss": 2.0737,
      "nll_loss": 2.0255472660064697,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19064341485500336,
      "rewards/margins": 0.043793708086013794,
      "rewards/rejected": -0.23443713784217834,
      "step": 343
    },
    {
      "epoch": 0.94994822229893,
      "grad_norm": 0.4325099587440491,
      "learning_rate": 9.258054469825972e-09,
      "log_odds_chosen": 0.49040326476097107,
      "log_odds_ratio": -0.48156440258026123,
      "logits/chosen": -0.17820492386817932,
      "logits/rejected": -0.17450091242790222,
      "logps/chosen": -1.8755344152450562,
      "logps/rejected": -2.3057727813720703,
      "loss": 2.056,
      "nll_loss": 2.0078787803649902,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18755343556404114,
      "rewards/margins": 0.04302385449409485,
      "rewards/rejected": -0.230577290058136,
      "step": 344
    },
    {
      "epoch": 0.9527096996893338,
      "grad_norm": 0.4622589647769928,
      "learning_rate": 8.3579178509528e-09,
      "log_odds_chosen": 0.5557539463043213,
      "log_odds_ratio": -0.4589051902294159,
      "logits/chosen": -0.11451496928930283,
      "logits/rejected": -0.1634458750486374,
      "logps/chosen": -1.9416611194610596,
      "logps/rejected": -2.4332499504089355,
      "loss": 2.1447,
      "nll_loss": 2.098790168762207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19416610896587372,
      "rewards/margins": 0.04915890470147133,
      "rewards/rejected": -0.24332500994205475,
      "step": 345
    },
    {
      "epoch": 0.9554711770797376,
      "grad_norm": 0.42654889822006226,
      "learning_rate": 7.503438532937168e-09,
      "log_odds_chosen": 0.5813964605331421,
      "log_odds_ratio": -0.4537157118320465,
      "logits/chosen": -0.3199605941772461,
      "logits/rejected": -0.22434376180171967,
      "logps/chosen": -1.7952463626861572,
      "logps/rejected": -2.298389434814453,
      "loss": 1.9615,
      "nll_loss": 1.9161052703857422,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17952464520931244,
      "rewards/margins": 0.050314296036958694,
      "rewards/rejected": -0.22983895242214203,
      "step": 346
    },
    {
      "epoch": 0.9582326544701415,
      "grad_norm": 0.4929260313510895,
      "learning_rate": 6.694695868667554e-09,
      "log_odds_chosen": 0.24821370840072632,
      "log_odds_ratio": -0.5799821615219116,
      "logits/chosen": -0.24652338027954102,
      "logits/rejected": -0.20021158456802368,
      "logps/chosen": -2.029514789581299,
      "logps/rejected": -2.2481038570404053,
      "loss": 2.1994,
      "nll_loss": 2.141369342803955,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20295149087905884,
      "rewards/margins": 0.021858885884284973,
      "rewards/rejected": -0.224810391664505,
      "step": 347
    },
    {
      "epoch": 0.9609941318605454,
      "grad_norm": 0.4227042496204376,
      "learning_rate": 5.931764963608865e-09,
      "log_odds_chosen": 0.38164404034614563,
      "log_odds_ratio": -0.5276705026626587,
      "logits/chosen": -0.20029394328594208,
      "logits/rejected": 0.02388381026685238,
      "logps/chosen": -1.8693372011184692,
      "logps/rejected": -2.2021918296813965,
      "loss": 2.0628,
      "nll_loss": 2.010028600692749,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1869337260723114,
      "rewards/margins": 0.03328544646501541,
      "rewards/rejected": -0.22021916508674622,
      "step": 348
    },
    {
      "epoch": 0.9637556092509493,
      "grad_norm": 0.40184086561203003,
      "learning_rate": 5.214716668827557e-09,
      "log_odds_chosen": 0.5314087271690369,
      "log_odds_ratio": -0.4719686806201935,
      "logits/chosen": -0.1988634169101715,
      "logits/rejected": -0.25241750478744507,
      "logps/chosen": -1.8679484128952026,
      "logps/rejected": -2.338563919067383,
      "loss": 2.0468,
      "nll_loss": 1.9996461868286133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18679484724998474,
      "rewards/margins": 0.04706154763698578,
      "rewards/rejected": -0.23385639488697052,
      "step": 349
    },
    {
      "epoch": 0.9665170866413532,
      "grad_norm": 0.4056241512298584,
      "learning_rate": 4.543617574412184e-09,
      "log_odds_chosen": 0.45314475893974304,
      "log_odds_ratio": -0.4986838698387146,
      "logits/chosen": -0.18360307812690735,
      "logits/rejected": -0.09356098622083664,
      "logps/chosen": -1.8170093297958374,
      "logps/rejected": -2.2090914249420166,
      "loss": 1.9971,
      "nll_loss": 1.947261095046997,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1817009598016739,
      "rewards/margins": 0.03920818120241165,
      "rewards/rejected": -0.22090913355350494,
      "step": 350
    },
    {
      "epoch": 0.9692785640317569,
      "grad_norm": 0.4642251133918762,
      "learning_rate": 3.9185300032889e-09,
      "log_odds_chosen": 0.3950684368610382,
      "log_odds_ratio": -0.5235860347747803,
      "logits/chosen": -0.2754601836204529,
      "logits/rejected": -0.2326173335313797,
      "logps/chosen": -1.9764922857284546,
      "logps/rejected": -2.325756788253784,
      "loss": 2.1481,
      "nll_loss": 2.0957164764404297,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19764922559261322,
      "rewards/margins": 0.034926433116197586,
      "rewards/rejected": -0.2325756698846817,
      "step": 351
    },
    {
      "epoch": 0.9720400414221608,
      "grad_norm": 0.4343326687812805,
      "learning_rate": 3.3395120054343086e-09,
      "log_odds_chosen": 0.3908619284629822,
      "log_odds_ratio": -0.525107741355896,
      "logits/chosen": -0.24127677083015442,
      "logits/rejected": -0.18407291173934937,
      "logps/chosen": -1.886029601097107,
      "logps/rejected": -2.232459545135498,
      "loss": 2.0549,
      "nll_loss": 2.002397060394287,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18860295414924622,
      "rewards/margins": 0.034643009305000305,
      "rewards/rejected": -0.22324594855308533,
      "step": 352
    },
    {
      "epoch": 0.9748015188125647,
      "grad_norm": 0.44272759556770325,
      "learning_rate": 2.8066173524839977e-09,
      "log_odds_chosen": 0.4167912006378174,
      "log_odds_ratio": -0.5094950199127197,
      "logits/chosen": -0.24375535547733307,
      "logits/rejected": -0.15310092270374298,
      "logps/chosen": -1.9302788972854614,
      "logps/rejected": -2.2968368530273438,
      "loss": 2.0918,
      "nll_loss": 2.040801525115967,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19302789866924286,
      "rewards/margins": 0.03665578365325928,
      "rewards/rejected": -0.22968368232250214,
      "step": 353
    },
    {
      "epoch": 0.9775629962029686,
      "grad_norm": 0.45615145564079285,
      "learning_rate": 2.3198955327393686e-09,
      "log_odds_chosen": 0.4656871557235718,
      "log_odds_ratio": -0.4895220994949341,
      "logits/chosen": -0.23265661299228668,
      "logits/rejected": -0.11756740510463715,
      "logps/chosen": -1.922983169555664,
      "logps/rejected": -2.3313231468200684,
      "loss": 2.1045,
      "nll_loss": 2.0555341243743896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1922983080148697,
      "rewards/margins": 0.04083399474620819,
      "rewards/rejected": -0.23313231766223907,
      "step": 354
    },
    {
      "epoch": 0.9803244735933725,
      "grad_norm": 0.4014591872692108,
      "learning_rate": 1.8793917465713684e-09,
      "log_odds_chosen": 0.6040939688682556,
      "log_odds_ratio": -0.44817936420440674,
      "logits/chosen": -0.1957464963197708,
      "logits/rejected": -0.1527189016342163,
      "logps/chosen": -1.7173447608947754,
      "logps/rejected": -2.236257314682007,
      "loss": 1.9018,
      "nll_loss": 1.8569546937942505,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17173448204994202,
      "rewards/margins": 0.0518912598490715,
      "rewards/rejected": -0.22362574934959412,
      "step": 355
    },
    {
      "epoch": 0.9830859509837763,
      "grad_norm": 0.40029025077819824,
      "learning_rate": 1.4851469022233997e-09,
      "log_odds_chosen": 0.3815869390964508,
      "log_odds_ratio": -0.529025673866272,
      "logits/chosen": -0.27820494771003723,
      "logits/rejected": -0.47735050320625305,
      "logps/chosen": -1.889628291130066,
      "logps/rejected": -2.225501775741577,
      "loss": 2.0743,
      "nll_loss": 2.021350860595703,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18896281719207764,
      "rewards/margins": 0.03358734771609306,
      "rewards/rejected": -0.222550168633461,
      "step": 356
    },
    {
      "epoch": 0.9858474283741802,
      "grad_norm": 0.40846526622772217,
      "learning_rate": 1.1371976120118088e-09,
      "log_odds_chosen": 0.4965031147003174,
      "log_odds_ratio": -0.4815439283847809,
      "logits/chosen": -0.09253039211034775,
      "logits/rejected": -0.14713740348815918,
      "logps/chosen": -1.9486680030822754,
      "logps/rejected": -2.388983726501465,
      "loss": 2.1139,
      "nll_loss": 2.0657119750976562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19486680626869202,
      "rewards/margins": 0.04403157904744148,
      "rewards/rejected": -0.2388983815908432,
      "step": 357
    },
    {
      "epoch": 0.988608905764584,
      "grad_norm": 0.4720138609409332,
      "learning_rate": 8.35576188926046e-10,
      "log_odds_chosen": 0.43012499809265137,
      "log_odds_ratio": -0.5054396390914917,
      "logits/chosen": -0.4898189902305603,
      "logits/rejected": 0.027382859960198402,
      "logps/chosen": -1.7912402153015137,
      "logps/rejected": -2.158855676651001,
      "loss": 1.9848,
      "nll_loss": 1.9342451095581055,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17912402749061584,
      "rewards/margins": 0.036761559545993805,
      "rewards/rejected": -0.21588559448719025,
      "step": 358
    },
    {
      "epoch": 0.9913703831549879,
      "grad_norm": 0.42207953333854675,
      "learning_rate": 5.80310643627957e-10,
      "log_odds_chosen": 0.4414942264556885,
      "log_odds_ratio": -0.5013801455497742,
      "logits/chosen": -0.212925985455513,
      "logits/rejected": -0.22522786259651184,
      "logps/chosen": -1.8451660871505737,
      "logps/rejected": -2.227259874343872,
      "loss": 2.0238,
      "nll_loss": 1.9736899137496948,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18451660871505737,
      "rewards/margins": 0.03820939362049103,
      "rewards/rejected": -0.2227260023355484,
      "step": 359
    },
    {
      "epoch": 0.9941318605453918,
      "grad_norm": 0.43415895104408264,
      "learning_rate": 3.71424681850141e-10,
      "log_odds_chosen": 0.6658082008361816,
      "log_odds_ratio": -0.42513564229011536,
      "logits/chosen": -0.20278087258338928,
      "logits/rejected": -0.3631182312965393,
      "logps/chosen": -1.8669443130493164,
      "logps/rejected": -2.45871639251709,
      "loss": 2.05,
      "nll_loss": 2.0075032711029053,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1866944432258606,
      "rewards/margins": 0.05917719751596451,
      "rewards/rejected": -0.2458716332912445,
      "step": 360
    },
    {
      "epoch": 0.9968933379357956,
      "grad_norm": 0.37619003653526306,
      "learning_rate": 2.0893770219493344e-10,
      "log_odds_chosen": 0.4615989327430725,
      "log_odds_ratio": -0.49466732144355774,
      "logits/chosen": -0.29233649373054504,
      "logits/rejected": -0.09165595471858978,
      "logps/chosen": -1.759258508682251,
      "logps/rejected": -2.154985189437866,
      "loss": 1.9504,
      "nll_loss": 1.900978446006775,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1759258508682251,
      "rewards/margins": 0.039572667330503464,
      "rewards/rejected": -0.21549852192401886,
      "step": 361
    },
    {
      "epoch": 0.9996548153261995,
      "grad_norm": 0.41035768389701843,
      "learning_rate": 9.286479433257e-11,
      "log_odds_chosen": 0.46994221210479736,
      "log_odds_ratio": -0.4945923686027527,
      "logits/chosen": -0.2844846844673157,
      "logits/rejected": -0.2605111002922058,
      "logps/chosen": -1.8897802829742432,
      "logps/rejected": -2.304417133331299,
      "loss": 2.0612,
      "nll_loss": 2.011777877807617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18897803127765656,
      "rewards/margins": 0.041463691741228104,
      "rewards/rejected": -0.23044171929359436,
      "step": 362
    },
    {
      "epoch": 1.0,
      "grad_norm": 0.5401268601417542,
      "learning_rate": 2.321673760002918e-11,
      "log_odds_chosen": 0.2301175892353058,
      "log_odds_ratio": -0.5846931338310242,
      "logits/chosen": -0.319807767868042,
      "logits/rejected": -0.25849246978759766,
      "logps/chosen": -1.9792284965515137,
      "logps/rejected": -2.1805315017700195,
      "loss": 2.1647,
      "nll_loss": 2.1062676906585693,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19792285561561584,
      "rewards/margins": 0.02013029158115387,
      "rewards/rejected": -0.21805314719676971,
      "step": 363
    }
  ],
  "logging_steps": 1,
  "max_steps": 363,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}